1 /*
   2  * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.util.Arrays;
  29 import java.util.Objects;
  30 import java.util.function.IntUnaryOperator;
  31 
  32 import jdk.internal.vm.annotation.ForceInline;
  33 import jdk.internal.vm.vector.VectorSupport;
  34 
  35 import static jdk.internal.vm.vector.VectorSupport.*;
  36 
  37 import static jdk.incubator.vector.VectorOperators.*;
  38 
  39 #warn This file is preprocessed before being compiled
  40 
  41 @SuppressWarnings("cast")  // warning: redundant cast
  42 final class $vectortype$ extends $abstractvectortype$ {
  43     static final $Type$Species VSPECIES =
  44         ($Type$Species) $Type$Vector.SPECIES_$BITS$;
  45 
  46     static final VectorShape VSHAPE =
  47         VSPECIES.vectorShape();
  48 
  49     static final Class<$vectortype$> VCLASS = $vectortype$.class;
  50 
  51     static final int VSIZE = VSPECIES.vectorBitSize();
  52 
  53     static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
  54 
  55     static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
  56 
  57     $vectortype$($type$[] v) {
  58         super(v);
  59     }
  60 
  61     // For compatibility as $vectortype$::new,
  62     // stored into species.vectorFactory.
  63     $vectortype$(Object v) {
  64         this(($type$[]) v);
  65     }
  66 
  67     static final $vectortype$ ZERO = new $vectortype$(new $type$[VLENGTH]);
  68     static final $vectortype$ IOTA = new $vectortype$(VSPECIES.iotaArray());
  69 
  70     static {
  71         // Warm up a few species caches.
  72         // If we do this too much we will
  73         // get NPEs from bootstrap circularity.
  74         VSPECIES.dummyVector();
  75         VSPECIES.withLanes(LaneType.BYTE);
  76     }
  77 
  78     // Specialized extractors
  79 
  80     @ForceInline
  81     final @Override
  82     public $Type$Species vspecies() {
  83         // ISSUE:  This should probably be a @Stable
  84         // field inside AbstractVector, rather than
  85         // a megamorphic method.
  86         return VSPECIES;
  87     }
  88 
  89     @ForceInline
  90     @Override
  91     public final Class<$Boxtype$> elementType() { return $type$.class; }
  92 
  93     @ForceInline
  94     @Override
  95     public final int elementSize() { return $Boxtype$.SIZE; }
  96 
  97     @ForceInline
  98     @Override
  99     public final VectorShape shape() { return VSHAPE; }
 100 
 101     @ForceInline
 102     @Override
 103     public final int length() { return VLENGTH; }
 104 
 105     @ForceInline
 106     @Override
 107     public final int bitSize() { return VSIZE; }
 108 
 109     @ForceInline
 110     @Override
 111     public final int byteSize() { return VSIZE / Byte.SIZE; }
 112 
 113     /*package-private*/
 114     @ForceInline
 115     final @Override
 116     $type$[] vec() {
 117         return ($type$[])getPayload();
 118     }
 119 
 120     // Virtualized constructors
 121 
 122     @Override
 123     @ForceInline
 124     public final $vectortype$ broadcast($type$ e) {
 125         return ($vectortype$) super.broadcastTemplate(e);  // specialize
 126     }
 127 
 128 #if[!long]
 129     @Override
 130     @ForceInline
 131     public final $vectortype$ broadcast(long e) {
 132         return ($vectortype$) super.broadcastTemplate(e);  // specialize
 133     }
 134 #end[!long]
 135 
 136     @Override
 137     @ForceInline
 138     $masktype$ maskFromArray(boolean[] bits) {
 139         return new $masktype$(bits);
 140     }
 141 
 142     @Override
 143     @ForceInline
 144     $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; }
 145 
 146     @ForceInline
 147     $shuffletype$ iotaShuffle(int start, int step, boolean wrap) {
 148       if (wrap) {
 149         return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1,
 150                 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
 151       } else {
 152         return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0,
 153                 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
 154       }
 155     }
 156 
 157     @Override
 158     @ForceInline
 159     $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); }
 160 
 161     @Override
 162     @ForceInline
 163     $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); }
 164 
 165     @Override
 166     @ForceInline
 167     $shuffletype$ shuffleFromOp(IntUnaryOperator fn) { return new $shuffletype$(fn); }
 168 
 169     // Make a vector of the same species but the given elements:
 170     @ForceInline
 171     final @Override
 172     $vectortype$ vectorFactory($type$[] vec) {
 173         return new $vectortype$(vec);
 174     }
 175 
 176     @ForceInline
 177     final @Override
 178     Byte$bits$Vector asByteVectorRaw() {
 179         return (Byte$bits$Vector) super.asByteVectorRawTemplate();  // specialize
 180     }
 181 
 182     @ForceInline
 183     final @Override
 184     AbstractVector<?> asVectorRaw(LaneType laneType) {
 185         return super.asVectorRawTemplate(laneType);  // specialize
 186     }
 187 
 188     // Unary operator
 189 
 190     @ForceInline
 191     final @Override
 192     $vectortype$ uOp(FUnOp f) {
 193         return ($vectortype$) super.uOpTemplate(f);  // specialize
 194     }
 195 
 196     @ForceInline
 197     final @Override
 198     $vectortype$ uOp(VectorMask<$Boxtype$> m, FUnOp f) {
 199         return ($vectortype$)
 200             super.uOpTemplate(($masktype$)m, f);  // specialize
 201     }
 202 
 203     // Binary operator
 204 
 205     @ForceInline
 206     final @Override
 207     $vectortype$ bOp(Vector<$Boxtype$> v, FBinOp f) {
 208         return ($vectortype$) super.bOpTemplate(($vectortype$)v, f);  // specialize
 209     }
 210 
 211     @ForceInline
 212     final @Override
 213     $vectortype$ bOp(Vector<$Boxtype$> v,
 214                      VectorMask<$Boxtype$> m, FBinOp f) {
 215         return ($vectortype$)
 216             super.bOpTemplate(($vectortype$)v, ($masktype$)m,
 217                               f);  // specialize
 218     }
 219 
 220     // Ternary operator
 221 
 222     @ForceInline
 223     final @Override
 224     $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, FTriOp f) {
 225         return ($vectortype$)
 226             super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
 227                               f);  // specialize
 228     }
 229 
 230     @ForceInline
 231     final @Override
 232     $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2,
 233                      VectorMask<$Boxtype$> m, FTriOp f) {
 234         return ($vectortype$)
 235             super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
 236                               ($masktype$)m, f);  // specialize
 237     }
 238 
 239     @ForceInline
 240     final @Override
 241     $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f) {
 242         return super.rOpTemplate(v, m, f);  // specialize
 243     }
 244 
 245     @Override
 246     @ForceInline
 247     public final <F>
 248     Vector<F> convertShape(VectorOperators.Conversion<$Boxtype$,F> conv,
 249                            VectorSpecies<F> rsp, int part) {
 250         return super.convertShapeTemplate(conv, rsp, part);  // specialize
 251     }
 252 
 253     @Override
 254     @ForceInline
 255     public final <F>
 256     Vector<F> reinterpretShape(VectorSpecies<F> toSpecies, int part) {
 257         return super.reinterpretShapeTemplate(toSpecies, part);  // specialize
 258     }
 259 
 260     // Specialized algebraic operations:
 261 
 262     // The following definition forces a specialized version of this
 263     // crucial method into the v-table of this class.  A call to add()
 264     // will inline to a call to lanewise(ADD,), at which point the JIT
 265     // intrinsic will have the opcode of ADD, plus all the metadata
 266     // for this particular class, enabling it to generate precise
 267     // code.
 268     //
 269     // There is probably no benefit to the JIT to specialize the
 270     // masked or broadcast versions of the lanewise method.
 271 
 272     @Override
 273     @ForceInline
 274     public $vectortype$ lanewise(Unary op) {
 275         return ($vectortype$) super.lanewiseTemplate(op);  // specialize
 276     }
 277 
 278     @Override
 279     @ForceInline
 280     public $vectortype$ lanewise(Unary op, VectorMask<$Boxtype$> m) {
 281         return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, ($masktype$) m);  // specialize
 282     }
 283 
 284     @Override
 285     @ForceInline
 286     public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v) {
 287         return ($vectortype$) super.lanewiseTemplate(op, v);  // specialize
 288     }
 289 
 290     @Override
 291     @ForceInline
 292     public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 293         return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v, ($masktype$) m);  // specialize
 294     }
 295 
 296 #if[!FP]
 297     /*package-private*/
 298     @Override
 299     @ForceInline $vectortype$
 300     lanewiseShift(VectorOperators.Binary op, int e) {
 301         return ($vectortype$) super.lanewiseShiftTemplate(op, e);  // specialize
 302     }
 303 
 304     /*package-private*/
 305     @Override
 306     @ForceInline $vectortype$
 307     lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m) {
 308         return ($vectortype$) super.lanewiseShiftTemplate(op, $masktype$.class, e, ($masktype$) m);  // specialize
 309     }
 310 #end[!FP]
 311 
 312     /*package-private*/
 313     @Override
 314     @ForceInline
 315     public final
 316     $vectortype$
 317     lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2) {
 318         return ($vectortype$) super.lanewiseTemplate(op, v1, v2);  // specialize
 319     }
 320 
 321     @Override
 322     @ForceInline
 323     public final
 324     $vectortype$
 325     lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, VectorMask<$Boxtype$> m) {
 326         return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v1, v2, ($masktype$) m);  // specialize
 327     }
 328 
 329     @Override
 330     @ForceInline
 331     public final
 332     $vectortype$ addIndex(int scale) {
 333         return ($vectortype$) super.addIndexTemplate(scale);  // specialize
 334     }
 335 
 336     // Type specific horizontal reductions
 337 
 338     @Override
 339     @ForceInline
 340     public final $type$ reduceLanes(VectorOperators.Associative op) {
 341         return super.reduceLanesTemplate(op);  // specialized
 342     }
 343 
 344     @Override
 345     @ForceInline
 346     public final $type$ reduceLanes(VectorOperators.Associative op,
 347                                     VectorMask<$Boxtype$> m) {
 348         return super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m);  // specialized
 349     }
 350 
 351     @Override
 352     @ForceInline
 353     public final long reduceLanesToLong(VectorOperators.Associative op) {
 354         return (long) super.reduceLanesTemplate(op);  // specialized
 355     }
 356 
 357     @Override
 358     @ForceInline
 359     public final long reduceLanesToLong(VectorOperators.Associative op,
 360                                         VectorMask<$Boxtype$> m) {
 361         return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m);  // specialized
 362     }
 363 
 364     @ForceInline
 365     public VectorShuffle<$Boxtype$> toShuffle() {
 366         return super.toShuffleTemplate($shuffletype$.class); // specialize
 367     }
 368 
 369     // Specialized unary testing
 370 
 371     @Override
 372     @ForceInline
 373     public final $masktype$ test(Test op) {
 374         return super.testTemplate($masktype$.class, op);  // specialize
 375     }
 376 
 377     // Specialized comparisons
 378 
 379     @Override
 380     @ForceInline
 381     public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v) {
 382         return super.compareTemplate($masktype$.class, op, v);  // specialize
 383     }
 384 
 385     @Override
 386     @ForceInline
 387     public final $masktype$ compare(Comparison op, $type$ s) {
 388         return super.compareTemplate($masktype$.class, op, s);  // specialize
 389     }
 390 
 391 #if[!long]
 392     @Override
 393     @ForceInline
 394     public final $masktype$ compare(Comparison op, long s) {
 395         return super.compareTemplate($masktype$.class, op, s);  // specialize
 396     }
 397 #end[!long]
 398 
 399     @Override
 400     @ForceInline
 401     public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 402         return super.compareTemplate($masktype$.class, op, v, ($masktype$) m);
 403     }
 404 
 405 
 406     @Override
 407     @ForceInline
 408     public $vectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 409         return ($vectortype$)
 410             super.blendTemplate($masktype$.class,
 411                                 ($vectortype$) v,
 412                                 ($masktype$) m);  // specialize
 413     }
 414 
 415     @Override
 416     @ForceInline
 417     public $vectortype$ slice(int origin, Vector<$Boxtype$> v) {
 418         return ($vectortype$) super.sliceTemplate(origin, v);  // specialize
 419     }
 420 
 421     @Override
 422     @ForceInline
 423     public $vectortype$ slice(int origin) {
 424         return ($vectortype$) super.sliceTemplate(origin);  // specialize
 425     }
 426 
 427     @Override
 428     @ForceInline
 429     public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part) {
 430         return ($vectortype$) super.unsliceTemplate(origin, w, part);  // specialize
 431     }
 432 
 433     @Override
 434     @ForceInline
 435     public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m) {
 436         return ($vectortype$)
 437             super.unsliceTemplate($masktype$.class,
 438                                   origin, w, part,
 439                                   ($masktype$) m);  // specialize
 440     }
 441 
 442     @Override
 443     @ForceInline
 444     public $vectortype$ unslice(int origin) {
 445         return ($vectortype$) super.unsliceTemplate(origin);  // specialize
 446     }
 447 
 448     @Override
 449     @ForceInline
 450     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s) {
 451         return ($vectortype$)
 452             super.rearrangeTemplate($shuffletype$.class,
 453                                     ($shuffletype$) s);  // specialize
 454     }
 455 
 456     @Override
 457     @ForceInline
 458     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle,
 459                                   VectorMask<$Boxtype$> m) {
 460         return ($vectortype$)
 461             super.rearrangeTemplate($shuffletype$.class,
 462                                     $masktype$.class,
 463                                     ($shuffletype$) shuffle,
 464                                     ($masktype$) m);  // specialize
 465     }
 466 
 467     @Override
 468     @ForceInline
 469     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s,
 470                                   Vector<$Boxtype$> v) {
 471         return ($vectortype$)
 472             super.rearrangeTemplate($shuffletype$.class,
 473                                     ($shuffletype$) s,
 474                                     ($vectortype$) v);  // specialize
 475     }
 476 
 477     @Override
 478     @ForceInline
 479     public $vectortype$ selectFrom(Vector<$Boxtype$> v) {
 480         return ($vectortype$)
 481             super.selectFromTemplate(($vectortype$) v);  // specialize
 482     }
 483 
 484     @Override
 485     @ForceInline
 486     public $vectortype$ selectFrom(Vector<$Boxtype$> v,
 487                                    VectorMask<$Boxtype$> m) {
 488         return ($vectortype$)
 489             super.selectFromTemplate(($vectortype$) v,
 490                                      ($masktype$) m);  // specialize
 491     }
 492 
 493 
 494 #if[FP]
 495     @ForceInline
 496     @Override
 497     public $type$ lane(int i) {
 498 #if[!Max]
 499         $bitstype$ bits;
 500         switch(i) {
 501             case 0: bits = laneHelper(0); break;
 502 #if[!1L]
 503             case 1: bits = laneHelper(1); break;
 504 #if[!2L]
 505             case 2: bits = laneHelper(2); break;
 506             case 3: bits = laneHelper(3); break;
 507 #if[!4L]
 508             case 4: bits = laneHelper(4); break;
 509             case 5: bits = laneHelper(5); break;
 510             case 6: bits = laneHelper(6); break;
 511             case 7: bits = laneHelper(7); break;
 512 #if[!8L]
 513             case 8: bits = laneHelper(8); break;
 514             case 9: bits = laneHelper(9); break;
 515             case 10: bits = laneHelper(10); break;
 516             case 11: bits = laneHelper(11); break;
 517             case 12: bits = laneHelper(12); break;
 518             case 13: bits = laneHelper(13); break;
 519             case 14: bits = laneHelper(14); break;
 520             case 15: bits = laneHelper(15); break;
 521 #end[!8L]
 522 #end[!4L]
 523 #end[!2L]
 524 #end[!1L]
 525             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 526         }
 527 #else[!Max]
 528         if (i < 0 || i >= VLENGTH) {
 529             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 530         }
 531         $bitstype$ bits = laneHelper(i);
 532 #end[!Max]
 533         return $Type$.$bitstype$BitsTo$Fptype$(bits);
 534     }
 535 
 536     public $bitstype$ laneHelper(int i) {
 537         return ($bitstype$) VectorSupport.extract(
 538                      VCLASS, ETYPE, VLENGTH,
 539                      this, i,
 540                      (vec, ix) -> {
 541                      $type$[] vecarr = vec.vec();
 542                      return (long)$Type$.$type$To$Bitstype$Bits(vecarr[ix]);
 543                      });
 544     }
 545 
 546     @ForceInline
 547     @Override
 548     public $vectortype$ withLane(int i, $type$ e) {
 549 #if[!Max]
 550         switch(i) {
 551             case 0: return withLaneHelper(0, e);
 552 #if[!1L]
 553             case 1: return withLaneHelper(1, e);
 554 #if[!2L]
 555             case 2: return withLaneHelper(2, e);
 556             case 3: return withLaneHelper(3, e);
 557 #if[!4L]
 558             case 4: return withLaneHelper(4, e);
 559             case 5: return withLaneHelper(5, e);
 560             case 6: return withLaneHelper(6, e);
 561             case 7: return withLaneHelper(7, e);
 562 #if[!8L]
 563             case 8: return withLaneHelper(8, e);
 564             case 9: return withLaneHelper(9, e);
 565             case 10: return withLaneHelper(10, e);
 566             case 11: return withLaneHelper(11, e);
 567             case 12: return withLaneHelper(12, e);
 568             case 13: return withLaneHelper(13, e);
 569             case 14: return withLaneHelper(14, e);
 570             case 15: return withLaneHelper(15, e);
 571 #end[!8L]
 572 #end[!4L]
 573 #end[!2L]
 574 #end[!1L]
 575             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 576         }
 577 #else[!Max]
 578         if (i < 0 || i >= VLENGTH) {
 579             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 580         }
 581         return withLaneHelper(i, e);
 582 #end[!Max]
 583     }
 584 
 585     public $vectortype$ withLaneHelper(int i, $type$ e) {
 586         return VectorSupport.insert(
 587                                 VCLASS, ETYPE, VLENGTH,
 588                                 this, i, (long)$Type$.$type$To$Bitstype$Bits(e),
 589                                 (v, ix, bits) -> {
 590                                     $type$[] res = v.vec().clone();
 591                                     res[ix] = $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits);
 592                                     return v.vectorFactory(res);
 593                                 });
 594     }
 595 #else[FP]
 596     @ForceInline
 597     @Override
 598     public $type$ lane(int i) {
 599 #if[!Max]
 600         switch(i) {
 601             case 0: return laneHelper(0);
 602 #if[!1L]
 603             case 1: return laneHelper(1);
 604 #if[!2L]
 605             case 2: return laneHelper(2);
 606             case 3: return laneHelper(3);
 607 #if[!4L]
 608             case 4: return laneHelper(4);
 609             case 5: return laneHelper(5);
 610             case 6: return laneHelper(6);
 611             case 7: return laneHelper(7);
 612 #if[!8L]
 613             case 8: return laneHelper(8);
 614             case 9: return laneHelper(9);
 615             case 10: return laneHelper(10);
 616             case 11: return laneHelper(11);
 617             case 12: return laneHelper(12);
 618             case 13: return laneHelper(13);
 619             case 14: return laneHelper(14);
 620             case 15: return laneHelper(15);
 621 #if[!16L]
 622             case 16: return laneHelper(16);
 623             case 17: return laneHelper(17);
 624             case 18: return laneHelper(18);
 625             case 19: return laneHelper(19);
 626             case 20: return laneHelper(20);
 627             case 21: return laneHelper(21);
 628             case 22: return laneHelper(22);
 629             case 23: return laneHelper(23);
 630             case 24: return laneHelper(24);
 631             case 25: return laneHelper(25);
 632             case 26: return laneHelper(26);
 633             case 27: return laneHelper(27);
 634             case 28: return laneHelper(28);
 635             case 29: return laneHelper(29);
 636             case 30: return laneHelper(30);
 637             case 31: return laneHelper(31);
 638 #if[!32L]
 639             case 32: return laneHelper(32);
 640             case 33: return laneHelper(33);
 641             case 34: return laneHelper(34);
 642             case 35: return laneHelper(35);
 643             case 36: return laneHelper(36);
 644             case 37: return laneHelper(37);
 645             case 38: return laneHelper(38);
 646             case 39: return laneHelper(39);
 647             case 40: return laneHelper(40);
 648             case 41: return laneHelper(41);
 649             case 42: return laneHelper(42);
 650             case 43: return laneHelper(43);
 651             case 44: return laneHelper(44);
 652             case 45: return laneHelper(45);
 653             case 46: return laneHelper(46);
 654             case 47: return laneHelper(47);
 655             case 48: return laneHelper(48);
 656             case 49: return laneHelper(49);
 657             case 50: return laneHelper(50);
 658             case 51: return laneHelper(51);
 659             case 52: return laneHelper(52);
 660             case 53: return laneHelper(53);
 661             case 54: return laneHelper(54);
 662             case 55: return laneHelper(55);
 663             case 56: return laneHelper(56);
 664             case 57: return laneHelper(57);
 665             case 58: return laneHelper(58);
 666             case 59: return laneHelper(59);
 667             case 60: return laneHelper(60);
 668             case 61: return laneHelper(61);
 669             case 62: return laneHelper(62);
 670             case 63: return laneHelper(63);
 671 #end[!32L]
 672 #end[!16L]
 673 #end[!8L]
 674 #end[!4L]
 675 #end[!2L]
 676 #end[!1L]
 677             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 678         }
 679 #else[!Max]
 680         if (i < 0 || i >= VLENGTH) {
 681             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 682         }
 683         return laneHelper(i);
 684 #end[!Max]
 685     }
 686 
 687     public $type$ laneHelper(int i) {
 688         return ($type$) VectorSupport.extract(
 689                                 VCLASS, ETYPE, VLENGTH,
 690                                 this, i,
 691                                 (vec, ix) -> {
 692                                     $type$[] vecarr = vec.vec();
 693                                     return (long)vecarr[ix];
 694                                 });
 695     }
 696 
 697     @ForceInline
 698     @Override
 699     public $vectortype$ withLane(int i, $type$ e) {
 700 #if[!Max]
 701         switch (i) {
 702             case 0: return withLaneHelper(0, e);
 703 #if[!1L]
 704             case 1: return withLaneHelper(1, e);
 705 #if[!2L]
 706             case 2: return withLaneHelper(2, e);
 707             case 3: return withLaneHelper(3, e);
 708 #if[!4L]
 709             case 4: return withLaneHelper(4, e);
 710             case 5: return withLaneHelper(5, e);
 711             case 6: return withLaneHelper(6, e);
 712             case 7: return withLaneHelper(7, e);
 713 #if[!8L]
 714             case 8: return withLaneHelper(8, e);
 715             case 9: return withLaneHelper(9, e);
 716             case 10: return withLaneHelper(10, e);
 717             case 11: return withLaneHelper(11, e);
 718             case 12: return withLaneHelper(12, e);
 719             case 13: return withLaneHelper(13, e);
 720             case 14: return withLaneHelper(14, e);
 721             case 15: return withLaneHelper(15, e);
 722 #if[!16L]
 723             case 16: return withLaneHelper(16, e);
 724             case 17: return withLaneHelper(17, e);
 725             case 18: return withLaneHelper(18, e);
 726             case 19: return withLaneHelper(19, e);
 727             case 20: return withLaneHelper(20, e);
 728             case 21: return withLaneHelper(21, e);
 729             case 22: return withLaneHelper(22, e);
 730             case 23: return withLaneHelper(23, e);
 731             case 24: return withLaneHelper(24, e);
 732             case 25: return withLaneHelper(25, e);
 733             case 26: return withLaneHelper(26, e);
 734             case 27: return withLaneHelper(27, e);
 735             case 28: return withLaneHelper(28, e);
 736             case 29: return withLaneHelper(29, e);
 737             case 30: return withLaneHelper(30, e);
 738             case 31: return withLaneHelper(31, e);
 739 #if[!32L]
 740             case 32: return withLaneHelper(32, e);
 741             case 33: return withLaneHelper(33, e);
 742             case 34: return withLaneHelper(34, e);
 743             case 35: return withLaneHelper(35, e);
 744             case 36: return withLaneHelper(36, e);
 745             case 37: return withLaneHelper(37, e);
 746             case 38: return withLaneHelper(38, e);
 747             case 39: return withLaneHelper(39, e);
 748             case 40: return withLaneHelper(40, e);
 749             case 41: return withLaneHelper(41, e);
 750             case 42: return withLaneHelper(42, e);
 751             case 43: return withLaneHelper(43, e);
 752             case 44: return withLaneHelper(44, e);
 753             case 45: return withLaneHelper(45, e);
 754             case 46: return withLaneHelper(46, e);
 755             case 47: return withLaneHelper(47, e);
 756             case 48: return withLaneHelper(48, e);
 757             case 49: return withLaneHelper(49, e);
 758             case 50: return withLaneHelper(50, e);
 759             case 51: return withLaneHelper(51, e);
 760             case 52: return withLaneHelper(52, e);
 761             case 53: return withLaneHelper(53, e);
 762             case 54: return withLaneHelper(54, e);
 763             case 55: return withLaneHelper(55, e);
 764             case 56: return withLaneHelper(56, e);
 765             case 57: return withLaneHelper(57, e);
 766             case 58: return withLaneHelper(58, e);
 767             case 59: return withLaneHelper(59, e);
 768             case 60: return withLaneHelper(60, e);
 769             case 61: return withLaneHelper(61, e);
 770             case 62: return withLaneHelper(62, e);
 771             case 63: return withLaneHelper(63, e);
 772 #end[!32L]
 773 #end[!16L]
 774 #end[!8L]
 775 #end[!4L]
 776 #end[!2L]
 777 #end[!1L]
 778             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 779         }
 780 #else[!Max]
 781         if (i < 0 || i >= VLENGTH) {
 782             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 783         }
 784         return withLaneHelper(i, e);
 785 #end[!Max]
 786     }
 787 
 788     public $vectortype$ withLaneHelper(int i, $type$ e) {
 789         return VectorSupport.insert(
 790                                 VCLASS, ETYPE, VLENGTH,
 791                                 this, i, (long)e,
 792                                 (v, ix, bits) -> {
 793                                     $type$[] res = v.vec().clone();
 794                                     res[ix] = ($type$)bits;
 795                                     return v.vectorFactory(res);
 796                                 });
 797     }
 798 #end[FP]
 799 
 800     // Mask
 801 
 802     static final class $masktype$ extends AbstractMask<$Boxtype$> {
 803         static final int VLENGTH = VSPECIES.laneCount();    // used by the JVM
 804         static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
 805 
 806         $masktype$(boolean[] bits) {
 807             this(bits, 0);
 808         }
 809 
 810         $masktype$(boolean[] bits, int offset) {
 811             super(prepare(bits, offset));
 812         }
 813 
 814         $masktype$(boolean val) {
 815             super(prepare(val));
 816         }
 817 
 818         private static boolean[] prepare(boolean[] bits, int offset) {
 819             boolean[] newBits = new boolean[VSPECIES.laneCount()];
 820             for (int i = 0; i < newBits.length; i++) {
 821                 newBits[i] = bits[offset + i];
 822             }
 823             return newBits;
 824         }
 825 
 826         private static boolean[] prepare(boolean val) {
 827             boolean[] bits = new boolean[VSPECIES.laneCount()];
 828             Arrays.fill(bits, val);
 829             return bits;
 830         }
 831 
 832         @ForceInline
 833         final @Override
 834         public $Type$Species vspecies() {
 835             // ISSUE:  This should probably be a @Stable
 836             // field inside AbstractMask, rather than
 837             // a megamorphic method.
 838             return VSPECIES;
 839         }
 840 
 841         @ForceInline
 842         boolean[] getBits() {
 843             return (boolean[])getPayload();
 844         }
 845 
 846         @Override
 847         $masktype$ uOp(MUnOp f) {
 848             boolean[] res = new boolean[vspecies().laneCount()];
 849             boolean[] bits = getBits();
 850             for (int i = 0; i < res.length; i++) {
 851                 res[i] = f.apply(i, bits[i]);
 852             }
 853             return new $masktype$(res);
 854         }
 855 
 856         @Override
 857         $masktype$ bOp(VectorMask<$Boxtype$> m, MBinOp f) {
 858             boolean[] res = new boolean[vspecies().laneCount()];
 859             boolean[] bits = getBits();
 860             boolean[] mbits = (($masktype$)m).getBits();
 861             for (int i = 0; i < res.length; i++) {
 862                 res[i] = f.apply(i, bits[i], mbits[i]);
 863             }
 864             return new $masktype$(res);
 865         }
 866 
 867         @ForceInline
 868         @Override
 869         public final
 870         $vectortype$ toVector() {
 871             return ($vectortype$) super.toVectorTemplate();  // specialize
 872         }
 873 
 874         /**
 875          * Helper function for lane-wise mask conversions.
 876          * This function kicks in after intrinsic failure.
 877          */
 878         @ForceInline
 879         private final <E>
 880         VectorMask<E> defaultMaskCast(AbstractSpecies<E> dsp) {
 881             if (length() != dsp.laneCount())
 882                 throw new IllegalArgumentException("VectorMask length and species length differ");
 883             boolean[] maskArray = toArray();
 884             return  dsp.maskFactory(maskArray).check(dsp);
 885         }
 886 
 887         @Override
 888         @ForceInline
 889         public <E> VectorMask<E> cast(VectorSpecies<E> dsp) {
 890             AbstractSpecies<E> species = (AbstractSpecies<E>) dsp;
 891             if (length() != species.laneCount())
 892                 throw new IllegalArgumentException("VectorMask length and species length differ");
 893 
 894             return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
 895                 this.getClass(), ETYPE, VLENGTH,
 896                 species.maskType(), species.elementType(), VLENGTH,
 897                 this, species,
 898                 (m, s) -> s.maskFactory(m.toArray()).check(s));
 899         }
 900 
 901         @Override
 902         @ForceInline
 903         public $masktype$ eq(VectorMask<$Boxtype$> mask) {
 904             Objects.requireNonNull(mask);
 905             $masktype$ m = ($masktype$)mask;
 906             return xor(m.not());
 907         }
 908 
 909         // Unary operations
 910 
 911         @Override
 912         @ForceInline
 913         public $masktype$ not() {
 914             return xor(maskAll(true));
 915         }
 916 
 917         // Binary operations
 918 
 919         @Override
 920         @ForceInline
 921         public $masktype$ and(VectorMask<$Boxtype$> mask) {
 922             Objects.requireNonNull(mask);
 923             $masktype$ m = ($masktype$)mask;
 924             return VectorSupport.binaryOp(VECTOR_OP_AND, $masktype$.class, null, $bitstype$.class, VLENGTH,
 925                                           this, m, null,
 926                                           (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a & b));
 927         }
 928 
 929         @Override
 930         @ForceInline
 931         public $masktype$ or(VectorMask<$Boxtype$> mask) {
 932             Objects.requireNonNull(mask);
 933             $masktype$ m = ($masktype$)mask;
 934             return VectorSupport.binaryOp(VECTOR_OP_OR, $masktype$.class, null, $bitstype$.class, VLENGTH,
 935                                           this, m, null,
 936                                           (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a | b));
 937         }
 938 
 939         @ForceInline
 940         /* package-private */
 941         $masktype$ xor(VectorMask<$Boxtype$> mask) {
 942             Objects.requireNonNull(mask);
 943             $masktype$ m = ($masktype$)mask;
 944             return VectorSupport.binaryOp(VECTOR_OP_XOR, $masktype$.class, null, $bitstype$.class, VLENGTH,
 945                                           this, m, null,
 946                                           (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a ^ b));
 947         }
 948 
 949         // Mask Query operations
 950 
 951         @Override
 952         @ForceInline
 953         public int trueCount() {
 954             return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, $masktype$.class, $bitstype$.class, VLENGTH, this,
 955                                                       (m) -> trueCountHelper(m.getBits()));
 956         }
 957 
 958         @Override
 959         @ForceInline
 960         public int firstTrue() {
 961             return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
 962                                                       (m) -> firstTrueHelper(m.getBits()));
 963         }
 964 
 965         @Override
 966         @ForceInline
 967         public int lastTrue() {
 968             return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
 969                                                       (m) -> lastTrueHelper(m.getBits()));
 970         }
 971 
 972         @Override
 973         @ForceInline
 974         public long toLong() {
 975             if (length() > Long.SIZE) {
 976                 throw new UnsupportedOperationException("too many lanes for one long");
 977             }
 978             return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TOLONG, $masktype$.class, $bitstype$.class, VLENGTH, this,
 979                                                       (m) -> toLongHelper(m.getBits()));
 980         }
 981 
 982         // Reductions
 983 
 984         @Override
 985         @ForceInline
 986         public boolean anyTrue() {
 987             return VectorSupport.test(BT_ne, $masktype$.class, $bitstype$.class, VLENGTH,
 988                                          this, vspecies().maskAll(true),
 989                                          (m, __) -> anyTrueHelper((($masktype$)m).getBits()));
 990         }
 991 
 992         @Override
 993         @ForceInline
 994         public boolean allTrue() {
 995             return VectorSupport.test(BT_overflow, $masktype$.class, $bitstype$.class, VLENGTH,
 996                                          this, vspecies().maskAll(true),
 997                                          (m, __) -> allTrueHelper((($masktype$)m).getBits()));
 998         }
 999 
1000         @ForceInline
1001         /*package-private*/
1002         static $masktype$ maskAll(boolean bit) {
1003             return VectorSupport.broadcastCoerced($masktype$.class, $bitstype$.class, VLENGTH,
1004                                                   (bit ? -1 : 0), null,
1005                                                   (v, __) -> (v != 0 ? TRUE_MASK : FALSE_MASK));
1006         }
1007         private static final $masktype$  TRUE_MASK = new $masktype$(true);
1008         private static final $masktype$ FALSE_MASK = new $masktype$(false);
1009 
1010 #if[intAndMax]
1011 
1012         static boolean[] maskLowerHalf() {
1013             boolean[] a = new boolean[VLENGTH];
1014             int len = a.length >> 1;
1015             for (int i = 0; i < len; i++) {
1016                 a[i] = true;
1017             }
1018             return a;
1019         }
1020 
1021 #end[intAndMax]
1022 #if[intAndMax]
1023         static final IntMaxMask LOWER_HALF_TRUE_MASK = new IntMaxMask(maskLowerHalf());
1024 #end[intAndMax]
1025     }
1026 
1027     // Shuffle
1028 
1029     static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> {
1030         static final int VLENGTH = VSPECIES.laneCount();    // used by the JVM
1031         static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
1032 
1033         $shuffletype$(byte[] reorder) {
1034             super(VLENGTH, reorder);
1035         }
1036 
1037         public $shuffletype$(int[] reorder) {
1038             super(VLENGTH, reorder);
1039         }
1040 
1041         public $shuffletype$(int[] reorder, int i) {
1042             super(VLENGTH, reorder, i);
1043         }
1044 
1045         public $shuffletype$(IntUnaryOperator fn) {
1046             super(VLENGTH, fn);
1047         }
1048 
1049         @Override
1050         public $Type$Species vspecies() {
1051             return VSPECIES;
1052         }
1053 
1054         static {
1055             // There must be enough bits in the shuffle lanes to encode
1056             // VLENGTH valid indexes and VLENGTH exceptional ones.
1057             assert(VLENGTH < Byte.MAX_VALUE);
1058             assert(Byte.MIN_VALUE <= -VLENGTH);
1059         }
1060         static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY);
1061 
1062         @Override
1063         @ForceInline
1064         public $vectortype$ toVector() {
1065             return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH,
1066                                                     (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate())));
1067         }
1068 
1069         @Override
1070         @ForceInline
1071         public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
1072             AbstractSpecies<F> species = (AbstractSpecies<F>) s;
1073             if (length() != species.laneCount())
1074                 throw new IllegalArgumentException("VectorShuffle length and species length differ");
1075             int[] shuffleArray = toArray();
1076             return s.shuffleFromArray(shuffleArray, 0).check(s);
1077         }
1078 
1079         @ForceInline
1080         @Override
1081         public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {
1082             $shuffletype$ s = ($shuffletype$) shuffle;
1083             byte[] reorder1 = reorder();
1084             byte[] reorder2 = s.reorder();
1085             byte[] r = new byte[reorder1.length];
1086             for (int i = 0; i < reorder1.length; i++) {
1087                 int ssi = reorder2[i];
1088                 r[i] = reorder1[ssi];  // throws on exceptional index
1089             }
1090             return new $shuffletype$(r);
1091         }
1092     }
1093 
1094     // ================================================
1095 
1096     // Specialized low-level memory operations.
1097 
1098     @ForceInline
1099     @Override
1100     final
1101     $abstractvectortype$ fromArray0($type$[] a, int offset) {
1102         return super.fromArray0Template(a, offset);  // specialize
1103     }
1104 
1105     @ForceInline
1106     @Override
1107     final
1108     $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1109         return super.fromArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1110     }
1111 
1112 #if[!byteOrShort]
1113     @ForceInline
1114     @Override
1115     final
1116     $abstractvectortype$ fromArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1117         return super.fromArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1118     }
1119 #end[!byteOrShort]
1120 
1121 #if[short]
1122     @ForceInline
1123     @Override
1124     final
1125     $abstractvectortype$ fromCharArray0(char[] a, int offset) {
1126         return super.fromCharArray0Template(a, offset);  // specialize
1127     }
1128 
1129     @ForceInline
1130     @Override
1131     final
1132     $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1133         return super.fromCharArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1134     }
1135 #end[short]
1136 
1137 #if[byte]
1138     @ForceInline
1139     @Override
1140     final
1141     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset) {
1142         return super.fromBooleanArray0Template(a, offset);  // specialize
1143     }
1144 
1145     @ForceInline
1146     @Override
1147     final
1148     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1149         return super.fromBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1150     }
1151 #end[byte]
1152 
1153     @ForceInline
1154     @Override
1155     final
1156     $abstractvectortype$ fromByteArray0(byte[] a, int offset) {
1157         return super.fromByteArray0Template(a, offset);  // specialize
1158     }
1159 
1160     @ForceInline
1161     @Override
1162     final
1163     $abstractvectortype$ fromByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m) {
1164         return super.fromByteArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1165     }
1166 
1167     @ForceInline
1168     @Override
1169     final
1170     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset) {
1171         return super.fromByteBuffer0Template(bb, offset);  // specialize
1172     }
1173 
1174     @ForceInline
1175     @Override
1176     final
1177     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m) {
1178         return super.fromByteBuffer0Template($masktype$.class, bb, offset, ($masktype$) m);  // specialize
1179     }
1180 
1181     @ForceInline
1182     @Override
1183     final
1184     void intoArray0($type$[] a, int offset) {
1185         super.intoArray0Template(a, offset);  // specialize
1186     }
1187 
1188     @ForceInline
1189     @Override
1190     final
1191     void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1192         super.intoArray0Template($masktype$.class, a, offset, ($masktype$) m);
1193     }
1194 
1195 #if[!byteOrShort]
1196     @ForceInline
1197     @Override
1198     final
1199     void intoArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1200         super.intoArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1201     }
1202 #end[!byteOrShort]
1203 
1204 #if[byte]
1205     @ForceInline
1206     @Override
1207     final
1208     void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1209         super.intoBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m);
1210     }
1211 #end[byte]
1212 
1213     @ForceInline
1214     @Override
1215     final
1216     void intoByteArray0(byte[] a, int offset) {
1217         super.intoByteArray0Template(a, offset);  // specialize
1218     }
1219 
1220     @ForceInline
1221     @Override
1222     final
1223     void intoByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m) {
1224         super.intoByteArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1225     }
1226 
1227     @ForceInline
1228     @Override
1229     final
1230     void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m) {
1231         super.intoByteBuffer0Template($masktype$.class, bb, offset, ($masktype$) m);
1232     }
1233 
1234 #if[short]
1235     @ForceInline
1236     @Override
1237     final
1238     void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1239         super.intoCharArray0Template($masktype$.class, a, offset, ($masktype$) m);
1240     }
1241 #end[short]
1242 
1243     // End of specialized low-level memory operations.
1244 
1245     // ================================================
1246 
1247 }