1 /*
   2  * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.util.Arrays;
  29 import java.util.Objects;
  30 import java.util.function.IntUnaryOperator;
  31 
  32 import jdk.internal.vm.annotation.ForceInline;
  33 import jdk.internal.vm.vector.VectorSupport;
  34 
  35 import static jdk.internal.vm.vector.VectorSupport.*;
  36 
  37 import static jdk.incubator.vector.VectorOperators.*;
  38 
  39 #warn This file is preprocessed before being compiled
  40 
  41 @SuppressWarnings("cast")  // warning: redundant cast
  42 final class $vectortype$ extends $abstractvectortype$ {
  43     static final $Type$Species VSPECIES =
  44         ($Type$Species) $Type$Vector.SPECIES_$BITS$;
  45 
  46     static final VectorShape VSHAPE =
  47         VSPECIES.vectorShape();
  48 
  49     static final Class<$vectortype$> VCLASS = $vectortype$.class;
  50 
  51     static final int VSIZE = VSPECIES.vectorBitSize();
  52 
  53     static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
  54 
  55     static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
  56 
  57     $vectortype$($type$[] v) {
  58         super(v);
  59     }
  60 
  61     // For compatibility as $vectortype$::new,
  62     // stored into species.vectorFactory.
  63     $vectortype$(Object v) {
  64         this(($type$[]) v);
  65     }
  66 
  67     static final $vectortype$ ZERO = new $vectortype$(new $type$[VLENGTH]);
  68     static final $vectortype$ IOTA = new $vectortype$(VSPECIES.iotaArray());
  69 
  70     static {
  71         // Warm up a few species caches.
  72         // If we do this too much we will
  73         // get NPEs from bootstrap circularity.
  74         VSPECIES.dummyVector();
  75         VSPECIES.withLanes(LaneType.BYTE);
  76     }
  77 
  78     // Specialized extractors
  79 
  80     @ForceInline
  81     final @Override
  82     public $Type$Species vspecies() {
  83         // ISSUE:  This should probably be a @Stable
  84         // field inside AbstractVector, rather than
  85         // a megamorphic method.
  86         return VSPECIES;
  87     }
  88 
  89     @ForceInline
  90     @Override
  91     public final Class<$Boxtype$> elementType() { return $type$.class; }
  92 
  93     @ForceInline
  94     @Override
  95     public final int elementSize() { return $Boxtype$.SIZE; }
  96 
  97     @ForceInline
  98     @Override
  99     public final VectorShape shape() { return VSHAPE; }
 100 
 101     @ForceInline
 102     @Override
 103     public final int length() { return VLENGTH; }
 104 
 105     @ForceInline
 106     @Override
 107     public final int bitSize() { return VSIZE; }
 108 
 109     @ForceInline
 110     @Override
 111     public final int byteSize() { return VSIZE / Byte.SIZE; }
 112 
 113     /*package-private*/
 114     @ForceInline
 115     final @Override
 116     $type$[] vec() {
 117         return ($type$[])getPayload();
 118     }
 119 
 120     // Virtualized constructors
 121 
 122     @Override
 123     @ForceInline
 124     public final $vectortype$ broadcast($type$ e) {
 125         return ($vectortype$) super.broadcastTemplate(e);  // specialize
 126     }
 127 
 128 #if[!long]
 129     @Override
 130     @ForceInline
 131     public final $vectortype$ broadcast(long e) {
 132         return ($vectortype$) super.broadcastTemplate(e);  // specialize
 133     }
 134 #end[!long]
 135 
 136     @Override
 137     @ForceInline
 138     $masktype$ maskFromArray(boolean[] bits) {
 139         return new $masktype$(bits);
 140     }
 141 
 142     @Override
 143     @ForceInline
 144     $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; }
 145 
 146     @ForceInline
 147     $shuffletype$ iotaShuffle(int start, int step, boolean wrap) {
 148       if (wrap) {
 149         return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1,
 150                 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
 151       } else {
 152         return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0,
 153                 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
 154       }
 155     }
 156 
 157     @Override
 158     @ForceInline
 159     $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); }
 160 
 161     @Override
 162     @ForceInline
 163     $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); }
 164 
 165     @Override
 166     @ForceInline
 167     $shuffletype$ shuffleFromOp(IntUnaryOperator fn) { return new $shuffletype$(fn); }
 168 
 169     // Make a vector of the same species but the given elements:
 170     @ForceInline
 171     final @Override
 172     $vectortype$ vectorFactory($type$[] vec) {
 173         return new $vectortype$(vec);
 174     }
 175 
 176     @ForceInline
 177     final @Override
 178     Byte$bits$Vector asByteVectorRaw() {
 179         return (Byte$bits$Vector) super.asByteVectorRawTemplate();  // specialize
 180     }
 181 
 182     @ForceInline
 183     final @Override
 184     AbstractVector<?> asVectorRaw(LaneType laneType) {
 185         return super.asVectorRawTemplate(laneType);  // specialize
 186     }
 187 
 188     // Unary operator
 189 
 190     @ForceInline
 191     final @Override
 192     $vectortype$ uOp(FUnOp f) {
 193         return ($vectortype$) super.uOpTemplate(f);  // specialize
 194     }
 195 
 196     @ForceInline
 197     final @Override
 198     $vectortype$ uOp(VectorMask<$Boxtype$> m, FUnOp f) {
 199         return ($vectortype$)
 200             super.uOpTemplate(($masktype$)m, f);  // specialize
 201     }
 202 
 203     // Binary operator
 204 
 205     @ForceInline
 206     final @Override
 207     $vectortype$ bOp(Vector<$Boxtype$> v, FBinOp f) {
 208         return ($vectortype$) super.bOpTemplate(($vectortype$)v, f);  // specialize
 209     }
 210 
 211     @ForceInline
 212     final @Override
 213     $vectortype$ bOp(Vector<$Boxtype$> v,
 214                      VectorMask<$Boxtype$> m, FBinOp f) {
 215         return ($vectortype$)
 216             super.bOpTemplate(($vectortype$)v, ($masktype$)m,
 217                               f);  // specialize
 218     }
 219 
 220     // Ternary operator
 221 
 222     @ForceInline
 223     final @Override
 224     $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, FTriOp f) {
 225         return ($vectortype$)
 226             super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
 227                               f);  // specialize
 228     }
 229 
 230     @ForceInline
 231     final @Override
 232     $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2,
 233                      VectorMask<$Boxtype$> m, FTriOp f) {
 234         return ($vectortype$)
 235             super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
 236                               ($masktype$)m, f);  // specialize
 237     }
 238 
 239     @ForceInline
 240     final @Override
 241     $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f) {
 242         return super.rOpTemplate(v, m, f);  // specialize
 243     }
 244 
 245     @Override
 246     @ForceInline
 247     public final <F>
 248     Vector<F> convertShape(VectorOperators.Conversion<$Boxtype$,F> conv,
 249                            VectorSpecies<F> rsp, int part) {
 250         return super.convertShapeTemplate(conv, rsp, part);  // specialize
 251     }
 252 
 253     @Override
 254     @ForceInline
 255     public final <F>
 256     Vector<F> reinterpretShape(VectorSpecies<F> toSpecies, int part) {
 257         return super.reinterpretShapeTemplate(toSpecies, part);  // specialize
 258     }
 259 
 260     // Specialized algebraic operations:
 261 
 262     // The following definition forces a specialized version of this
 263     // crucial method into the v-table of this class.  A call to add()
 264     // will inline to a call to lanewise(ADD,), at which point the JIT
 265     // intrinsic will have the opcode of ADD, plus all the metadata
 266     // for this particular class, enabling it to generate precise
 267     // code.
 268     //
 269     // There is probably no benefit to the JIT to specialize the
 270     // masked or broadcast versions of the lanewise method.
 271 
 272     @Override
 273     @ForceInline
 274     public $vectortype$ lanewise(Unary op) {
 275         return ($vectortype$) super.lanewiseTemplate(op);  // specialize
 276     }
 277 
 278     @Override
 279     @ForceInline
 280     public $vectortype$ lanewise(Unary op, VectorMask<$Boxtype$> m) {
 281         return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, ($masktype$) m);  // specialize
 282     }
 283 
 284     @Override
 285     @ForceInline
 286     public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v) {
 287         return ($vectortype$) super.lanewiseTemplate(op, v);  // specialize
 288     }
 289 
 290     @Override
 291     @ForceInline
 292     public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 293         return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v, ($masktype$) m);  // specialize
 294     }
 295 
 296 #if[!FP]
 297     /*package-private*/
 298     @Override
 299     @ForceInline $vectortype$
 300     lanewiseShift(VectorOperators.Binary op, int e) {
 301         return ($vectortype$) super.lanewiseShiftTemplate(op, e);  // specialize
 302     }
 303 
 304     /*package-private*/
 305     @Override
 306     @ForceInline $vectortype$
 307     lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m) {
 308         return ($vectortype$) super.lanewiseShiftTemplate(op, $masktype$.class, e, ($masktype$) m);  // specialize
 309     }
 310 #end[!FP]
 311 
 312     /*package-private*/
 313     @Override
 314     @ForceInline
 315     public final
 316     $vectortype$
 317     lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2) {
 318         return ($vectortype$) super.lanewiseTemplate(op, v1, v2);  // specialize
 319     }
 320 
 321     @Override
 322     @ForceInline
 323     public final
 324     $vectortype$
 325     lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, VectorMask<$Boxtype$> m) {
 326         return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v1, v2, ($masktype$) m);  // specialize
 327     }
 328 
 329     @Override
 330     @ForceInline
 331     public final
 332     $vectortype$ addIndex(int scale) {
 333         return ($vectortype$) super.addIndexTemplate(scale);  // specialize
 334     }
 335 
 336     // Type specific horizontal reductions
 337 
 338     @Override
 339     @ForceInline
 340     public final $type$ reduceLanes(VectorOperators.Associative op) {
 341         return super.reduceLanesTemplate(op);  // specialized
 342     }
 343 
 344     @Override
 345     @ForceInline
 346     public final $type$ reduceLanes(VectorOperators.Associative op,
 347                                     VectorMask<$Boxtype$> m) {
 348         return super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m);  // specialized
 349     }
 350 
 351     @Override
 352     @ForceInline
 353     public final long reduceLanesToLong(VectorOperators.Associative op) {
 354         return (long) super.reduceLanesTemplate(op);  // specialized
 355     }
 356 
 357     @Override
 358     @ForceInline
 359     public final long reduceLanesToLong(VectorOperators.Associative op,
 360                                         VectorMask<$Boxtype$> m) {
 361         return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m);  // specialized
 362     }
 363 
 364     @ForceInline
 365     public VectorShuffle<$Boxtype$> toShuffle() {
 366         return super.toShuffleTemplate($shuffletype$.class); // specialize
 367     }
 368 
 369     // Specialized unary testing
 370 
 371     @Override
 372     @ForceInline
 373     public final $masktype$ test(Test op) {
 374         return super.testTemplate($masktype$.class, op);  // specialize
 375     }
 376 
 377     // Specialized comparisons
 378 
 379     @Override
 380     @ForceInline
 381     public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v) {
 382         return super.compareTemplate($masktype$.class, op, v);  // specialize
 383     }
 384 
 385     @Override
 386     @ForceInline
 387     public final $masktype$ compare(Comparison op, $type$ s) {
 388         return super.compareTemplate($masktype$.class, op, s);  // specialize
 389     }
 390 
 391 #if[!long]
 392     @Override
 393     @ForceInline
 394     public final $masktype$ compare(Comparison op, long s) {
 395         return super.compareTemplate($masktype$.class, op, s);  // specialize
 396     }
 397 #end[!long]
 398 
 399     @Override
 400     @ForceInline
 401     public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 402         return super.compareTemplate($masktype$.class, op, v, ($masktype$) m);
 403     }
 404 
 405 
 406     @Override
 407     @ForceInline
 408     public $vectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 409         return ($vectortype$)
 410             super.blendTemplate($masktype$.class,
 411                                 ($vectortype$) v,
 412                                 ($masktype$) m);  // specialize
 413     }
 414 
 415     @Override
 416     @ForceInline
 417     public $vectortype$ slice(int origin, Vector<$Boxtype$> v) {
 418         return ($vectortype$) super.sliceTemplate(origin, v);  // specialize
 419     }
 420 
 421     @Override
 422     @ForceInline
 423     public $vectortype$ slice(int origin) {
 424         return ($vectortype$) super.sliceTemplate(origin);  // specialize
 425     }
 426 
 427     @Override
 428     @ForceInline
 429     public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part) {
 430         return ($vectortype$) super.unsliceTemplate(origin, w, part);  // specialize
 431     }
 432 
 433     @Override
 434     @ForceInline
 435     public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m) {
 436         return ($vectortype$)
 437             super.unsliceTemplate($masktype$.class,
 438                                   origin, w, part,
 439                                   ($masktype$) m);  // specialize
 440     }
 441 
 442     @Override
 443     @ForceInline
 444     public $vectortype$ unslice(int origin) {
 445         return ($vectortype$) super.unsliceTemplate(origin);  // specialize
 446     }
 447 
 448     @Override
 449     @ForceInline
 450     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s) {
 451         return ($vectortype$)
 452             super.rearrangeTemplate($shuffletype$.class,
 453                                     ($shuffletype$) s);  // specialize
 454     }
 455 
 456     @Override
 457     @ForceInline
 458     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle,
 459                                   VectorMask<$Boxtype$> m) {
 460         return ($vectortype$)
 461             super.rearrangeTemplate($shuffletype$.class,
 462                                     $masktype$.class,
 463                                     ($shuffletype$) shuffle,
 464                                     ($masktype$) m);  // specialize
 465     }
 466 
 467     @Override
 468     @ForceInline
 469     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s,
 470                                   Vector<$Boxtype$> v) {
 471         return ($vectortype$)
 472             super.rearrangeTemplate($shuffletype$.class,
 473                                     ($shuffletype$) s,
 474                                     ($vectortype$) v);  // specialize
 475     }
 476 
 477     @Override
 478     @ForceInline
 479     public $vectortype$ compress(VectorMask<$Boxtype$> m) {
 480         return ($vectortype$)
 481             super.compressTemplate($masktype$.class,
 482                                    ($masktype$) m);  // specialize
 483     }
 484 
 485     @Override
 486     @ForceInline
 487     public $vectortype$ expand(VectorMask<$Boxtype$> m) {
 488         return ($vectortype$)
 489             super.expandTemplate($masktype$.class,
 490                                    ($masktype$) m);  // specialize
 491     }
 492 
 493     @Override
 494     @ForceInline
 495     public $vectortype$ selectFrom(Vector<$Boxtype$> v) {
 496         return ($vectortype$)
 497             super.selectFromTemplate(($vectortype$) v);  // specialize
 498     }
 499 
 500     @Override
 501     @ForceInline
 502     public $vectortype$ selectFrom(Vector<$Boxtype$> v,
 503                                    VectorMask<$Boxtype$> m) {
 504         return ($vectortype$)
 505             super.selectFromTemplate(($vectortype$) v,
 506                                      ($masktype$) m);  // specialize
 507     }
 508 
 509 
 510 #if[FP]
 511     @ForceInline
 512     @Override
 513     public $type$ lane(int i) {
 514 #if[!Max]
 515         $bitstype$ bits;
 516         switch(i) {
 517             case 0: bits = laneHelper(0); break;
 518 #if[!1L]
 519             case 1: bits = laneHelper(1); break;
 520 #if[!2L]
 521             case 2: bits = laneHelper(2); break;
 522             case 3: bits = laneHelper(3); break;
 523 #if[!4L]
 524             case 4: bits = laneHelper(4); break;
 525             case 5: bits = laneHelper(5); break;
 526             case 6: bits = laneHelper(6); break;
 527             case 7: bits = laneHelper(7); break;
 528 #if[!8L]
 529             case 8: bits = laneHelper(8); break;
 530             case 9: bits = laneHelper(9); break;
 531             case 10: bits = laneHelper(10); break;
 532             case 11: bits = laneHelper(11); break;
 533             case 12: bits = laneHelper(12); break;
 534             case 13: bits = laneHelper(13); break;
 535             case 14: bits = laneHelper(14); break;
 536             case 15: bits = laneHelper(15); break;
 537 #end[!8L]
 538 #end[!4L]
 539 #end[!2L]
 540 #end[!1L]
 541             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 542         }
 543 #else[!Max]
 544         if (i < 0 || i >= VLENGTH) {
 545             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 546         }
 547         $bitstype$ bits = laneHelper(i);
 548 #end[!Max]
 549         return $Type$.$bitstype$BitsTo$Fptype$(bits);
 550     }
 551 
 552     public $bitstype$ laneHelper(int i) {
 553         return ($bitstype$) VectorSupport.extract(
 554                      VCLASS, ETYPE, VLENGTH,
 555                      this, i,
 556                      (vec, ix) -> {
 557                      $type$[] vecarr = vec.vec();
 558                      return (long)$Type$.$type$To$Bitstype$Bits(vecarr[ix]);
 559                      });
 560     }
 561 
 562     @ForceInline
 563     @Override
 564     public $vectortype$ withLane(int i, $type$ e) {
 565 #if[!Max]
 566         switch(i) {
 567             case 0: return withLaneHelper(0, e);
 568 #if[!1L]
 569             case 1: return withLaneHelper(1, e);
 570 #if[!2L]
 571             case 2: return withLaneHelper(2, e);
 572             case 3: return withLaneHelper(3, e);
 573 #if[!4L]
 574             case 4: return withLaneHelper(4, e);
 575             case 5: return withLaneHelper(5, e);
 576             case 6: return withLaneHelper(6, e);
 577             case 7: return withLaneHelper(7, e);
 578 #if[!8L]
 579             case 8: return withLaneHelper(8, e);
 580             case 9: return withLaneHelper(9, e);
 581             case 10: return withLaneHelper(10, e);
 582             case 11: return withLaneHelper(11, e);
 583             case 12: return withLaneHelper(12, e);
 584             case 13: return withLaneHelper(13, e);
 585             case 14: return withLaneHelper(14, e);
 586             case 15: return withLaneHelper(15, e);
 587 #end[!8L]
 588 #end[!4L]
 589 #end[!2L]
 590 #end[!1L]
 591             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 592         }
 593 #else[!Max]
 594         if (i < 0 || i >= VLENGTH) {
 595             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 596         }
 597         return withLaneHelper(i, e);
 598 #end[!Max]
 599     }
 600 
 601     public $vectortype$ withLaneHelper(int i, $type$ e) {
 602         return VectorSupport.insert(
 603                                 VCLASS, ETYPE, VLENGTH,
 604                                 this, i, (long)$Type$.$type$To$Bitstype$Bits(e),
 605                                 (v, ix, bits) -> {
 606                                     $type$[] res = v.vec().clone();
 607                                     res[ix] = $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits);
 608                                     return v.vectorFactory(res);
 609                                 });
 610     }
 611 #else[FP]
 612     @ForceInline
 613     @Override
 614     public $type$ lane(int i) {
 615 #if[!Max]
 616         switch(i) {
 617             case 0: return laneHelper(0);
 618 #if[!1L]
 619             case 1: return laneHelper(1);
 620 #if[!2L]
 621             case 2: return laneHelper(2);
 622             case 3: return laneHelper(3);
 623 #if[!4L]
 624             case 4: return laneHelper(4);
 625             case 5: return laneHelper(5);
 626             case 6: return laneHelper(6);
 627             case 7: return laneHelper(7);
 628 #if[!8L]
 629             case 8: return laneHelper(8);
 630             case 9: return laneHelper(9);
 631             case 10: return laneHelper(10);
 632             case 11: return laneHelper(11);
 633             case 12: return laneHelper(12);
 634             case 13: return laneHelper(13);
 635             case 14: return laneHelper(14);
 636             case 15: return laneHelper(15);
 637 #if[!16L]
 638             case 16: return laneHelper(16);
 639             case 17: return laneHelper(17);
 640             case 18: return laneHelper(18);
 641             case 19: return laneHelper(19);
 642             case 20: return laneHelper(20);
 643             case 21: return laneHelper(21);
 644             case 22: return laneHelper(22);
 645             case 23: return laneHelper(23);
 646             case 24: return laneHelper(24);
 647             case 25: return laneHelper(25);
 648             case 26: return laneHelper(26);
 649             case 27: return laneHelper(27);
 650             case 28: return laneHelper(28);
 651             case 29: return laneHelper(29);
 652             case 30: return laneHelper(30);
 653             case 31: return laneHelper(31);
 654 #if[!32L]
 655             case 32: return laneHelper(32);
 656             case 33: return laneHelper(33);
 657             case 34: return laneHelper(34);
 658             case 35: return laneHelper(35);
 659             case 36: return laneHelper(36);
 660             case 37: return laneHelper(37);
 661             case 38: return laneHelper(38);
 662             case 39: return laneHelper(39);
 663             case 40: return laneHelper(40);
 664             case 41: return laneHelper(41);
 665             case 42: return laneHelper(42);
 666             case 43: return laneHelper(43);
 667             case 44: return laneHelper(44);
 668             case 45: return laneHelper(45);
 669             case 46: return laneHelper(46);
 670             case 47: return laneHelper(47);
 671             case 48: return laneHelper(48);
 672             case 49: return laneHelper(49);
 673             case 50: return laneHelper(50);
 674             case 51: return laneHelper(51);
 675             case 52: return laneHelper(52);
 676             case 53: return laneHelper(53);
 677             case 54: return laneHelper(54);
 678             case 55: return laneHelper(55);
 679             case 56: return laneHelper(56);
 680             case 57: return laneHelper(57);
 681             case 58: return laneHelper(58);
 682             case 59: return laneHelper(59);
 683             case 60: return laneHelper(60);
 684             case 61: return laneHelper(61);
 685             case 62: return laneHelper(62);
 686             case 63: return laneHelper(63);
 687 #end[!32L]
 688 #end[!16L]
 689 #end[!8L]
 690 #end[!4L]
 691 #end[!2L]
 692 #end[!1L]
 693             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 694         }
 695 #else[!Max]
 696         if (i < 0 || i >= VLENGTH) {
 697             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 698         }
 699         return laneHelper(i);
 700 #end[!Max]
 701     }
 702 
 703     public $type$ laneHelper(int i) {
 704         return ($type$) VectorSupport.extract(
 705                                 VCLASS, ETYPE, VLENGTH,
 706                                 this, i,
 707                                 (vec, ix) -> {
 708                                     $type$[] vecarr = vec.vec();
 709                                     return (long)vecarr[ix];
 710                                 });
 711     }
 712 
 713     @ForceInline
 714     @Override
 715     public $vectortype$ withLane(int i, $type$ e) {
 716 #if[!Max]
 717         switch (i) {
 718             case 0: return withLaneHelper(0, e);
 719 #if[!1L]
 720             case 1: return withLaneHelper(1, e);
 721 #if[!2L]
 722             case 2: return withLaneHelper(2, e);
 723             case 3: return withLaneHelper(3, e);
 724 #if[!4L]
 725             case 4: return withLaneHelper(4, e);
 726             case 5: return withLaneHelper(5, e);
 727             case 6: return withLaneHelper(6, e);
 728             case 7: return withLaneHelper(7, e);
 729 #if[!8L]
 730             case 8: return withLaneHelper(8, e);
 731             case 9: return withLaneHelper(9, e);
 732             case 10: return withLaneHelper(10, e);
 733             case 11: return withLaneHelper(11, e);
 734             case 12: return withLaneHelper(12, e);
 735             case 13: return withLaneHelper(13, e);
 736             case 14: return withLaneHelper(14, e);
 737             case 15: return withLaneHelper(15, e);
 738 #if[!16L]
 739             case 16: return withLaneHelper(16, e);
 740             case 17: return withLaneHelper(17, e);
 741             case 18: return withLaneHelper(18, e);
 742             case 19: return withLaneHelper(19, e);
 743             case 20: return withLaneHelper(20, e);
 744             case 21: return withLaneHelper(21, e);
 745             case 22: return withLaneHelper(22, e);
 746             case 23: return withLaneHelper(23, e);
 747             case 24: return withLaneHelper(24, e);
 748             case 25: return withLaneHelper(25, e);
 749             case 26: return withLaneHelper(26, e);
 750             case 27: return withLaneHelper(27, e);
 751             case 28: return withLaneHelper(28, e);
 752             case 29: return withLaneHelper(29, e);
 753             case 30: return withLaneHelper(30, e);
 754             case 31: return withLaneHelper(31, e);
 755 #if[!32L]
 756             case 32: return withLaneHelper(32, e);
 757             case 33: return withLaneHelper(33, e);
 758             case 34: return withLaneHelper(34, e);
 759             case 35: return withLaneHelper(35, e);
 760             case 36: return withLaneHelper(36, e);
 761             case 37: return withLaneHelper(37, e);
 762             case 38: return withLaneHelper(38, e);
 763             case 39: return withLaneHelper(39, e);
 764             case 40: return withLaneHelper(40, e);
 765             case 41: return withLaneHelper(41, e);
 766             case 42: return withLaneHelper(42, e);
 767             case 43: return withLaneHelper(43, e);
 768             case 44: return withLaneHelper(44, e);
 769             case 45: return withLaneHelper(45, e);
 770             case 46: return withLaneHelper(46, e);
 771             case 47: return withLaneHelper(47, e);
 772             case 48: return withLaneHelper(48, e);
 773             case 49: return withLaneHelper(49, e);
 774             case 50: return withLaneHelper(50, e);
 775             case 51: return withLaneHelper(51, e);
 776             case 52: return withLaneHelper(52, e);
 777             case 53: return withLaneHelper(53, e);
 778             case 54: return withLaneHelper(54, e);
 779             case 55: return withLaneHelper(55, e);
 780             case 56: return withLaneHelper(56, e);
 781             case 57: return withLaneHelper(57, e);
 782             case 58: return withLaneHelper(58, e);
 783             case 59: return withLaneHelper(59, e);
 784             case 60: return withLaneHelper(60, e);
 785             case 61: return withLaneHelper(61, e);
 786             case 62: return withLaneHelper(62, e);
 787             case 63: return withLaneHelper(63, e);
 788 #end[!32L]
 789 #end[!16L]
 790 #end[!8L]
 791 #end[!4L]
 792 #end[!2L]
 793 #end[!1L]
 794             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 795         }
 796 #else[!Max]
 797         if (i < 0 || i >= VLENGTH) {
 798             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 799         }
 800         return withLaneHelper(i, e);
 801 #end[!Max]
 802     }
 803 
 804     public $vectortype$ withLaneHelper(int i, $type$ e) {
 805         return VectorSupport.insert(
 806                                 VCLASS, ETYPE, VLENGTH,
 807                                 this, i, (long)e,
 808                                 (v, ix, bits) -> {
 809                                     $type$[] res = v.vec().clone();
 810                                     res[ix] = ($type$)bits;
 811                                     return v.vectorFactory(res);
 812                                 });
 813     }
 814 #end[FP]
 815 
 816     // Mask
 817 
 818     static final class $masktype$ extends AbstractMask<$Boxtype$> {
 819         static final int VLENGTH = VSPECIES.laneCount();    // used by the JVM
 820         static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
 821 
 822         $masktype$(boolean[] bits) {
 823             this(bits, 0);
 824         }
 825 
 826         $masktype$(boolean[] bits, int offset) {
 827             super(prepare(bits, offset));
 828         }
 829 
 830         $masktype$(boolean val) {
 831             super(prepare(val));
 832         }
 833 
 834         private static boolean[] prepare(boolean[] bits, int offset) {
 835             boolean[] newBits = new boolean[VSPECIES.laneCount()];
 836             for (int i = 0; i < newBits.length; i++) {
 837                 newBits[i] = bits[offset + i];
 838             }
 839             return newBits;
 840         }
 841 
 842         private static boolean[] prepare(boolean val) {
 843             boolean[] bits = new boolean[VSPECIES.laneCount()];
 844             Arrays.fill(bits, val);
 845             return bits;
 846         }
 847 
 848         @ForceInline
 849         final @Override
 850         public $Type$Species vspecies() {
 851             // ISSUE:  This should probably be a @Stable
 852             // field inside AbstractMask, rather than
 853             // a megamorphic method.
 854             return VSPECIES;
 855         }
 856 
 857         @ForceInline
 858         boolean[] getBits() {
 859             return (boolean[])getPayload();
 860         }
 861 
 862         @Override
 863         $masktype$ uOp(MUnOp f) {
 864             boolean[] res = new boolean[vspecies().laneCount()];
 865             boolean[] bits = getBits();
 866             for (int i = 0; i < res.length; i++) {
 867                 res[i] = f.apply(i, bits[i]);
 868             }
 869             return new $masktype$(res);
 870         }
 871 
 872         @Override
 873         $masktype$ bOp(VectorMask<$Boxtype$> m, MBinOp f) {
 874             boolean[] res = new boolean[vspecies().laneCount()];
 875             boolean[] bits = getBits();
 876             boolean[] mbits = (($masktype$)m).getBits();
 877             for (int i = 0; i < res.length; i++) {
 878                 res[i] = f.apply(i, bits[i], mbits[i]);
 879             }
 880             return new $masktype$(res);
 881         }
 882 
 883         @ForceInline
 884         @Override
 885         public final
 886         $vectortype$ toVector() {
 887             return ($vectortype$) super.toVectorTemplate();  // specialize
 888         }
 889 
 890         /**
 891          * Helper function for lane-wise mask conversions.
 892          * This function kicks in after intrinsic failure.
 893          */
 894         @ForceInline
 895         private final <E>
 896         VectorMask<E> defaultMaskCast(AbstractSpecies<E> dsp) {
 897             if (length() != dsp.laneCount())
 898                 throw new IllegalArgumentException("VectorMask length and species length differ");
 899             boolean[] maskArray = toArray();
 900             return  dsp.maskFactory(maskArray).check(dsp);
 901         }
 902 
 903         @Override
 904         @ForceInline
 905         public <E> VectorMask<E> cast(VectorSpecies<E> dsp) {
 906             AbstractSpecies<E> species = (AbstractSpecies<E>) dsp;
 907             if (length() != species.laneCount())
 908                 throw new IllegalArgumentException("VectorMask length and species length differ");
 909 
 910             return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
 911                 this.getClass(), ETYPE, VLENGTH,
 912                 species.maskType(), species.elementType(), VLENGTH,
 913                 this, species,
 914                 (m, s) -> s.maskFactory(m.toArray()).check(s));
 915         }
 916 
 917         @Override
 918         @ForceInline
 919         public $masktype$ eq(VectorMask<$Boxtype$> mask) {
 920             Objects.requireNonNull(mask);
 921             $masktype$ m = ($masktype$)mask;
 922             return xor(m.not());
 923         }
 924 
 925         // Unary operations
 926 
 927         @Override
 928         @ForceInline
 929         public $masktype$ not() {
 930             return xor(maskAll(true));
 931         }
 932 
 933         @Override
 934         @ForceInline
 935         public $masktype$ compress() {
 936             return ($masktype$)VectorSupport.comExpOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
 937                 $vectortype$.class, $masktype$.class, ETYPE, VLENGTH, null, this,
 938                 (v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
 939         }
 940 
 941 
 942         // Binary operations
 943 
 944         @Override
 945         @ForceInline
 946         public $masktype$ and(VectorMask<$Boxtype$> mask) {
 947             Objects.requireNonNull(mask);
 948             $masktype$ m = ($masktype$)mask;
 949             return VectorSupport.binaryOp(VECTOR_OP_AND, $masktype$.class, null, $bitstype$.class, VLENGTH,
 950                                           this, m, null,
 951                                           (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a & b));
 952         }
 953 
 954         @Override
 955         @ForceInline
 956         public $masktype$ or(VectorMask<$Boxtype$> mask) {
 957             Objects.requireNonNull(mask);
 958             $masktype$ m = ($masktype$)mask;
 959             return VectorSupport.binaryOp(VECTOR_OP_OR, $masktype$.class, null, $bitstype$.class, VLENGTH,
 960                                           this, m, null,
 961                                           (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a | b));
 962         }
 963 
 964         @ForceInline
 965         /* package-private */
 966         $masktype$ xor(VectorMask<$Boxtype$> mask) {
 967             Objects.requireNonNull(mask);
 968             $masktype$ m = ($masktype$)mask;
 969             return VectorSupport.binaryOp(VECTOR_OP_XOR, $masktype$.class, null, $bitstype$.class, VLENGTH,
 970                                           this, m, null,
 971                                           (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a ^ b));
 972         }
 973 
 974         // Mask Query operations
 975 
 976         @Override
 977         @ForceInline
 978         public int trueCount() {
 979             return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, $masktype$.class, $bitstype$.class, VLENGTH, this,
 980                                                       (m) -> trueCountHelper(m.getBits()));
 981         }
 982 
 983         @Override
 984         @ForceInline
 985         public int firstTrue() {
 986             return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
 987                                                       (m) -> firstTrueHelper(m.getBits()));
 988         }
 989 
 990         @Override
 991         @ForceInline
 992         public int lastTrue() {
 993             return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
 994                                                       (m) -> lastTrueHelper(m.getBits()));
 995         }
 996 
 997         @Override
 998         @ForceInline
 999         public long toLong() {
1000             if (length() > Long.SIZE) {
1001                 throw new UnsupportedOperationException("too many lanes for one long");
1002             }
1003             return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TOLONG, $masktype$.class, $bitstype$.class, VLENGTH, this,
1004                                                       (m) -> toLongHelper(m.getBits()));
1005         }
1006 
1007         // Reductions
1008 
1009         @Override
1010         @ForceInline
1011         public boolean anyTrue() {
1012             return VectorSupport.test(BT_ne, $masktype$.class, $bitstype$.class, VLENGTH,
1013                                          this, vspecies().maskAll(true),
1014                                          (m, __) -> anyTrueHelper((($masktype$)m).getBits()));
1015         }
1016 
1017         @Override
1018         @ForceInline
1019         public boolean allTrue() {
1020             return VectorSupport.test(BT_overflow, $masktype$.class, $bitstype$.class, VLENGTH,
1021                                          this, vspecies().maskAll(true),
1022                                          (m, __) -> allTrueHelper((($masktype$)m).getBits()));
1023         }
1024 
1025         @ForceInline
1026         /*package-private*/
1027         static $masktype$ maskAll(boolean bit) {
1028             return VectorSupport.broadcastCoerced($masktype$.class, $bitstype$.class, VLENGTH,
1029                                                   (bit ? -1 : 0), null,
1030                                                   (v, __) -> (v != 0 ? TRUE_MASK : FALSE_MASK));
1031         }
1032         private static final $masktype$  TRUE_MASK = new $masktype$(true);
1033         private static final $masktype$ FALSE_MASK = new $masktype$(false);
1034 
1035 #if[intAndMax]
1036 
1037         static boolean[] maskLowerHalf() {
1038             boolean[] a = new boolean[VLENGTH];
1039             int len = a.length >> 1;
1040             for (int i = 0; i < len; i++) {
1041                 a[i] = true;
1042             }
1043             return a;
1044         }
1045 
1046 #end[intAndMax]
1047 #if[intAndMax]
1048         static final IntMaxMask LOWER_HALF_TRUE_MASK = new IntMaxMask(maskLowerHalf());
1049 #end[intAndMax]
1050     }
1051 
1052     // Shuffle
1053 
1054     static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> {
1055         static final int VLENGTH = VSPECIES.laneCount();    // used by the JVM
1056         static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
1057 
1058         $shuffletype$(byte[] reorder) {
1059             super(VLENGTH, reorder);
1060         }
1061 
1062         public $shuffletype$(int[] reorder) {
1063             super(VLENGTH, reorder);
1064         }
1065 
1066         public $shuffletype$(int[] reorder, int i) {
1067             super(VLENGTH, reorder, i);
1068         }
1069 
1070         public $shuffletype$(IntUnaryOperator fn) {
1071             super(VLENGTH, fn);
1072         }
1073 
1074         @Override
1075         public $Type$Species vspecies() {
1076             return VSPECIES;
1077         }
1078 
1079         static {
1080             // There must be enough bits in the shuffle lanes to encode
1081             // VLENGTH valid indexes and VLENGTH exceptional ones.
1082             assert(VLENGTH < Byte.MAX_VALUE);
1083             assert(Byte.MIN_VALUE <= -VLENGTH);
1084         }
1085         static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY);
1086 
1087         @Override
1088         @ForceInline
1089         public $vectortype$ toVector() {
1090             return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH,
1091                                                     (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate())));
1092         }
1093 
1094         @Override
1095         @ForceInline
1096         public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
1097             AbstractSpecies<F> species = (AbstractSpecies<F>) s;
1098             if (length() != species.laneCount())
1099                 throw new IllegalArgumentException("VectorShuffle length and species length differ");
1100             int[] shuffleArray = toArray();
1101             return s.shuffleFromArray(shuffleArray, 0).check(s);
1102         }
1103 
1104         @ForceInline
1105         @Override
1106         public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {
1107             $shuffletype$ s = ($shuffletype$) shuffle;
1108             byte[] reorder1 = reorder();
1109             byte[] reorder2 = s.reorder();
1110             byte[] r = new byte[reorder1.length];
1111             for (int i = 0; i < reorder1.length; i++) {
1112                 int ssi = reorder2[i];
1113                 r[i] = reorder1[ssi];  // throws on exceptional index
1114             }
1115             return new $shuffletype$(r);
1116         }
1117     }
1118 
1119     // ================================================
1120 
1121     // Specialized low-level memory operations.
1122 
1123     @ForceInline
1124     @Override
1125     final
1126     $abstractvectortype$ fromArray0($type$[] a, int offset) {
1127         return super.fromArray0Template(a, offset);  // specialize
1128     }
1129 
1130     @ForceInline
1131     @Override
1132     final
1133     $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1134         return super.fromArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1135     }
1136 
1137 #if[!byteOrShort]
1138     @ForceInline
1139     @Override
1140     final
1141     $abstractvectortype$ fromArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1142         return super.fromArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1143     }
1144 #end[!byteOrShort]
1145 
1146 #if[short]
1147     @ForceInline
1148     @Override
1149     final
1150     $abstractvectortype$ fromCharArray0(char[] a, int offset) {
1151         return super.fromCharArray0Template(a, offset);  // specialize
1152     }
1153 
1154     @ForceInline
1155     @Override
1156     final
1157     $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1158         return super.fromCharArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1159     }
1160 #end[short]
1161 
1162 #if[byte]
1163     @ForceInline
1164     @Override
1165     final
1166     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset) {
1167         return super.fromBooleanArray0Template(a, offset);  // specialize
1168     }
1169 
1170     @ForceInline
1171     @Override
1172     final
1173     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1174         return super.fromBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1175     }
1176 #end[byte]
1177 
1178     @ForceInline
1179     @Override
1180     final
1181     $abstractvectortype$ fromByteArray0(byte[] a, int offset) {
1182         return super.fromByteArray0Template(a, offset);  // specialize
1183     }
1184 
1185     @ForceInline
1186     @Override
1187     final
1188     $abstractvectortype$ fromByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m) {
1189         return super.fromByteArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1190     }
1191 
1192     @ForceInline
1193     @Override
1194     final
1195     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset) {
1196         return super.fromByteBuffer0Template(bb, offset);  // specialize
1197     }
1198 
1199     @ForceInline
1200     @Override
1201     final
1202     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m) {
1203         return super.fromByteBuffer0Template($masktype$.class, bb, offset, ($masktype$) m);  // specialize
1204     }
1205 
1206     @ForceInline
1207     @Override
1208     final
1209     void intoArray0($type$[] a, int offset) {
1210         super.intoArray0Template(a, offset);  // specialize
1211     }
1212 
1213     @ForceInline
1214     @Override
1215     final
1216     void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1217         super.intoArray0Template($masktype$.class, a, offset, ($masktype$) m);
1218     }
1219 
1220 #if[!byteOrShort]
1221     @ForceInline
1222     @Override
1223     final
1224     void intoArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1225         super.intoArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1226     }
1227 #end[!byteOrShort]
1228 
1229 #if[byte]
1230     @ForceInline
1231     @Override
1232     final
1233     void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1234         super.intoBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m);
1235     }
1236 #end[byte]
1237 
1238     @ForceInline
1239     @Override
1240     final
1241     void intoByteArray0(byte[] a, int offset) {
1242         super.intoByteArray0Template(a, offset);  // specialize
1243     }
1244 
1245     @ForceInline
1246     @Override
1247     final
1248     void intoByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m) {
1249         super.intoByteArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1250     }
1251 
1252     @ForceInline
1253     @Override
1254     final
1255     void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m) {
1256         super.intoByteBuffer0Template($masktype$.class, bb, offset, ($masktype$) m);
1257     }
1258 
1259 #if[short]
1260     @ForceInline
1261     @Override
1262     final
1263     void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1264         super.intoCharArray0Template($masktype$.class, a, offset, ($masktype$) m);
1265     }
1266 #end[short]
1267 
1268     // End of specialized low-level memory operations.
1269 
1270     // ================================================
1271 
1272 }
--- EOF ---