1 /*
   2  * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.nio.ByteOrder;
  29 import java.nio.ReadOnlyBufferException;
  30 import java.util.Arrays;
  31 import java.util.Objects;
  32 import java.util.function.BinaryOperator;
  33 import java.util.function.Function;
  34 import java.util.function.UnaryOperator;
  35 
  36 import jdk.internal.misc.ScopedMemoryAccess;
  37 import jdk.internal.misc.Unsafe;
  38 import jdk.internal.vm.annotation.ForceInline;
  39 import jdk.internal.vm.vector.VectorSupport;
  40 
  41 import static jdk.internal.vm.vector.VectorSupport.*;
  42 import static jdk.incubator.vector.VectorIntrinsics.*;
  43 
  44 import static jdk.incubator.vector.VectorOperators.*;
  45 
  46 #warn This file is preprocessed before being compiled
  47 
  48 /**
  49  * A specialized {@link Vector} representing an ordered immutable sequence of
  50  * {@code $type$} values.
  51  */
  52 @SuppressWarnings("cast")  // warning: redundant cast
  53 public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
  54 
  55     $abstractvectortype$($type$[] vec) {
  56         super(vec);
  57     }
  58 
  59 #if[FP]
  60     static final int FORBID_OPCODE_KIND = VO_NOFP;
  61 #else[FP]
  62     static final int FORBID_OPCODE_KIND = VO_ONLYFP;
  63 #end[FP]
  64 
  65     @ForceInline
  66     static int opCode(Operator op) {
  67         return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
  68     }
  69     @ForceInline
  70     static int opCode(Operator op, int requireKind) {
  71         requireKind |= VO_OPCODE_VALID;
  72         return VectorOperators.opCode(op, requireKind, FORBID_OPCODE_KIND);
  73     }
  74     @ForceInline
  75     static boolean opKind(Operator op, int bit) {
  76         return VectorOperators.opKind(op, bit);
  77     }
  78 
  79     // Virtualized factories and operators,
  80     // coded with portable definitions.
  81     // These are all @ForceInline in case
  82     // they need to be used performantly.
  83     // The various shape-specific subclasses
  84     // also specialize them by wrapping
  85     // them in a call like this:
  86     //    return (Byte128Vector)
  87     //       super.bOp((Byte128Vector) o);
  88     // The purpose of that is to forcibly inline
  89     // the generic definition from this file
  90     // into a sharply type- and size-specific
  91     // wrapper in the subclass file, so that
  92     // the JIT can specialize the code.
  93     // The code is only inlined and expanded
  94     // if it gets hot.  Think of it as a cheap
  95     // and lazy version of C++ templates.
  96 
  97     // Virtualized getter
  98 
  99     /*package-private*/
 100     abstract $type$[] vec();
 101 
 102     // Virtualized constructors
 103 
 104     /**
 105      * Build a vector directly using my own constructor.
 106      * It is an error if the array is aliased elsewhere.
 107      */
 108     /*package-private*/
 109     abstract $abstractvectortype$ vectorFactory($type$[] vec);
 110 
 111     /**
 112      * Build a mask directly using my species.
 113      * It is an error if the array is aliased elsewhere.
 114      */
 115     /*package-private*/
 116     @ForceInline
 117     final
 118     AbstractMask<$Boxtype$> maskFactory(boolean[] bits) {
 119         return vspecies().maskFactory(bits);
 120     }
 121 
 122     // Constant loader (takes dummy as vector arg)
 123     interface FVOp {
 124         $type$ apply(int i);
 125     }
 126 
 127     /*package-private*/
 128     @ForceInline
 129     final
 130     $abstractvectortype$ vOp(FVOp f) {
 131         $type$[] res = new $type$[length()];
 132         for (int i = 0; i < res.length; i++) {
 133             res[i] = f.apply(i);
 134         }
 135         return vectorFactory(res);
 136     }
 137 
 138     @ForceInline
 139     final
 140     $abstractvectortype$ vOp(VectorMask<$Boxtype$> m, FVOp f) {
 141         $type$[] res = new $type$[length()];
 142         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 143         for (int i = 0; i < res.length; i++) {
 144             if (mbits[i]) {
 145                 res[i] = f.apply(i);
 146             }
 147         }
 148         return vectorFactory(res);
 149     }
 150 
 151     // Unary operator
 152 
 153     /*package-private*/
 154     interface FUnOp {
 155         $type$ apply(int i, $type$ a);
 156     }
 157 
 158     /*package-private*/
 159     abstract
 160     $abstractvectortype$ uOp(FUnOp f);
 161     @ForceInline
 162     final
 163     $abstractvectortype$ uOpTemplate(FUnOp f) {
 164         $type$[] vec = vec();
 165         $type$[] res = new $type$[length()];
 166         for (int i = 0; i < res.length; i++) {
 167             res[i] = f.apply(i, vec[i]);
 168         }
 169         return vectorFactory(res);
 170     }
 171 
 172     /*package-private*/
 173     abstract
 174     $abstractvectortype$ uOp(VectorMask<$Boxtype$> m,
 175                              FUnOp f);
 176     @ForceInline
 177     final
 178     $abstractvectortype$ uOpTemplate(VectorMask<$Boxtype$> m,
 179                                      FUnOp f) {
 180         $type$[] vec = vec();
 181         $type$[] res = new $type$[length()];
 182         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 183         for (int i = 0; i < res.length; i++) {
 184             res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
 185         }
 186         return vectorFactory(res);
 187     }
 188 
 189     // Binary operator
 190 
 191     /*package-private*/
 192     interface FBinOp {
 193         $type$ apply(int i, $type$ a, $type$ b);
 194     }
 195 
 196     /*package-private*/
 197     abstract
 198     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 199                              FBinOp f);
 200     @ForceInline
 201     final
 202     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 203                                      FBinOp f) {
 204         $type$[] res = new $type$[length()];
 205         $type$[] vec1 = this.vec();
 206         $type$[] vec2 = (($abstractvectortype$)o).vec();
 207         for (int i = 0; i < res.length; i++) {
 208             res[i] = f.apply(i, vec1[i], vec2[i]);
 209         }
 210         return vectorFactory(res);
 211     }
 212 
 213     /*package-private*/
 214     abstract
 215     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 216                              VectorMask<$Boxtype$> m,
 217                              FBinOp f);
 218     @ForceInline
 219     final
 220     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 221                                      VectorMask<$Boxtype$> m,
 222                                      FBinOp f) {
 223         $type$[] res = new $type$[length()];
 224         $type$[] vec1 = this.vec();
 225         $type$[] vec2 = (($abstractvectortype$)o).vec();
 226         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 227         for (int i = 0; i < res.length; i++) {
 228             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
 229         }
 230         return vectorFactory(res);
 231     }
 232 
 233     // Ternary operator
 234 
 235     /*package-private*/
 236     interface FTriOp {
 237         $type$ apply(int i, $type$ a, $type$ b, $type$ c);
 238     }
 239 
 240     /*package-private*/
 241     abstract
 242     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 243                              Vector<$Boxtype$> o2,
 244                              FTriOp f);
 245     @ForceInline
 246     final
 247     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 248                                      Vector<$Boxtype$> o2,
 249                                      FTriOp f) {
 250         $type$[] res = new $type$[length()];
 251         $type$[] vec1 = this.vec();
 252         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 253         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 254         for (int i = 0; i < res.length; i++) {
 255             res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
 256         }
 257         return vectorFactory(res);
 258     }
 259 
 260     /*package-private*/
 261     abstract
 262     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 263                              Vector<$Boxtype$> o2,
 264                              VectorMask<$Boxtype$> m,
 265                              FTriOp f);
 266     @ForceInline
 267     final
 268     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 269                                      Vector<$Boxtype$> o2,
 270                                      VectorMask<$Boxtype$> m,
 271                                      FTriOp f) {
 272         $type$[] res = new $type$[length()];
 273         $type$[] vec1 = this.vec();
 274         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 275         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 276         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 277         for (int i = 0; i < res.length; i++) {
 278             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
 279         }
 280         return vectorFactory(res);
 281     }
 282 
 283     // Reduction operator
 284 
 285     /*package-private*/
 286     abstract
 287     $type$ rOp($type$ v, FBinOp f);
 288     @ForceInline
 289     final
 290     $type$ rOpTemplate($type$ v, FBinOp f) {
 291         $type$[] vec = vec();
 292         for (int i = 0; i < vec.length; i++) {
 293             v = f.apply(i, v, vec[i]);
 294         }
 295         return v;
 296     }
 297 
 298     // Memory reference
 299 
 300     /*package-private*/
 301     interface FLdOp<M> {
 302         $type$ apply(M memory, int offset, int i);
 303     }
 304 
 305     /*package-private*/
 306     @ForceInline
 307     final
 308     <M> $abstractvectortype$ ldOp(M memory, int offset,
 309                                   FLdOp<M> f) {
 310         //dummy; no vec = vec();
 311         $type$[] res = new $type$[length()];
 312         for (int i = 0; i < res.length; i++) {
 313             res[i] = f.apply(memory, offset, i);
 314         }
 315         return vectorFactory(res);
 316     }
 317 
 318     /*package-private*/
 319     @ForceInline
 320     final
 321     <M> $abstractvectortype$ ldOp(M memory, int offset,
 322                                   VectorMask<$Boxtype$> m,
 323                                   FLdOp<M> f) {
 324         //$type$[] vec = vec();
 325         $type$[] res = new $type$[length()];
 326         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 327         for (int i = 0; i < res.length; i++) {
 328             if (mbits[i]) {
 329                 res[i] = f.apply(memory, offset, i);
 330             }
 331         }
 332         return vectorFactory(res);
 333     }
 334 
 335     interface FStOp<M> {
 336         void apply(M memory, int offset, int i, $type$ a);
 337     }
 338 
 339     /*package-private*/
 340     @ForceInline
 341     final
 342     <M> void stOp(M memory, int offset,
 343                   FStOp<M> f) {
 344         $type$[] vec = vec();
 345         for (int i = 0; i < vec.length; i++) {
 346             f.apply(memory, offset, i, vec[i]);
 347         }
 348     }
 349 
 350     /*package-private*/
 351     @ForceInline
 352     final
 353     <M> void stOp(M memory, int offset,
 354                   VectorMask<$Boxtype$> m,
 355                   FStOp<M> f) {
 356         $type$[] vec = vec();
 357         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 358         for (int i = 0; i < vec.length; i++) {
 359             if (mbits[i]) {
 360                 f.apply(memory, offset, i, vec[i]);
 361             }
 362         }
 363     }
 364 
 365     // Binary test
 366 
 367     /*package-private*/
 368     interface FBinTest {
 369         boolean apply(int cond, int i, $type$ a, $type$ b);
 370     }
 371 
 372     /*package-private*/
 373     @ForceInline
 374     final
 375     AbstractMask<$Boxtype$> bTest(int cond,
 376                                   Vector<$Boxtype$> o,
 377                                   FBinTest f) {
 378         $type$[] vec1 = vec();
 379         $type$[] vec2 = (($abstractvectortype$)o).vec();
 380         boolean[] bits = new boolean[length()];
 381         for (int i = 0; i < length(); i++){
 382             bits[i] = f.apply(cond, i, vec1[i], vec2[i]);
 383         }
 384         return maskFactory(bits);
 385     }
 386 
 387 #if[BITWISE]
 388     /*package-private*/
 389     @ForceInline
 390     static $type$ rotateLeft($type$ a, int n) {
 391 #if[intOrLong]
 392         return $Boxtype$.rotateLeft(a, n);
 393 #else[intOrLong]
 394         return ($type$)((((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) << (n & $Boxtype$.SIZE-1)) | (((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) >>> ($Boxtype$.SIZE - (n & $Boxtype$.SIZE-1))));
 395 #end[intOrLong]
 396     }
 397 
 398     /*package-private*/
 399     @ForceInline
 400     static $type$ rotateRight($type$ a, int n) {
 401 #if[intOrLong]
 402         return $Boxtype$.rotateRight(a, n);
 403 #else[intOrLong]
 404         return ($type$)((((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) >>> (n & $Boxtype$.SIZE-1)) | (((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) << ($Boxtype$.SIZE - (n & $Boxtype$.SIZE-1))));
 405 #end[intOrLong]
 406     }
 407 #end[BITWISE]
 408 
 409     /*package-private*/
 410     @Override
 411     abstract $Type$Species vspecies();
 412 
 413     /*package-private*/
 414     @ForceInline
 415     static long toBits($type$ e) {
 416         return {#if[FP]? $Type$.$type$ToRaw$Bitstype$Bits(e): e};
 417     }
 418 
 419     /*package-private*/
 420     @ForceInline
 421     static $type$ fromBits(long bits) {
 422         return {#if[FP]?$Type$.$bitstype$BitsTo$Type$}(($bitstype$)bits);
 423     }
 424 
 425     // Static factories (other than memory operations)
 426 
 427     // Note: A surprising behavior in javadoc
 428     // sometimes makes a lone /** {@inheritDoc} */
 429     // comment drop the method altogether,
 430     // apparently if the method mentions an
 431     // parameter or return type of Vector<$Boxtype$>
 432     // instead of Vector<E> as originally specified.
 433     // Adding an empty HTML fragment appears to
 434     // nudge javadoc into providing the desired
 435     // inherited documentation.  We use the HTML
 436     // comment <!--workaround--> for this.
 437 
 438     /**
 439      * Returns a vector of the given species
 440      * where all lane elements are set to
 441      * zero, the default primitive value.
 442      *
 443      * @param species species of the desired zero vector
 444      * @return a zero vector
 445      */
 446     @ForceInline
 447     public static $abstractvectortype$ zero(VectorSpecies<$Boxtype$> species) {
 448         $Type$Species vsp = ($Type$Species) species;
 449 #if[FP]
 450         return VectorSupport.broadcastCoerced(vsp.vectorType(), $type$.class, species.length(),
 451                         toBits(0.0f), vsp,
 452                         ((bits_, s_) -> s_.rvOp(i -> bits_)));
 453 #else[FP]
 454         return VectorSupport.broadcastCoerced(vsp.vectorType(), $type$.class, species.length(),
 455                                 0, vsp,
 456                                 ((bits_, s_) -> s_.rvOp(i -> bits_)));
 457 #end[FP]
 458     }
 459 
 460     /**
 461      * Returns a vector of the same species as this one
 462      * where all lane elements are set to
 463      * the primitive value {@code e}.
 464      *
 465      * The contents of the current vector are discarded;
 466      * only the species is relevant to this operation.
 467      *
 468      * <p> This method returns the value of this expression:
 469      * {@code $abstractvectortype$.broadcast(this.species(), e)}.
 470      *
 471      * @apiNote
 472      * Unlike the similar method named {@code broadcast()}
 473      * in the supertype {@code Vector}, this method does not
 474      * need to validate its argument, and cannot throw
 475      * {@code IllegalArgumentException}.  This method is
 476      * therefore preferable to the supertype method.
 477      *
 478      * @param e the value to broadcast
 479      * @return a vector where all lane elements are set to
 480      *         the primitive value {@code e}
 481      * @see #broadcast(VectorSpecies,long)
 482      * @see Vector#broadcast(long)
 483      * @see VectorSpecies#broadcast(long)
 484      */
 485     public abstract $abstractvectortype$ broadcast($type$ e);
 486 
 487     /**
 488      * Returns a vector of the given species
 489      * where all lane elements are set to
 490      * the primitive value {@code e}.
 491      *
 492      * @param species species of the desired vector
 493      * @param e the value to broadcast
 494      * @return a vector where all lane elements are set to
 495      *         the primitive value {@code e}
 496      * @see #broadcast(long)
 497      * @see Vector#broadcast(long)
 498      * @see VectorSpecies#broadcast(long)
 499      */
 500     @ForceInline
 501     public static $abstractvectortype$ broadcast(VectorSpecies<$Boxtype$> species, $type$ e) {
 502         $Type$Species vsp = ($Type$Species) species;
 503         return vsp.broadcast(e);
 504     }
 505 
 506     /*package-private*/
 507     @ForceInline
 508     final $abstractvectortype$ broadcastTemplate($type$ e) {
 509         $Type$Species vsp = vspecies();
 510         return vsp.broadcast(e);
 511     }
 512 
 513 #if[!long]
 514     /**
 515      * {@inheritDoc} <!--workaround-->
 516      * @apiNote
 517      * When working with vector subtypes like {@code $abstractvectortype$},
 518      * {@linkplain #broadcast($type$) the more strongly typed method}
 519      * is typically selected.  It can be explicitly selected
 520      * using a cast: {@code v.broadcast(($type$)e)}.
 521      * The two expressions will produce numerically identical results.
 522      */
 523     @Override
 524     public abstract $abstractvectortype$ broadcast(long e);
 525 
 526     /**
 527      * Returns a vector of the given species
 528      * where all lane elements are set to
 529      * the primitive value {@code e}.
 530      *
 531      * The {@code long} value must be accurately representable
 532      * by the {@code ETYPE} of the vector species, so that
 533      * {@code e==(long)(ETYPE)e}.
 534      *
 535      * @param species species of the desired vector
 536      * @param e the value to broadcast
 537      * @return a vector where all lane elements are set to
 538      *         the primitive value {@code e}
 539      * @throws IllegalArgumentException
 540      *         if the given {@code long} value cannot
 541      *         be represented by the vector's {@code ETYPE}
 542      * @see #broadcast(VectorSpecies,$type$)
 543      * @see VectorSpecies#checkValue(long)
 544      */
 545     @ForceInline
 546     public static $abstractvectortype$ broadcast(VectorSpecies<$Boxtype$> species, long e) {
 547         $Type$Species vsp = ($Type$Species) species;
 548         return vsp.broadcast(e);
 549     }
 550 
 551     /*package-private*/
 552     @ForceInline
 553     final $abstractvectortype$ broadcastTemplate(long e) {
 554         return vspecies().broadcast(e);
 555     }
 556 #end[!long]
 557 
 558     // Unary lanewise support
 559 
 560     /**
 561      * {@inheritDoc} <!--workaround-->
 562      */
 563     public abstract
 564     $abstractvectortype$ lanewise(VectorOperators.Unary op);
 565 
 566     @ForceInline
 567     final
 568     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op) {
 569         if (opKind(op, VO_SPECIAL)) {
 570             if (op == ZOMO) {
 571                 return blend(broadcast(-1), compare(NE, 0));
 572             }
 573 #if[BITWISE]
 574             if (op == NOT) {
 575                 return broadcast(-1).lanewiseTemplate(XOR, this);
 576             } else if (op == NEG) {
 577                 // FIXME: Support this in the JIT.
 578                 return broadcast(0).lanewiseTemplate(SUB, this);
 579             }
 580 #end[BITWISE]
 581         }
 582         int opc = opCode(op);
 583         return VectorSupport.unaryOp(
 584             opc, getClass(), $type$.class, length(),
 585             this,
 586             UN_IMPL.find(op, opc, (opc_) -> {
 587               switch (opc_) {
 588                 case VECTOR_OP_NEG: return v0 ->
 589                         v0.uOp((i, a) -> ($type$) -a);
 590                 case VECTOR_OP_ABS: return v0 ->
 591                         v0.uOp((i, a) -> ($type$) Math.abs(a));
 592 #if[FP]
 593                 case VECTOR_OP_SIN: return v0 ->
 594                         v0.uOp((i, a) -> ($type$) Math.sin(a));
 595                 case VECTOR_OP_COS: return v0 ->
 596                         v0.uOp((i, a) -> ($type$) Math.cos(a));
 597                 case VECTOR_OP_TAN: return v0 ->
 598                         v0.uOp((i, a) -> ($type$) Math.tan(a));
 599                 case VECTOR_OP_ASIN: return v0 ->
 600                         v0.uOp((i, a) -> ($type$) Math.asin(a));
 601                 case VECTOR_OP_ACOS: return v0 ->
 602                         v0.uOp((i, a) -> ($type$) Math.acos(a));
 603                 case VECTOR_OP_ATAN: return v0 ->
 604                         v0.uOp((i, a) -> ($type$) Math.atan(a));
 605                 case VECTOR_OP_EXP: return v0 ->
 606                         v0.uOp((i, a) -> ($type$) Math.exp(a));
 607                 case VECTOR_OP_LOG: return v0 ->
 608                         v0.uOp((i, a) -> ($type$) Math.log(a));
 609                 case VECTOR_OP_LOG10: return v0 ->
 610                         v0.uOp((i, a) -> ($type$) Math.log10(a));
 611                 case VECTOR_OP_SQRT: return v0 ->
 612                         v0.uOp((i, a) -> ($type$) Math.sqrt(a));
 613                 case VECTOR_OP_CBRT: return v0 ->
 614                         v0.uOp((i, a) -> ($type$) Math.cbrt(a));
 615                 case VECTOR_OP_SINH: return v0 ->
 616                         v0.uOp((i, a) -> ($type$) Math.sinh(a));
 617                 case VECTOR_OP_COSH: return v0 ->
 618                         v0.uOp((i, a) -> ($type$) Math.cosh(a));
 619                 case VECTOR_OP_TANH: return v0 ->
 620                         v0.uOp((i, a) -> ($type$) Math.tanh(a));
 621                 case VECTOR_OP_EXPM1: return v0 ->
 622                         v0.uOp((i, a) -> ($type$) Math.expm1(a));
 623                 case VECTOR_OP_LOG1P: return v0 ->
 624                         v0.uOp((i, a) -> ($type$) Math.log1p(a));
 625 #end[FP]
 626                 default: return null;
 627               }}));
 628     }
 629     private static final
 630     ImplCache<Unary,UnaryOperator<$abstractvectortype$>> UN_IMPL
 631         = new ImplCache<>(Unary.class, $Type$Vector.class);
 632 
 633     /**
 634      * {@inheritDoc} <!--workaround-->
 635      */
 636     @ForceInline
 637     public final
 638     $abstractvectortype$ lanewise(VectorOperators.Unary op,
 639                                   VectorMask<$Boxtype$> m) {
 640         return blend(lanewise(op), m);
 641     }
 642 
 643     // Binary lanewise support
 644 
 645     /**
 646      * {@inheritDoc} <!--workaround-->
 647      * @see #lanewise(VectorOperators.Binary,$type$)
 648      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 649      */
 650     @Override
 651     public abstract
 652     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 653                                   Vector<$Boxtype$> v);
 654     @ForceInline
 655     final
 656     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 657                                           Vector<$Boxtype$> v) {
 658         $abstractvectortype$ that = ($abstractvectortype$) v;
 659         that.check(this);
 660         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 661             if (op == FIRST_NONZERO) {
 662                 // FIXME: Support this in the JIT.
 663                 VectorMask<$Boxbitstype$> thisNZ
 664                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 665                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 666                 op = OR_UNCHECKED;
 667 #if[FP]
 668                 // FIXME: Support OR_UNCHECKED on float/double also!
 669                 return this.viewAsIntegralLanes()
 670                     .lanewise(op, that.viewAsIntegralLanes())
 671                     .viewAsFloatingLanes();
 672 #end[FP]
 673             }
 674 #if[BITWISE]
 675 #if[!FP]
 676             if (opKind(op, VO_SHIFT)) {
 677                 // As per shift specification for Java, mask the shift count.
 678                 // This allows the JIT to ignore some ISA details.
 679                 that = that.lanewise(AND, SHIFT_MASK);
 680             }
 681 #end[!FP]
 682             if (op == AND_NOT) {
 683                 // FIXME: Support this in the JIT.
 684                 that = that.lanewise(NOT);
 685                 op = AND;
 686             } else if (op == DIV) {
 687                 VectorMask<$Boxtype$> eqz = that.eq(($type$)0);
 688                 if (eqz.anyTrue()) {
 689                     throw that.divZeroException();
 690                 }
 691             }
 692 #end[BITWISE]
 693         }
 694         int opc = opCode(op);
 695         return VectorSupport.binaryOp(
 696             opc, getClass(), $type$.class, length(),
 697             this, that,
 698             BIN_IMPL.find(op, opc, (opc_) -> {
 699               switch (opc_) {
 700                 case VECTOR_OP_ADD: return (v0, v1) ->
 701                         v0.bOp(v1, (i, a, b) -> ($type$)(a + b));
 702                 case VECTOR_OP_SUB: return (v0, v1) ->
 703                         v0.bOp(v1, (i, a, b) -> ($type$)(a - b));
 704                 case VECTOR_OP_MUL: return (v0, v1) ->
 705                         v0.bOp(v1, (i, a, b) -> ($type$)(a * b));
 706                 case VECTOR_OP_DIV: return (v0, v1) ->
 707                         v0.bOp(v1, (i, a, b) -> ($type$)(a / b));
 708                 case VECTOR_OP_MAX: return (v0, v1) ->
 709                         v0.bOp(v1, (i, a, b) -> ($type$)Math.max(a, b));
 710                 case VECTOR_OP_MIN: return (v0, v1) ->
 711                         v0.bOp(v1, (i, a, b) -> ($type$)Math.min(a, b));
 712 #if[BITWISE]
 713                 case VECTOR_OP_AND: return (v0, v1) ->
 714                         v0.bOp(v1, (i, a, b) -> ($type$)(a & b));
 715                 case VECTOR_OP_OR: return (v0, v1) ->
 716                         v0.bOp(v1, (i, a, b) -> ($type$)(a | b));
 717                 case VECTOR_OP_XOR: return (v0, v1) ->
 718                         v0.bOp(v1, (i, a, b) -> ($type$)(a ^ b));
 719                 case VECTOR_OP_LSHIFT: return (v0, v1) ->
 720                         v0.bOp(v1, (i, a, n) -> ($type$)(a << n));
 721                 case VECTOR_OP_RSHIFT: return (v0, v1) ->
 722                         v0.bOp(v1, (i, a, n) -> ($type$)(a >> n));
 723                 case VECTOR_OP_URSHIFT: return (v0, v1) ->
 724                         v0.bOp(v1, (i, a, n) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 725                 case VECTOR_OP_LROTATE: return (v0, v1) ->
 726                         v0.bOp(v1, (i, a, n) -> rotateLeft(a, (int)n));
 727                 case VECTOR_OP_RROTATE: return (v0, v1) ->
 728                         v0.bOp(v1, (i, a, n) -> rotateRight(a, (int)n));
 729 #end[BITWISE]
 730 #if[FP]
 731                 case VECTOR_OP_ATAN2: return (v0, v1) ->
 732                         v0.bOp(v1, (i, a, b) -> ($type$) Math.atan2(a, b));
 733                 case VECTOR_OP_POW: return (v0, v1) ->
 734                         v0.bOp(v1, (i, a, b) -> ($type$) Math.pow(a, b));
 735                 case VECTOR_OP_HYPOT: return (v0, v1) ->
 736                         v0.bOp(v1, (i, a, b) -> ($type$) Math.hypot(a, b));
 737 #end[FP]
 738                 default: return null;
 739                 }}));
 740     }
 741     private static final
 742     ImplCache<Binary,BinaryOperator<$abstractvectortype$>> BIN_IMPL
 743         = new ImplCache<>(Binary.class, $Type$Vector.class);
 744 
 745     /**
 746      * {@inheritDoc} <!--workaround-->
 747      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 748      */
 749     @ForceInline
 750     public final
 751     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 752                                   Vector<$Boxtype$> v,
 753                                   VectorMask<$Boxtype$> m) {
 754 #if[BITWISE]
 755         $abstractvectortype$ that = ($abstractvectortype$) v;
 756         if (op == DIV) {
 757             VectorMask<$Boxtype$> eqz = that.eq(($type$)0);
 758             if (eqz.and(m).anyTrue()) {
 759                 throw that.divZeroException();
 760             }
 761             // suppress div/0 exceptions in unset lanes
 762             that = that.lanewise(NOT, eqz);
 763             return blend(lanewise(DIV, that), m);
 764         }
 765 #end[BITWISE]
 766         return blend(lanewise(op, v), m);
 767     }
 768     // FIXME: Maybe all of the public final methods in this file (the
 769     // simple ones that just call lanewise) should be pushed down to
 770     // the X-VectorBits template.  They can't optimize properly at
 771     // this level, and must rely on inlining.  Does it work?
 772     // (If it works, of course keep the code here.)
 773 
 774     /**
 775      * Combines the lane values of this vector
 776      * with the value of a broadcast scalar.
 777      *
 778      * This is a lane-wise binary operation which applies
 779      * the selected operation to each lane.
 780      * The return value will be equal to this expression:
 781      * {@code this.lanewise(op, this.broadcast(e))}.
 782      *
 783      * @param op the operation used to process lane values
 784      * @param e the input scalar
 785      * @return the result of applying the operation lane-wise
 786      *         to the two input vectors
 787      * @throws UnsupportedOperationException if this vector does
 788      *         not support the requested operation
 789      * @see #lanewise(VectorOperators.Binary,Vector)
 790      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 791      */
 792     @ForceInline
 793     public final
 794     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 795                                   $type$ e) {
 796 #if[BITWISE]
 797         if (opKind(op, VO_SHIFT) && ($type$)(int)e == e) {
 798             return lanewiseShift(op, (int) e);
 799         }
 800         if (op == AND_NOT) {
 801             op = AND; e = ($type$) ~e;
 802         }
 803 #end[BITWISE]
 804         return lanewise(op, broadcast(e));
 805     }
 806 
 807     /**
 808      * Combines the lane values of this vector
 809      * with the value of a broadcast scalar,
 810      * with selection of lane elements controlled by a mask.
 811      *
 812      * This is a masked lane-wise binary operation which applies
 813      * the selected operation to each lane.
 814      * The return value will be equal to this expression:
 815      * {@code this.lanewise(op, this.broadcast(e), m)}.
 816      *
 817      * @param op the operation used to process lane values
 818      * @param e the input scalar
 819      * @param m the mask controlling lane selection
 820      * @return the result of applying the operation lane-wise
 821      *         to the input vector and the scalar
 822      * @throws UnsupportedOperationException if this vector does
 823      *         not support the requested operation
 824      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
 825      * @see #lanewise(VectorOperators.Binary,$type$)
 826      */
 827     @ForceInline
 828     public final
 829     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 830                                   $type$ e,
 831                                   VectorMask<$Boxtype$> m) {
 832         return blend(lanewise(op, e), m);
 833     }
 834 
 835 #if[!long]
 836     /**
 837      * {@inheritDoc} <!--workaround-->
 838      * @apiNote
 839      * When working with vector subtypes like {@code $abstractvectortype$},
 840      * {@linkplain #lanewise(VectorOperators.Binary,$type$)
 841      * the more strongly typed method}
 842      * is typically selected.  It can be explicitly selected
 843      * using a cast: {@code v.lanewise(op,($type$)e)}.
 844      * The two expressions will produce numerically identical results.
 845      */
 846     @ForceInline
 847     public final
 848     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 849                                   long e) {
 850         $type$ e1 = ($type$) e;
 851         if ((long)e1 != e
 852 #if[BITWISE]
 853             // allow shift ops to clip down their int parameters
 854             && !(opKind(op, VO_SHIFT) && (int)e1 == e)
 855 #end[BITWISE]
 856             ) {
 857             vspecies().checkValue(e);  // for exception
 858         }
 859         return lanewise(op, e1);
 860     }
 861 
 862     /**
 863      * {@inheritDoc} <!--workaround-->
 864      * @apiNote
 865      * When working with vector subtypes like {@code $abstractvectortype$},
 866      * {@linkplain #lanewise(VectorOperators.Binary,$type$,VectorMask)
 867      * the more strongly typed method}
 868      * is typically selected.  It can be explicitly selected
 869      * using a cast: {@code v.lanewise(op,($type$)e,m)}.
 870      * The two expressions will produce numerically identical results.
 871      */
 872     @ForceInline
 873     public final
 874     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 875                                   long e, VectorMask<$Boxtype$> m) {
 876         return blend(lanewise(op, e), m);
 877     }
 878 #end[!long]
 879 
 880 #if[BITWISE]
 881     /*package-private*/
 882     abstract $abstractvectortype$
 883     lanewiseShift(VectorOperators.Binary op, int e);
 884 
 885     /*package-private*/
 886     @ForceInline
 887     final $abstractvectortype$
 888     lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
 889         // Special handling for these.  FIXME: Refactor?
 890         assert(opKind(op, VO_SHIFT));
 891         // As per shift specification for Java, mask the shift count.
 892         e &= SHIFT_MASK;
 893         int opc = opCode(op);
 894         return VectorSupport.broadcastInt(
 895             opc, getClass(), $type$.class, length(),
 896             this, e,
 897             BIN_INT_IMPL.find(op, opc, (opc_) -> {
 898               switch (opc_) {
 899                 case VECTOR_OP_LSHIFT: return (v, n) ->
 900                         v.uOp((i, a) -> ($type$)(a << n));
 901                 case VECTOR_OP_RSHIFT: return (v, n) ->
 902                         v.uOp((i, a) -> ($type$)(a >> n));
 903                 case VECTOR_OP_URSHIFT: return (v, n) ->
 904                         v.uOp((i, a) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 905                 case VECTOR_OP_LROTATE: return (v, n) ->
 906                         v.uOp((i, a) -> rotateLeft(a, (int)n));
 907                 case VECTOR_OP_RROTATE: return (v, n) ->
 908                         v.uOp((i, a) -> rotateRight(a, (int)n));
 909                 default: return null;
 910                 }}));
 911     }
 912     private static final
 913     ImplCache<Binary,VectorBroadcastIntOp<$abstractvectortype$>> BIN_INT_IMPL
 914         = new ImplCache<>(Binary.class, $Type$Vector.class);
 915 
 916     // As per shift specification for Java, mask the shift count.
 917     // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
 918     // The latter two maskings go beyond the JLS, but seem reasonable
 919     // since our lane types are first-class types, not just dressed
 920     // up ints.
 921     private static final int SHIFT_MASK = ($Boxtype$.SIZE - 1);
 922 #if[byteOrShort]
 923     // Also simulate >>> on sub-word variables with a mask.
 924     private static final int LSHR_SETUP_MASK = ((1 << $Boxtype$.SIZE) - 1);
 925 #else[byteOrShort]
 926     private static final $type$ LSHR_SETUP_MASK = -1;
 927 #end[byteOrShort]
 928 #end[BITWISE]
 929 
 930     // Ternary lanewise support
 931 
 932     // Ternary operators come in eight variations:
 933     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
 934     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
 935 
 936     // It is annoying to support all of these variations of masking
 937     // and broadcast, but it would be more surprising not to continue
 938     // the obvious pattern started by unary and binary.
 939 
 940    /**
 941      * {@inheritDoc} <!--workaround-->
 942      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
 943      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
 944      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
 945      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
 946      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
 947      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
 948      */
 949     @Override
 950     public abstract
 951     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
 952                                                   Vector<$Boxtype$> v1,
 953                                                   Vector<$Boxtype$> v2);
 954     @ForceInline
 955     final
 956     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
 957                                           Vector<$Boxtype$> v1,
 958                                           Vector<$Boxtype$> v2) {
 959         $abstractvectortype$ that = ($abstractvectortype$) v1;
 960         $abstractvectortype$ tother = ($abstractvectortype$) v2;
 961         // It's a word: https://www.dictionary.com/browse/tother
 962         // See also Chapter 11 of Dickens, Our Mutual Friend:
 963         // "Totherest Governor," replied Mr Riderhood...
 964         that.check(this);
 965         tother.check(this);
 966 #if[BITWISE]
 967         if (op == BITWISE_BLEND) {
 968             // FIXME: Support this in the JIT.
 969             that = this.lanewise(XOR, that).lanewise(AND, tother);
 970             return this.lanewise(XOR, that);
 971         }
 972 #end[BITWISE]
 973         int opc = opCode(op);
 974         return VectorSupport.ternaryOp(
 975             opc, getClass(), $type$.class, length(),
 976             this, that, tother,
 977             TERN_IMPL.find(op, opc, (opc_) -> {
 978               switch (opc_) {
 979 #if[FP]
 980                 case VECTOR_OP_FMA: return (v0, v1_, v2_) ->
 981                         v0.tOp(v1_, v2_, (i, a, b, c) -> Math.fma(a, b, c));
 982 #end[FP]
 983                 default: return null;
 984                 }}));
 985     }
 986     private static final
 987     ImplCache<Ternary,TernaryOperation<$abstractvectortype$>> TERN_IMPL
 988         = new ImplCache<>(Ternary.class, $Type$Vector.class);
 989 
 990     /**
 991      * {@inheritDoc} <!--workaround-->
 992      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
 993      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
 994      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
 995      */
 996     @ForceInline
 997     public final
 998     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
 999                                   Vector<$Boxtype$> v1,
1000                                   Vector<$Boxtype$> v2,
1001                                   VectorMask<$Boxtype$> m) {
1002         return blend(lanewise(op, v1, v2), m);
1003     }
1004 
1005     /**
1006      * Combines the lane values of this vector
1007      * with the values of two broadcast scalars.
1008      *
1009      * This is a lane-wise ternary operation which applies
1010      * the selected operation to each lane.
1011      * The return value will be equal to this expression:
1012      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
1013      *
1014      * @param op the operation used to combine lane values
1015      * @param e1 the first input scalar
1016      * @param e2 the second input scalar
1017      * @return the result of applying the operation lane-wise
1018      *         to the input vector and the scalars
1019      * @throws UnsupportedOperationException if this vector does
1020      *         not support the requested operation
1021      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1022      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1023      */
1024     @ForceInline
1025     public final
1026     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2)
1027                                   $type$ e1,
1028                                   $type$ e2) {
1029         return lanewise(op, broadcast(e1), broadcast(e2));
1030     }
1031 
1032     /**
1033      * Combines the lane values of this vector
1034      * with the values of two broadcast scalars,
1035      * with selection of lane elements controlled by a mask.
1036      *
1037      * This is a masked lane-wise ternary operation which applies
1038      * the selected operation to each lane.
1039      * The return value will be equal to this expression:
1040      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
1041      *
1042      * @param op the operation used to combine lane values
1043      * @param e1 the first input scalar
1044      * @param e2 the second input scalar
1045      * @param m the mask controlling lane selection
1046      * @return the result of applying the operation lane-wise
1047      *         to the input vector and the scalars
1048      * @throws UnsupportedOperationException if this vector does
1049      *         not support the requested operation
1050      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1051      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1052      */
1053     @ForceInline
1054     public final
1055     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
1056                                   $type$ e1,
1057                                   $type$ e2,
1058                                   VectorMask<$Boxtype$> m) {
1059         return blend(lanewise(op, e1, e2), m);
1060     }
1061 
1062     /**
1063      * Combines the lane values of this vector
1064      * with the values of another vector and a broadcast scalar.
1065      *
1066      * This is a lane-wise ternary operation which applies
1067      * the selected operation to each lane.
1068      * The return value will be equal to this expression:
1069      * {@code this.lanewise(op, v1, this.broadcast(e2))}.
1070      *
1071      * @param op the operation used to combine lane values
1072      * @param v1 the other input vector
1073      * @param e2 the input scalar
1074      * @return the result of applying the operation lane-wise
1075      *         to the input vectors and the scalar
1076      * @throws UnsupportedOperationException if this vector does
1077      *         not support the requested operation
1078      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1079      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1080      */
1081     @ForceInline
1082     public final
1083     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2)
1084                                   Vector<$Boxtype$> v1,
1085                                   $type$ e2) {
1086         return lanewise(op, v1, broadcast(e2));
1087     }
1088 
1089     /**
1090      * Combines the lane values of this vector
1091      * with the values of another vector and a broadcast scalar,
1092      * with selection of lane elements controlled by a mask.
1093      *
1094      * This is a masked lane-wise ternary operation which applies
1095      * the selected operation to each lane.
1096      * The return value will be equal to this expression:
1097      * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
1098      *
1099      * @param op the operation used to combine lane values
1100      * @param v1 the other input vector
1101      * @param e2 the input scalar
1102      * @param m the mask controlling lane selection
1103      * @return the result of applying the operation lane-wise
1104      *         to the input vectors and the scalar
1105      * @throws UnsupportedOperationException if this vector does
1106      *         not support the requested operation
1107      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1108      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1109      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
1110      */
1111     @ForceInline
1112     public final
1113     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
1114                                   Vector<$Boxtype$> v1,
1115                                   $type$ e2,
1116                                   VectorMask<$Boxtype$> m) {
1117         return blend(lanewise(op, v1, e2), m);
1118     }
1119 
1120     /**
1121      * Combines the lane values of this vector
1122      * with the values of another vector and a broadcast scalar.
1123      *
1124      * This is a lane-wise ternary operation which applies
1125      * the selected operation to each lane.
1126      * The return value will be equal to this expression:
1127      * {@code this.lanewise(op, this.broadcast(e1), v2)}.
1128      *
1129      * @param op the operation used to combine lane values
1130      * @param e1 the input scalar
1131      * @param v2 the other input vector
1132      * @return the result of applying the operation lane-wise
1133      *         to the input vectors and the scalar
1134      * @throws UnsupportedOperationException if this vector does
1135      *         not support the requested operation
1136      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1137      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1138      */
1139     @ForceInline
1140     public final
1141     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2)
1142                                   $type$ e1,
1143                                   Vector<$Boxtype$> v2) {
1144         return lanewise(op, broadcast(e1), v2);
1145     }
1146 
1147     /**
1148      * Combines the lane values of this vector
1149      * with the values of another vector and a broadcast scalar,
1150      * with selection of lane elements controlled by a mask.
1151      *
1152      * This is a masked lane-wise ternary operation which applies
1153      * the selected operation to each lane.
1154      * The return value will be equal to this expression:
1155      * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
1156      *
1157      * @param op the operation used to combine lane values
1158      * @param e1 the input scalar
1159      * @param v2 the other input vector
1160      * @param m the mask controlling lane selection
1161      * @return the result of applying the operation lane-wise
1162      *         to the input vectors and the scalar
1163      * @throws UnsupportedOperationException if this vector does
1164      *         not support the requested operation
1165      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1166      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
1167      */
1168     @ForceInline
1169     public final
1170     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
1171                                   $type$ e1,
1172                                   Vector<$Boxtype$> v2,
1173                                   VectorMask<$Boxtype$> m) {
1174         return blend(lanewise(op, e1, v2), m);
1175     }
1176 
1177     // (Thus endeth the Great and Mighty Ternary Ogdoad.)
1178     // https://en.wikipedia.org/wiki/Ogdoad
1179 
1180     /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1181     //
1182     // These include masked and non-masked versions.
1183     // This subclass adds broadcast (masked or not).
1184 
1185     /**
1186      * {@inheritDoc} <!--workaround-->
1187      * @see #add($type$)
1188      */
1189     @Override
1190     @ForceInline
1191     public final $abstractvectortype$ add(Vector<$Boxtype$> v) {
1192         return lanewise(ADD, v);
1193     }
1194 
1195     /**
1196      * Adds this vector to the broadcast of an input scalar.
1197      *
1198      * This is a lane-wise binary operation which applies
1199      * the primitive addition operation ({@code +}) to each lane.
1200      *
1201      * This method is also equivalent to the expression
1202      * {@link #lanewise(VectorOperators.Binary,$type$)
1203      *    lanewise}{@code (}{@link VectorOperators#ADD
1204      *    ADD}{@code , e)}.
1205      *
1206      * @param e the input scalar
1207      * @return the result of adding each lane of this vector to the scalar
1208      * @see #add(Vector)
1209      * @see #broadcast($type$)
1210      * @see #add($type$,VectorMask)
1211      * @see VectorOperators#ADD
1212      * @see #lanewise(VectorOperators.Binary,Vector)
1213      * @see #lanewise(VectorOperators.Binary,$type$)
1214      */
1215     @ForceInline
1216     public final
1217     $abstractvectortype$ add($type$ e) {
1218         return lanewise(ADD, e);
1219     }
1220 
1221     /**
1222      * {@inheritDoc} <!--workaround-->
1223      * @see #add($type$,VectorMask)
1224      */
1225     @Override
1226     @ForceInline
1227     public final $abstractvectortype$ add(Vector<$Boxtype$> v,
1228                                           VectorMask<$Boxtype$> m) {
1229         return lanewise(ADD, v, m);
1230     }
1231 
1232     /**
1233      * Adds this vector to the broadcast of an input scalar,
1234      * selecting lane elements controlled by a mask.
1235      *
1236      * This is a masked lane-wise binary operation which applies
1237      * the primitive addition operation ({@code +}) to each lane.
1238      *
1239      * This method is also equivalent to the expression
1240      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1241      *    lanewise}{@code (}{@link VectorOperators#ADD
1242      *    ADD}{@code , s, m)}.
1243      *
1244      * @param e the input scalar
1245      * @param m the mask controlling lane selection
1246      * @return the result of adding each lane of this vector to the scalar
1247      * @see #add(Vector,VectorMask)
1248      * @see #broadcast($type$)
1249      * @see #add($type$)
1250      * @see VectorOperators#ADD
1251      * @see #lanewise(VectorOperators.Binary,Vector)
1252      * @see #lanewise(VectorOperators.Binary,$type$)
1253      */
1254     @ForceInline
1255     public final $abstractvectortype$ add($type$ e,
1256                                           VectorMask<$Boxtype$> m) {
1257         return lanewise(ADD, e, m);
1258     }
1259 
1260     /**
1261      * {@inheritDoc} <!--workaround-->
1262      * @see #sub($type$)
1263      */
1264     @Override
1265     @ForceInline
1266     public final $abstractvectortype$ sub(Vector<$Boxtype$> v) {
1267         return lanewise(SUB, v);
1268     }
1269 
1270     /**
1271      * Subtracts an input scalar from this vector.
1272      *
1273      * This is a masked lane-wise binary operation which applies
1274      * the primitive subtraction operation ({@code -}) to each lane.
1275      *
1276      * This method is also equivalent to the expression
1277      * {@link #lanewise(VectorOperators.Binary,$type$)
1278      *    lanewise}{@code (}{@link VectorOperators#SUB
1279      *    SUB}{@code , e)}.
1280      *
1281      * @param e the input scalar
1282      * @return the result of subtracting the scalar from each lane of this vector
1283      * @see #sub(Vector)
1284      * @see #broadcast($type$)
1285      * @see #sub($type$,VectorMask)
1286      * @see VectorOperators#SUB
1287      * @see #lanewise(VectorOperators.Binary,Vector)
1288      * @see #lanewise(VectorOperators.Binary,$type$)
1289      */
1290     @ForceInline
1291     public final $abstractvectortype$ sub($type$ e) {
1292         return lanewise(SUB, e);
1293     }
1294 
1295     /**
1296      * {@inheritDoc} <!--workaround-->
1297      * @see #sub($type$,VectorMask)
1298      */
1299     @Override
1300     @ForceInline
1301     public final $abstractvectortype$ sub(Vector<$Boxtype$> v,
1302                                           VectorMask<$Boxtype$> m) {
1303         return lanewise(SUB, v, m);
1304     }
1305 
1306     /**
1307      * Subtracts an input scalar from this vector
1308      * under the control of a mask.
1309      *
1310      * This is a masked lane-wise binary operation which applies
1311      * the primitive subtraction operation ({@code -}) to each lane.
1312      *
1313      * This method is also equivalent to the expression
1314      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1315      *    lanewise}{@code (}{@link VectorOperators#SUB
1316      *    SUB}{@code , s, m)}.
1317      *
1318      * @param e the input scalar
1319      * @param m the mask controlling lane selection
1320      * @return the result of subtracting the scalar from each lane of this vector
1321      * @see #sub(Vector,VectorMask)
1322      * @see #broadcast($type$)
1323      * @see #sub($type$)
1324      * @see VectorOperators#SUB
1325      * @see #lanewise(VectorOperators.Binary,Vector)
1326      * @see #lanewise(VectorOperators.Binary,$type$)
1327      */
1328     @ForceInline
1329     public final $abstractvectortype$ sub($type$ e,
1330                                           VectorMask<$Boxtype$> m) {
1331         return lanewise(SUB, e, m);
1332     }
1333 
1334     /**
1335      * {@inheritDoc} <!--workaround-->
1336      * @see #mul($type$)
1337      */
1338     @Override
1339     @ForceInline
1340     public final $abstractvectortype$ mul(Vector<$Boxtype$> v) {
1341         return lanewise(MUL, v);
1342     }
1343 
1344     /**
1345      * Multiplies this vector by the broadcast of an input scalar.
1346      *
1347      * This is a lane-wise binary operation which applies
1348      * the primitive multiplication operation ({@code *}) to each lane.
1349      *
1350      * This method is also equivalent to the expression
1351      * {@link #lanewise(VectorOperators.Binary,$type$)
1352      *    lanewise}{@code (}{@link VectorOperators#MUL
1353      *    MUL}{@code , e)}.
1354      *
1355      * @param e the input scalar
1356      * @return the result of multiplying this vector by the given scalar
1357      * @see #mul(Vector)
1358      * @see #broadcast($type$)
1359      * @see #mul($type$,VectorMask)
1360      * @see VectorOperators#MUL
1361      * @see #lanewise(VectorOperators.Binary,Vector)
1362      * @see #lanewise(VectorOperators.Binary,$type$)
1363      */
1364     @ForceInline
1365     public final $abstractvectortype$ mul($type$ e) {
1366         return lanewise(MUL, e);
1367     }
1368 
1369     /**
1370      * {@inheritDoc} <!--workaround-->
1371      * @see #mul($type$,VectorMask)
1372      */
1373     @Override
1374     @ForceInline
1375     public final $abstractvectortype$ mul(Vector<$Boxtype$> v,
1376                                           VectorMask<$Boxtype$> m) {
1377         return lanewise(MUL, v, m);
1378     }
1379 
1380     /**
1381      * Multiplies this vector by the broadcast of an input scalar,
1382      * selecting lane elements controlled by a mask.
1383      *
1384      * This is a masked lane-wise binary operation which applies
1385      * the primitive multiplication operation ({@code *}) to each lane.
1386      *
1387      * This method is also equivalent to the expression
1388      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1389      *    lanewise}{@code (}{@link VectorOperators#MUL
1390      *    MUL}{@code , s, m)}.
1391      *
1392      * @param e the input scalar
1393      * @param m the mask controlling lane selection
1394      * @return the result of muling each lane of this vector to the scalar
1395      * @see #mul(Vector,VectorMask)
1396      * @see #broadcast($type$)
1397      * @see #mul($type$)
1398      * @see VectorOperators#MUL
1399      * @see #lanewise(VectorOperators.Binary,Vector)
1400      * @see #lanewise(VectorOperators.Binary,$type$)
1401      */
1402     @ForceInline
1403     public final $abstractvectortype$ mul($type$ e,
1404                                           VectorMask<$Boxtype$> m) {
1405         return lanewise(MUL, e, m);
1406     }
1407 
1408     /**
1409      * {@inheritDoc} <!--workaround-->
1410 #if[FP]
1411      * @apiNote Because the underlying scalar operator is an IEEE
1412      * floating point number, division by zero in fact will
1413      * not throw an exception, but will yield a signed
1414      * infinity or NaN.
1415 #else[FP]
1416      * @apiNote If there is a zero divisor, {@code
1417      * ArithmeticException} will be thrown.
1418 #end[FP]
1419      */
1420     @Override
1421     @ForceInline
1422     public final $abstractvectortype$ div(Vector<$Boxtype$> v) {
1423         return lanewise(DIV, v);
1424     }
1425 
1426     /**
1427      * Divides this vector by the broadcast of an input scalar.
1428      *
1429      * This is a lane-wise binary operation which applies
1430      * the primitive division operation ({@code /}) to each lane.
1431      *
1432      * This method is also equivalent to the expression
1433      * {@link #lanewise(VectorOperators.Binary,$type$)
1434      *    lanewise}{@code (}{@link VectorOperators#DIV
1435      *    DIV}{@code , e)}.
1436      *
1437 #if[FP]
1438      * @apiNote Because the underlying scalar operator is an IEEE
1439      * floating point number, division by zero in fact will
1440      * not throw an exception, but will yield a signed
1441      * infinity or NaN.
1442 #else[FP]
1443      * @apiNote If there is a zero divisor, {@code
1444      * ArithmeticException} will be thrown.
1445 #end[FP]
1446      *
1447      * @param e the input scalar
1448      * @return the result of dividing each lane of this vector by the scalar
1449      * @see #div(Vector)
1450      * @see #broadcast($type$)
1451      * @see #div($type$,VectorMask)
1452      * @see VectorOperators#DIV
1453      * @see #lanewise(VectorOperators.Binary,Vector)
1454      * @see #lanewise(VectorOperators.Binary,$type$)
1455      */
1456     @ForceInline
1457     public final $abstractvectortype$ div($type$ e) {
1458         return lanewise(DIV, e);
1459     }
1460 
1461     /**
1462      * {@inheritDoc} <!--workaround-->
1463      * @see #div($type$,VectorMask)
1464 #if[FP]
1465      * @apiNote Because the underlying scalar operator is an IEEE
1466      * floating point number, division by zero in fact will
1467      * not throw an exception, but will yield a signed
1468      * infinity or NaN.
1469 #else[FP]
1470      * @apiNote If there is a zero divisor, {@code
1471      * ArithmeticException} will be thrown.
1472 #end[FP]
1473      */
1474     @Override
1475     @ForceInline
1476     public final $abstractvectortype$ div(Vector<$Boxtype$> v,
1477                                           VectorMask<$Boxtype$> m) {
1478         return lanewise(DIV, v, m);
1479     }
1480 
1481     /**
1482      * Divides this vector by the broadcast of an input scalar,
1483      * selecting lane elements controlled by a mask.
1484      *
1485      * This is a masked lane-wise binary operation which applies
1486      * the primitive division operation ({@code /}) to each lane.
1487      *
1488      * This method is also equivalent to the expression
1489      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1490      *    lanewise}{@code (}{@link VectorOperators#DIV
1491      *    DIV}{@code , s, m)}.
1492      *
1493 #if[FP]
1494      * @apiNote Because the underlying scalar operator is an IEEE
1495      * floating point number, division by zero in fact will
1496      * not throw an exception, but will yield a signed
1497      * infinity or NaN.
1498 #else[FP]
1499      * @apiNote If there is a zero divisor, {@code
1500      * ArithmeticException} will be thrown.
1501 #end[FP]
1502      *
1503      * @param e the input scalar
1504      * @param m the mask controlling lane selection
1505      * @return the result of dividing each lane of this vector by the scalar
1506      * @see #div(Vector,VectorMask)
1507      * @see #broadcast($type$)
1508      * @see #div($type$)
1509      * @see VectorOperators#DIV
1510      * @see #lanewise(VectorOperators.Binary,Vector)
1511      * @see #lanewise(VectorOperators.Binary,$type$)
1512      */
1513     @ForceInline
1514     public final $abstractvectortype$ div($type$ e,
1515                                           VectorMask<$Boxtype$> m) {
1516         return lanewise(DIV, e, m);
1517     }
1518 
1519     /// END OF FULL-SERVICE BINARY METHODS
1520 
1521     /// SECOND-TIER BINARY METHODS
1522     //
1523     // There are no masked versions.
1524 
1525     /**
1526      * {@inheritDoc} <!--workaround-->
1527 #if[FP]
1528      * @apiNote
1529      * For this method, floating point negative
1530      * zero {@code -0.0} is treated as a value distinct from, and less
1531      * than the default value (positive zero).
1532 #end[FP]
1533      */
1534     @Override
1535     @ForceInline
1536     public final $abstractvectortype$ min(Vector<$Boxtype$> v) {
1537         return lanewise(MIN, v);
1538     }
1539 
1540     // FIXME:  "broadcast of an input scalar" is really wordy.  Reduce?
1541     /**
1542      * Computes the smaller of this vector and the broadcast of an input scalar.
1543      *
1544      * This is a lane-wise binary operation which applies the
1545      * operation {@code Math.min()} to each pair of
1546      * corresponding lane values.
1547      *
1548      * This method is also equivalent to the expression
1549      * {@link #lanewise(VectorOperators.Binary,$type$)
1550      *    lanewise}{@code (}{@link VectorOperators#MIN
1551      *    MIN}{@code , e)}.
1552      *
1553      * @param e the input scalar
1554      * @return the result of multiplying this vector by the given scalar
1555      * @see #min(Vector)
1556      * @see #broadcast($type$)
1557      * @see VectorOperators#MIN
1558      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1559 #if[FP]
1560      * @apiNote
1561      * For this method, floating point negative
1562      * zero {@code -0.0} is treated as a value distinct from, and less
1563      * than the default value (positive zero).
1564 #end[FP]
1565      */
1566     @ForceInline
1567     public final $abstractvectortype$ min($type$ e) {
1568         return lanewise(MIN, e);
1569     }
1570 
1571     /**
1572      * {@inheritDoc} <!--workaround-->
1573 #if[FP]
1574      * @apiNote
1575      * For this method, floating point negative
1576      * zero {@code -0.0} is treated as a value distinct from, and less
1577      * than the default value (positive zero).
1578 #end[FP]
1579      */
1580     @Override
1581     @ForceInline
1582     public final $abstractvectortype$ max(Vector<$Boxtype$> v) {
1583         return lanewise(MAX, v);
1584     }
1585 
1586     /**
1587      * Computes the larger of this vector and the broadcast of an input scalar.
1588      *
1589      * This is a lane-wise binary operation which applies the
1590      * operation {@code Math.max()} to each pair of
1591      * corresponding lane values.
1592      *
1593      * This method is also equivalent to the expression
1594      * {@link #lanewise(VectorOperators.Binary,$type$)
1595      *    lanewise}{@code (}{@link VectorOperators#MAX
1596      *    MAX}{@code , e)}.
1597      *
1598      * @param e the input scalar
1599      * @return the result of multiplying this vector by the given scalar
1600      * @see #max(Vector)
1601      * @see #broadcast($type$)
1602      * @see VectorOperators#MAX
1603      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1604 #if[FP]
1605      * @apiNote
1606      * For this method, floating point negative
1607      * zero {@code -0.0} is treated as a value distinct from, and less
1608      * than the default value (positive zero).
1609 #end[FP]
1610      */
1611     @ForceInline
1612     public final $abstractvectortype$ max($type$ e) {
1613         return lanewise(MAX, e);
1614     }
1615 
1616 #if[BITWISE]
1617     // common bitwise operators: and, or, not (with scalar versions)
1618     /**
1619      * Computes the bitwise logical conjunction ({@code &})
1620      * of this vector and a second input vector.
1621      *
1622      * This is a lane-wise binary operation which applies the
1623      * the primitive bitwise "and" operation ({@code &})
1624      * to each pair of corresponding lane values.
1625      *
1626      * This method is also equivalent to the expression
1627      * {@link #lanewise(VectorOperators.Binary,Vector)
1628      *    lanewise}{@code (}{@link VectorOperators#AND
1629      *    AND}{@code , v)}.
1630      *
1631      * <p>
1632      * This is not a full-service named operation like
1633      * {@link #add(Vector) add}.  A masked version of
1634      * this operation is not directly available
1635      * but may be obtained via the masked version of
1636      * {@code lanewise}.
1637      *
1638      * @param v a second input vector
1639      * @return the bitwise {@code &} of this vector and the second input vector
1640      * @see #and($type$)
1641      * @see #or(Vector)
1642      * @see #not()
1643      * @see VectorOperators#AND
1644      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1645      */
1646     @ForceInline
1647     public final $abstractvectortype$ and(Vector<$Boxtype$> v) {
1648         return lanewise(AND, v);
1649     }
1650 
1651     /**
1652      * Computes the bitwise logical conjunction ({@code &})
1653      * of this vector and a scalar.
1654      *
1655      * This is a lane-wise binary operation which applies the
1656      * the primitive bitwise "and" operation ({@code &})
1657      * to each pair of corresponding lane values.
1658      *
1659      * This method is also equivalent to the expression
1660      * {@link #lanewise(VectorOperators.Binary,Vector)
1661      *    lanewise}{@code (}{@link VectorOperators#AND
1662      *    AND}{@code , e)}.
1663      *
1664      * @param e an input scalar
1665      * @return the bitwise {@code &} of this vector and scalar
1666      * @see #and(Vector)
1667      * @see VectorOperators#AND
1668      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1669      */
1670     @ForceInline
1671     public final $abstractvectortype$ and($type$ e) {
1672         return lanewise(AND, e);
1673     }
1674 
1675     /**
1676      * Computes the bitwise logical disjunction ({@code |})
1677      * of this vector and a second input vector.
1678      *
1679      * This is a lane-wise binary operation which applies the
1680      * the primitive bitwise "or" operation ({@code |})
1681      * to each pair of corresponding lane values.
1682      *
1683      * This method is also equivalent to the expression
1684      * {@link #lanewise(VectorOperators.Binary,Vector)
1685      *    lanewise}{@code (}{@link VectorOperators#OR
1686      *    AND}{@code , v)}.
1687      *
1688      * <p>
1689      * This is not a full-service named operation like
1690      * {@link #add(Vector) add}.  A masked version of
1691      * this operation is not directly available
1692      * but may be obtained via the masked version of
1693      * {@code lanewise}.
1694      *
1695      * @param v a second input vector
1696      * @return the bitwise {@code |} of this vector and the second input vector
1697      * @see #or($type$)
1698      * @see #and(Vector)
1699      * @see #not()
1700      * @see VectorOperators#OR
1701      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1702      */
1703     @ForceInline
1704     public final $abstractvectortype$ or(Vector<$Boxtype$> v) {
1705         return lanewise(OR, v);
1706     }
1707 
1708     /**
1709      * Computes the bitwise logical disjunction ({@code |})
1710      * of this vector and a scalar.
1711      *
1712      * This is a lane-wise binary operation which applies the
1713      * the primitive bitwise "or" operation ({@code |})
1714      * to each pair of corresponding lane values.
1715      *
1716      * This method is also equivalent to the expression
1717      * {@link #lanewise(VectorOperators.Binary,Vector)
1718      *    lanewise}{@code (}{@link VectorOperators#OR
1719      *    OR}{@code , e)}.
1720      *
1721      * @param e an input scalar
1722      * @return the bitwise {@code |} of this vector and scalar
1723      * @see #or(Vector)
1724      * @see VectorOperators#OR
1725      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1726      */
1727     @ForceInline
1728     public final $abstractvectortype$ or($type$ e) {
1729         return lanewise(OR, e);
1730     }
1731 
1732 #end[BITWISE]
1733 
1734 #if[FP]
1735     // common FP operator: pow
1736     /**
1737      * Raises this vector to the power of a second input vector.
1738      *
1739      * This is a lane-wise binary operation which applies an operation
1740      * conforming to the specification of
1741      * {@link Math#pow Math.pow(a,b)}
1742      * to each pair of corresponding lane values.
1743 #if[intOrFloat]
1744      * The operation is adapted to cast the operands and the result,
1745      * specifically widening {@code float} operands to {@code double}
1746      * operands and narrowing the {@code double} result to a {@code float}
1747      * result.
1748 #end[intOrFloat]
1749      *
1750      * This method is also equivalent to the expression
1751      * {@link #lanewise(VectorOperators.Binary,Vector)
1752      *    lanewise}{@code (}{@link VectorOperators#POW
1753      *    POW}{@code , b)}.
1754      *
1755      * <p>
1756      * This is not a full-service named operation like
1757      * {@link #add(Vector) add}.  A masked version of
1758      * this operation is not directly available
1759      * but may be obtained via the masked version of
1760      * {@code lanewise}.
1761      *
1762      * @param b a vector exponent by which to raise this vector
1763      * @return the {@code b}-th power of this vector
1764      * @see #pow($type$)
1765      * @see VectorOperators#POW
1766      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1767      */
1768     @ForceInline
1769     public final $abstractvectortype$ pow(Vector<$Boxtype$> b) {
1770         return lanewise(POW, b);
1771     }
1772 
1773     /**
1774      * Raises this vector to a scalar power.
1775      *
1776      * This is a lane-wise binary operation which applies an operation
1777      * conforming to the specification of
1778      * {@link Math#pow Math.pow(a,b)}
1779      * to each pair of corresponding lane values.
1780 #if[intOrFloat]
1781      * The operation is adapted to cast the operands and the result,
1782      * specifically widening {@code float} operands to {@code double}
1783      * operands and narrowing the {@code double} result to a {@code float}
1784      * result.
1785 #end[intOrFloat]
1786      *
1787      * This method is also equivalent to the expression
1788      * {@link #lanewise(VectorOperators.Binary,Vector)
1789      *    lanewise}{@code (}{@link VectorOperators#POW
1790      *    POW}{@code , b)}.
1791      *
1792      * @param b a scalar exponent by which to raise this vector
1793      * @return the {@code b}-th power of this vector
1794      * @see #pow(Vector)
1795      * @see VectorOperators#POW
1796      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1797      */
1798     @ForceInline
1799     public final $abstractvectortype$ pow($type$ b) {
1800         return lanewise(POW, b);
1801     }
1802 #end[FP]
1803 
1804     /// UNARY METHODS
1805 
1806     /**
1807      * {@inheritDoc} <!--workaround-->
1808      */
1809     @Override
1810     @ForceInline
1811     public final
1812     $abstractvectortype$ neg() {
1813         return lanewise(NEG);
1814     }
1815 
1816     /**
1817      * {@inheritDoc} <!--workaround-->
1818      */
1819     @Override
1820     @ForceInline
1821     public final
1822     $abstractvectortype$ abs() {
1823         return lanewise(ABS);
1824     }
1825 
1826 #if[BITWISE]
1827     // not (~)
1828     /**
1829      * Computes the bitwise logical complement ({@code ~})
1830      * of this vector.
1831      *
1832      * This is a lane-wise binary operation which applies the
1833      * the primitive bitwise "not" operation ({@code ~})
1834      * to each lane value.
1835      *
1836      * This method is also equivalent to the expression
1837      * {@link #lanewise(VectorOperators.Unary)
1838      *    lanewise}{@code (}{@link VectorOperators#NOT
1839      *    NOT}{@code )}.
1840      *
1841      * <p>
1842      * This is not a full-service named operation like
1843      * {@link #add(Vector) add}.  A masked version of
1844      * this operation is not directly available
1845      * but may be obtained via the masked version of
1846      * {@code lanewise}.
1847      *
1848      * @return the bitwise complement {@code ~} of this vector
1849      * @see #and(Vector)
1850      * @see VectorOperators#NOT
1851      * @see #lanewise(VectorOperators.Unary,VectorMask)
1852      */
1853     @ForceInline
1854     public final $abstractvectortype$ not() {
1855         return lanewise(NOT);
1856     }
1857 #end[BITWISE]
1858 
1859 #if[FP]
1860     // sqrt
1861     /**
1862      * Computes the square root of this vector.
1863      *
1864      * This is a lane-wise unary operation which applies an operation
1865      * conforming to the specification of
1866      * {@link Math#sqrt Math.sqrt(a)}
1867      * to each lane value.
1868 #if[intOrFloat]
1869      * The operation is adapted to cast the operand and the result,
1870      * specifically widening the {@code float} operand to a {@code double}
1871      * operand and narrowing the {@code double} result to a {@code float}
1872      * result.
1873 #end[intOrFloat]
1874      *
1875      * This method is also equivalent to the expression
1876      * {@link #lanewise(VectorOperators.Unary)
1877      *    lanewise}{@code (}{@link VectorOperators#SQRT
1878      *    SQRT}{@code )}.
1879      *
1880      * @return the square root of this vector
1881      * @see VectorOperators#SQRT
1882      * @see #lanewise(VectorOperators.Unary,VectorMask)
1883      */
1884     @ForceInline
1885     public final $abstractvectortype$ sqrt() {
1886         return lanewise(SQRT);
1887     }
1888 #end[FP]
1889 
1890     /// COMPARISONS
1891 
1892     /**
1893      * {@inheritDoc} <!--workaround-->
1894      */
1895     @Override
1896     @ForceInline
1897     public final
1898     VectorMask<$Boxtype$> eq(Vector<$Boxtype$> v) {
1899         return compare(EQ, v);
1900     }
1901 
1902     /**
1903      * Tests if this vector is equal to an input scalar.
1904      *
1905      * This is a lane-wise binary test operation which applies
1906      * the primitive equals operation ({@code ==}) to each lane.
1907      * The result is the same as {@code compare(VectorOperators.Comparison.EQ, e)}.
1908      *
1909      * @param e the input scalar
1910      * @return the result mask of testing if this vector
1911      *         is equal to {@code e}
1912      * @see #compare(VectorOperators.Comparison,$type$)
1913      */
1914     @ForceInline
1915     public final
1916     VectorMask<$Boxtype$> eq($type$ e) {
1917         return compare(EQ, e);
1918     }
1919 
1920     /**
1921      * {@inheritDoc} <!--workaround-->
1922      */
1923     @Override
1924     @ForceInline
1925     public final
1926     VectorMask<$Boxtype$> lt(Vector<$Boxtype$> v) {
1927         return compare(LT, v);
1928     }
1929 
1930     /**
1931      * Tests if this vector is less than an input scalar.
1932      *
1933      * This is a lane-wise binary test operation which applies
1934      * the primitive less than operation ({@code <}) to each lane.
1935      * The result is the same as {@code compare(VectorOperators.LT, e)}.
1936      *
1937      * @param e the input scalar
1938      * @return the mask result of testing if this vector
1939      *         is less than the input scalar
1940      * @see #compare(VectorOperators.Comparison,$type$)
1941      */
1942     @ForceInline
1943     public final
1944     VectorMask<$Boxtype$> lt($type$ e) {
1945         return compare(LT, e);
1946     }
1947 
1948     /**
1949      * {@inheritDoc} <!--workaround-->
1950      */
1951     @Override
1952     public abstract
1953     VectorMask<$Boxtype$> test(VectorOperators.Test op);
1954 
1955     /*package-private*/
1956     @ForceInline
1957     final
1958     <M extends VectorMask<$Boxtype$>>
1959     M testTemplate(Class<M> maskType, Test op) {
1960         $Type$Species vsp = vspecies();
1961         if (opKind(op, VO_SPECIAL)) {
1962             $Bitstype$Vector bits = this.viewAsIntegralLanes();
1963             VectorMask<$Boxbitstype$> m;
1964             if (op == IS_DEFAULT) {
1965                 m = bits.compare(EQ, ($bitstype$) 0);
1966             } else if (op == IS_NEGATIVE) {
1967                 m = bits.compare(LT, ($bitstype$) 0);
1968             }
1969 #if[FP]
1970             else if (op == IS_FINITE ||
1971                      op == IS_NAN ||
1972                      op == IS_INFINITE) {
1973                 // first kill the sign:
1974                 bits = bits.and($Boxbitstype$.MAX_VALUE);
1975                 // next find the bit pattern for infinity:
1976                 $bitstype$ infbits = ($bitstype$) toBits($Boxtype$.POSITIVE_INFINITY);
1977                 // now compare:
1978                 if (op == IS_FINITE) {
1979                     m = bits.compare(LT, infbits);
1980                 } else if (op == IS_NAN) {
1981                     m = bits.compare(GT, infbits);
1982                 } else {
1983                     m = bits.compare(EQ, infbits);
1984                 }
1985             }
1986 #end[FP]
1987             else {
1988                 throw new AssertionError(op);
1989             }
1990             return maskType.cast(m{#if[FP]?.cast(this.vspecies())});
1991         }
1992         int opc = opCode(op);
1993         throw new AssertionError(op);
1994     }
1995 
1996     /**
1997      * {@inheritDoc} <!--workaround-->
1998      */
1999     @Override
2000     @ForceInline
2001     public final
2002     VectorMask<$Boxtype$> test(VectorOperators.Test op,
2003                                   VectorMask<$Boxtype$> m) {
2004         return test(op).and(m);
2005     }
2006 
2007     /**
2008      * {@inheritDoc} <!--workaround-->
2009      */
2010     @Override
2011     public abstract
2012     VectorMask<$Boxtype$> compare(VectorOperators.Comparison op, Vector<$Boxtype$> v);
2013 
2014     /*package-private*/
2015     @ForceInline
2016     final
2017     <M extends VectorMask<$Boxtype$>>
2018     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v) {
2019         Objects.requireNonNull(v);
2020         $Type$Species vsp = vspecies();
2021         $abstractvectortype$ that = ($abstractvectortype$) v;
2022         that.check(this);
2023         int opc = opCode(op);
2024         return VectorSupport.compare(
2025             opc, getClass(), maskType, $type$.class, length(),
2026             this, that,
2027             (cond, v0, v1) -> {
2028                 AbstractMask<$Boxtype$> m
2029                     = v0.bTest(cond, v1, (cond_, i, a, b)
2030                                -> compareWithOp(cond, a, b));
2031                 @SuppressWarnings("unchecked")
2032                 M m2 = (M) m;
2033                 return m2;
2034             });
2035     }
2036 
2037     @ForceInline
2038     private static boolean compareWithOp(int cond, $type$ a, $type$ b) {
2039         return switch (cond) {
2040             case BT_eq -> a == b;
2041             case BT_ne -> a != b;
2042             case BT_lt -> a < b;
2043             case BT_le -> a <= b;
2044             case BT_gt -> a > b;
2045             case BT_ge -> a >= b;
2046 #if[!FP]
2047             case BT_ult -> $Boxtype$.compareUnsigned(a, b) < 0;
2048             case BT_ule -> $Boxtype$.compareUnsigned(a, b) <= 0;
2049             case BT_ugt -> $Boxtype$.compareUnsigned(a, b) > 0;
2050             case BT_uge -> $Boxtype$.compareUnsigned(a, b) >= 0;
2051 #end[!FP]
2052             default -> throw new AssertionError();
2053         };
2054     }
2055 
2056     /**
2057      * {@inheritDoc} <!--workaround-->
2058      */
2059     @Override
2060     @ForceInline
2061     public final
2062     VectorMask<$Boxtype$> compare(VectorOperators.Comparison op,
2063                                   Vector<$Boxtype$> v,
2064                                   VectorMask<$Boxtype$> m) {
2065         return compare(op, v).and(m);
2066     }
2067 
2068     /**
2069      * Tests this vector by comparing it with an input scalar,
2070      * according to the given comparison operation.
2071      *
2072      * This is a lane-wise binary test operation which applies
2073      * the comparison operation to each lane.
2074      * <p>
2075      * The result is the same as
2076      * {@code compare(op, broadcast(species(), e))}.
2077      * That is, the scalar may be regarded as broadcast to
2078      * a vector of the same species, and then compared
2079      * against the original vector, using the selected
2080      * comparison operation.
2081      *
2082      * @param op the operation used to compare lane values
2083      * @param e the input scalar
2084      * @return the mask result of testing lane-wise if this vector
2085      *         compares to the input, according to the selected
2086      *         comparison operator
2087      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector)
2088      * @see #eq($type$)
2089      * @see #lt($type$)
2090      */
2091     public abstract
2092     VectorMask<$Boxtype$> compare(Comparison op, $type$ e);
2093 
2094     /*package-private*/
2095     @ForceInline
2096     final
2097     <M extends VectorMask<$Boxtype$>>
2098     M compareTemplate(Class<M> maskType, Comparison op, $type$ e) {
2099         return compareTemplate(maskType, op, broadcast(e));
2100     }
2101 
2102     /**
2103      * Tests this vector by comparing it with an input scalar,
2104      * according to the given comparison operation,
2105      * in lanes selected by a mask.
2106      *
2107      * This is a masked lane-wise binary test operation which applies
2108      * to each pair of corresponding lane values.
2109      *
2110      * The returned result is equal to the expression
2111      * {@code compare(op,s).and(m)}.
2112      *
2113      * @param op the operation used to compare lane values
2114      * @param e the input scalar
2115      * @param m the mask controlling lane selection
2116      * @return the mask result of testing lane-wise if this vector
2117      *         compares to the input, according to the selected
2118      *         comparison operator,
2119      *         and only in the lanes selected by the mask
2120      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector,VectorMask)
2121      */
2122     @ForceInline
2123     public final VectorMask<$Boxtype$> compare(VectorOperators.Comparison op,
2124                                                $type$ e,
2125                                                VectorMask<$Boxtype$> m) {
2126         return compare(op, e).and(m);
2127     }
2128 
2129 #if[!long]
2130     /**
2131      * {@inheritDoc} <!--workaround-->
2132      */
2133     @Override
2134     public abstract
2135     VectorMask<$Boxtype$> compare(Comparison op, long e);
2136 
2137     /*package-private*/
2138     @ForceInline
2139     final
2140     <M extends VectorMask<$Boxtype$>>
2141     M compareTemplate(Class<M> maskType, Comparison op, long e) {
2142         return compareTemplate(maskType, op, broadcast(e));
2143     }
2144 
2145     /**
2146      * {@inheritDoc} <!--workaround-->
2147      */
2148     @Override
2149     @ForceInline
2150     public final
2151     VectorMask<$Boxtype$> compare(Comparison op, long e, VectorMask<$Boxtype$> m) {
2152         return compare(op, broadcast(e), m);
2153     }
2154 
2155 
2156 #end[!long]
2157 
2158     /**
2159      * {@inheritDoc} <!--workaround-->
2160      */
2161     @Override public abstract
2162     $abstractvectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m);
2163 
2164     /*package-private*/
2165     @ForceInline
2166     final
2167     <M extends VectorMask<$Boxtype$>>
2168     $abstractvectortype$
2169     blendTemplate(Class<M> maskType, $abstractvectortype$ v, M m) {
2170         v.check(this);
2171         return VectorSupport.blend(
2172             getClass(), maskType, $type$.class, length(),
2173             this, v, m,
2174             (v0, v1, m_) -> v0.bOp(v1, m_, (i, a, b) -> b));
2175     }
2176 
2177     /**
2178      * {@inheritDoc} <!--workaround-->
2179      */
2180     @Override public abstract $abstractvectortype$ addIndex(int scale);
2181 
2182     /*package-private*/
2183     @ForceInline
2184     final $abstractvectortype$ addIndexTemplate(int scale) {
2185         $Type$Species vsp = vspecies();
2186         // make sure VLENGTH*scale doesn't overflow:
2187         vsp.checkScale(scale);
2188         return VectorSupport.indexVector(
2189             getClass(), $type$.class, length(),
2190             this, scale, vsp,
2191             (v, scale_, s)
2192             -> {
2193                 // If the platform doesn't support an INDEX
2194                 // instruction directly, load IOTA from memory
2195                 // and multiply.
2196                 $abstractvectortype$ iota = s.iota();
2197                 $type$ sc = ($type$) scale_;
2198                 return v.add(sc == 1 ? iota : iota.mul(sc));
2199             });
2200     }
2201 
2202     /**
2203      * Replaces selected lanes of this vector with
2204      * a scalar value
2205      * under the control of a mask.
2206      *
2207      * This is a masked lane-wise binary operation which
2208      * selects each lane value from one or the other input.
2209      *
2210      * The returned result is equal to the expression
2211      * {@code blend(broadcast(e),m)}.
2212      *
2213      * @param e the input scalar, containing the replacement lane value
2214      * @param m the mask controlling lane selection of the scalar
2215      * @return the result of blending the lane elements of this vector with
2216      *         the scalar value
2217      */
2218     @ForceInline
2219     public final $abstractvectortype$ blend($type$ e,
2220                                             VectorMask<$Boxtype$> m) {
2221         return blend(broadcast(e), m);
2222     }
2223 
2224 #if[!long]
2225     /**
2226      * Replaces selected lanes of this vector with
2227      * a scalar value
2228      * under the control of a mask.
2229      *
2230      * This is a masked lane-wise binary operation which
2231      * selects each lane value from one or the other input.
2232      *
2233      * The returned result is equal to the expression
2234      * {@code blend(broadcast(e),m)}.
2235      *
2236      * @param e the input scalar, containing the replacement lane value
2237      * @param m the mask controlling lane selection of the scalar
2238      * @return the result of blending the lane elements of this vector with
2239      *         the scalar value
2240      */
2241     @ForceInline
2242     public final $abstractvectortype$ blend(long e,
2243                                             VectorMask<$Boxtype$> m) {
2244         return blend(broadcast(e), m);
2245     }
2246 #end[!long]
2247 
2248     /**
2249      * {@inheritDoc} <!--workaround-->
2250      */
2251     @Override
2252     public abstract
2253     $abstractvectortype$ slice(int origin, Vector<$Boxtype$> v1);
2254 
2255     /*package-private*/
2256     final
2257     @ForceInline
2258     $abstractvectortype$ sliceTemplate(int origin, Vector<$Boxtype$> v1) {
2259         $abstractvectortype$ that = ($abstractvectortype$) v1;
2260         that.check(this);
2261         Objects.checkIndex(origin, length() + 1);
2262         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2263         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin))));
2264         iota = iotaShuffle(origin, 1, true);
2265         return that.rearrange(iota).blend(this.rearrange(iota), blendMask);
2266     }
2267 
2268     /**
2269      * {@inheritDoc} <!--workaround-->
2270      */
2271     @Override
2272     @ForceInline
2273     public final
2274     $abstractvectortype$ slice(int origin,
2275                                Vector<$Boxtype$> w,
2276                                VectorMask<$Boxtype$> m) {
2277         return broadcast(0).blend(slice(origin, w), m);
2278     }
2279 
2280     /**
2281      * {@inheritDoc} <!--workaround-->
2282      */
2283     @Override
2284     public abstract
2285     $abstractvectortype$ slice(int origin);
2286 
2287     /*package-private*/
2288     final
2289     @ForceInline
2290     $abstractvectortype$ sliceTemplate(int origin) {
2291         Objects.checkIndex(origin, length() + 1);
2292         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2293         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin))));
2294         iota = iotaShuffle(origin, 1, true);
2295         return vspecies().zero().blend(this.rearrange(iota), blendMask);
2296     }
2297 
2298     /**
2299      * {@inheritDoc} <!--workaround-->
2300      */
2301     @Override
2302     public abstract
2303     $abstractvectortype$ unslice(int origin, Vector<$Boxtype$> w, int part);
2304 
2305     /*package-private*/
2306     final
2307     @ForceInline
2308     $abstractvectortype$
2309     unsliceTemplate(int origin, Vector<$Boxtype$> w, int part) {
2310         $abstractvectortype$ that = ($abstractvectortype$) w;
2311         that.check(this);
2312         Objects.checkIndex(origin, length() + 1);
2313         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2314         VectorMask<$Boxtype$> blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT,
2315                                                                   (broadcast(($type$)(origin))));
2316         iota = iotaShuffle(-origin, 1, true);
2317         return that.blend(this.rearrange(iota), blendMask);
2318     }
2319 
2320     /*package-private*/
2321     final
2322     @ForceInline
2323     <M extends VectorMask<$Boxtype$>>
2324     $abstractvectortype$
2325     unsliceTemplate(Class<M> maskType, int origin, Vector<$Boxtype$> w, int part, M m) {
2326         $abstractvectortype$ that = ($abstractvectortype$) w;
2327         that.check(this);
2328         $abstractvectortype$ slice = that.sliceTemplate(origin, that);
2329         slice = slice.blendTemplate(maskType, this, m);
2330         return slice.unsliceTemplate(origin, w, part);
2331     }
2332 
2333     /**
2334      * {@inheritDoc} <!--workaround-->
2335      */
2336     @Override
2337     public abstract
2338     $abstractvectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m);
2339 
2340     /**
2341      * {@inheritDoc} <!--workaround-->
2342      */
2343     @Override
2344     public abstract
2345     $abstractvectortype$ unslice(int origin);
2346 
2347     /*package-private*/
2348     final
2349     @ForceInline
2350     $abstractvectortype$
2351     unsliceTemplate(int origin) {
2352         Objects.checkIndex(origin, length() + 1);
2353         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2354         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.GE,
2355                                                                   (broadcast(($type$)(origin))));
2356         iota = iotaShuffle(-origin, 1, true);
2357         return vspecies().zero().blend(this.rearrange(iota), blendMask);
2358     }
2359 
2360     private ArrayIndexOutOfBoundsException
2361     wrongPartForSlice(int part) {
2362         String msg = String.format("bad part number %d for slice operation",
2363                                    part);
2364         return new ArrayIndexOutOfBoundsException(msg);
2365     }
2366 
2367     /**
2368      * {@inheritDoc} <!--workaround-->
2369      */
2370     @Override
2371     public abstract
2372     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> m);
2373 
2374     /*package-private*/
2375     @ForceInline
2376     final
2377     <S extends VectorShuffle<$Boxtype$>>
2378     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype, S shuffle) {
2379         shuffle.checkIndexes();
2380         return VectorSupport.rearrangeOp(
2381             getClass(), shuffletype, $type$.class, length(),
2382             this, shuffle,
2383             (v1, s_) -> v1.uOp((i, a) -> {
2384                 int ei = s_.laneSource(i);
2385                 return v1.lane(ei);
2386             }));
2387     }
2388 
2389     /**
2390      * {@inheritDoc} <!--workaround-->
2391      */
2392     @Override
2393     public abstract
2394     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2395                                    VectorMask<$Boxtype$> m);
2396 
2397     /*package-private*/
2398     @ForceInline
2399     final
2400     <S extends VectorShuffle<$Boxtype$>>
2401     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2402                                            S shuffle,
2403                                            VectorMask<$Boxtype$> m) {
2404         $abstractvectortype$ unmasked =
2405             VectorSupport.rearrangeOp(
2406                 getClass(), shuffletype, $type$.class, length(),
2407                 this, shuffle,
2408                 (v1, s_) -> v1.uOp((i, a) -> {
2409                     int ei = s_.laneSource(i);
2410                     return ei < 0 ? 0 : v1.lane(ei);
2411                 }));
2412         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2413         if (m.andNot(valid).anyTrue()) {
2414             shuffle.checkIndexes();
2415             throw new AssertionError();
2416         }
2417         return broadcast(($type$)0).blend(unmasked, m);
2418     }
2419 
2420     /**
2421      * {@inheritDoc} <!--workaround-->
2422      */
2423     @Override
2424     public abstract
2425     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2426                                    Vector<$Boxtype$> v);
2427 
2428     /*package-private*/
2429     @ForceInline
2430     final
2431     <S extends VectorShuffle<$Boxtype$>>
2432     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2433                                            S shuffle,
2434                                            $abstractvectortype$ v) {
2435         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2436         @SuppressWarnings("unchecked")
2437         S ws = (S) shuffle.wrapIndexes();
2438         $abstractvectortype$ r0 =
2439             VectorSupport.rearrangeOp(
2440                 getClass(), shuffletype, $type$.class, length(),
2441                 this, ws,
2442                 (v0, s_) -> v0.uOp((i, a) -> {
2443                     int ei = s_.laneSource(i);
2444                     return v0.lane(ei);
2445                 }));
2446         $abstractvectortype$ r1 =
2447             VectorSupport.rearrangeOp(
2448                 getClass(), shuffletype, $type$.class, length(),
2449                 v, ws,
2450                 (v1, s_) -> v1.uOp((i, a) -> {
2451                     int ei = s_.laneSource(i);
2452                     return v1.lane(ei);
2453                 }));
2454         return r1.blend(r0, valid);
2455     }
2456 
2457     @ForceInline
2458     private final
2459     VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) {
2460         $type$[] a = toArray();
2461         int[] sa = new int[a.length];
2462         for (int i = 0; i < a.length; i++) {
2463             sa[i] = (int) a[i];
2464         }
2465         return VectorShuffle.fromArray(dsp, sa, 0);
2466     }
2467 
2468     /*package-private*/
2469     @ForceInline
2470     final
2471     VectorShuffle<$Boxtype$> toShuffleTemplate(Class<?> shuffleType) {
2472         $Type$Species vsp = vspecies();
2473         return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
2474                                      getClass(), $type$.class, length(),
2475                                      shuffleType, byte.class, length(),
2476                                      this, vsp,
2477                                      $Type$Vector::toShuffle0);
2478     }
2479 
2480     /**
2481      * {@inheritDoc} <!--workaround-->
2482      */
2483     @Override
2484     public abstract
2485     $abstractvectortype$ selectFrom(Vector<$Boxtype$> v);
2486 
2487     /*package-private*/
2488     @ForceInline
2489     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v) {
2490         return v.rearrange(this.toShuffle());
2491     }
2492 
2493     /**
2494      * {@inheritDoc} <!--workaround-->
2495      */
2496     @Override
2497     public abstract
2498     $abstractvectortype$ selectFrom(Vector<$Boxtype$> s, VectorMask<$Boxtype$> m);
2499 
2500     /*package-private*/
2501     @ForceInline
2502     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v,
2503                                                   AbstractMask<$Boxtype$> m) {
2504         return v.rearrange(this.toShuffle(), m);
2505     }
2506 
2507     /// Ternary operations
2508 
2509 #if[BITWISE]
2510     /**
2511      * Blends together the bits of two vectors under
2512      * the control of a third, which supplies mask bits.
2513      *
2514      * This is a lane-wise ternary operation which performs
2515      * a bitwise blending operation {@code (a&~c)|(b&c)}
2516      * to each lane.
2517      *
2518      * This method is also equivalent to the expression
2519      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2520      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2521      *    BITWISE_BLEND}{@code , bits, mask)}.
2522      *
2523      * @param bits input bits to blend into the current vector
2524      * @param mask a bitwise mask to enable blending of the input bits
2525      * @return the bitwise blend of the given bits into the current vector,
2526      *         under control of the bitwise mask
2527      * @see #bitwiseBlend($type$,$type$)
2528      * @see #bitwiseBlend($type$,Vector)
2529      * @see #bitwiseBlend(Vector,$type$)
2530      * @see VectorOperators#BITWISE_BLEND
2531      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
2532      */
2533     @ForceInline
2534     public final
2535     $abstractvectortype$ bitwiseBlend(Vector<$Boxtype$> bits, Vector<$Boxtype$> mask) {
2536         return lanewise(BITWISE_BLEND, bits, mask);
2537     }
2538 
2539     /**
2540      * Blends together the bits of a vector and a scalar under
2541      * the control of another scalar, which supplies mask bits.
2542      *
2543      * This is a lane-wise ternary operation which performs
2544      * a bitwise blending operation {@code (a&~c)|(b&c)}
2545      * to each lane.
2546      *
2547      * This method is also equivalent to the expression
2548      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2549      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2550      *    BITWISE_BLEND}{@code , bits, mask)}.
2551      *
2552      * @param bits input bits to blend into the current vector
2553      * @param mask a bitwise mask to enable blending of the input bits
2554      * @return the bitwise blend of the given bits into the current vector,
2555      *         under control of the bitwise mask
2556      * @see #bitwiseBlend(Vector,Vector)
2557      * @see VectorOperators#BITWISE_BLEND
2558      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
2559      */
2560     @ForceInline
2561     public final
2562     $abstractvectortype$ bitwiseBlend($type$ bits, $type$ mask) {
2563         return lanewise(BITWISE_BLEND, bits, mask);
2564     }
2565 
2566     /**
2567      * Blends together the bits of a vector and a scalar under
2568      * the control of another vector, which supplies mask bits.
2569      *
2570      * This is a lane-wise ternary operation which performs
2571      * a bitwise blending operation {@code (a&~c)|(b&c)}
2572      * to each lane.
2573      *
2574      * This method is also equivalent to the expression
2575      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2576      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2577      *    BITWISE_BLEND}{@code , bits, mask)}.
2578      *
2579      * @param bits input bits to blend into the current vector
2580      * @param mask a bitwise mask to enable blending of the input bits
2581      * @return the bitwise blend of the given bits into the current vector,
2582      *         under control of the bitwise mask
2583      * @see #bitwiseBlend(Vector,Vector)
2584      * @see VectorOperators#BITWISE_BLEND
2585      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
2586      */
2587     @ForceInline
2588     public final
2589     $abstractvectortype$ bitwiseBlend($type$ bits, Vector<$Boxtype$> mask) {
2590         return lanewise(BITWISE_BLEND, bits, mask);
2591     }
2592 
2593     /**
2594      * Blends together the bits of two vectors under
2595      * the control of a scalar, which supplies mask bits.
2596      *
2597      * This is a lane-wise ternary operation which performs
2598      * a bitwise blending operation {@code (a&~c)|(b&c)}
2599      * to each lane.
2600      *
2601      * This method is also equivalent to the expression
2602      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2603      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2604      *    BITWISE_BLEND}{@code , bits, mask)}.
2605      *
2606      * @param bits input bits to blend into the current vector
2607      * @param mask a bitwise mask to enable blending of the input bits
2608      * @return the bitwise blend of the given bits into the current vector,
2609      *         under control of the bitwise mask
2610      * @see #bitwiseBlend(Vector,Vector)
2611      * @see VectorOperators#BITWISE_BLEND
2612      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
2613      */
2614     @ForceInline
2615     public final
2616     $abstractvectortype$ bitwiseBlend(Vector<$Boxtype$> bits, $type$ mask) {
2617         return lanewise(BITWISE_BLEND, bits, mask);
2618     }
2619 #end[BITWISE]
2620 
2621 #if[FP]
2622     /**
2623      * Multiplies this vector by a second input vector, and sums
2624      * the result with a third.
2625      *
2626      * Extended precision is used for the intermediate result,
2627      * avoiding possible loss of precision from rounding once
2628      * for each of the two operations.
2629      * The result is numerically close to {@code this.mul(b).add(c)},
2630      * and is typically closer to the true mathematical result.
2631      *
2632      * This is a lane-wise ternary operation which applies an operation
2633      * conforming to the specification of
2634      * {@link Math#fma($type$,$type$,$type$) Math.fma(a,b,c)}
2635      * to each lane.
2636 #if[intOrFloat]
2637      * The operation is adapted to cast the operands and the result,
2638      * specifically widening {@code float} operands to {@code double}
2639      * operands and narrowing the {@code double} result to a {@code float}
2640      * result.
2641 #end[intOrFloat]
2642      *
2643      * This method is also equivalent to the expression
2644      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2645      *    lanewise}{@code (}{@link VectorOperators#FMA
2646      *    FMA}{@code , b, c)}.
2647      *
2648      * @param b the second input vector, supplying multiplier values
2649      * @param c the third input vector, supplying addend values
2650      * @return the product of this vector and the second input vector
2651      *         summed with the third input vector, using extended precision
2652      *         for the intermediate result
2653      * @see #fma($type$,$type$)
2654      * @see VectorOperators#FMA
2655      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
2656      */
2657     @ForceInline
2658     public final
2659     $abstractvectortype$ fma(Vector<$Boxtype$> b, Vector<$Boxtype$> c) {
2660         return lanewise(FMA, b, c);
2661     }
2662 
2663     /**
2664      * Multiplies this vector by a scalar multiplier, and sums
2665      * the result with a scalar addend.
2666      *
2667      * Extended precision is used for the intermediate result,
2668      * avoiding possible loss of precision from rounding once
2669      * for each of the two operations.
2670      * The result is numerically close to {@code this.mul(b).add(c)},
2671      * and is typically closer to the true mathematical result.
2672      *
2673      * This is a lane-wise ternary operation which applies an operation
2674      * conforming to the specification of
2675      * {@link Math#fma($type$,$type$,$type$) Math.fma(a,b,c)}
2676      * to each lane.
2677 #if[intOrFloat]
2678      * The operation is adapted to cast the operands and the result,
2679      * specifically widening {@code float} operands to {@code double}
2680      * operands and narrowing the {@code double} result to a {@code float}
2681      * result.
2682 #end[intOrFloat]
2683      *
2684      * This method is also equivalent to the expression
2685      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2686      *    lanewise}{@code (}{@link VectorOperators#FMA
2687      *    FMA}{@code , b, c)}.
2688      *
2689      * @param b the scalar multiplier
2690      * @param c the scalar addend
2691      * @return the product of this vector and the scalar multiplier
2692      *         summed with scalar addend, using extended precision
2693      *         for the intermediate result
2694      * @see #fma(Vector,Vector)
2695      * @see VectorOperators#FMA
2696      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
2697      */
2698     @ForceInline
2699     public final
2700     $abstractvectortype$ fma($type$ b, $type$ c) {
2701         return lanewise(FMA, b, c);
2702     }
2703 
2704     // Don't bother with (Vector,$type$) and ($type$,Vector) overloadings.
2705 #end[FP]
2706 
2707     // Type specific horizontal reductions
2708 
2709     /**
2710      * Returns a value accumulated from all the lanes of this vector.
2711      *
2712      * This is an associative cross-lane reduction operation which
2713      * applies the specified operation to all the lane elements.
2714      * <p>
2715      * A few reduction operations do not support arbitrary reordering
2716      * of their operands, yet are included here because of their
2717      * usefulness.
2718      * <ul>
2719      * <li>
2720      * In the case of {@code FIRST_NONZERO}, the reduction returns
2721      * the value from the lowest-numbered non-zero lane.
2722 #if[FP]
2723      * (As with {@code MAX} and {@code MIN}, floating point negative
2724      * zero {@code -0.0} is treated as a value distinct from
2725      * the default value, positive zero. So a first-nonzero lane reduction
2726      * might return {@code -0.0} even in the presence of non-zero
2727      * lane values.)
2728      * <li>
2729      * In the case of {@code ADD} and {@code MUL}, the
2730      * precise result will reflect the choice of an arbitrary order
2731      * of operations, which may even vary over time.
2732      * For further details see the section
2733      * <a href="VectorOperators.html#fp_assoc">Operations on floating point vectors</a>.
2734 #end[FP]
2735      * <li>
2736      * All other reduction operations are fully commutative and
2737      * associative.  The implementation can choose any order of
2738      * processing, yet it will always produce the same result.
2739      * </ul>
2740      *
2741      * @param op the operation used to combine lane values
2742      * @return the accumulated result
2743      * @throws UnsupportedOperationException if this vector does
2744      *         not support the requested operation
2745      * @see #reduceLanes(VectorOperators.Associative,VectorMask)
2746      * @see #add(Vector)
2747      * @see #mul(Vector)
2748      * @see #min(Vector)
2749      * @see #max(Vector)
2750 #if[BITWISE]
2751      * @see #and(Vector)
2752      * @see #or(Vector)
2753      * @see VectorOperators#XOR
2754 #end[BITWISE]
2755      * @see VectorOperators#FIRST_NONZERO
2756      */
2757     public abstract $type$ reduceLanes(VectorOperators.Associative op);
2758 
2759     /**
2760      * Returns a value accumulated from selected lanes of this vector,
2761      * controlled by a mask.
2762      *
2763      * This is an associative cross-lane reduction operation which
2764      * applies the specified operation to the selected lane elements.
2765      * <p>
2766      * If no elements are selected, an operation-specific identity
2767      * value is returned.
2768      * <ul>
2769      * <li>
2770      * If the operation is
2771 #if[BITWISE]
2772      *  {@code ADD}, {@code XOR}, {@code OR},
2773 #else[BITWISE]
2774      *  {@code ADD}
2775 #end[BITWISE]
2776      * or {@code FIRST_NONZERO},
2777      * then the identity value is {#if[FP]?positive }zero, the default {@code $type$} value.
2778      * <li>
2779      * If the operation is {@code MUL},
2780      * then the identity value is one.
2781 #if[BITWISE]
2782      * <li>
2783      * If the operation is {@code AND},
2784      * then the identity value is minus one (all bits set).
2785      * <li>
2786      * If the operation is {@code MAX},
2787      * then the identity value is {@code $Boxtype$.MIN_VALUE}.
2788      * <li>
2789      * If the operation is {@code MIN},
2790      * then the identity value is {@code $Boxtype$.MAX_VALUE}.
2791 #end[BITWISE]
2792 #if[FP]
2793      * <li>
2794      * If the operation is {@code MAX},
2795      * then the identity value is {@code $Boxtype$.NEGATIVE_INFINITY}.
2796      * <li>
2797      * If the operation is {@code MIN},
2798      * then the identity value is {@code $Boxtype$.POSITIVE_INFINITY}.
2799 #end[FP]
2800      * </ul>
2801      * <p>
2802      * A few reduction operations do not support arbitrary reordering
2803      * of their operands, yet are included here because of their
2804      * usefulness.
2805      * <ul>
2806      * <li>
2807      * In the case of {@code FIRST_NONZERO}, the reduction returns
2808      * the value from the lowest-numbered non-zero lane.
2809 #if[FP]
2810      * (As with {@code MAX} and {@code MIN}, floating point negative
2811      * zero {@code -0.0} is treated as a value distinct from
2812      * the default value, positive zero. So a first-nonzero lane reduction
2813      * might return {@code -0.0} even in the presence of non-zero
2814      * lane values.)
2815      * <li>
2816      * In the case of {@code ADD} and {@code MUL}, the
2817      * precise result will reflect the choice of an arbitrary order
2818      * of operations, which may even vary over time.
2819      * For further details see the section
2820      * <a href="VectorOperators.html#fp_assoc">Operations on floating point vectors</a>.
2821 #end[FP]
2822      * <li>
2823      * All other reduction operations are fully commutative and
2824      * associative.  The implementation can choose any order of
2825      * processing, yet it will always produce the same result.
2826      * </ul>
2827      *
2828      * @param op the operation used to combine lane values
2829      * @param m the mask controlling lane selection
2830      * @return the reduced result accumulated from the selected lane values
2831      * @throws UnsupportedOperationException if this vector does
2832      *         not support the requested operation
2833      * @see #reduceLanes(VectorOperators.Associative)
2834      */
2835     public abstract $type$ reduceLanes(VectorOperators.Associative op,
2836                                        VectorMask<$Boxtype$> m);
2837 
2838     /*package-private*/
2839     @ForceInline
2840     final
2841     $type$ reduceLanesTemplate(VectorOperators.Associative op,
2842                                VectorMask<$Boxtype$> m) {
2843         $abstractvectortype$ v = reduceIdentityVector(op).blend(this, m);
2844         return v.reduceLanesTemplate(op);
2845     }
2846 
2847     /*package-private*/
2848     @ForceInline
2849     final
2850     $type$ reduceLanesTemplate(VectorOperators.Associative op) {
2851         if (op == FIRST_NONZERO) {
2852             // FIXME:  The JIT should handle this, and other scan ops alos.
2853             VectorMask<$Boxbitstype$> thisNZ
2854                 = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
2855             return this.lane(thisNZ.firstTrue());
2856         }
2857         int opc = opCode(op);
2858         return fromBits(VectorSupport.reductionCoerced(
2859             opc, getClass(), $type$.class, length(),
2860             this,
2861             REDUCE_IMPL.find(op, opc, (opc_) -> {
2862               switch (opc_) {
2863               case VECTOR_OP_ADD: return v ->
2864                       toBits(v.rOp(($type$)0, (i, a, b) -> ($type$)(a + b)));
2865               case VECTOR_OP_MUL: return v ->
2866                       toBits(v.rOp(($type$)1, (i, a, b) -> ($type$)(a * b)));
2867               case VECTOR_OP_MIN: return v ->
2868                       toBits(v.rOp(MAX_OR_INF, (i, a, b) -> ($type$) Math.min(a, b)));
2869               case VECTOR_OP_MAX: return v ->
2870                       toBits(v.rOp(MIN_OR_INF, (i, a, b) -> ($type$) Math.max(a, b)));
2871 #if[BITWISE]
2872               case VECTOR_OP_AND: return v ->
2873                       toBits(v.rOp(($type$)-1, (i, a, b) -> ($type$)(a & b)));
2874               case VECTOR_OP_OR: return v ->
2875                       toBits(v.rOp(($type$)0, (i, a, b) -> ($type$)(a | b)));
2876               case VECTOR_OP_XOR: return v ->
2877                       toBits(v.rOp(($type$)0, (i, a, b) -> ($type$)(a ^ b)));
2878 #end[BITWISE]
2879               default: return null;
2880               }})));
2881     }
2882     private static final
2883     ImplCache<Associative,Function<$abstractvectortype$,Long>> REDUCE_IMPL
2884         = new ImplCache<>(Associative.class, $Type$Vector.class);
2885 
2886     private
2887     @ForceInline
2888     $abstractvectortype$ reduceIdentityVector(VectorOperators.Associative op) {
2889         int opc = opCode(op);
2890         UnaryOperator<$abstractvectortype$> fn
2891             = REDUCE_ID_IMPL.find(op, opc, (opc_) -> {
2892                 switch (opc_) {
2893                 case VECTOR_OP_ADD:
2894 #if[BITWISE]
2895                 case VECTOR_OP_OR:
2896                 case VECTOR_OP_XOR:
2897 #end[BITWISE]
2898                     return v -> v.broadcast(0);
2899                 case VECTOR_OP_MUL:
2900                     return v -> v.broadcast(1);
2901 #if[BITWISE]
2902                 case VECTOR_OP_AND:
2903                     return v -> v.broadcast(-1);
2904 #end[BITWISE]
2905                 case VECTOR_OP_MIN:
2906                     return v -> v.broadcast(MAX_OR_INF);
2907                 case VECTOR_OP_MAX:
2908                     return v -> v.broadcast(MIN_OR_INF);
2909                 default: return null;
2910                 }
2911             });
2912         return fn.apply(this);
2913     }
2914     private static final
2915     ImplCache<Associative,UnaryOperator<$abstractvectortype$>> REDUCE_ID_IMPL
2916         = new ImplCache<>(Associative.class, $Type$Vector.class);
2917 
2918 #if[FP]
2919     private static final $type$ MIN_OR_INF = $Boxtype$.NEGATIVE_INFINITY;
2920     private static final $type$ MAX_OR_INF = $Boxtype$.POSITIVE_INFINITY;
2921 #else[FP]
2922     private static final $type$ MIN_OR_INF = $Boxtype$.MIN_VALUE;
2923     private static final $type$ MAX_OR_INF = $Boxtype$.MAX_VALUE;
2924 #end[FP]
2925 
2926     public @Override abstract long reduceLanesToLong(VectorOperators.Associative op);
2927     public @Override abstract long reduceLanesToLong(VectorOperators.Associative op,
2928                                                      VectorMask<$Boxtype$> m);
2929 
2930     // Type specific accessors
2931 
2932     /**
2933      * Gets the lane element at lane index {@code i}
2934      *
2935      * @param i the lane index
2936      * @return the lane element at lane index {@code i}
2937      * @throws IllegalArgumentException if the index is is out of range
2938      * ({@code < 0 || >= length()})
2939      */
2940     public abstract $type$ lane(int i);
2941 
2942     /**
2943      * Replaces the lane element of this vector at lane index {@code i} with
2944      * value {@code e}.
2945      *
2946      * This is a cross-lane operation and behaves as if it returns the result
2947      * of blending this vector with an input vector that is the result of
2948      * broadcasting {@code e} and a mask that has only one lane set at lane
2949      * index {@code i}.
2950      *
2951      * @param i the lane index of the lane element to be replaced
2952      * @param e the value to be placed
2953      * @return the result of replacing the lane element of this vector at lane
2954      * index {@code i} with value {@code e}.
2955      * @throws IllegalArgumentException if the index is is out of range
2956      * ({@code < 0 || >= length()})
2957      */
2958     public abstract $abstractvectortype$ withLane(int i, $type$ e);
2959 
2960     // Memory load operations
2961 
2962     /**
2963      * Returns an array of type {@code $type$[]}
2964      * containing all the lane values.
2965      * The array length is the same as the vector length.
2966      * The array elements are stored in lane order.
2967      * <p>
2968      * This method behaves as if it stores
2969      * this vector into an allocated array
2970      * (using {@link #intoArray($type$[], int) intoArray})
2971      * and returns the array as follows:
2972      * <pre>{@code
2973      *   $type$[] a = new $type$[this.length()];
2974      *   this.intoArray(a, 0);
2975      *   return a;
2976      * }</pre>
2977      *
2978      * @return an array containing the lane values of this vector
2979      */
2980     @ForceInline
2981     @Override
2982     public final $type$[] toArray() {
2983         $type$[] a = new $type$[vspecies().laneCount()];
2984         intoArray(a, 0);
2985         return a;
2986     }
2987 
2988 #if[int]
2989     /**
2990      * {@inheritDoc} <!--workaround-->
2991      * This is an alias for {@link #toArray()}
2992      * When this method is used on used on vectors
2993      * of type {@code $abstractvectortype$},
2994      * there will be no loss of range or precision.
2995      */
2996     @ForceInline
2997     @Override
2998     public final int[] toIntArray() {
2999         return toArray();
3000     }
3001 #else[int]
3002     /** {@inheritDoc} <!--workaround-->
3003 #if[!FP]
3004 #if[!long]
3005      * @implNote
3006      * When this method is used on used on vectors
3007      * of type {@code $abstractvectortype$},
3008      * there will be no loss of precision or range,
3009      * and so no {@code UnsupportedOperationException} will
3010      * be thrown.
3011 #end[!long]
3012 #end[!FP]
3013      */
3014     @ForceInline
3015     @Override
3016     public final int[] toIntArray() {
3017         $type$[] a = toArray();
3018         int[] res = new int[a.length];
3019         for (int i = 0; i < a.length; i++) {
3020             $type$ e = a[i];
3021             res[i] = (int) $Type$Species.toIntegralChecked(e, true);
3022         }
3023         return res;
3024     }
3025 #end[int]
3026 
3027 #if[long]
3028     /**
3029      * {@inheritDoc} <!--workaround-->
3030      * This is an alias for {@link #toArray()}
3031      * When this method is used on used on vectors
3032      * of type {@code $abstractvectortype$},
3033      * there will be no loss of range or precision.
3034      */
3035     @ForceInline
3036     @Override
3037     public final long[] toLongArray() {
3038         return toArray();
3039     }
3040 #else[long]
3041     /** {@inheritDoc} <!--workaround-->
3042 #if[!FP]
3043      * @implNote
3044      * When this method is used on used on vectors
3045      * of type {@code $abstractvectortype$},
3046      * there will be no loss of precision or range,
3047      * and so no {@code UnsupportedOperationException} will
3048      * be thrown.
3049 #end[!FP]
3050      */
3051     @ForceInline
3052     @Override
3053     public final long[] toLongArray() {
3054         $type$[] a = toArray();
3055         long[] res = new long[a.length];
3056         for (int i = 0; i < a.length; i++) {
3057             $type$ e = a[i];
3058             res[i] = $Type$Species.toIntegralChecked(e, false);
3059         }
3060         return res;
3061     }
3062 #end[long]
3063 
3064 #if[double]
3065     /** {@inheritDoc} <!--workaround-->
3066      * @implNote
3067      * This is an alias for {@link #toArray()}
3068      * When this method is used on used on vectors
3069      * of type {@code $abstractvectortype$},
3070      * there will be no loss of precision.
3071      */
3072     @ForceInline
3073     @Override
3074     public final double[] toDoubleArray() {
3075         return toArray();
3076     }
3077 #else[double]
3078     /** {@inheritDoc} <!--workaround-->
3079 #if[long]
3080      * @implNote
3081      * When this method is used on used on vectors
3082      * of type {@code $abstractvectortype$},
3083      * up to nine bits of precision may be lost
3084      * for lane values of large magnitude.
3085 #else[long]
3086      * @implNote
3087      * When this method is used on used on vectors
3088      * of type {@code $abstractvectortype$},
3089      * there will be no loss of precision.
3090 #end[long]
3091      */
3092     @ForceInline
3093     @Override
3094     public final double[] toDoubleArray() {
3095         $type$[] a = toArray();
3096         double[] res = new double[a.length];
3097         for (int i = 0; i < a.length; i++) {
3098             res[i] = (double) a[i];
3099         }
3100         return res;
3101     }
3102 #end[double]
3103 
3104     /**
3105      * Loads a vector from a byte array starting at an offset.
3106      * Bytes are composed into primitive lane elements according
3107      * to the specified byte order.
3108      * The vector is arranged into lanes according to
3109      * <a href="Vector.html#lane-order">memory ordering</a>.
3110      * <p>
3111      * This method behaves as if it returns the result of calling
3112      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3113      * fromByteBuffer()} as follows:
3114      * <pre>{@code
3115      * var bb = ByteBuffer.wrap(a);
3116      * var m = species.maskAll(true);
3117      * return fromByteBuffer(species, bb, offset, bo, m);
3118      * }</pre>
3119      *
3120      * @param species species of desired vector
3121      * @param a the byte array
3122      * @param offset the offset into the array
3123      * @param bo the intended byte order
3124      * @return a vector loaded from a byte array
3125      * @throws IndexOutOfBoundsException
3126      *         if {@code offset+N*ESIZE < 0}
3127      *         or {@code offset+(N+1)*ESIZE > a.length}
3128      *         for any lane {@code N} in the vector
3129      */
3130     @ForceInline
3131     public static
3132     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3133                                        byte[] a, int offset,
3134                                        ByteOrder bo) {
3135         offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
3136         $Type$Species vsp = ($Type$Species) species;
3137         return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
3138     }
3139 
3140     /**
3141      * Loads a vector from a byte array starting at an offset
3142      * and using a mask.
3143      * Lanes where the mask is unset are filled with the default
3144      * value of {@code $type$} ({#if[FP]?positive }zero).
3145      * Bytes are composed into primitive lane elements according
3146      * to the specified byte order.
3147      * The vector is arranged into lanes according to
3148      * <a href="Vector.html#lane-order">memory ordering</a>.
3149      * <p>
3150      * This method behaves as if it returns the result of calling
3151      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3152      * fromByteBuffer()} as follows:
3153      * <pre>{@code
3154      * var bb = ByteBuffer.wrap(a);
3155      * return fromByteBuffer(species, bb, offset, bo, m);
3156      * }</pre>
3157      *
3158      * @param species species of desired vector
3159      * @param a the byte array
3160      * @param offset the offset into the array
3161      * @param bo the intended byte order
3162      * @param m the mask controlling lane selection
3163      * @return a vector loaded from a byte array
3164      * @throws IndexOutOfBoundsException
3165      *         if {@code offset+N*ESIZE < 0}
3166      *         or {@code offset+(N+1)*ESIZE > a.length}
3167      *         for any lane {@code N} in the vector
3168      *         where the mask is set
3169      */
3170     @ForceInline
3171     public static
3172     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3173                                        byte[] a, int offset,
3174                                        ByteOrder bo,
3175                                        VectorMask<$Boxtype$> m) {
3176         $Type$Species vsp = ($Type$Species) species;
3177         if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
3178             $abstractvectortype$ zero = vsp.zero();
3179             $abstractvectortype$ v = zero.fromByteArray0(a, offset);
3180             return zero.blend(v.maybeSwap(bo), m);
3181         }
3182 
3183         // FIXME: optimize
3184         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
3185         ByteBuffer wb = wrapper(a, bo);
3186         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
3187                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
3188     }
3189 
3190     /**
3191      * Loads a vector from an array of type {@code $type$[]}
3192      * starting at an offset.
3193      * For each vector lane, where {@code N} is the vector lane index, the
3194      * array element at index {@code offset + N} is placed into the
3195      * resulting vector at lane index {@code N}.
3196      *
3197      * @param species species of desired vector
3198      * @param a the array
3199      * @param offset the offset into the array
3200      * @return the vector loaded from an array
3201      * @throws IndexOutOfBoundsException
3202      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3203      *         for any lane {@code N} in the vector
3204      */
3205     @ForceInline
3206     public static
3207     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3208                                    $type$[] a, int offset) {
3209         offset = checkFromIndexSize(offset, species.length(), a.length);
3210         $Type$Species vsp = ($Type$Species) species;
3211         return vsp.dummyVector().fromArray0(a, offset);
3212     }
3213 
3214     /**
3215      * Loads a vector from an array of type {@code $type$[]}
3216      * starting at an offset and using a mask.
3217      * Lanes where the mask is unset are filled with the default
3218      * value of {@code $type$} ({#if[FP]?positive }zero).
3219      * For each vector lane, where {@code N} is the vector lane index,
3220      * if the mask lane at index {@code N} is set then the array element at
3221      * index {@code offset + N} is placed into the resulting vector at lane index
3222      * {@code N}, otherwise the default element value is placed into the
3223      * resulting vector at lane index {@code N}.
3224      *
3225      * @param species species of desired vector
3226      * @param a the array
3227      * @param offset the offset into the array
3228      * @param m the mask controlling lane selection
3229      * @return the vector loaded from an array
3230      * @throws IndexOutOfBoundsException
3231      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3232      *         for any lane {@code N} in the vector
3233      *         where the mask is set
3234      */
3235     @ForceInline
3236     public static
3237     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3238                                    $type$[] a, int offset,
3239                                    VectorMask<$Boxtype$> m) {
3240         $Type$Species vsp = ($Type$Species) species;
3241         if (offset >= 0 && offset <= (a.length - species.length())) {
3242             $abstractvectortype$ zero = vsp.zero();
3243             return zero.blend(zero.fromArray0(a, offset), m);
3244         }
3245 
3246         // FIXME: optimize
3247         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3248         return vsp.vOp(m, i -> a[offset + i]);
3249     }
3250 
3251     /**
3252      * Gathers a new vector composed of elements from an array of type
3253      * {@code $type$[]},
3254      * using indexes obtained by adding a fixed {@code offset} to a
3255      * series of secondary offsets from an <em>index map</em>.
3256      * The index map is a contiguous sequence of {@code VLENGTH}
3257      * elements in a second array of {@code int}s, starting at a given
3258      * {@code mapOffset}.
3259      * <p>
3260      * For each vector lane, where {@code N} is the vector lane index,
3261      * the lane is loaded from the array
3262      * element {@code a[f(N)]}, where {@code f(N)} is the
3263      * index mapping expression
3264      * {@code offset + indexMap[mapOffset + N]]}.
3265      *
3266      * @param species species of desired vector
3267      * @param a the array
3268      * @param offset the offset into the array, may be negative if relative
3269      * indexes in the index map compensate to produce a value within the
3270      * array bounds
3271      * @param indexMap the index map
3272      * @param mapOffset the offset into the index map
3273      * @return the vector loaded from the indexed elements of the array
3274      * @throws IndexOutOfBoundsException
3275      *         if {@code mapOffset+N < 0}
3276      *         or if {@code mapOffset+N >= indexMap.length},
3277      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3278      *         is an invalid index into {@code a},
3279      *         for any lane {@code N} in the vector
3280      * @see $abstractvectortype$#toIntArray()
3281      */
3282 #if[byteOrShort]
3283     @ForceInline
3284     public static
3285     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3286                                    $type$[] a, int offset,
3287                                    int[] indexMap, int mapOffset) {
3288         $Type$Species vsp = ($Type$Species) species;
3289         return vsp.vOp(n -> a[offset + indexMap[mapOffset + n]]);
3290     }
3291 #else[byteOrShort]
3292     @ForceInline
3293     public static
3294     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3295                                    $type$[] a, int offset,
3296                                    int[] indexMap, int mapOffset) {
3297         $Type$Species vsp = ($Type$Species) species;
3298         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3299         Objects.requireNonNull(a);
3300         Objects.requireNonNull(indexMap);
3301         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();
3302 
3303 #if[longOrDouble]
3304         if (vsp.laneCount() == 1) {
3305           return $abstractvectortype$.fromArray(vsp, a, offset + indexMap[mapOffset]);
3306         }
3307 
3308         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3309         IntVector vix;
3310         if (isp.laneCount() != vsp.laneCount()) {
3311             // For $Type$MaxVector,  if vector length is non-power-of-two or
3312             // 2048 bits, indexShape of $Type$ species is S_MAX_BIT.
3313             // Assume that vector length is 2048, then the lane count of $Type$
3314             // vector is 32. When converting $Type$ species to int species,
3315             // indexShape is still S_MAX_BIT, but the lane count of int vector
3316             // is 64. So when loading index vector (IntVector), only lower half
3317             // of index data is needed.
3318             vix = IntVector
3319                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3320                 .add(offset);
3321         } else {
3322             vix = IntVector
3323                 .fromArray(isp, indexMap, mapOffset)
3324                 .add(offset);
3325         }
3326 #else[longOrDouble]
3327         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3328         IntVector vix = IntVector
3329             .fromArray(isp, indexMap, mapOffset)
3330             .add(offset);
3331 #end[longOrDouble]
3332 
3333         vix = VectorIntrinsics.checkIndex(vix, a.length);
3334 
3335         return VectorSupport.loadWithMap(
3336             vectorType, $type$.class, vsp.laneCount(),
3337             IntVector.species(vsp.indexShape()).vectorType(),
3338             a, ARRAY_BASE, vix,
3339             a, offset, indexMap, mapOffset, vsp,
3340             ($type$[] c, int idx, int[] iMap, int idy, $Type$Species s) ->
3341             s.vOp(n -> c[idx + iMap[idy+n]]));
3342         }
3343 #end[byteOrShort]
3344 
3345     /**
3346      * Gathers a new vector composed of elements from an array of type
3347      * {@code $type$[]},
3348      * under the control of a mask, and
3349      * using indexes obtained by adding a fixed {@code offset} to a
3350      * series of secondary offsets from an <em>index map</em>.
3351      * The index map is a contiguous sequence of {@code VLENGTH}
3352      * elements in a second array of {@code int}s, starting at a given
3353      * {@code mapOffset}.
3354      * <p>
3355      * For each vector lane, where {@code N} is the vector lane index,
3356      * if the lane is set in the mask,
3357      * the lane is loaded from the array
3358      * element {@code a[f(N)]}, where {@code f(N)} is the
3359      * index mapping expression
3360      * {@code offset + indexMap[mapOffset + N]]}.
3361      * Unset lanes in the resulting vector are set to zero.
3362      *
3363      * @param species species of desired vector
3364      * @param a the array
3365      * @param offset the offset into the array, may be negative if relative
3366      * indexes in the index map compensate to produce a value within the
3367      * array bounds
3368      * @param indexMap the index map
3369      * @param mapOffset the offset into the index map
3370      * @param m the mask controlling lane selection
3371      * @return the vector loaded from the indexed elements of the array
3372      * @throws IndexOutOfBoundsException
3373      *         if {@code mapOffset+N < 0}
3374      *         or if {@code mapOffset+N >= indexMap.length},
3375      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3376      *         is an invalid index into {@code a},
3377      *         for any lane {@code N} in the vector
3378      *         where the mask is set
3379      * @see $abstractvectortype$#toIntArray()
3380      */
3381 #if[byteOrShort]
3382     @ForceInline
3383     public static
3384     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3385                                    $type$[] a, int offset,
3386                                    int[] indexMap, int mapOffset,
3387                                    VectorMask<$Boxtype$> m) {
3388         $Type$Species vsp = ($Type$Species) species;
3389         return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
3390     }
3391 #else[byteOrShort]
3392     @ForceInline
3393     public static
3394     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3395                                    $type$[] a, int offset,
3396                                    int[] indexMap, int mapOffset,
3397                                    VectorMask<$Boxtype$> m) {
3398         if (m.allTrue()) {
3399             return fromArray(species, a, offset, indexMap, mapOffset);
3400         }
3401         else {
3402             // FIXME: Cannot vectorize yet, if there's a mask.
3403             $Type$Species vsp = ($Type$Species) species;
3404             return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
3405         }
3406     }
3407 #end[byteOrShort]
3408 
3409 #if[short]
3410     /**
3411      * Loads a vector from an array of type {@code char[]}
3412      * starting at an offset.
3413      * For each vector lane, where {@code N} is the vector lane index, the
3414      * array element at index {@code offset + N}
3415      * is first cast to a {@code short} value and then
3416      * placed into the resulting vector at lane index {@code N}.
3417      *
3418      * @param species species of desired vector
3419      * @param a the array
3420      * @param offset the offset into the array
3421      * @return the vector loaded from an array
3422      * @throws IndexOutOfBoundsException
3423      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3424      *         for any lane {@code N} in the vector
3425      */
3426     @ForceInline
3427     public static
3428     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3429                                        char[] a, int offset) {
3430         offset = checkFromIndexSize(offset, species.length(), a.length);
3431         $Type$Species vsp = ($Type$Species) species;
3432         return vsp.dummyVector().fromCharArray0(a, offset);
3433     }
3434 
3435     /**
3436      * Loads a vector from an array of type {@code char[]}
3437      * starting at an offset and using a mask.
3438      * Lanes where the mask is unset are filled with the default
3439      * value of {@code $type$} ({#if[FP]?positive }zero).
3440      * For each vector lane, where {@code N} is the vector lane index,
3441      * if the mask lane at index {@code N} is set then the array element at
3442      * index {@code offset + N}
3443      * is first cast to a {@code short} value and then
3444      * placed into the resulting vector at lane index
3445      * {@code N}, otherwise the default element value is placed into the
3446      * resulting vector at lane index {@code N}.
3447      *
3448      * @param species species of desired vector
3449      * @param a the array
3450      * @param offset the offset into the array
3451      * @param m the mask controlling lane selection
3452      * @return the vector loaded from an array
3453      * @throws IndexOutOfBoundsException
3454      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3455      *         for any lane {@code N} in the vector
3456      *         where the mask is set
3457      */
3458     @ForceInline
3459     public static
3460     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3461                                        char[] a, int offset,
3462                                        VectorMask<$Boxtype$> m) {
3463         $Type$Species vsp = ($Type$Species) species;
3464         if (offset >= 0 && offset <= (a.length - species.length())) {
3465             $abstractvectortype$ zero = vsp.zero();
3466             return zero.blend(zero.fromCharArray0(a, offset), m);
3467         }
3468 
3469         // FIXME: optimize
3470         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3471         return vsp.vOp(m, i -> (short) a[offset + i]);
3472     }
3473 
3474     /**
3475      * Gathers a new vector composed of elements from an array of type
3476      * {@code char[]},
3477      * using indexes obtained by adding a fixed {@code offset} to a
3478      * series of secondary offsets from an <em>index map</em>.
3479      * The index map is a contiguous sequence of {@code VLENGTH}
3480      * elements in a second array of {@code int}s, starting at a given
3481      * {@code mapOffset}.
3482      * <p>
3483      * For each vector lane, where {@code N} is the vector lane index,
3484      * the lane is loaded from the expression
3485      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3486      * index mapping expression
3487      * {@code offset + indexMap[mapOffset + N]]}.
3488      *
3489      * @param species species of desired vector
3490      * @param a the array
3491      * @param offset the offset into the array, may be negative if relative
3492      * indexes in the index map compensate to produce a value within the
3493      * array bounds
3494      * @param indexMap the index map
3495      * @param mapOffset the offset into the index map
3496      * @return the vector loaded from the indexed elements of the array
3497      * @throws IndexOutOfBoundsException
3498      *         if {@code mapOffset+N < 0}
3499      *         or if {@code mapOffset+N >= indexMap.length},
3500      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3501      *         is an invalid index into {@code a},
3502      *         for any lane {@code N} in the vector
3503      * @see $abstractvectortype$#toIntArray()
3504      */
3505     @ForceInline
3506     public static
3507     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3508                                        char[] a, int offset,
3509                                        int[] indexMap, int mapOffset) {
3510         // FIXME: optimize
3511         $Type$Species vsp = ($Type$Species) species;
3512         return vsp.vOp(n -> (short) a[offset + indexMap[mapOffset + n]]);
3513     }
3514 
3515     /**
3516      * Gathers a new vector composed of elements from an array of type
3517      * {@code char[]},
3518      * under the control of a mask, and
3519      * using indexes obtained by adding a fixed {@code offset} to a
3520      * series of secondary offsets from an <em>index map</em>.
3521      * The index map is a contiguous sequence of {@code VLENGTH}
3522      * elements in a second array of {@code int}s, starting at a given
3523      * {@code mapOffset}.
3524      * <p>
3525      * For each vector lane, where {@code N} is the vector lane index,
3526      * if the lane is set in the mask,
3527      * the lane is loaded from the expression
3528      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3529      * index mapping expression
3530      * {@code offset + indexMap[mapOffset + N]]}.
3531      * Unset lanes in the resulting vector are set to zero.
3532      *
3533      * @param species species of desired vector
3534      * @param a the array
3535      * @param offset the offset into the array, may be negative if relative
3536      * indexes in the index map compensate to produce a value within the
3537      * array bounds
3538      * @param indexMap the index map
3539      * @param mapOffset the offset into the index map
3540      * @param m the mask controlling lane selection
3541      * @return the vector loaded from the indexed elements of the array
3542      * @throws IndexOutOfBoundsException
3543      *         if {@code mapOffset+N < 0}
3544      *         or if {@code mapOffset+N >= indexMap.length},
3545      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3546      *         is an invalid index into {@code a},
3547      *         for any lane {@code N} in the vector
3548      *         where the mask is set
3549      * @see $abstractvectortype$#toIntArray()
3550      */
3551     @ForceInline
3552     public static
3553     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3554                                        char[] a, int offset,
3555                                        int[] indexMap, int mapOffset,
3556                                        VectorMask<$Boxtype$> m) {
3557         // FIXME: optimize
3558         $Type$Species vsp = ($Type$Species) species;
3559         return vsp.vOp(m, n -> (short) a[offset + indexMap[mapOffset + n]]);
3560     }
3561 #end[short]
3562 
3563 #if[byte]
3564     /**
3565      * Loads a vector from an array of type {@code boolean[]}
3566      * starting at an offset.
3567      * For each vector lane, where {@code N} is the vector lane index, the
3568      * array element at index {@code offset + N}
3569      * is first converted to a {@code byte} value and then
3570      * placed into the resulting vector at lane index {@code N}.
3571      * <p>
3572      * A {@code boolean} value is converted to a {@code byte} value by applying the
3573      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3574      *
3575      * @param species species of desired vector
3576      * @param a the array
3577      * @param offset the offset into the array
3578      * @return the vector loaded from an array
3579      * @throws IndexOutOfBoundsException
3580      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3581      *         for any lane {@code N} in the vector
3582      */
3583     @ForceInline
3584     public static
3585     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3586                                           boolean[] a, int offset) {
3587         offset = checkFromIndexSize(offset, species.length(), a.length);
3588         $Type$Species vsp = ($Type$Species) species;
3589         return vsp.dummyVector().fromBooleanArray0(a, offset);
3590     }
3591 
3592     /**
3593      * Loads a vector from an array of type {@code boolean[]}
3594      * starting at an offset and using a mask.
3595      * Lanes where the mask is unset are filled with the default
3596      * value of {@code $type$} ({#if[FP]?positive }zero).
3597      * For each vector lane, where {@code N} is the vector lane index,
3598      * if the mask lane at index {@code N} is set then the array element at
3599      * index {@code offset + N}
3600      * is first converted to a {@code byte} value and then
3601      * placed into the resulting vector at lane index
3602      * {@code N}, otherwise the default element value is placed into the
3603      * resulting vector at lane index {@code N}.
3604      * <p>
3605      * A {@code boolean} value is converted to a {@code byte} value by applying the
3606      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3607      *
3608      * @param species species of desired vector
3609      * @param a the array
3610      * @param offset the offset into the array
3611      * @param m the mask controlling lane selection
3612      * @return the vector loaded from an array
3613      * @throws IndexOutOfBoundsException
3614      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3615      *         for any lane {@code N} in the vector
3616      *         where the mask is set
3617      */
3618     @ForceInline
3619     public static
3620     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3621                                           boolean[] a, int offset,
3622                                           VectorMask<$Boxtype$> m) {
3623         $Type$Species vsp = ($Type$Species) species;
3624         if (offset >= 0 && offset <= (a.length - species.length())) {
3625             $abstractvectortype$ zero = vsp.zero();
3626             return zero.blend(zero.fromBooleanArray0(a, offset), m);
3627         }
3628 
3629         // FIXME: optimize
3630         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3631         return vsp.vOp(m, i -> (byte) (a[offset + i] ? 1 : 0));
3632     }
3633 
3634     /**
3635      * Gathers a new vector composed of elements from an array of type
3636      * {@code boolean[]},
3637      * using indexes obtained by adding a fixed {@code offset} to a
3638      * series of secondary offsets from an <em>index map</em>.
3639      * The index map is a contiguous sequence of {@code VLENGTH}
3640      * elements in a second array of {@code int}s, starting at a given
3641      * {@code mapOffset}.
3642      * <p>
3643      * For each vector lane, where {@code N} is the vector lane index,
3644      * the lane is loaded from the expression
3645      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
3646      * index mapping expression
3647      * {@code offset + indexMap[mapOffset + N]]}.
3648      *
3649      * @param species species of desired vector
3650      * @param a the array
3651      * @param offset the offset into the array, may be negative if relative
3652      * indexes in the index map compensate to produce a value within the
3653      * array bounds
3654      * @param indexMap the index map
3655      * @param mapOffset the offset into the index map
3656      * @return the vector loaded from the indexed elements of the array
3657      * @throws IndexOutOfBoundsException
3658      *         if {@code mapOffset+N < 0}
3659      *         or if {@code mapOffset+N >= indexMap.length},
3660      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3661      *         is an invalid index into {@code a},
3662      *         for any lane {@code N} in the vector
3663      * @see $abstractvectortype$#toIntArray()
3664      */
3665     @ForceInline
3666     public static
3667     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3668                                           boolean[] a, int offset,
3669                                           int[] indexMap, int mapOffset) {
3670         // FIXME: optimize
3671         $Type$Species vsp = ($Type$Species) species;
3672         return vsp.vOp(n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
3673     }
3674 
3675     /**
3676      * Gathers a new vector composed of elements from an array of type
3677      * {@code boolean[]},
3678      * under the control of a mask, and
3679      * using indexes obtained by adding a fixed {@code offset} to a
3680      * series of secondary offsets from an <em>index map</em>.
3681      * The index map is a contiguous sequence of {@code VLENGTH}
3682      * elements in a second array of {@code int}s, starting at a given
3683      * {@code mapOffset}.
3684      * <p>
3685      * For each vector lane, where {@code N} is the vector lane index,
3686      * if the lane is set in the mask,
3687      * the lane is loaded from the expression
3688      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
3689      * index mapping expression
3690      * {@code offset + indexMap[mapOffset + N]]}.
3691      * Unset lanes in the resulting vector are set to zero.
3692      *
3693      * @param species species of desired vector
3694      * @param a the array
3695      * @param offset the offset into the array, may be negative if relative
3696      * indexes in the index map compensate to produce a value within the
3697      * array bounds
3698      * @param indexMap the index map
3699      * @param mapOffset the offset into the index map
3700      * @param m the mask controlling lane selection
3701      * @return the vector loaded from the indexed elements of the array
3702      * @throws IndexOutOfBoundsException
3703      *         if {@code mapOffset+N < 0}
3704      *         or if {@code mapOffset+N >= indexMap.length},
3705      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3706      *         is an invalid index into {@code a},
3707      *         for any lane {@code N} in the vector
3708      *         where the mask is set
3709      * @see $abstractvectortype$#toIntArray()
3710      */
3711     @ForceInline
3712     public static
3713     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3714                                           boolean[] a, int offset,
3715                                           int[] indexMap, int mapOffset,
3716                                           VectorMask<$Boxtype$> m) {
3717         // FIXME: optimize
3718         $Type$Species vsp = ($Type$Species) species;
3719         return vsp.vOp(m, n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
3720     }
3721 #end[byte]
3722 
3723     /**
3724      * Loads a vector from a {@linkplain ByteBuffer byte buffer}
3725      * starting at an offset into the byte buffer.
3726      * Bytes are composed into primitive lane elements according
3727      * to the specified byte order.
3728      * The vector is arranged into lanes according to
3729      * <a href="Vector.html#lane-order">memory ordering</a>.
3730      * <p>
3731      * This method behaves as if it returns the result of calling
3732      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3733      * fromByteBuffer()} as follows:
3734      * <pre>{@code
3735      * var m = species.maskAll(true);
3736      * return fromByteBuffer(species, bb, offset, bo, m);
3737      * }</pre>
3738      *
3739      * @param species species of desired vector
3740      * @param bb the byte buffer
3741      * @param offset the offset into the byte buffer
3742      * @param bo the intended byte order
3743      * @return a vector loaded from a byte buffer
3744      * @throws IndexOutOfBoundsException
3745      *         if {@code offset+N*$sizeInBytes$ < 0}
3746      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
3747      *         for any lane {@code N} in the vector
3748      */
3749     @ForceInline
3750     public static
3751     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
3752                                         ByteBuffer bb, int offset,
3753                                         ByteOrder bo) {
3754         offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
3755         $Type$Species vsp = ($Type$Species) species;
3756         return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
3757     }
3758 
3759     /**
3760      * Loads a vector from a {@linkplain ByteBuffer byte buffer}
3761      * starting at an offset into the byte buffer
3762      * and using a mask.
3763      * Lanes where the mask is unset are filled with the default
3764      * value of {@code $type$} ({#if[FP]?positive }zero).
3765      * Bytes are composed into primitive lane elements according
3766      * to the specified byte order.
3767      * The vector is arranged into lanes according to
3768      * <a href="Vector.html#lane-order">memory ordering</a>.
3769      * <p>
3770      * The following pseudocode illustrates the behavior:
3771      * <pre>{@code
3772      * $Type$Buffer eb = bb.duplicate()
3773      *     .position(offset){#if[byte]?;}
3774 #if[!byte]
3775      *     .order(bo).as$Type$Buffer();
3776 #end[!byte]
3777      * $type$[] ar = new $type$[species.length()];
3778      * for (int n = 0; n < ar.length; n++) {
3779      *     if (m.laneIsSet(n)) {
3780      *         ar[n] = eb.get(n);
3781      *     }
3782      * }
3783      * $abstractvectortype$ r = $abstractvectortype$.fromArray(species, ar, 0);
3784      * }</pre>
3785      * @implNote
3786 #if[!byte]
3787      * This operation is likely to be more efficient if
3788      * the specified byte order is the same as
3789      * {@linkplain ByteOrder#nativeOrder()
3790      * the platform native order},
3791      * since this method will not need to reorder
3792      * the bytes of lane values.
3793 #else[!byte]
3794      * The byte order argument is ignored.
3795 #end[!byte]
3796      *
3797      * @param species species of desired vector
3798      * @param bb the byte buffer
3799      * @param offset the offset into the byte buffer
3800      * @param bo the intended byte order
3801      * @param m the mask controlling lane selection
3802      * @return a vector loaded from a byte buffer
3803      * @throws IndexOutOfBoundsException
3804      *         if {@code offset+N*$sizeInBytes$ < 0}
3805      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
3806      *         for any lane {@code N} in the vector
3807      *         where the mask is set
3808      */
3809     @ForceInline
3810     public static
3811     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
3812                                         ByteBuffer bb, int offset,
3813                                         ByteOrder bo,
3814                                         VectorMask<$Boxtype$> m) {
3815         $Type$Species vsp = ($Type$Species) species;
3816         if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
3817             $abstractvectortype$ zero = vsp.zero();
3818             $abstractvectortype$ v = zero.fromByteBuffer0(bb, offset);
3819             return zero.blend(v.maybeSwap(bo), m);
3820         }
3821 
3822         // FIXME: optimize
3823         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
3824         ByteBuffer wb = wrapper(bb, bo);
3825         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
3826                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
3827     }
3828 
3829     // Memory store operations
3830 
3831     /**
3832      * Stores this vector into an array of type {@code $type$[]}
3833      * starting at an offset.
3834      * <p>
3835      * For each vector lane, where {@code N} is the vector lane index,
3836      * the lane element at index {@code N} is stored into the array
3837      * element {@code a[offset+N]}.
3838      *
3839      * @param a the array, of type {@code $type$[]}
3840      * @param offset the offset into the array
3841      * @throws IndexOutOfBoundsException
3842      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3843      *         for any lane {@code N} in the vector
3844      */
3845     @ForceInline
3846     public final
3847     void intoArray($type$[] a, int offset) {
3848         offset = checkFromIndexSize(offset, length(), a.length);
3849         $Type$Species vsp = vspecies();
3850         VectorSupport.store(
3851             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3852             a, arrayAddress(a, offset),
3853             this,
3854             a, offset,
3855             (arr, off, v)
3856             -> v.stOp(arr, off,
3857                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
3858     }
3859 
3860     /**
3861      * Stores this vector into an array of type {@code $type$[]}
3862      * starting at offset and using a mask.
3863      * <p>
3864      * For each vector lane, where {@code N} is the vector lane index,
3865      * the lane element at index {@code N} is stored into the array
3866      * element {@code a[offset+N]}.
3867      * If the mask lane at {@code N} is unset then the corresponding
3868      * array element {@code a[offset+N]} is left unchanged.
3869      * <p>
3870      * Array range checking is done for lanes where the mask is set.
3871      * Lanes where the mask is unset are not stored and do not need
3872      * to correspond to legitimate elements of {@code a}.
3873      * That is, unset lanes may correspond to array indexes less than
3874      * zero or beyond the end of the array.
3875      *
3876      * @param a the array, of type {@code $type$[]}
3877      * @param offset the offset into the array
3878      * @param m the mask controlling lane storage
3879      * @throws IndexOutOfBoundsException
3880      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3881      *         for any lane {@code N} in the vector
3882      *         where the mask is set
3883      */
3884     @ForceInline
3885     public final
3886     void intoArray($type$[] a, int offset,
3887                    VectorMask<$Boxtype$> m) {
3888         if (m.allTrue()) {
3889             intoArray(a, offset);
3890         } else {
3891             // FIXME: optimize
3892             $Type$Species vsp = vspecies();
3893             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3894             stOp(a, offset, m, (arr, off, i, v) -> arr[off+i] = v);
3895         }
3896     }
3897 
3898     /**
3899      * Scatters this vector into an array of type {@code $type$[]}
3900      * using indexes obtained by adding a fixed {@code offset} to a
3901      * series of secondary offsets from an <em>index map</em>.
3902      * The index map is a contiguous sequence of {@code VLENGTH}
3903      * elements in a second array of {@code int}s, starting at a given
3904      * {@code mapOffset}.
3905      * <p>
3906      * For each vector lane, where {@code N} is the vector lane index,
3907      * the lane element at index {@code N} is stored into the array
3908      * element {@code a[f(N)]}, where {@code f(N)} is the
3909      * index mapping expression
3910      * {@code offset + indexMap[mapOffset + N]]}.
3911      *
3912      * @param a the array
3913      * @param offset an offset to combine with the index map offsets
3914      * @param indexMap the index map
3915      * @param mapOffset the offset into the index map
3916      * @throws IndexOutOfBoundsException
3917      *         if {@code mapOffset+N < 0}
3918      *         or if {@code mapOffset+N >= indexMap.length},
3919      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3920      *         is an invalid index into {@code a},
3921      *         for any lane {@code N} in the vector
3922      * @see $abstractvectortype$#toIntArray()
3923      */
3924 #if[byteOrShort]
3925     @ForceInline
3926     public final
3927     void intoArray($type$[] a, int offset,
3928                    int[] indexMap, int mapOffset) {
3929         stOp(a, offset,
3930              (arr, off, i, e) -> {
3931                  int j = indexMap[mapOffset + i];
3932                  arr[off + j] = e;
3933              });
3934     }
3935 #else[byteOrShort]
3936     @ForceInline
3937     public final
3938     void intoArray($type$[] a, int offset,
3939                    int[] indexMap, int mapOffset) {
3940         $Type$Species vsp = vspecies();
3941         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3942 #if[longOrDouble]
3943         if (vsp.laneCount() == 1) {
3944             intoArray(a, offset + indexMap[mapOffset]);
3945             return;
3946         }
3947 
3948         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
3949         IntVector vix;
3950         if (isp.laneCount() != vsp.laneCount()) {
3951             // For $Type$MaxVector,  if vector length  is 2048 bits, indexShape
3952             // of $Type$ species is S_MAX_BIT. and the lane count of $Type$
3953             // vector is 32. When converting $Type$ species to int species,
3954             // indexShape is still S_MAX_BIT, but the lane count of int vector
3955             // is 64. So when loading index vector (IntVector), only lower half
3956             // of index data is needed.
3957             vix = IntVector
3958                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3959                 .add(offset);
3960         } else {
3961             vix = IntVector
3962                 .fromArray(isp, indexMap, mapOffset)
3963                 .add(offset);
3964         }
3965 
3966 #else[longOrDouble]
3967         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
3968         IntVector vix = IntVector
3969             .fromArray(isp, indexMap, mapOffset)
3970             .add(offset);
3971 #end[longOrDouble]
3972 
3973         vix = VectorIntrinsics.checkIndex(vix, a.length);
3974 
3975         VectorSupport.storeWithMap(
3976             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3977             isp.vectorType(),
3978             a, arrayAddress(a, 0), vix,
3979             this,
3980             a, offset, indexMap, mapOffset,
3981             (arr, off, v, map, mo)
3982             -> v.stOp(arr, off,
3983                       (arr_, off_, i, e) -> {
3984                           int j = map[mo + i];
3985                           arr[off + j] = e;
3986                       }));
3987     }
3988 #end[byteOrShort]
3989 
3990     /**
3991      * Scatters this vector into an array of type {@code $type$[]},
3992      * under the control of a mask, and
3993      * using indexes obtained by adding a fixed {@code offset} to a
3994      * series of secondary offsets from an <em>index map</em>.
3995      * The index map is a contiguous sequence of {@code VLENGTH}
3996      * elements in a second array of {@code int}s, starting at a given
3997      * {@code mapOffset}.
3998      * <p>
3999      * For each vector lane, where {@code N} is the vector lane index,
4000      * if the mask lane at index {@code N} is set then
4001      * the lane element at index {@code N} is stored into the array
4002      * element {@code a[f(N)]}, where {@code f(N)} is the
4003      * index mapping expression
4004      * {@code offset + indexMap[mapOffset + N]]}.
4005      *
4006      * @param a the array
4007      * @param offset an offset to combine with the index map offsets
4008      * @param indexMap the index map
4009      * @param mapOffset the offset into the index map
4010      * @param m the mask
4011      * @throws IndexOutOfBoundsException
4012      *         if {@code mapOffset+N < 0}
4013      *         or if {@code mapOffset+N >= indexMap.length},
4014      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4015      *         is an invalid index into {@code a},
4016      *         for any lane {@code N} in the vector
4017      *         where the mask is set
4018      * @see $abstractvectortype$#toIntArray()
4019      */
4020 #if[byteOrShort]
4021     @ForceInline
4022     public final
4023     void intoArray($type$[] a, int offset,
4024                    int[] indexMap, int mapOffset,
4025                    VectorMask<$Boxtype$> m) {
4026         stOp(a, offset, m,
4027              (arr, off, i, e) -> {
4028                  int j = indexMap[mapOffset + i];
4029                  arr[off + j] = e;
4030              });
4031     }
4032 #else[byteOrShort]
4033     @ForceInline
4034     public final
4035     void intoArray($type$[] a, int offset,
4036                    int[] indexMap, int mapOffset,
4037                    VectorMask<$Boxtype$> m) {
4038         if (m.allTrue()) {
4039             intoArray(a, offset, indexMap, mapOffset);
4040         }
4041         else {
4042             // FIXME: Cannot vectorize yet, if there's a mask.
4043             stOp(a, offset, m,
4044                  (arr, off, i, e) -> {
4045                      int j = indexMap[mapOffset + i];
4046                      arr[off + j] = e;
4047                  });
4048         }
4049     }
4050 #end[byteOrShort]
4051 
4052 #if[short]
4053     /**
4054      * Stores this vector into an array of type {@code char[]}
4055      * starting at an offset.
4056      * <p>
4057      * For each vector lane, where {@code N} is the vector lane index,
4058      * the lane element at index {@code N}
4059      * is first cast to a {@code char} value and then
4060      * stored into the array element {@code a[offset+N]}.
4061      *
4062      * @param a the array, of type {@code char[]}
4063      * @param offset the offset into the array
4064      * @throws IndexOutOfBoundsException
4065      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4066      *         for any lane {@code N} in the vector
4067      */
4068     @ForceInline
4069     public final
4070     void intoCharArray(char[] a, int offset) {
4071         offset = checkFromIndexSize(offset, length(), a.length);
4072         $Type$Species vsp = vspecies();
4073         VectorSupport.store(
4074             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4075             a, charArrayAddress(a, offset),
4076             this,
4077             a, offset,
4078             (arr, off, v)
4079             -> v.stOp(arr, off,
4080                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
4081     }
4082 
4083     /**
4084      * Stores this vector into an array of type {@code char[]}
4085      * starting at offset and using a mask.
4086      * <p>
4087      * For each vector lane, where {@code N} is the vector lane index,
4088      * the lane element at index {@code N}
4089      * is first cast to a {@code char} value and then
4090      * stored into the array element {@code a[offset+N]}.
4091      * If the mask lane at {@code N} is unset then the corresponding
4092      * array element {@code a[offset+N]} is left unchanged.
4093      * <p>
4094      * Array range checking is done for lanes where the mask is set.
4095      * Lanes where the mask is unset are not stored and do not need
4096      * to correspond to legitimate elements of {@code a}.
4097      * That is, unset lanes may correspond to array indexes less than
4098      * zero or beyond the end of the array.
4099      *
4100      * @param a the array, of type {@code char[]}
4101      * @param offset the offset into the array
4102      * @param m the mask controlling lane storage
4103      * @throws IndexOutOfBoundsException
4104      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4105      *         for any lane {@code N} in the vector
4106      *         where the mask is set
4107      */
4108     @ForceInline
4109     public final
4110     void intoCharArray(char[] a, int offset,
4111                        VectorMask<$Boxtype$> m) {
4112         if (m.allTrue()) {
4113             intoCharArray(a, offset);
4114         } else {
4115             // FIXME: optimize
4116             $Type$Species vsp = vspecies();
4117             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4118             stOp(a, offset, m, (arr, off, i, v) -> arr[off+i] = (char) v);
4119         }
4120     }
4121 
4122     /**
4123      * Scatters this vector into an array of type {@code char[]}
4124      * using indexes obtained by adding a fixed {@code offset} to a
4125      * series of secondary offsets from an <em>index map</em>.
4126      * The index map is a contiguous sequence of {@code VLENGTH}
4127      * elements in a second array of {@code int}s, starting at a given
4128      * {@code mapOffset}.
4129      * <p>
4130      * For each vector lane, where {@code N} is the vector lane index,
4131      * the lane element at index {@code N}
4132      * is first cast to a {@code char} value and then
4133      * stored into the array
4134      * element {@code a[f(N)]}, where {@code f(N)} is the
4135      * index mapping expression
4136      * {@code offset + indexMap[mapOffset + N]]}.
4137      *
4138      * @param a the array
4139      * @param offset an offset to combine with the index map offsets
4140      * @param indexMap the index map
4141      * @param mapOffset the offset into the index map
4142      * @throws IndexOutOfBoundsException
4143      *         if {@code mapOffset+N < 0}
4144      *         or if {@code mapOffset+N >= indexMap.length},
4145      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4146      *         is an invalid index into {@code a},
4147      *         for any lane {@code N} in the vector
4148      * @see $abstractvectortype$#toIntArray()
4149      */
4150     @ForceInline
4151     public final
4152     void intoCharArray(char[] a, int offset,
4153                        int[] indexMap, int mapOffset) {
4154         // FIXME: optimize
4155         stOp(a, offset,
4156              (arr, off, i, e) -> {
4157                  int j = indexMap[mapOffset + i];
4158                  arr[off + j] = (char) e;
4159              });
4160     }
4161 
4162     /**
4163      * Scatters this vector into an array of type {@code char[]},
4164      * under the control of a mask, and
4165      * using indexes obtained by adding a fixed {@code offset} to a
4166      * series of secondary offsets from an <em>index map</em>.
4167      * The index map is a contiguous sequence of {@code VLENGTH}
4168      * elements in a second array of {@code int}s, starting at a given
4169      * {@code mapOffset}.
4170      * <p>
4171      * For each vector lane, where {@code N} is the vector lane index,
4172      * if the mask lane at index {@code N} is set then
4173      * the lane element at index {@code N}
4174      * is first cast to a {@code char} value and then
4175      * stored into the array
4176      * element {@code a[f(N)]}, where {@code f(N)} is the
4177      * index mapping expression
4178      * {@code offset + indexMap[mapOffset + N]]}.
4179      *
4180      * @param a the array
4181      * @param offset an offset to combine with the index map offsets
4182      * @param indexMap the index map
4183      * @param mapOffset the offset into the index map
4184      * @param m the mask
4185      * @throws IndexOutOfBoundsException
4186      *         if {@code mapOffset+N < 0}
4187      *         or if {@code mapOffset+N >= indexMap.length},
4188      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4189      *         is an invalid index into {@code a},
4190      *         for any lane {@code N} in the vector
4191      *         where the mask is set
4192      * @see $abstractvectortype$#toIntArray()
4193      */
4194     @ForceInline
4195     public final
4196     void intoCharArray(char[] a, int offset,
4197                        int[] indexMap, int mapOffset,
4198                        VectorMask<$Boxtype$> m) {
4199         // FIXME: optimize
4200         stOp(a, offset, m,
4201              (arr, off, i, e) -> {
4202                  int j = indexMap[mapOffset + i];
4203                  arr[off + j] = (char) e;
4204              });
4205     }
4206 #end[short]
4207 
4208 #if[byte]
4209     /**
4210      * Stores this vector into an array of type {@code boolean[]}
4211      * starting at an offset.
4212      * <p>
4213      * For each vector lane, where {@code N} is the vector lane index,
4214      * the lane element at index {@code N}
4215      * is first converted to a {@code boolean} value and then
4216      * stored into the array element {@code a[offset+N]}.
4217      * <p>
4218      * A {@code byte} value is converted to a {@code boolean} value by applying the
4219      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4220      *
4221      * @param a the array
4222      * @param offset the offset into the array
4223      * @throws IndexOutOfBoundsException
4224      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4225      *         for any lane {@code N} in the vector
4226      */
4227     @ForceInline
4228     public final
4229     void intoBooleanArray(boolean[] a, int offset) {
4230         offset = checkFromIndexSize(offset, length(), a.length);
4231         $Type$Species vsp = vspecies();
4232         ByteVector normalized = this.and((byte) 1);
4233         VectorSupport.store(
4234             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4235             a, booleanArrayAddress(a, offset),
4236             normalized,
4237             a, offset,
4238             (arr, off, v)
4239             -> v.stOp(arr, off,
4240                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4241     }
4242 
4243     /**
4244      * Stores this vector into an array of type {@code boolean[]}
4245      * starting at offset and using a mask.
4246      * <p>
4247      * For each vector lane, where {@code N} is the vector lane index,
4248      * the lane element at index {@code N}
4249      * is first converted to a {@code boolean} value and then
4250      * stored into the array element {@code a[offset+N]}.
4251      * If the mask lane at {@code N} is unset then the corresponding
4252      * array element {@code a[offset+N]} is left unchanged.
4253      * <p>
4254      * A {@code byte} value is converted to a {@code boolean} value by applying the
4255      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4256      * <p>
4257      * Array range checking is done for lanes where the mask is set.
4258      * Lanes where the mask is unset are not stored and do not need
4259      * to correspond to legitimate elements of {@code a}.
4260      * That is, unset lanes may correspond to array indexes less than
4261      * zero or beyond the end of the array.
4262      *
4263      * @param a the array
4264      * @param offset the offset into the array
4265      * @param m the mask controlling lane storage
4266      * @throws IndexOutOfBoundsException
4267      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4268      *         for any lane {@code N} in the vector
4269      *         where the mask is set
4270      */
4271     @ForceInline
4272     public final
4273     void intoBooleanArray(boolean[] a, int offset,
4274                           VectorMask<$Boxtype$> m) {
4275         if (m.allTrue()) {
4276             intoBooleanArray(a, offset);
4277         } else {
4278             // FIXME: optimize
4279             $Type$Species vsp = vspecies();
4280             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4281             stOp(a, offset, m, (arr, off, i, e) -> arr[off+i] = (e & 1) != 0);
4282         }
4283     }
4284 
4285     /**
4286      * Scatters this vector into an array of type {@code boolean[]}
4287      * using indexes obtained by adding a fixed {@code offset} to a
4288      * series of secondary offsets from an <em>index map</em>.
4289      * The index map is a contiguous sequence of {@code VLENGTH}
4290      * elements in a second array of {@code int}s, starting at a given
4291      * {@code mapOffset}.
4292      * <p>
4293      * For each vector lane, where {@code N} is the vector lane index,
4294      * the lane element at index {@code N}
4295      * is first converted to a {@code boolean} value and then
4296      * stored into the array
4297      * element {@code a[f(N)]}, where {@code f(N)} is the
4298      * index mapping expression
4299      * {@code offset + indexMap[mapOffset + N]]}.
4300      * <p>
4301      * A {@code byte} value is converted to a {@code boolean} value by applying the
4302      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4303      *
4304      * @param a the array
4305      * @param offset an offset to combine with the index map offsets
4306      * @param indexMap the index map
4307      * @param mapOffset the offset into the index map
4308      * @throws IndexOutOfBoundsException
4309      *         if {@code mapOffset+N < 0}
4310      *         or if {@code mapOffset+N >= indexMap.length},
4311      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4312      *         is an invalid index into {@code a},
4313      *         for any lane {@code N} in the vector
4314      * @see $abstractvectortype$#toIntArray()
4315      */
4316     @ForceInline
4317     public final
4318     void intoBooleanArray(boolean[] a, int offset,
4319                           int[] indexMap, int mapOffset) {
4320         // FIXME: optimize
4321         stOp(a, offset,
4322              (arr, off, i, e) -> {
4323                  int j = indexMap[mapOffset + i];
4324                  arr[off + j] = (e & 1) != 0;
4325              });
4326     }
4327 
4328     /**
4329      * Scatters this vector into an array of type {@code boolean[]},
4330      * under the control of a mask, and
4331      * using indexes obtained by adding a fixed {@code offset} to a
4332      * series of secondary offsets from an <em>index map</em>.
4333      * The index map is a contiguous sequence of {@code VLENGTH}
4334      * elements in a second array of {@code int}s, starting at a given
4335      * {@code mapOffset}.
4336      * <p>
4337      * For each vector lane, where {@code N} is the vector lane index,
4338      * if the mask lane at index {@code N} is set then
4339      * the lane element at index {@code N}
4340      * is first converted to a {@code boolean} value and then
4341      * stored into the array
4342      * element {@code a[f(N)]}, where {@code f(N)} is the
4343      * index mapping expression
4344      * {@code offset + indexMap[mapOffset + N]]}.
4345      * <p>
4346      * A {@code byte} value is converted to a {@code boolean} value by applying the
4347      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4348      *
4349      * @param a the array
4350      * @param offset an offset to combine with the index map offsets
4351      * @param indexMap the index map
4352      * @param mapOffset the offset into the index map
4353      * @param m the mask
4354      * @throws IndexOutOfBoundsException
4355      *         if {@code mapOffset+N < 0}
4356      *         or if {@code mapOffset+N >= indexMap.length},
4357      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4358      *         is an invalid index into {@code a},
4359      *         for any lane {@code N} in the vector
4360      *         where the mask is set
4361      * @see $abstractvectortype$#toIntArray()
4362      */
4363     @ForceInline
4364     public final
4365     void intoBooleanArray(boolean[] a, int offset,
4366                           int[] indexMap, int mapOffset,
4367                           VectorMask<$Boxtype$> m) {
4368         // FIXME: optimize
4369         stOp(a, offset, m,
4370              (arr, off, i, e) -> {
4371                  int j = indexMap[mapOffset + i];
4372                  arr[off + j] = (e & 1) != 0;
4373              });
4374     }
4375 #end[byte]
4376 
4377     /**
4378      * {@inheritDoc} <!--workaround-->
4379      */
4380     @Override
4381     @ForceInline
4382     public final
4383     void intoByteArray(byte[] a, int offset,
4384                        ByteOrder bo) {
4385         offset = checkFromIndexSize(offset, byteSize(), a.length);
4386         maybeSwap(bo).intoByteArray0(a, offset);
4387     }
4388 
4389     /**
4390      * {@inheritDoc} <!--workaround-->
4391      */
4392     @Override
4393     @ForceInline
4394     public final
4395     void intoByteArray(byte[] a, int offset,
4396                        ByteOrder bo,
4397                        VectorMask<$Boxtype$> m) {
4398         if (m.allTrue()) {
4399             intoByteArray(a, offset, bo);
4400         } else {
4401             // FIXME: optimize
4402             $Type$Species vsp = vspecies();
4403             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
4404             ByteBuffer wb = wrapper(a, bo);
4405             this.stOp(wb, offset, m,
4406                     (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
4407         }
4408     }
4409 
4410     /**
4411      * {@inheritDoc} <!--workaround-->
4412      */
4413     @Override
4414     @ForceInline
4415     public final
4416     void intoByteBuffer(ByteBuffer bb, int offset,
4417                         ByteOrder bo) {
4418         if (bb.isReadOnly()) {
4419             throw new ReadOnlyBufferException();
4420         }
4421         offset = checkFromIndexSize(offset, byteSize(), bb.limit());
4422         maybeSwap(bo).intoByteBuffer0(bb, offset);
4423     }
4424 
4425     /**
4426      * {@inheritDoc} <!--workaround-->
4427      */
4428     @Override
4429     @ForceInline
4430     public final
4431     void intoByteBuffer(ByteBuffer bb, int offset,
4432                         ByteOrder bo,
4433                         VectorMask<$Boxtype$> m) {
4434         if (m.allTrue()) {
4435             intoByteBuffer(bb, offset, bo);
4436         } else {
4437             // FIXME: optimize
4438             if (bb.isReadOnly()) {
4439                 throw new ReadOnlyBufferException();
4440             }
4441             $Type$Species vsp = vspecies();
4442             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4443             ByteBuffer wb = wrapper(bb, bo);
4444             this.stOp(wb, offset, m,
4445                     (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
4446         }
4447     }
4448 
4449     // ================================================
4450 
4451     // Low-level memory operations.
4452     //
4453     // Note that all of these operations *must* inline into a context
4454     // where the exact species of the involved vector is a
4455     // compile-time constant.  Otherwise, the intrinsic generation
4456     // will fail and performance will suffer.
4457     //
4458     // In many cases this is achieved by re-deriving a version of the
4459     // method in each concrete subclass (per species).  The re-derived
4460     // method simply calls one of these generic methods, with exact
4461     // parameters for the controlling metadata, which is either a
4462     // typed vector or constant species instance.
4463 
4464     // Unchecked loading operations in native byte order.
4465     // Caller is responsible for applying index checks, masking, and
4466     // byte swapping.
4467 
4468     /*package-private*/
4469     abstract
4470     $abstractvectortype$ fromArray0($type$[] a, int offset);
4471     @ForceInline
4472     final
4473     $abstractvectortype$ fromArray0Template($type$[] a, int offset) {
4474         $Type$Species vsp = vspecies();
4475         return VectorSupport.load(
4476             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4477             a, arrayAddress(a, offset),
4478             a, offset, vsp,
4479             (arr, off, s) -> s.ldOp(arr, off,
4480                                     (arr_, off_, i) -> arr_[off_ + i]));
4481     }
4482 
4483 #if[short]
4484     /*package-private*/
4485     abstract
4486     $abstractvectortype$ fromCharArray0(char[] a, int offset);
4487     @ForceInline
4488     final
4489     $abstractvectortype$ fromCharArray0Template(char[] a, int offset) {
4490         $Type$Species vsp = vspecies();
4491         return VectorSupport.load(
4492             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4493             a, charArrayAddress(a, offset),
4494             a, offset, vsp,
4495             (arr, off, s) -> s.ldOp(arr, off,
4496                                     (arr_, off_, i) -> (short) arr_[off_ + i]));
4497     }
4498 #end[short]
4499 
4500 #if[byte]
4501     /*package-private*/
4502     abstract
4503     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset);
4504     @ForceInline
4505     final
4506     $abstractvectortype$ fromBooleanArray0Template(boolean[] a, int offset) {
4507         $Type$Species vsp = vspecies();
4508         return VectorSupport.load(
4509             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4510             a, booleanArrayAddress(a, offset),
4511             a, offset, vsp,
4512             (arr, off, s) -> s.ldOp(arr, off,
4513                                     (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4514     }
4515 #end[byte]
4516 
4517     @Override
4518     abstract
4519     $abstractvectortype$ fromByteArray0(byte[] a, int offset);
4520     @ForceInline
4521     final
4522     $abstractvectortype$ fromByteArray0Template(byte[] a, int offset) {
4523         $Type$Species vsp = vspecies();
4524         return VectorSupport.load(
4525             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4526             a, byteArrayAddress(a, offset),
4527             a, offset, vsp,
4528             (arr, off, s) -> {
4529                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4530                 return s.ldOp(wb, off,
4531                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4532             });
4533     }
4534 
4535     abstract
4536     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset);
4537     @ForceInline
4538     final
4539     $abstractvectortype$ fromByteBuffer0Template(ByteBuffer bb, int offset) {
4540         $Type$Species vsp = vspecies();
4541         return ScopedMemoryAccess.loadFromByteBuffer(
4542                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4543                 bb, offset, vsp,
4544                 (buf, off, s) -> {
4545                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4546                     return s.ldOp(wb, off,
4547                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4548                 });
4549     }
4550 
4551     // Unchecked storing operations in native byte order.
4552     // Caller is responsible for applying index checks, masking, and
4553     // byte swapping.
4554 
4555     abstract
4556     void intoArray0($type$[] a, int offset);
4557     @ForceInline
4558     final
4559     void intoArray0Template($type$[] a, int offset) {
4560         $Type$Species vsp = vspecies();
4561         VectorSupport.store(
4562             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4563             a, arrayAddress(a, offset),
4564             this, a, offset,
4565             (arr, off, v)
4566             -> v.stOp(arr, off,
4567                       (arr_, off_, i, e) -> arr_[off_+i] = e));
4568     }
4569 
4570     abstract
4571     void intoByteArray0(byte[] a, int offset);
4572     @ForceInline
4573     final
4574     void intoByteArray0Template(byte[] a, int offset) {
4575         $Type$Species vsp = vspecies();
4576         VectorSupport.store(
4577             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4578             a, byteArrayAddress(a, offset),
4579             this, a, offset,
4580             (arr, off, v) -> {
4581                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4582                 v.stOp(wb, off,
4583                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
4584             });
4585     }
4586 
4587     @ForceInline
4588     final
4589     void intoByteBuffer0(ByteBuffer bb, int offset) {
4590         $Type$Species vsp = vspecies();
4591         ScopedMemoryAccess.storeIntoByteBuffer(
4592                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4593                 this, bb, offset,
4594                 (buf, off, v) -> {
4595                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4596                     v.stOp(wb, off,
4597                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
4598                 });
4599     }
4600 
4601     // End of low-level memory operations.
4602 
4603     private static
4604     void checkMaskFromIndexSize(int offset,
4605                                 $Type$Species vsp,
4606                                 VectorMask<$Boxtype$> m,
4607                                 int scale,
4608                                 int limit) {
4609         ((AbstractMask<$Boxtype$>)m)
4610             .checkIndexByLane(offset, limit, vsp.iota(), scale);
4611     }
4612 
4613     @ForceInline
4614     private void conditionalStoreNYI(int offset,
4615                                      $Type$Species vsp,
4616                                      VectorMask<$Boxtype$> m,
4617                                      int scale,
4618                                      int limit) {
4619         if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
4620             String msg =
4621                 String.format("unimplemented: store @%d in [0..%d), %s in %s",
4622                               offset, limit, m, vsp);
4623             throw new AssertionError(msg);
4624         }
4625     }
4626 
4627     /*package-private*/
4628     @Override
4629     @ForceInline
4630     final
4631     $abstractvectortype$ maybeSwap(ByteOrder bo) {
4632 #if[!byte]
4633         if (bo != NATIVE_ENDIAN) {
4634             return this.reinterpretAsBytes()
4635                 .rearrange(swapBytesShuffle())
4636                 .reinterpretAs$Type$s();
4637         }
4638 #end[!byte]
4639         return this;
4640     }
4641 
4642     static final int ARRAY_SHIFT =
4643         31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_$TYPE$_INDEX_SCALE);
4644     static final long ARRAY_BASE =
4645         Unsafe.ARRAY_$TYPE$_BASE_OFFSET;
4646 
4647     @ForceInline
4648     static long arrayAddress($type$[] a, int index) {
4649         return ARRAY_BASE + (((long)index) << ARRAY_SHIFT);
4650     }
4651 
4652 #if[short]
4653     static final int ARRAY_CHAR_SHIFT =
4654             31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_CHAR_INDEX_SCALE);
4655     static final long ARRAY_CHAR_BASE =
4656             Unsafe.ARRAY_CHAR_BASE_OFFSET;
4657 
4658     @ForceInline
4659     static long charArrayAddress(char[] a, int index) {
4660         return ARRAY_CHAR_BASE + (((long)index) << ARRAY_CHAR_SHIFT);
4661     }
4662 #end[short]
4663 
4664 #if[byte]
4665     static final int ARRAY_BOOLEAN_SHIFT =
4666             31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_BOOLEAN_INDEX_SCALE);
4667     static final long ARRAY_BOOLEAN_BASE =
4668             Unsafe.ARRAY_BOOLEAN_BASE_OFFSET;
4669 
4670     @ForceInline
4671     static long booleanArrayAddress(boolean[] a, int index) {
4672         return ARRAY_BOOLEAN_BASE + (((long)index) << ARRAY_BOOLEAN_SHIFT);
4673     }
4674 #end[byte]
4675 
4676     @ForceInline
4677     static long byteArrayAddress(byte[] a, int index) {
4678         return Unsafe.ARRAY_BYTE_BASE_OFFSET + index;
4679     }
4680 
4681     // ================================================
4682 
4683     /// Reinterpreting view methods:
4684     //   lanewise reinterpret: viewAsXVector()
4685     //   keep shape, redraw lanes: reinterpretAsEs()
4686 
4687     /**
4688      * {@inheritDoc} <!--workaround-->
4689      */
4690     @ForceInline
4691     @Override
4692     public final ByteVector reinterpretAsBytes() {
4693 #if[byte]
4694         return this;
4695 #else[byte]
4696          // Going to ByteVector, pay close attention to byte order.
4697          assert(REGISTER_ENDIAN == ByteOrder.LITTLE_ENDIAN);
4698          return asByteVectorRaw();
4699          //return asByteVectorRaw().rearrange(swapBytesShuffle());
4700 #end[byte]
4701     }
4702 
4703     /**
4704      * {@inheritDoc} <!--workaround-->
4705      */
4706     @ForceInline
4707     @Override
4708     public final $Bitstype$Vector viewAsIntegralLanes() {
4709 #if[BITWISE]
4710         return this;
4711 #else[BITWISE]
4712         LaneType ilt = LaneType.$TYPE$.asIntegral();
4713         return ($Bitstype$Vector) asVectorRaw(ilt);
4714 #end[BITWISE]
4715     }
4716 
4717     /**
4718      * {@inheritDoc} <!--workaround-->
4719 #if[byteOrShort]
4720      *
4721      * @implNote This method always throws
4722      * {@code UnsupportedOperationException}, because there is no floating
4723      * point type of the same size as {@code $type$}.  The return type
4724      * of this method is arbitrarily designated as
4725      * {@code Vector<?>}.  Future versions of this API may change the return
4726      * type if additional floating point types become available.
4727 #end[byteOrShort]
4728      */
4729     @ForceInline
4730     @Override
4731     public final
4732     {#if[byteOrShort]?Vector<?>:$Fptype$Vector}
4733     viewAsFloatingLanes() {
4734 #if[FP]
4735         return this;
4736 #else[FP]
4737         LaneType flt = LaneType.$TYPE$.asFloating();
4738 #if[!byteOrShort]
4739         return ($Fptype$Vector) asVectorRaw(flt);
4740 #else[!byteOrShort]
4741         // asFloating() will throw UnsupportedOperationException for the unsupported type $type$
4742         throw new AssertionError("Cannot reach here");
4743 #end[!byteOrShort]
4744 #end[FP]
4745     }
4746 
4747     // ================================================
4748 
4749     /// Object methods: toString, equals, hashCode
4750     //
4751     // Object methods are defined as if via Arrays.toString, etc.,
4752     // is applied to the array of elements.  Two equal vectors
4753     // are required to have equal species and equal lane values.
4754 
4755     /**
4756      * Returns a string representation of this vector, of the form
4757      * {@code "[0,1,2...]"}, reporting the lane values of this vector,
4758      * in lane order.
4759      *
4760      * The string is produced as if by a call to {@link
4761      * java.util.Arrays#toString($type$[]) Arrays.toString()},
4762      * as appropriate to the {@code $type$} array returned by
4763      * {@link #toArray this.toArray()}.
4764      *
4765      * @return a string of the form {@code "[0,1,2...]"}
4766      * reporting the lane values of this vector
4767      */
4768     @Override
4769     @ForceInline
4770     public final
4771     String toString() {
4772         // now that toArray is strongly typed, we can define this
4773         return Arrays.toString(toArray());
4774     }
4775 
4776     /**
4777      * {@inheritDoc} <!--workaround-->
4778      */
4779     @Override
4780     @ForceInline
4781     public final
4782     boolean equals(Object obj) {
4783         if (obj instanceof Vector) {
4784             Vector<?> that = (Vector<?>) obj;
4785             if (this.species().equals(that.species())) {
4786                 return this.eq(that.check(this.species())).allTrue();
4787             }
4788         }
4789         return false;
4790     }
4791 
4792     /**
4793      * {@inheritDoc} <!--workaround-->
4794      */
4795     @Override
4796     @ForceInline
4797     public final
4798     int hashCode() {
4799         // now that toArray is strongly typed, we can define this
4800         return Objects.hash(species(), Arrays.hashCode(toArray()));
4801     }
4802 
4803     // ================================================
4804 
4805     // Species
4806 
4807     /**
4808      * Class representing {@link $abstractvectortype$}'s of the same {@link VectorShape VectorShape}.
4809      */
4810     /*package-private*/
4811     static final class $Type$Species extends AbstractSpecies<$Boxtype$> {
4812         private $Type$Species(VectorShape shape,
4813                 Class<? extends $abstractvectortype$> vectorType,
4814                 Class<? extends AbstractMask<$Boxtype$>> maskType,
4815                 Function<Object, $abstractvectortype$> vectorFactory) {
4816             super(shape, LaneType.of($type$.class),
4817                   vectorType, maskType,
4818                   vectorFactory);
4819             assert(this.elementSize() == $Boxtype$.SIZE);
4820         }
4821 
4822         // Specializing overrides:
4823 
4824         @Override
4825         @ForceInline
4826         public final Class<$Boxtype$> elementType() {
4827             return $type$.class;
4828         }
4829 
4830         @Override
4831         @ForceInline
4832         final Class<$Boxtype$> genericElementType() {
4833             return $Boxtype$.class;
4834         }
4835 
4836         @SuppressWarnings("unchecked")
4837         @Override
4838         @ForceInline
4839         public final Class<? extends $Type$Vector> vectorType() {
4840             return (Class<? extends $Type$Vector>) vectorType;
4841         }
4842 
4843         @Override
4844         @ForceInline
4845         public final long checkValue(long e) {
4846             longToElementBits(e);  // only for exception
4847             return e;
4848         }
4849 
4850         /*package-private*/
4851         @Override
4852         @ForceInline
4853         final $abstractvectortype$ broadcastBits(long bits) {
4854             return ($abstractvectortype$)
4855                 VectorSupport.broadcastCoerced(
4856                     vectorType, $type$.class, laneCount,
4857                     bits, this,
4858                     (bits_, s_) -> s_.rvOp(i -> bits_));
4859         }
4860 
4861         /*package-private*/
4862         @ForceInline
4863         {#if[long]?public }final $abstractvectortype$ broadcast($type$ e) {
4864             return broadcastBits(toBits(e));
4865         }
4866 
4867 #if[!long]
4868         @Override
4869         @ForceInline
4870         public final $abstractvectortype$ broadcast(long e) {
4871             return broadcastBits(longToElementBits(e));
4872         }
4873 #end[!long]
4874 
4875         /*package-private*/
4876         final @Override
4877         @ForceInline
4878         long longToElementBits(long value) {
4879 #if[long]
4880             // In this case, the conversion can never fail.
4881             return value;
4882 #else[long]
4883             // Do the conversion, and then test it for failure.
4884             $type$ e = ($type$) value;
4885             if ((long) e != value) {
4886                 throw badElementBits(value, e);
4887             }
4888             return toBits(e);
4889 #end[long]
4890         }
4891 
4892         /*package-private*/
4893         @ForceInline
4894         static long toIntegralChecked($type$ e, boolean convertToInt) {
4895             long value = convertToInt ? (int) e : (long) e;
4896             if (($type$) value != e) {
4897                 throw badArrayBits(e, convertToInt, value);
4898             }
4899             return value;
4900         }
4901 
4902         /* this non-public one is for internal conversions */
4903         @Override
4904         @ForceInline
4905         final $abstractvectortype$ fromIntValues(int[] values) {
4906             VectorIntrinsics.requireLength(values.length, laneCount);
4907             $type$[] va = new $type$[laneCount()];
4908             for (int i = 0; i < va.length; i++) {
4909                 int lv = values[i];
4910                 $type$ v = ($type$) lv;
4911                 va[i] = v;
4912                 if ((int)v != lv) {
4913                     throw badElementBits(lv, v);
4914                 }
4915             }
4916             return dummyVector().fromArray0(va, 0);
4917         }
4918 
4919         // Virtual constructors
4920 
4921         @ForceInline
4922         @Override final
4923         public $abstractvectortype$ fromArray(Object a, int offset) {
4924             // User entry point:  Be careful with inputs.
4925             return $abstractvectortype$
4926                 .fromArray(this, ($type$[]) a, offset);
4927         }
4928 
4929         @ForceInline
4930         @Override final
4931         $abstractvectortype$ dummyVector() {
4932             return ($abstractvectortype$) super.dummyVector();
4933         }
4934 
4935         /*package-private*/
4936         final @Override
4937         @ForceInline
4938         $abstractvectortype$ rvOp(RVOp f) {
4939             $type$[] res = new $type$[laneCount()];
4940             for (int i = 0; i < res.length; i++) {
4941                 $bitstype$ bits = {#if[!long]?($bitstype$)} f.apply(i);
4942                 res[i] = fromBits(bits);
4943             }
4944             return dummyVector().vectorFactory(res);
4945         }
4946 
4947         $Type$Vector vOp(FVOp f) {
4948             $type$[] res = new $type$[laneCount()];
4949             for (int i = 0; i < res.length; i++) {
4950                 res[i] = f.apply(i);
4951             }
4952             return dummyVector().vectorFactory(res);
4953         }
4954 
4955         $Type$Vector vOp(VectorMask<$Boxtype$> m, FVOp f) {
4956             $type$[] res = new $type$[laneCount()];
4957             boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
4958             for (int i = 0; i < res.length; i++) {
4959                 if (mbits[i]) {
4960                     res[i] = f.apply(i);
4961                 }
4962             }
4963             return dummyVector().vectorFactory(res);
4964         }
4965 
4966         /*package-private*/
4967         @ForceInline
4968         <M> $abstractvectortype$ ldOp(M memory, int offset,
4969                                       FLdOp<M> f) {
4970             return dummyVector().ldOp(memory, offset, f);
4971         }
4972 
4973         /*package-private*/
4974         @ForceInline
4975         <M> $abstractvectortype$ ldOp(M memory, int offset,
4976                                       AbstractMask<$Boxtype$> m,
4977                                       FLdOp<M> f) {
4978             return dummyVector().ldOp(memory, offset, m, f);
4979         }
4980 
4981         /*package-private*/
4982         @ForceInline
4983         <M> void stOp(M memory, int offset, FStOp<M> f) {
4984             dummyVector().stOp(memory, offset, f);
4985         }
4986 
4987         /*package-private*/
4988         @ForceInline
4989         <M> void stOp(M memory, int offset,
4990                       AbstractMask<$Boxtype$> m,
4991                       FStOp<M> f) {
4992             dummyVector().stOp(memory, offset, m, f);
4993         }
4994 
4995         // N.B. Make sure these constant vectors and
4996         // masks load up correctly into registers.
4997         //
4998         // Also, see if we can avoid all that switching.
4999         // Could we cache both vectors and both masks in
5000         // this species object?
5001 
5002         // Zero and iota vector access
5003         @Override
5004         @ForceInline
5005         public final $abstractvectortype$ zero() {
5006             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5007                 return $Type$MaxVector.ZERO;
5008             switch (vectorBitSize()) {
5009                 case 64: return $Type$64Vector.ZERO;
5010                 case 128: return $Type$128Vector.ZERO;
5011                 case 256: return $Type$256Vector.ZERO;
5012                 case 512: return $Type$512Vector.ZERO;
5013             }
5014             throw new AssertionError();
5015         }
5016 
5017         @Override
5018         @ForceInline
5019         public final $abstractvectortype$ iota() {
5020             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5021                 return $Type$MaxVector.IOTA;
5022             switch (vectorBitSize()) {
5023                 case 64: return $Type$64Vector.IOTA;
5024                 case 128: return $Type$128Vector.IOTA;
5025                 case 256: return $Type$256Vector.IOTA;
5026                 case 512: return $Type$512Vector.IOTA;
5027             }
5028             throw new AssertionError();
5029         }
5030 
5031         // Mask access
5032         @Override
5033         @ForceInline
5034         public final VectorMask<$Boxtype$> maskAll(boolean bit) {
5035             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5036                 return $Type$MaxVector.$Type$MaxMask.maskAll(bit);
5037             switch (vectorBitSize()) {
5038                 case 64: return $Type$64Vector.$Type$64Mask.maskAll(bit);
5039                 case 128: return $Type$128Vector.$Type$128Mask.maskAll(bit);
5040                 case 256: return $Type$256Vector.$Type$256Mask.maskAll(bit);
5041                 case 512: return $Type$512Vector.$Type$512Mask.maskAll(bit);
5042             }
5043             throw new AssertionError();
5044         }
5045     }
5046 
5047     /**
5048      * Finds a species for an element type of {@code $type$} and shape.
5049      *
5050      * @param s the shape
5051      * @return a species for an element type of {@code $type$} and shape
5052      * @throws IllegalArgumentException if no such species exists for the shape
5053      */
5054     static $Type$Species species(VectorShape s) {
5055         Objects.requireNonNull(s);
5056         switch (s) {
5057             case S_64_BIT: return ($Type$Species) SPECIES_64;
5058             case S_128_BIT: return ($Type$Species) SPECIES_128;
5059             case S_256_BIT: return ($Type$Species) SPECIES_256;
5060             case S_512_BIT: return ($Type$Species) SPECIES_512;
5061             case S_Max_BIT: return ($Type$Species) SPECIES_MAX;
5062             default: throw new IllegalArgumentException("Bad shape: " + s);
5063         }
5064     }
5065 
5066     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_64_BIT VectorShape.S_64_BIT}. */
5067     public static final VectorSpecies<$Boxtype$> SPECIES_64
5068         = new $Type$Species(VectorShape.S_64_BIT,
5069                             $Type$64Vector.class,
5070                             $Type$64Vector.$Type$64Mask.class,
5071                             $Type$64Vector::new);
5072 
5073     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
5074     public static final VectorSpecies<$Boxtype$> SPECIES_128
5075         = new $Type$Species(VectorShape.S_128_BIT,
5076                             $Type$128Vector.class,
5077                             $Type$128Vector.$Type$128Mask.class,
5078                             $Type$128Vector::new);
5079 
5080     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
5081     public static final VectorSpecies<$Boxtype$> SPECIES_256
5082         = new $Type$Species(VectorShape.S_256_BIT,
5083                             $Type$256Vector.class,
5084                             $Type$256Vector.$Type$256Mask.class,
5085                             $Type$256Vector::new);
5086 
5087     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
5088     public static final VectorSpecies<$Boxtype$> SPECIES_512
5089         = new $Type$Species(VectorShape.S_512_BIT,
5090                             $Type$512Vector.class,
5091                             $Type$512Vector.$Type$512Mask.class,
5092                             $Type$512Vector::new);
5093 
5094     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
5095     public static final VectorSpecies<$Boxtype$> SPECIES_MAX
5096         = new $Type$Species(VectorShape.S_Max_BIT,
5097                             $Type$MaxVector.class,
5098                             $Type$MaxVector.$Type$MaxMask.class,
5099                             $Type$MaxVector::new);
5100 
5101     /**
5102      * Preferred species for {@link $Type$Vector}s.
5103      * A preferred species is a species of maximal bit-size for the platform.
5104      */
5105     public static final VectorSpecies<$Boxtype$> SPECIES_PREFERRED
5106         = ($Type$Species) VectorSpecies.ofPreferred($type$.class);
5107 }