1 /*
   2  * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.nio.ByteOrder;
  29 import java.nio.ReadOnlyBufferException;
  30 import java.util.Arrays;
  31 import java.util.Objects;
  32 import java.util.function.Function;
  33 import java.util.function.UnaryOperator;
  34 
  35 import jdk.internal.misc.ScopedMemoryAccess;
  36 import jdk.internal.misc.Unsafe;
  37 import jdk.internal.vm.annotation.ForceInline;
  38 import jdk.internal.vm.vector.VectorSupport;
  39 
  40 import static jdk.internal.vm.vector.VectorSupport.*;
  41 import static jdk.incubator.vector.VectorIntrinsics.*;
  42 
  43 import static jdk.incubator.vector.VectorOperators.*;
  44 
  45 #warn This file is preprocessed before being compiled
  46 
  47 /**
  48  * A specialized {@link Vector} representing an ordered immutable sequence of
  49  * {@code $type$} values.
  50  */
  51 @SuppressWarnings("cast")  // warning: redundant cast
  52 public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
  53 
  54     $abstractvectortype$($type$[] vec) {
  55         super(vec);
  56     }
  57 
  58 #if[FP]
  59     static final int FORBID_OPCODE_KIND = VO_NOFP;
  60 #else[FP]
  61     static final int FORBID_OPCODE_KIND = VO_ONLYFP;
  62 #end[FP]
  63 
  64     @ForceInline
  65     static int opCode(Operator op) {
  66         return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
  67     }
  68     @ForceInline
  69     static int opCode(Operator op, int requireKind) {
  70         requireKind |= VO_OPCODE_VALID;
  71         return VectorOperators.opCode(op, requireKind, FORBID_OPCODE_KIND);
  72     }
  73     @ForceInline
  74     static boolean opKind(Operator op, int bit) {
  75         return VectorOperators.opKind(op, bit);
  76     }
  77 
  78     // Virtualized factories and operators,
  79     // coded with portable definitions.
  80     // These are all @ForceInline in case
  81     // they need to be used performantly.
  82     // The various shape-specific subclasses
  83     // also specialize them by wrapping
  84     // them in a call like this:
  85     //    return (Byte128Vector)
  86     //       super.bOp((Byte128Vector) o);
  87     // The purpose of that is to forcibly inline
  88     // the generic definition from this file
  89     // into a sharply type- and size-specific
  90     // wrapper in the subclass file, so that
  91     // the JIT can specialize the code.
  92     // The code is only inlined and expanded
  93     // if it gets hot.  Think of it as a cheap
  94     // and lazy version of C++ templates.
  95 
  96     // Virtualized getter
  97 
  98     /*package-private*/
  99     abstract $type$[] vec();
 100 
 101     // Virtualized constructors
 102 
 103     /**
 104      * Build a vector directly using my own constructor.
 105      * It is an error if the array is aliased elsewhere.
 106      */
 107     /*package-private*/
 108     abstract $abstractvectortype$ vectorFactory($type$[] vec);
 109 
 110     /**
 111      * Build a mask directly using my species.
 112      * It is an error if the array is aliased elsewhere.
 113      */
 114     /*package-private*/
 115     @ForceInline
 116     final
 117     AbstractMask<$Boxtype$> maskFactory(boolean[] bits) {
 118         return vspecies().maskFactory(bits);
 119     }
 120 
 121     // Constant loader (takes dummy as vector arg)
 122     interface FVOp {
 123         $type$ apply(int i);
 124     }
 125 
 126     /*package-private*/
 127     @ForceInline
 128     final
 129     $abstractvectortype$ vOp(FVOp f) {
 130         $type$[] res = new $type$[length()];
 131         for (int i = 0; i < res.length; i++) {
 132             res[i] = f.apply(i);
 133         }
 134         return vectorFactory(res);
 135     }
 136 
 137     @ForceInline
 138     final
 139     $abstractvectortype$ vOp(VectorMask<$Boxtype$> m, FVOp f) {
 140         $type$[] res = new $type$[length()];
 141         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 142         for (int i = 0; i < res.length; i++) {
 143             if (mbits[i]) {
 144                 res[i] = f.apply(i);
 145             }
 146         }
 147         return vectorFactory(res);
 148     }
 149 
 150     // Unary operator
 151 
 152     /*package-private*/
 153     interface FUnOp {
 154         $type$ apply(int i, $type$ a);
 155     }
 156 
 157     /*package-private*/
 158     abstract
 159     $abstractvectortype$ uOp(FUnOp f);
 160     @ForceInline
 161     final
 162     $abstractvectortype$ uOpTemplate(FUnOp f) {
 163         $type$[] vec = vec();
 164         $type$[] res = new $type$[length()];
 165         for (int i = 0; i < res.length; i++) {
 166             res[i] = f.apply(i, vec[i]);
 167         }
 168         return vectorFactory(res);
 169     }
 170 
 171     /*package-private*/
 172     abstract
 173     $abstractvectortype$ uOp(VectorMask<$Boxtype$> m,
 174                              FUnOp f);
 175     @ForceInline
 176     final
 177     $abstractvectortype$ uOpTemplate(VectorMask<$Boxtype$> m,
 178                                      FUnOp f) {
 179         if (m == null) {
 180             return uOpTemplate(f);
 181         }
 182         $type$[] vec = vec();
 183         $type$[] res = new $type$[length()];
 184         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 185         for (int i = 0; i < res.length; i++) {
 186             res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
 187         }
 188         return vectorFactory(res);
 189     }
 190 
 191     // Binary operator
 192 
 193     /*package-private*/
 194     interface FBinOp {
 195         $type$ apply(int i, $type$ a, $type$ b);
 196     }
 197 
 198     /*package-private*/
 199     abstract
 200     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 201                              FBinOp f);
 202     @ForceInline
 203     final
 204     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 205                                      FBinOp f) {
 206         $type$[] res = new $type$[length()];
 207         $type$[] vec1 = this.vec();
 208         $type$[] vec2 = (($abstractvectortype$)o).vec();
 209         for (int i = 0; i < res.length; i++) {
 210             res[i] = f.apply(i, vec1[i], vec2[i]);
 211         }
 212         return vectorFactory(res);
 213     }
 214 
 215     /*package-private*/
 216     abstract
 217     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 218                              VectorMask<$Boxtype$> m,
 219                              FBinOp f);
 220     @ForceInline
 221     final
 222     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 223                                      VectorMask<$Boxtype$> m,
 224                                      FBinOp f) {
 225         if (m == null) {
 226             return bOpTemplate(o, f);
 227         }
 228         $type$[] res = new $type$[length()];
 229         $type$[] vec1 = this.vec();
 230         $type$[] vec2 = (($abstractvectortype$)o).vec();
 231         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 232         for (int i = 0; i < res.length; i++) {
 233             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
 234         }
 235         return vectorFactory(res);
 236     }
 237 
 238     // Ternary operator
 239 
 240     /*package-private*/
 241     interface FTriOp {
 242         $type$ apply(int i, $type$ a, $type$ b, $type$ c);
 243     }
 244 
 245     /*package-private*/
 246     abstract
 247     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 248                              Vector<$Boxtype$> o2,
 249                              FTriOp f);
 250     @ForceInline
 251     final
 252     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 253                                      Vector<$Boxtype$> o2,
 254                                      FTriOp f) {
 255         $type$[] res = new $type$[length()];
 256         $type$[] vec1 = this.vec();
 257         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 258         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 259         for (int i = 0; i < res.length; i++) {
 260             res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
 261         }
 262         return vectorFactory(res);
 263     }
 264 
 265     /*package-private*/
 266     abstract
 267     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 268                              Vector<$Boxtype$> o2,
 269                              VectorMask<$Boxtype$> m,
 270                              FTriOp f);
 271     @ForceInline
 272     final
 273     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 274                                      Vector<$Boxtype$> o2,
 275                                      VectorMask<$Boxtype$> m,
 276                                      FTriOp f) {
 277         if (m == null) {
 278             return tOpTemplate(o1, o2, f);
 279         }
 280         $type$[] res = new $type$[length()];
 281         $type$[] vec1 = this.vec();
 282         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 283         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 284         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 285         for (int i = 0; i < res.length; i++) {
 286             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
 287         }
 288         return vectorFactory(res);
 289     }
 290 
 291     // Reduction operator
 292 
 293     /*package-private*/
 294     abstract
 295     $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f);
 296 
 297     @ForceInline
 298     final
 299     $type$ rOpTemplate($type$ v, VectorMask<$Boxtype$> m, FBinOp f) {
 300         if (m == null) {
 301             return rOpTemplate(v, f);
 302         }
 303         $type$[] vec = vec();
 304         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 305         for (int i = 0; i < vec.length; i++) {
 306             v = mbits[i] ? f.apply(i, v, vec[i]) : v;
 307         }
 308         return v;
 309     }
 310 
 311     @ForceInline
 312     final
 313     $type$ rOpTemplate($type$ v, FBinOp f) {
 314         $type$[] vec = vec();
 315         for (int i = 0; i < vec.length; i++) {
 316             v = f.apply(i, v, vec[i]);
 317         }
 318         return v;
 319     }
 320 
 321     // Memory reference
 322 
 323     /*package-private*/
 324     interface FLdOp<M> {
 325         $type$ apply(M memory, int offset, int i);
 326     }
 327 
 328     /*package-private*/
 329     @ForceInline
 330     final
 331     <M> $abstractvectortype$ ldOp(M memory, int offset,
 332                                   FLdOp<M> f) {
 333         //dummy; no vec = vec();
 334         $type$[] res = new $type$[length()];
 335         for (int i = 0; i < res.length; i++) {
 336             res[i] = f.apply(memory, offset, i);
 337         }
 338         return vectorFactory(res);
 339     }
 340 
 341     /*package-private*/
 342     @ForceInline
 343     final
 344     <M> $abstractvectortype$ ldOp(M memory, int offset,
 345                                   VectorMask<$Boxtype$> m,
 346                                   FLdOp<M> f) {
 347         //$type$[] vec = vec();
 348         $type$[] res = new $type$[length()];
 349         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 350         for (int i = 0; i < res.length; i++) {
 351             if (mbits[i]) {
 352                 res[i] = f.apply(memory, offset, i);
 353             }
 354         }
 355         return vectorFactory(res);
 356     }
 357 
 358     interface FStOp<M> {
 359         void apply(M memory, int offset, int i, $type$ a);
 360     }
 361 
 362     /*package-private*/
 363     @ForceInline
 364     final
 365     <M> void stOp(M memory, int offset,
 366                   FStOp<M> f) {
 367         $type$[] vec = vec();
 368         for (int i = 0; i < vec.length; i++) {
 369             f.apply(memory, offset, i, vec[i]);
 370         }
 371     }
 372 
 373     /*package-private*/
 374     @ForceInline
 375     final
 376     <M> void stOp(M memory, int offset,
 377                   VectorMask<$Boxtype$> m,
 378                   FStOp<M> f) {
 379         $type$[] vec = vec();
 380         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 381         for (int i = 0; i < vec.length; i++) {
 382             if (mbits[i]) {
 383                 f.apply(memory, offset, i, vec[i]);
 384             }
 385         }
 386     }
 387 
 388     // Binary test
 389 
 390     /*package-private*/
 391     interface FBinTest {
 392         boolean apply(int cond, int i, $type$ a, $type$ b);
 393     }
 394 
 395     /*package-private*/
 396     @ForceInline
 397     final
 398     AbstractMask<$Boxtype$> bTest(int cond,
 399                                   Vector<$Boxtype$> o,
 400                                   FBinTest f) {
 401         $type$[] vec1 = vec();
 402         $type$[] vec2 = (($abstractvectortype$)o).vec();
 403         boolean[] bits = new boolean[length()];
 404         for (int i = 0; i < length(); i++){
 405             bits[i] = f.apply(cond, i, vec1[i], vec2[i]);
 406         }
 407         return maskFactory(bits);
 408     }
 409 
 410 #if[BITWISE]
 411     /*package-private*/
 412     @ForceInline
 413     static $type$ rotateLeft($type$ a, int n) {
 414 #if[intOrLong]
 415         return $Boxtype$.rotateLeft(a, n);
 416 #else[intOrLong]
 417         return ($type$)((((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) << (n & $Boxtype$.SIZE-1)) | (((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) >>> ($Boxtype$.SIZE - (n & $Boxtype$.SIZE-1))));
 418 #end[intOrLong]
 419     }
 420 
 421     /*package-private*/
 422     @ForceInline
 423     static $type$ rotateRight($type$ a, int n) {
 424 #if[intOrLong]
 425         return $Boxtype$.rotateRight(a, n);
 426 #else[intOrLong]
 427         return ($type$)((((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) >>> (n & $Boxtype$.SIZE-1)) | (((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) << ($Boxtype$.SIZE - (n & $Boxtype$.SIZE-1))));
 428 #end[intOrLong]
 429     }
 430 #end[BITWISE]
 431 
 432     /*package-private*/
 433     @Override
 434     abstract $Type$Species vspecies();
 435 
 436     /*package-private*/
 437     @ForceInline
 438     static long toBits($type$ e) {
 439         return {#if[FP]? $Type$.$type$ToRaw$Bitstype$Bits(e): e};
 440     }
 441 
 442     /*package-private*/
 443     @ForceInline
 444     static $type$ fromBits(long bits) {
 445         return {#if[FP]?$Type$.$bitstype$BitsTo$Type$}(($bitstype$)bits);
 446     }
 447 
 448     // Static factories (other than memory operations)
 449 
 450     // Note: A surprising behavior in javadoc
 451     // sometimes makes a lone /** {@inheritDoc} */
 452     // comment drop the method altogether,
 453     // apparently if the method mentions an
 454     // parameter or return type of Vector<$Boxtype$>
 455     // instead of Vector<E> as originally specified.
 456     // Adding an empty HTML fragment appears to
 457     // nudge javadoc into providing the desired
 458     // inherited documentation.  We use the HTML
 459     // comment <!--workaround--> for this.
 460 
 461     /**
 462      * Returns a vector of the given species
 463      * where all lane elements are set to
 464      * zero, the default primitive value.
 465      *
 466      * @param species species of the desired zero vector
 467      * @return a zero vector
 468      */
 469     @ForceInline
 470     public static $abstractvectortype$ zero(VectorSpecies<$Boxtype$> species) {
 471         $Type$Species vsp = ($Type$Species) species;
 472 #if[FP]
 473         return VectorSupport.fromBitsCoerced(vsp.vectorType(), $type$.class, species.length(),
 474                         toBits(0.0f), MODE_BROADCAST, vsp,
 475                         ((bits_, s_) -> s_.rvOp(i -> bits_)));
 476 #else[FP]
 477         return VectorSupport.fromBitsCoerced(vsp.vectorType(), $type$.class, species.length(),
 478                                 0, MODE_BROADCAST, vsp,
 479                                 ((bits_, s_) -> s_.rvOp(i -> bits_)));
 480 #end[FP]
 481     }
 482 
 483     /**
 484      * Returns a vector of the same species as this one
 485      * where all lane elements are set to
 486      * the primitive value {@code e}.
 487      *
 488      * The contents of the current vector are discarded;
 489      * only the species is relevant to this operation.
 490      *
 491      * <p> This method returns the value of this expression:
 492      * {@code $abstractvectortype$.broadcast(this.species(), e)}.
 493      *
 494      * @apiNote
 495      * Unlike the similar method named {@code broadcast()}
 496      * in the supertype {@code Vector}, this method does not
 497      * need to validate its argument, and cannot throw
 498      * {@code IllegalArgumentException}.  This method is
 499      * therefore preferable to the supertype method.
 500      *
 501      * @param e the value to broadcast
 502      * @return a vector where all lane elements are set to
 503      *         the primitive value {@code e}
 504      * @see #broadcast(VectorSpecies,long)
 505      * @see Vector#broadcast(long)
 506      * @see VectorSpecies#broadcast(long)
 507      */
 508     public abstract $abstractvectortype$ broadcast($type$ e);
 509 
 510     /**
 511      * Returns a vector of the given species
 512      * where all lane elements are set to
 513      * the primitive value {@code e}.
 514      *
 515      * @param species species of the desired vector
 516      * @param e the value to broadcast
 517      * @return a vector where all lane elements are set to
 518      *         the primitive value {@code e}
 519      * @see #broadcast(long)
 520      * @see Vector#broadcast(long)
 521      * @see VectorSpecies#broadcast(long)
 522      */
 523     @ForceInline
 524     public static $abstractvectortype$ broadcast(VectorSpecies<$Boxtype$> species, $type$ e) {
 525         $Type$Species vsp = ($Type$Species) species;
 526         return vsp.broadcast(e);
 527     }
 528 
 529     /*package-private*/
 530     @ForceInline
 531     final $abstractvectortype$ broadcastTemplate($type$ e) {
 532         $Type$Species vsp = vspecies();
 533         return vsp.broadcast(e);
 534     }
 535 
 536 #if[!long]
 537     /**
 538      * {@inheritDoc} <!--workaround-->
 539      * @apiNote
 540      * When working with vector subtypes like {@code $abstractvectortype$},
 541      * {@linkplain #broadcast($type$) the more strongly typed method}
 542      * is typically selected.  It can be explicitly selected
 543      * using a cast: {@code v.broadcast(($type$)e)}.
 544      * The two expressions will produce numerically identical results.
 545      */
 546     @Override
 547     public abstract $abstractvectortype$ broadcast(long e);
 548 
 549     /**
 550      * Returns a vector of the given species
 551      * where all lane elements are set to
 552      * the primitive value {@code e}.
 553      *
 554      * The {@code long} value must be accurately representable
 555      * by the {@code ETYPE} of the vector species, so that
 556      * {@code e==(long)(ETYPE)e}.
 557      *
 558      * @param species species of the desired vector
 559      * @param e the value to broadcast
 560      * @return a vector where all lane elements are set to
 561      *         the primitive value {@code e}
 562      * @throws IllegalArgumentException
 563      *         if the given {@code long} value cannot
 564      *         be represented by the vector's {@code ETYPE}
 565      * @see #broadcast(VectorSpecies,$type$)
 566      * @see VectorSpecies#checkValue(long)
 567      */
 568     @ForceInline
 569     public static $abstractvectortype$ broadcast(VectorSpecies<$Boxtype$> species, long e) {
 570         $Type$Species vsp = ($Type$Species) species;
 571         return vsp.broadcast(e);
 572     }
 573 
 574     /*package-private*/
 575     @ForceInline
 576     final $abstractvectortype$ broadcastTemplate(long e) {
 577         return vspecies().broadcast(e);
 578     }
 579 #end[!long]
 580 
 581     // Unary lanewise support
 582 
 583     /**
 584      * {@inheritDoc} <!--workaround-->
 585      */
 586     public abstract
 587     $abstractvectortype$ lanewise(VectorOperators.Unary op);
 588 
 589     @ForceInline
 590     final
 591     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op) {
 592         if (opKind(op, VO_SPECIAL)) {
 593             if (op == ZOMO) {
 594                 return blend(broadcast(-1), compare(NE, 0));
 595             }
 596 #if[BITWISE]
 597             if (op == NOT) {
 598                 return broadcast(-1).lanewise(XOR, this);
 599             }
 600 #end[BITWISE]
 601         }
 602         int opc = opCode(op);
 603         return VectorSupport.unaryOp(
 604             opc, getClass(), null, $type$.class, length(),
 605             this, null,
 606             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));
 607     }
 608 
 609     /**
 610      * {@inheritDoc} <!--workaround-->
 611      */
 612     @Override
 613     public abstract
 614     $abstractvectortype$ lanewise(VectorOperators.Unary op,
 615                                   VectorMask<$Boxtype$> m);
 616     @ForceInline
 617     final
 618     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op,
 619                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
 620                                           VectorMask<$Boxtype$> m) {
 621         m.check(maskClass, this);
 622         if (opKind(op, VO_SPECIAL)) {
 623             if (op == ZOMO) {
 624                 return blend(broadcast(-1), compare(NE, 0, m));
 625             }
 626 #if[BITWISE]
 627             if (op == NOT) {
 628                 return lanewise(XOR, broadcast(-1), m);
 629             }
 630 #end[BITWISE]
 631         }
 632         int opc = opCode(op);
 633         return VectorSupport.unaryOp(
 634             opc, getClass(), maskClass, $type$.class, length(),
 635             this, m,
 636             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));
 637     }
 638 
 639     private static final
 640     ImplCache<Unary, UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 641         UN_IMPL = new ImplCache<>(Unary.class, $Type$Vector.class);
 642 
 643     private static UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> unaryOperations(int opc_) {
 644         switch (opc_) {
 645             case VECTOR_OP_NEG: return (v0, m) ->
 646                     v0.uOp(m, (i, a) -> ($type$) -a);
 647             case VECTOR_OP_ABS: return (v0, m) ->
 648                     v0.uOp(m, (i, a) -> ($type$) Math.abs(a));
 649 #if[FP]
 650             case VECTOR_OP_SIN: return (v0, m) ->
 651                     v0.uOp(m, (i, a) -> ($type$) Math.sin(a));
 652             case VECTOR_OP_COS: return (v0, m) ->
 653                     v0.uOp(m, (i, a) -> ($type$) Math.cos(a));
 654             case VECTOR_OP_TAN: return (v0, m) ->
 655                     v0.uOp(m, (i, a) -> ($type$) Math.tan(a));
 656             case VECTOR_OP_ASIN: return (v0, m) ->
 657                     v0.uOp(m, (i, a) -> ($type$) Math.asin(a));
 658             case VECTOR_OP_ACOS: return (v0, m) ->
 659                     v0.uOp(m, (i, a) -> ($type$) Math.acos(a));
 660             case VECTOR_OP_ATAN: return (v0, m) ->
 661                     v0.uOp(m, (i, a) -> ($type$) Math.atan(a));
 662             case VECTOR_OP_EXP: return (v0, m) ->
 663                     v0.uOp(m, (i, a) -> ($type$) Math.exp(a));
 664             case VECTOR_OP_LOG: return (v0, m) ->
 665                     v0.uOp(m, (i, a) -> ($type$) Math.log(a));
 666             case VECTOR_OP_LOG10: return (v0, m) ->
 667                     v0.uOp(m, (i, a) -> ($type$) Math.log10(a));
 668             case VECTOR_OP_SQRT: return (v0, m) ->
 669                     v0.uOp(m, (i, a) -> ($type$) Math.sqrt(a));
 670             case VECTOR_OP_CBRT: return (v0, m) ->
 671                     v0.uOp(m, (i, a) -> ($type$) Math.cbrt(a));
 672             case VECTOR_OP_SINH: return (v0, m) ->
 673                     v0.uOp(m, (i, a) -> ($type$) Math.sinh(a));
 674             case VECTOR_OP_COSH: return (v0, m) ->
 675                     v0.uOp(m, (i, a) -> ($type$) Math.cosh(a));
 676             case VECTOR_OP_TANH: return (v0, m) ->
 677                     v0.uOp(m, (i, a) -> ($type$) Math.tanh(a));
 678             case VECTOR_OP_EXPM1: return (v0, m) ->
 679                     v0.uOp(m, (i, a) -> ($type$) Math.expm1(a));
 680             case VECTOR_OP_LOG1P: return (v0, m) ->
 681                     v0.uOp(m, (i, a) -> ($type$) Math.log1p(a));
 682 #end[FP]
 683             default: return null;
 684         }
 685     }
 686 
 687     // Binary lanewise support
 688 
 689     /**
 690      * {@inheritDoc} <!--workaround-->
 691      * @see #lanewise(VectorOperators.Binary,$type$)
 692      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 693      */
 694     @Override
 695     public abstract
 696     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 697                                   Vector<$Boxtype$> v);
 698     @ForceInline
 699     final
 700     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 701                                           Vector<$Boxtype$> v) {
 702         $abstractvectortype$ that = ($abstractvectortype$) v;
 703         that.check(this);
 704 
 705         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 706             if (op == FIRST_NONZERO) {
 707                 // FIXME: Support this in the JIT.
 708                 VectorMask<$Boxbitstype$> thisNZ
 709                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 710                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 711                 op = OR_UNCHECKED;
 712 #if[FP]
 713                 // FIXME: Support OR_UNCHECKED on float/double also!
 714                 return this.viewAsIntegralLanes()
 715                     .lanewise(op, that.viewAsIntegralLanes())
 716                     .viewAsFloatingLanes();
 717 #end[FP]
 718             }
 719 #if[BITWISE]
 720 #if[!FP]
 721             if (opKind(op, VO_SHIFT)) {
 722                 // As per shift specification for Java, mask the shift count.
 723                 // This allows the JIT to ignore some ISA details.
 724                 that = that.lanewise(AND, SHIFT_MASK);
 725             }
 726 #end[!FP]
 727             if (op == AND_NOT) {
 728                 // FIXME: Support this in the JIT.
 729                 that = that.lanewise(NOT);
 730                 op = AND;
 731             } else if (op == DIV) {
 732                 VectorMask<$Boxtype$> eqz = that.eq(($type$) 0);
 733                 if (eqz.anyTrue()) {
 734                     throw that.divZeroException();
 735                 }
 736             }
 737 #end[BITWISE]
 738         }
 739 
 740         int opc = opCode(op);
 741         return VectorSupport.binaryOp(
 742             opc, getClass(), null, $type$.class, length(),
 743             this, that, null,
 744             BIN_IMPL.find(op, opc, $abstractvectortype$::binaryOperations));
 745     }
 746 
 747     /**
 748      * {@inheritDoc} <!--workaround-->
 749      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 750      */
 751     @Override
 752     public abstract
 753     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 754                                   Vector<$Boxtype$> v,
 755                                   VectorMask<$Boxtype$> m);
 756     @ForceInline
 757     final
 758     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 759                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
 760                                           Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 761         $abstractvectortype$ that = ($abstractvectortype$) v;
 762         that.check(this);
 763         m.check(maskClass, this);
 764 
 765         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 766             if (op == FIRST_NONZERO) {
 767 #if[FP]
 768                 return blend(lanewise(op, v), m);
 769 #else[FP]
 770                 // FIXME: Support this in the JIT.
 771                 VectorMask<$Boxbitstype$> thisNZ
 772                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 773                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 774                 op = OR_UNCHECKED;
 775 #end[FP]
 776             }
 777 #if[BITWISE]
 778 #if[!FP]
 779             if (opKind(op, VO_SHIFT)) {
 780                 // As per shift specification for Java, mask the shift count.
 781                 // This allows the JIT to ignore some ISA details.
 782                 that = that.lanewise(AND, SHIFT_MASK);
 783             }
 784 #end[!FP]
 785             if (op == AND_NOT) {
 786                 // FIXME: Support this in the JIT.
 787                 that = that.lanewise(NOT);
 788                 op = AND;
 789             } else if (op == DIV) {
 790                 VectorMask<$Boxtype$> eqz = that.eq(($type$)0);
 791                 if (eqz.and(m).anyTrue()) {
 792                     throw that.divZeroException();
 793                 }
 794                 // suppress div/0 exceptions in unset lanes
 795                 that = that.lanewise(NOT, eqz);
 796             }
 797 #end[BITWISE]
 798         }
 799 
 800         int opc = opCode(op);
 801         return VectorSupport.binaryOp(
 802             opc, getClass(), maskClass, $type$.class, length(),
 803             this, that, m,
 804             BIN_IMPL.find(op, opc, $abstractvectortype$::binaryOperations));
 805     }
 806 
 807     private static final
 808     ImplCache<Binary, BinaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 809         BIN_IMPL = new ImplCache<>(Binary.class, $Type$Vector.class);
 810 
 811     private static BinaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> binaryOperations(int opc_) {
 812         switch (opc_) {
 813             case VECTOR_OP_ADD: return (v0, v1, vm) ->
 814                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a + b));
 815             case VECTOR_OP_SUB: return (v0, v1, vm) ->
 816                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a - b));
 817             case VECTOR_OP_MUL: return (v0, v1, vm) ->
 818                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a * b));
 819             case VECTOR_OP_DIV: return (v0, v1, vm) ->
 820                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a / b));
 821             case VECTOR_OP_MAX: return (v0, v1, vm) ->
 822                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.max(a, b));
 823             case VECTOR_OP_MIN: return (v0, v1, vm) ->
 824                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.min(a, b));
 825 #if[BITWISE]
 826             case VECTOR_OP_AND: return (v0, v1, vm) ->
 827                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a & b));
 828             case VECTOR_OP_OR: return (v0, v1, vm) ->
 829                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a | b));
 830             case VECTOR_OP_XOR: return (v0, v1, vm) ->
 831                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a ^ b));
 832             case VECTOR_OP_LSHIFT: return (v0, v1, vm) ->
 833                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a << n));
 834             case VECTOR_OP_RSHIFT: return (v0, v1, vm) ->
 835                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a >> n));
 836             case VECTOR_OP_URSHIFT: return (v0, v1, vm) ->
 837                     v0.bOp(v1, vm, (i, a, n) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 838             case VECTOR_OP_LROTATE: return (v0, v1, vm) ->
 839                     v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
 840             case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
 841                     v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
 842 #end[BITWISE]
 843 #if[FP]
 844             case VECTOR_OP_OR: return (v0, v1, vm) ->
 845                     v0.bOp(v1, vm, (i, a, b) -> fromBits(toBits(a) | toBits(b)));
 846             case VECTOR_OP_ATAN2: return (v0, v1, vm) ->
 847                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.atan2(a, b));
 848             case VECTOR_OP_POW: return (v0, v1, vm) ->
 849                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.pow(a, b));
 850             case VECTOR_OP_HYPOT: return (v0, v1, vm) ->
 851                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.hypot(a, b));
 852 #end[FP]
 853             default: return null;
 854         }
 855     }
 856 
 857     // FIXME: Maybe all of the public final methods in this file (the
 858     // simple ones that just call lanewise) should be pushed down to
 859     // the X-VectorBits template.  They can't optimize properly at
 860     // this level, and must rely on inlining.  Does it work?
 861     // (If it works, of course keep the code here.)
 862 
 863     /**
 864      * Combines the lane values of this vector
 865      * with the value of a broadcast scalar.
 866      *
 867      * This is a lane-wise binary operation which applies
 868      * the selected operation to each lane.
 869      * The return value will be equal to this expression:
 870      * {@code this.lanewise(op, this.broadcast(e))}.
 871      *
 872      * @param op the operation used to process lane values
 873      * @param e the input scalar
 874      * @return the result of applying the operation lane-wise
 875      *         to the two input vectors
 876      * @throws UnsupportedOperationException if this vector does
 877      *         not support the requested operation
 878      * @see #lanewise(VectorOperators.Binary,Vector)
 879      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 880      */
 881     @ForceInline
 882     public final
 883     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 884                                   $type$ e) {
 885 #if[BITWISE]
 886         if (opKind(op, VO_SHIFT) && ($type$)(int)e == e) {
 887             return lanewiseShift(op, (int) e);
 888         }
 889         if (op == AND_NOT) {
 890             op = AND; e = ($type$) ~e;
 891         }
 892 #end[BITWISE]
 893         return lanewise(op, broadcast(e));
 894     }
 895 
 896     /**
 897      * Combines the lane values of this vector
 898      * with the value of a broadcast scalar,
 899      * with selection of lane elements controlled by a mask.
 900      *
 901      * This is a masked lane-wise binary operation which applies
 902      * the selected operation to each lane.
 903      * The return value will be equal to this expression:
 904      * {@code this.lanewise(op, this.broadcast(e), m)}.
 905      *
 906      * @param op the operation used to process lane values
 907      * @param e the input scalar
 908      * @param m the mask controlling lane selection
 909      * @return the result of applying the operation lane-wise
 910      *         to the input vector and the scalar
 911      * @throws UnsupportedOperationException if this vector does
 912      *         not support the requested operation
 913      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
 914      * @see #lanewise(VectorOperators.Binary,$type$)
 915      */
 916     @ForceInline
 917     public final
 918     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 919                                   $type$ e,
 920                                   VectorMask<$Boxtype$> m) {
 921 #if[BITWISE]
 922         if (opKind(op, VO_SHIFT) && ($type$)(int)e == e) {
 923             return lanewiseShift(op, (int) e, m);
 924         }
 925         if (op == AND_NOT) {
 926             op = AND; e = ($type$) ~e;
 927         }
 928 #end[BITWISE]
 929         return lanewise(op, broadcast(e), m);
 930     }
 931 
 932 #if[!long]
 933     /**
 934      * {@inheritDoc} <!--workaround-->
 935      * @apiNote
 936      * When working with vector subtypes like {@code $abstractvectortype$},
 937      * {@linkplain #lanewise(VectorOperators.Binary,$type$)
 938      * the more strongly typed method}
 939      * is typically selected.  It can be explicitly selected
 940      * using a cast: {@code v.lanewise(op,($type$)e)}.
 941      * The two expressions will produce numerically identical results.
 942      */
 943     @ForceInline
 944     public final
 945     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 946                                   long e) {
 947         $type$ e1 = ($type$) e;
 948 #if[BITWISE]
 949         if ((long)e1 != e
 950             // allow shift ops to clip down their int parameters
 951             && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
 952 #else[BITWISE]
 953         if ((long)e1 != e) {
 954 #end[BITWISE]
 955             vspecies().checkValue(e);  // for exception
 956         }
 957         return lanewise(op, e1);
 958     }
 959 
 960     /**
 961      * {@inheritDoc} <!--workaround-->
 962      * @apiNote
 963      * When working with vector subtypes like {@code $abstractvectortype$},
 964      * {@linkplain #lanewise(VectorOperators.Binary,$type$,VectorMask)
 965      * the more strongly typed method}
 966      * is typically selected.  It can be explicitly selected
 967      * using a cast: {@code v.lanewise(op,($type$)e,m)}.
 968      * The two expressions will produce numerically identical results.
 969      */
 970     @ForceInline
 971     public final
 972     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 973                                   long e, VectorMask<$Boxtype$> m) {
 974         $type$ e1 = ($type$) e;
 975 #if[BITWISE]
 976         if ((long)e1 != e
 977             // allow shift ops to clip down their int parameters
 978             && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
 979 #else[BITWISE]
 980         if ((long)e1 != e) {
 981 #end[BITWISE]
 982             vspecies().checkValue(e);  // for exception
 983         }
 984         return lanewise(op, e1, m);
 985     }
 986 #end[!long]
 987 
 988 #if[BITWISE]
 989     /*package-private*/
 990     abstract $abstractvectortype$
 991     lanewiseShift(VectorOperators.Binary op, int e);
 992 
 993     /*package-private*/
 994     @ForceInline
 995     final $abstractvectortype$
 996     lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
 997         // Special handling for these.  FIXME: Refactor?
 998         assert(opKind(op, VO_SHIFT));
 999         // As per shift specification for Java, mask the shift count.
1000         e &= SHIFT_MASK;
1001         int opc = opCode(op);
1002         return VectorSupport.broadcastInt(
1003             opc, getClass(), null, $type$.class, length(),
1004             this, e, null,
1005             BIN_INT_IMPL.find(op, opc, $abstractvectortype$::broadcastIntOperations));
1006     }
1007 
1008     /*package-private*/
1009     abstract $abstractvectortype$
1010     lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m);
1011 
1012     /*package-private*/
1013     @ForceInline
1014     final $abstractvectortype$
1015     lanewiseShiftTemplate(VectorOperators.Binary op,
1016                           Class<? extends VectorMask<$Boxtype$>> maskClass,
1017                           int e, VectorMask<$Boxtype$> m) {
1018         m.check(maskClass, this);
1019         assert(opKind(op, VO_SHIFT));
1020         // As per shift specification for Java, mask the shift count.
1021         e &= SHIFT_MASK;
1022         int opc = opCode(op);
1023         return VectorSupport.broadcastInt(
1024             opc, getClass(), maskClass, $type$.class, length(),
1025             this, e, m,
1026             BIN_INT_IMPL.find(op, opc, $abstractvectortype$::broadcastIntOperations));
1027     }
1028 
1029     private static final
1030     ImplCache<Binary,VectorBroadcastIntOp<$abstractvectortype$, VectorMask<$Boxtype$>>> BIN_INT_IMPL
1031         = new ImplCache<>(Binary.class, $Type$Vector.class);
1032 
1033     private static VectorBroadcastIntOp<$abstractvectortype$, VectorMask<$Boxtype$>> broadcastIntOperations(int opc_) {
1034         switch (opc_) {
1035             case VECTOR_OP_LSHIFT: return (v, n, m) ->
1036                     v.uOp(m, (i, a) -> ($type$)(a << n));
1037             case VECTOR_OP_RSHIFT: return (v, n, m) ->
1038                     v.uOp(m, (i, a) -> ($type$)(a >> n));
1039             case VECTOR_OP_URSHIFT: return (v, n, m) ->
1040                     v.uOp(m, (i, a) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
1041             case VECTOR_OP_LROTATE: return (v, n, m) ->
1042                     v.uOp(m, (i, a) -> rotateLeft(a, (int)n));
1043             case VECTOR_OP_RROTATE: return (v, n, m) ->
1044                     v.uOp(m, (i, a) -> rotateRight(a, (int)n));
1045             default: return null;
1046         }
1047     }
1048 
1049     // As per shift specification for Java, mask the shift count.
1050     // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
1051     // The latter two maskings go beyond the JLS, but seem reasonable
1052     // since our lane types are first-class types, not just dressed
1053     // up ints.
1054     private static final int SHIFT_MASK = ($Boxtype$.SIZE - 1);
1055 #if[byteOrShort]
1056     // Also simulate >>> on sub-word variables with a mask.
1057     private static final int LSHR_SETUP_MASK = ((1 << $Boxtype$.SIZE) - 1);
1058 #else[byteOrShort]
1059     private static final $type$ LSHR_SETUP_MASK = -1;
1060 #end[byteOrShort]
1061 #end[BITWISE]
1062 
1063     // Ternary lanewise support
1064 
1065     // Ternary operators come in eight variations:
1066     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
1067     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
1068 
1069     // It is annoying to support all of these variations of masking
1070     // and broadcast, but it would be more surprising not to continue
1071     // the obvious pattern started by unary and binary.
1072 
1073    /**
1074      * {@inheritDoc} <!--workaround-->
1075      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1076      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1077      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1078      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1079      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
1080      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
1081      */
1082     @Override
1083     public abstract
1084     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
1085                                                   Vector<$Boxtype$> v1,
1086                                                   Vector<$Boxtype$> v2);
1087     @ForceInline
1088     final
1089     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
1090                                           Vector<$Boxtype$> v1,
1091                                           Vector<$Boxtype$> v2) {
1092         $abstractvectortype$ that = ($abstractvectortype$) v1;
1093         $abstractvectortype$ tother = ($abstractvectortype$) v2;
1094         // It's a word: https://www.dictionary.com/browse/tother
1095         // See also Chapter 11 of Dickens, Our Mutual Friend:
1096         // "Totherest Governor," replied Mr Riderhood...
1097         that.check(this);
1098         tother.check(this);
1099 #if[BITWISE]
1100         if (op == BITWISE_BLEND) {
1101             // FIXME: Support this in the JIT.
1102             that = this.lanewise(XOR, that).lanewise(AND, tother);
1103             return this.lanewise(XOR, that);
1104         }
1105 #end[BITWISE]
1106         int opc = opCode(op);
1107         return VectorSupport.ternaryOp(
1108             opc, getClass(), null, $type$.class, length(),
1109             this, that, tother, null,
1110             TERN_IMPL.find(op, opc, $abstractvectortype$::ternaryOperations));
1111     }
1112 
1113     /**
1114      * {@inheritDoc} <!--workaround-->
1115      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1116      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1117      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1118      */
1119     @Override
1120     public abstract
1121     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
1122                                   Vector<$Boxtype$> v1,
1123                                   Vector<$Boxtype$> v2,
1124                                   VectorMask<$Boxtype$> m);
1125     @ForceInline
1126     final
1127     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
1128                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
1129                                           Vector<$Boxtype$> v1,
1130                                           Vector<$Boxtype$> v2,
1131                                           VectorMask<$Boxtype$> m) {
1132         $abstractvectortype$ that = ($abstractvectortype$) v1;
1133         $abstractvectortype$ tother = ($abstractvectortype$) v2;
1134         // It's a word: https://www.dictionary.com/browse/tother
1135         // See also Chapter 11 of Dickens, Our Mutual Friend:
1136         // "Totherest Governor," replied Mr Riderhood...
1137         that.check(this);
1138         tother.check(this);
1139         m.check(maskClass, this);
1140 
1141 #if[BITWISE]
1142         if (op == BITWISE_BLEND) {
1143             // FIXME: Support this in the JIT.
1144             that = this.lanewise(XOR, that).lanewise(AND, tother);
1145             return this.lanewise(XOR, that, m);
1146         }
1147 #end[BITWISE]
1148         int opc = opCode(op);
1149         return VectorSupport.ternaryOp(
1150             opc, getClass(), maskClass, $type$.class, length(),
1151             this, that, tother, m,
1152             TERN_IMPL.find(op, opc, $abstractvectortype$::ternaryOperations));
1153     }
1154 
1155     private static final
1156     ImplCache<Ternary, TernaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
1157         TERN_IMPL = new ImplCache<>(Ternary.class, $Type$Vector.class);
1158 
1159     private static TernaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> ternaryOperations(int opc_) {
1160         switch (opc_) {
1161 #if[FP]
1162             case VECTOR_OP_FMA: return (v0, v1_, v2_, m) ->
1163                     v0.tOp(v1_, v2_, m, (i, a, b, c) -> Math.fma(a, b, c));
1164 #end[FP]
1165             default: return null;
1166         }
1167     }
1168 
1169     /**
1170      * Combines the lane values of this vector
1171      * with the values of two broadcast scalars.
1172      *
1173      * This is a lane-wise ternary operation which applies
1174      * the selected operation to each lane.
1175      * The return value will be equal to this expression:
1176      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
1177      *
1178      * @param op the operation used to combine lane values
1179      * @param e1 the first input scalar
1180      * @param e2 the second input scalar
1181      * @return the result of applying the operation lane-wise
1182      *         to the input vector and the scalars
1183      * @throws UnsupportedOperationException if this vector does
1184      *         not support the requested operation
1185      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1186      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1187      */
1188     @ForceInline
1189     public final
1190     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2)
1191                                   $type$ e1,
1192                                   $type$ e2) {
1193         return lanewise(op, broadcast(e1), broadcast(e2));
1194     }
1195 
1196     /**
1197      * Combines the lane values of this vector
1198      * with the values of two broadcast scalars,
1199      * with selection of lane elements controlled by a mask.
1200      *
1201      * This is a masked lane-wise ternary operation which applies
1202      * the selected operation to each lane.
1203      * The return value will be equal to this expression:
1204      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
1205      *
1206      * @param op the operation used to combine lane values
1207      * @param e1 the first input scalar
1208      * @param e2 the second input scalar
1209      * @param m the mask controlling lane selection
1210      * @return the result of applying the operation lane-wise
1211      *         to the input vector and the scalars
1212      * @throws UnsupportedOperationException if this vector does
1213      *         not support the requested operation
1214      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1215      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1216      */
1217     @ForceInline
1218     public final
1219     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
1220                                   $type$ e1,
1221                                   $type$ e2,
1222                                   VectorMask<$Boxtype$> m) {
1223         return lanewise(op, broadcast(e1), broadcast(e2), m);
1224     }
1225 
1226     /**
1227      * Combines the lane values of this vector
1228      * with the values of another vector and a broadcast scalar.
1229      *
1230      * This is a lane-wise ternary operation which applies
1231      * the selected operation to each lane.
1232      * The return value will be equal to this expression:
1233      * {@code this.lanewise(op, v1, this.broadcast(e2))}.
1234      *
1235      * @param op the operation used to combine lane values
1236      * @param v1 the other input vector
1237      * @param e2 the input scalar
1238      * @return the result of applying the operation lane-wise
1239      *         to the input vectors and the scalar
1240      * @throws UnsupportedOperationException if this vector does
1241      *         not support the requested operation
1242      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1243      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1244      */
1245     @ForceInline
1246     public final
1247     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2)
1248                                   Vector<$Boxtype$> v1,
1249                                   $type$ e2) {
1250         return lanewise(op, v1, broadcast(e2));
1251     }
1252 
1253     /**
1254      * Combines the lane values of this vector
1255      * with the values of another vector and a broadcast scalar,
1256      * with selection of lane elements controlled by a mask.
1257      *
1258      * This is a masked lane-wise ternary operation which applies
1259      * the selected operation to each lane.
1260      * The return value will be equal to this expression:
1261      * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
1262      *
1263      * @param op the operation used to combine lane values
1264      * @param v1 the other input vector
1265      * @param e2 the input scalar
1266      * @param m the mask controlling lane selection
1267      * @return the result of applying the operation lane-wise
1268      *         to the input vectors and the scalar
1269      * @throws UnsupportedOperationException if this vector does
1270      *         not support the requested operation
1271      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1272      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1273      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
1274      */
1275     @ForceInline
1276     public final
1277     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
1278                                   Vector<$Boxtype$> v1,
1279                                   $type$ e2,
1280                                   VectorMask<$Boxtype$> m) {
1281         return lanewise(op, v1, broadcast(e2), m);
1282     }
1283 
1284     /**
1285      * Combines the lane values of this vector
1286      * with the values of another vector and a broadcast scalar.
1287      *
1288      * This is a lane-wise ternary operation which applies
1289      * the selected operation to each lane.
1290      * The return value will be equal to this expression:
1291      * {@code this.lanewise(op, this.broadcast(e1), v2)}.
1292      *
1293      * @param op the operation used to combine lane values
1294      * @param e1 the input scalar
1295      * @param v2 the other input vector
1296      * @return the result of applying the operation lane-wise
1297      *         to the input vectors and the scalar
1298      * @throws UnsupportedOperationException if this vector does
1299      *         not support the requested operation
1300      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1301      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1302      */
1303     @ForceInline
1304     public final
1305     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2)
1306                                   $type$ e1,
1307                                   Vector<$Boxtype$> v2) {
1308         return lanewise(op, broadcast(e1), v2);
1309     }
1310 
1311     /**
1312      * Combines the lane values of this vector
1313      * with the values of another vector and a broadcast scalar,
1314      * with selection of lane elements controlled by a mask.
1315      *
1316      * This is a masked lane-wise ternary operation which applies
1317      * the selected operation to each lane.
1318      * The return value will be equal to this expression:
1319      * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
1320      *
1321      * @param op the operation used to combine lane values
1322      * @param e1 the input scalar
1323      * @param v2 the other input vector
1324      * @param m the mask controlling lane selection
1325      * @return the result of applying the operation lane-wise
1326      *         to the input vectors and the scalar
1327      * @throws UnsupportedOperationException if this vector does
1328      *         not support the requested operation
1329      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1330      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
1331      */
1332     @ForceInline
1333     public final
1334     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
1335                                   $type$ e1,
1336                                   Vector<$Boxtype$> v2,
1337                                   VectorMask<$Boxtype$> m) {
1338         return lanewise(op, broadcast(e1), v2, m);
1339     }
1340 
1341     // (Thus endeth the Great and Mighty Ternary Ogdoad.)
1342     // https://en.wikipedia.org/wiki/Ogdoad
1343 
1344     /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1345     //
1346     // These include masked and non-masked versions.
1347     // This subclass adds broadcast (masked or not).
1348 
1349     /**
1350      * {@inheritDoc} <!--workaround-->
1351      * @see #add($type$)
1352      */
1353     @Override
1354     @ForceInline
1355     public final $abstractvectortype$ add(Vector<$Boxtype$> v) {
1356         return lanewise(ADD, v);
1357     }
1358 
1359     /**
1360      * Adds this vector to the broadcast of an input scalar.
1361      *
1362      * This is a lane-wise binary operation which applies
1363      * the primitive addition operation ({@code +}) to each lane.
1364      *
1365      * This method is also equivalent to the expression
1366      * {@link #lanewise(VectorOperators.Binary,$type$)
1367      *    lanewise}{@code (}{@link VectorOperators#ADD
1368      *    ADD}{@code , e)}.
1369      *
1370      * @param e the input scalar
1371      * @return the result of adding each lane of this vector to the scalar
1372      * @see #add(Vector)
1373      * @see #broadcast($type$)
1374      * @see #add($type$,VectorMask)
1375      * @see VectorOperators#ADD
1376      * @see #lanewise(VectorOperators.Binary,Vector)
1377      * @see #lanewise(VectorOperators.Binary,$type$)
1378      */
1379     @ForceInline
1380     public final
1381     $abstractvectortype$ add($type$ e) {
1382         return lanewise(ADD, e);
1383     }
1384 
1385     /**
1386      * {@inheritDoc} <!--workaround-->
1387      * @see #add($type$,VectorMask)
1388      */
1389     @Override
1390     @ForceInline
1391     public final $abstractvectortype$ add(Vector<$Boxtype$> v,
1392                                           VectorMask<$Boxtype$> m) {
1393         return lanewise(ADD, v, m);
1394     }
1395 
1396     /**
1397      * Adds this vector to the broadcast of an input scalar,
1398      * selecting lane elements controlled by a mask.
1399      *
1400      * This is a masked lane-wise binary operation which applies
1401      * the primitive addition operation ({@code +}) to each lane.
1402      *
1403      * This method is also equivalent to the expression
1404      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1405      *    lanewise}{@code (}{@link VectorOperators#ADD
1406      *    ADD}{@code , s, m)}.
1407      *
1408      * @param e the input scalar
1409      * @param m the mask controlling lane selection
1410      * @return the result of adding each lane of this vector to the scalar
1411      * @see #add(Vector,VectorMask)
1412      * @see #broadcast($type$)
1413      * @see #add($type$)
1414      * @see VectorOperators#ADD
1415      * @see #lanewise(VectorOperators.Binary,Vector)
1416      * @see #lanewise(VectorOperators.Binary,$type$)
1417      */
1418     @ForceInline
1419     public final $abstractvectortype$ add($type$ e,
1420                                           VectorMask<$Boxtype$> m) {
1421         return lanewise(ADD, e, m);
1422     }
1423 
1424     /**
1425      * {@inheritDoc} <!--workaround-->
1426      * @see #sub($type$)
1427      */
1428     @Override
1429     @ForceInline
1430     public final $abstractvectortype$ sub(Vector<$Boxtype$> v) {
1431         return lanewise(SUB, v);
1432     }
1433 
1434     /**
1435      * Subtracts an input scalar from this vector.
1436      *
1437      * This is a masked lane-wise binary operation which applies
1438      * the primitive subtraction operation ({@code -}) to each lane.
1439      *
1440      * This method is also equivalent to the expression
1441      * {@link #lanewise(VectorOperators.Binary,$type$)
1442      *    lanewise}{@code (}{@link VectorOperators#SUB
1443      *    SUB}{@code , e)}.
1444      *
1445      * @param e the input scalar
1446      * @return the result of subtracting the scalar from each lane of this vector
1447      * @see #sub(Vector)
1448      * @see #broadcast($type$)
1449      * @see #sub($type$,VectorMask)
1450      * @see VectorOperators#SUB
1451      * @see #lanewise(VectorOperators.Binary,Vector)
1452      * @see #lanewise(VectorOperators.Binary,$type$)
1453      */
1454     @ForceInline
1455     public final $abstractvectortype$ sub($type$ e) {
1456         return lanewise(SUB, e);
1457     }
1458 
1459     /**
1460      * {@inheritDoc} <!--workaround-->
1461      * @see #sub($type$,VectorMask)
1462      */
1463     @Override
1464     @ForceInline
1465     public final $abstractvectortype$ sub(Vector<$Boxtype$> v,
1466                                           VectorMask<$Boxtype$> m) {
1467         return lanewise(SUB, v, m);
1468     }
1469 
1470     /**
1471      * Subtracts an input scalar from this vector
1472      * under the control of a mask.
1473      *
1474      * This is a masked lane-wise binary operation which applies
1475      * the primitive subtraction operation ({@code -}) to each lane.
1476      *
1477      * This method is also equivalent to the expression
1478      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1479      *    lanewise}{@code (}{@link VectorOperators#SUB
1480      *    SUB}{@code , s, m)}.
1481      *
1482      * @param e the input scalar
1483      * @param m the mask controlling lane selection
1484      * @return the result of subtracting the scalar from each lane of this vector
1485      * @see #sub(Vector,VectorMask)
1486      * @see #broadcast($type$)
1487      * @see #sub($type$)
1488      * @see VectorOperators#SUB
1489      * @see #lanewise(VectorOperators.Binary,Vector)
1490      * @see #lanewise(VectorOperators.Binary,$type$)
1491      */
1492     @ForceInline
1493     public final $abstractvectortype$ sub($type$ e,
1494                                           VectorMask<$Boxtype$> m) {
1495         return lanewise(SUB, e, m);
1496     }
1497 
1498     /**
1499      * {@inheritDoc} <!--workaround-->
1500      * @see #mul($type$)
1501      */
1502     @Override
1503     @ForceInline
1504     public final $abstractvectortype$ mul(Vector<$Boxtype$> v) {
1505         return lanewise(MUL, v);
1506     }
1507 
1508     /**
1509      * Multiplies this vector by the broadcast of an input scalar.
1510      *
1511      * This is a lane-wise binary operation which applies
1512      * the primitive multiplication operation ({@code *}) to each lane.
1513      *
1514      * This method is also equivalent to the expression
1515      * {@link #lanewise(VectorOperators.Binary,$type$)
1516      *    lanewise}{@code (}{@link VectorOperators#MUL
1517      *    MUL}{@code , e)}.
1518      *
1519      * @param e the input scalar
1520      * @return the result of multiplying this vector by the given scalar
1521      * @see #mul(Vector)
1522      * @see #broadcast($type$)
1523      * @see #mul($type$,VectorMask)
1524      * @see VectorOperators#MUL
1525      * @see #lanewise(VectorOperators.Binary,Vector)
1526      * @see #lanewise(VectorOperators.Binary,$type$)
1527      */
1528     @ForceInline
1529     public final $abstractvectortype$ mul($type$ e) {
1530         return lanewise(MUL, e);
1531     }
1532 
1533     /**
1534      * {@inheritDoc} <!--workaround-->
1535      * @see #mul($type$,VectorMask)
1536      */
1537     @Override
1538     @ForceInline
1539     public final $abstractvectortype$ mul(Vector<$Boxtype$> v,
1540                                           VectorMask<$Boxtype$> m) {
1541         return lanewise(MUL, v, m);
1542     }
1543 
1544     /**
1545      * Multiplies this vector by the broadcast of an input scalar,
1546      * selecting lane elements controlled by a mask.
1547      *
1548      * This is a masked lane-wise binary operation which applies
1549      * the primitive multiplication operation ({@code *}) to each lane.
1550      *
1551      * This method is also equivalent to the expression
1552      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1553      *    lanewise}{@code (}{@link VectorOperators#MUL
1554      *    MUL}{@code , s, m)}.
1555      *
1556      * @param e the input scalar
1557      * @param m the mask controlling lane selection
1558      * @return the result of muling each lane of this vector to the scalar
1559      * @see #mul(Vector,VectorMask)
1560      * @see #broadcast($type$)
1561      * @see #mul($type$)
1562      * @see VectorOperators#MUL
1563      * @see #lanewise(VectorOperators.Binary,Vector)
1564      * @see #lanewise(VectorOperators.Binary,$type$)
1565      */
1566     @ForceInline
1567     public final $abstractvectortype$ mul($type$ e,
1568                                           VectorMask<$Boxtype$> m) {
1569         return lanewise(MUL, e, m);
1570     }
1571 
1572     /**
1573      * {@inheritDoc} <!--workaround-->
1574 #if[FP]
1575      * @apiNote Because the underlying scalar operator is an IEEE
1576      * floating point number, division by zero in fact will
1577      * not throw an exception, but will yield a signed
1578      * infinity or NaN.
1579 #else[FP]
1580      * @apiNote If there is a zero divisor, {@code
1581      * ArithmeticException} will be thrown.
1582 #end[FP]
1583      */
1584     @Override
1585     @ForceInline
1586     public final $abstractvectortype$ div(Vector<$Boxtype$> v) {
1587         return lanewise(DIV, v);
1588     }
1589 
1590     /**
1591      * Divides this vector by the broadcast of an input scalar.
1592      *
1593      * This is a lane-wise binary operation which applies
1594      * the primitive division operation ({@code /}) to each lane.
1595      *
1596      * This method is also equivalent to the expression
1597      * {@link #lanewise(VectorOperators.Binary,$type$)
1598      *    lanewise}{@code (}{@link VectorOperators#DIV
1599      *    DIV}{@code , e)}.
1600      *
1601 #if[FP]
1602      * @apiNote Because the underlying scalar operator is an IEEE
1603      * floating point number, division by zero in fact will
1604      * not throw an exception, but will yield a signed
1605      * infinity or NaN.
1606 #else[FP]
1607      * @apiNote If there is a zero divisor, {@code
1608      * ArithmeticException} will be thrown.
1609 #end[FP]
1610      *
1611      * @param e the input scalar
1612      * @return the result of dividing each lane of this vector by the scalar
1613      * @see #div(Vector)
1614      * @see #broadcast($type$)
1615      * @see #div($type$,VectorMask)
1616      * @see VectorOperators#DIV
1617      * @see #lanewise(VectorOperators.Binary,Vector)
1618      * @see #lanewise(VectorOperators.Binary,$type$)
1619      */
1620     @ForceInline
1621     public final $abstractvectortype$ div($type$ e) {
1622         return lanewise(DIV, e);
1623     }
1624 
1625     /**
1626      * {@inheritDoc} <!--workaround-->
1627      * @see #div($type$,VectorMask)
1628 #if[FP]
1629      * @apiNote Because the underlying scalar operator is an IEEE
1630      * floating point number, division by zero in fact will
1631      * not throw an exception, but will yield a signed
1632      * infinity or NaN.
1633 #else[FP]
1634      * @apiNote If there is a zero divisor, {@code
1635      * ArithmeticException} will be thrown.
1636 #end[FP]
1637      */
1638     @Override
1639     @ForceInline
1640     public final $abstractvectortype$ div(Vector<$Boxtype$> v,
1641                                           VectorMask<$Boxtype$> m) {
1642         return lanewise(DIV, v, m);
1643     }
1644 
1645     /**
1646      * Divides this vector by the broadcast of an input scalar,
1647      * selecting lane elements controlled by a mask.
1648      *
1649      * This is a masked lane-wise binary operation which applies
1650      * the primitive division operation ({@code /}) to each lane.
1651      *
1652      * This method is also equivalent to the expression
1653      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1654      *    lanewise}{@code (}{@link VectorOperators#DIV
1655      *    DIV}{@code , s, m)}.
1656      *
1657 #if[FP]
1658      * @apiNote Because the underlying scalar operator is an IEEE
1659      * floating point number, division by zero in fact will
1660      * not throw an exception, but will yield a signed
1661      * infinity or NaN.
1662 #else[FP]
1663      * @apiNote If there is a zero divisor, {@code
1664      * ArithmeticException} will be thrown.
1665 #end[FP]
1666      *
1667      * @param e the input scalar
1668      * @param m the mask controlling lane selection
1669      * @return the result of dividing each lane of this vector by the scalar
1670      * @see #div(Vector,VectorMask)
1671      * @see #broadcast($type$)
1672      * @see #div($type$)
1673      * @see VectorOperators#DIV
1674      * @see #lanewise(VectorOperators.Binary,Vector)
1675      * @see #lanewise(VectorOperators.Binary,$type$)
1676      */
1677     @ForceInline
1678     public final $abstractvectortype$ div($type$ e,
1679                                           VectorMask<$Boxtype$> m) {
1680         return lanewise(DIV, e, m);
1681     }
1682 
1683     /// END OF FULL-SERVICE BINARY METHODS
1684 
1685     /// SECOND-TIER BINARY METHODS
1686     //
1687     // There are no masked versions.
1688 
1689     /**
1690      * {@inheritDoc} <!--workaround-->
1691 #if[FP]
1692      * @apiNote
1693      * For this method, floating point negative
1694      * zero {@code -0.0} is treated as a value distinct from, and less
1695      * than the default value (positive zero).
1696 #end[FP]
1697      */
1698     @Override
1699     @ForceInline
1700     public final $abstractvectortype$ min(Vector<$Boxtype$> v) {
1701         return lanewise(MIN, v);
1702     }
1703 
1704     // FIXME:  "broadcast of an input scalar" is really wordy.  Reduce?
1705     /**
1706      * Computes the smaller of this vector and the broadcast of an input scalar.
1707      *
1708      * This is a lane-wise binary operation which applies the
1709      * operation {@code Math.min()} to each pair of
1710      * corresponding lane values.
1711      *
1712      * This method is also equivalent to the expression
1713      * {@link #lanewise(VectorOperators.Binary,$type$)
1714      *    lanewise}{@code (}{@link VectorOperators#MIN
1715      *    MIN}{@code , e)}.
1716      *
1717      * @param e the input scalar
1718      * @return the result of multiplying this vector by the given scalar
1719      * @see #min(Vector)
1720      * @see #broadcast($type$)
1721      * @see VectorOperators#MIN
1722      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1723 #if[FP]
1724      * @apiNote
1725      * For this method, floating point negative
1726      * zero {@code -0.0} is treated as a value distinct from, and less
1727      * than the default value (positive zero).
1728 #end[FP]
1729      */
1730     @ForceInline
1731     public final $abstractvectortype$ min($type$ e) {
1732         return lanewise(MIN, e);
1733     }
1734 
1735     /**
1736      * {@inheritDoc} <!--workaround-->
1737 #if[FP]
1738      * @apiNote
1739      * For this method, floating point negative
1740      * zero {@code -0.0} is treated as a value distinct from, and less
1741      * than the default value (positive zero).
1742 #end[FP]
1743      */
1744     @Override
1745     @ForceInline
1746     public final $abstractvectortype$ max(Vector<$Boxtype$> v) {
1747         return lanewise(MAX, v);
1748     }
1749 
1750     /**
1751      * Computes the larger of this vector and the broadcast of an input scalar.
1752      *
1753      * This is a lane-wise binary operation which applies the
1754      * operation {@code Math.max()} to each pair of
1755      * corresponding lane values.
1756      *
1757      * This method is also equivalent to the expression
1758      * {@link #lanewise(VectorOperators.Binary,$type$)
1759      *    lanewise}{@code (}{@link VectorOperators#MAX
1760      *    MAX}{@code , e)}.
1761      *
1762      * @param e the input scalar
1763      * @return the result of multiplying this vector by the given scalar
1764      * @see #max(Vector)
1765      * @see #broadcast($type$)
1766      * @see VectorOperators#MAX
1767      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1768 #if[FP]
1769      * @apiNote
1770      * For this method, floating point negative
1771      * zero {@code -0.0} is treated as a value distinct from, and less
1772      * than the default value (positive zero).
1773 #end[FP]
1774      */
1775     @ForceInline
1776     public final $abstractvectortype$ max($type$ e) {
1777         return lanewise(MAX, e);
1778     }
1779 
1780 #if[BITWISE]
1781     // common bitwise operators: and, or, not (with scalar versions)
1782     /**
1783      * Computes the bitwise logical conjunction ({@code &})
1784      * of this vector and a second input vector.
1785      *
1786      * This is a lane-wise binary operation which applies the
1787      * the primitive bitwise "and" operation ({@code &})
1788      * to each pair of corresponding lane values.
1789      *
1790      * This method is also equivalent to the expression
1791      * {@link #lanewise(VectorOperators.Binary,Vector)
1792      *    lanewise}{@code (}{@link VectorOperators#AND
1793      *    AND}{@code , v)}.
1794      *
1795      * <p>
1796      * This is not a full-service named operation like
1797      * {@link #add(Vector) add}.  A masked version of
1798      * this operation is not directly available
1799      * but may be obtained via the masked version of
1800      * {@code lanewise}.
1801      *
1802      * @param v a second input vector
1803      * @return the bitwise {@code &} of this vector and the second input vector
1804      * @see #and($type$)
1805      * @see #or(Vector)
1806      * @see #not()
1807      * @see VectorOperators#AND
1808      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1809      */
1810     @ForceInline
1811     public final $abstractvectortype$ and(Vector<$Boxtype$> v) {
1812         return lanewise(AND, v);
1813     }
1814 
1815     /**
1816      * Computes the bitwise logical conjunction ({@code &})
1817      * of this vector and a scalar.
1818      *
1819      * This is a lane-wise binary operation which applies the
1820      * the primitive bitwise "and" operation ({@code &})
1821      * to each pair of corresponding lane values.
1822      *
1823      * This method is also equivalent to the expression
1824      * {@link #lanewise(VectorOperators.Binary,Vector)
1825      *    lanewise}{@code (}{@link VectorOperators#AND
1826      *    AND}{@code , e)}.
1827      *
1828      * @param e an input scalar
1829      * @return the bitwise {@code &} of this vector and scalar
1830      * @see #and(Vector)
1831      * @see VectorOperators#AND
1832      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1833      */
1834     @ForceInline
1835     public final $abstractvectortype$ and($type$ e) {
1836         return lanewise(AND, e);
1837     }
1838 
1839     /**
1840      * Computes the bitwise logical disjunction ({@code |})
1841      * of this vector and a second input vector.
1842      *
1843      * This is a lane-wise binary operation which applies the
1844      * the primitive bitwise "or" operation ({@code |})
1845      * to each pair of corresponding lane values.
1846      *
1847      * This method is also equivalent to the expression
1848      * {@link #lanewise(VectorOperators.Binary,Vector)
1849      *    lanewise}{@code (}{@link VectorOperators#OR
1850      *    AND}{@code , v)}.
1851      *
1852      * <p>
1853      * This is not a full-service named operation like
1854      * {@link #add(Vector) add}.  A masked version of
1855      * this operation is not directly available
1856      * but may be obtained via the masked version of
1857      * {@code lanewise}.
1858      *
1859      * @param v a second input vector
1860      * @return the bitwise {@code |} of this vector and the second input vector
1861      * @see #or($type$)
1862      * @see #and(Vector)
1863      * @see #not()
1864      * @see VectorOperators#OR
1865      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1866      */
1867     @ForceInline
1868     public final $abstractvectortype$ or(Vector<$Boxtype$> v) {
1869         return lanewise(OR, v);
1870     }
1871 
1872     /**
1873      * Computes the bitwise logical disjunction ({@code |})
1874      * of this vector and a scalar.
1875      *
1876      * This is a lane-wise binary operation which applies the
1877      * the primitive bitwise "or" operation ({@code |})
1878      * to each pair of corresponding lane values.
1879      *
1880      * This method is also equivalent to the expression
1881      * {@link #lanewise(VectorOperators.Binary,Vector)
1882      *    lanewise}{@code (}{@link VectorOperators#OR
1883      *    OR}{@code , e)}.
1884      *
1885      * @param e an input scalar
1886      * @return the bitwise {@code |} of this vector and scalar
1887      * @see #or(Vector)
1888      * @see VectorOperators#OR
1889      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1890      */
1891     @ForceInline
1892     public final $abstractvectortype$ or($type$ e) {
1893         return lanewise(OR, e);
1894     }
1895 
1896 #end[BITWISE]
1897 
1898 #if[FP]
1899     // common FP operator: pow
1900     /**
1901      * Raises this vector to the power of a second input vector.
1902      *
1903      * This is a lane-wise binary operation which applies an operation
1904      * conforming to the specification of
1905      * {@link Math#pow Math.pow(a,b)}
1906      * to each pair of corresponding lane values.
1907 #if[intOrFloat]
1908      * The operation is adapted to cast the operands and the result,
1909      * specifically widening {@code float} operands to {@code double}
1910      * operands and narrowing the {@code double} result to a {@code float}
1911      * result.
1912 #end[intOrFloat]
1913      *
1914      * This method is also equivalent to the expression
1915      * {@link #lanewise(VectorOperators.Binary,Vector)
1916      *    lanewise}{@code (}{@link VectorOperators#POW
1917      *    POW}{@code , b)}.
1918      *
1919      * <p>
1920      * This is not a full-service named operation like
1921      * {@link #add(Vector) add}.  A masked version of
1922      * this operation is not directly available
1923      * but may be obtained via the masked version of
1924      * {@code lanewise}.
1925      *
1926      * @param b a vector exponent by which to raise this vector
1927      * @return the {@code b}-th power of this vector
1928      * @see #pow($type$)
1929      * @see VectorOperators#POW
1930      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1931      */
1932     @ForceInline
1933     public final $abstractvectortype$ pow(Vector<$Boxtype$> b) {
1934         return lanewise(POW, b);
1935     }
1936 
1937     /**
1938      * Raises this vector to a scalar power.
1939      *
1940      * This is a lane-wise binary operation which applies an operation
1941      * conforming to the specification of
1942      * {@link Math#pow Math.pow(a,b)}
1943      * to each pair of corresponding lane values.
1944 #if[intOrFloat]
1945      * The operation is adapted to cast the operands and the result,
1946      * specifically widening {@code float} operands to {@code double}
1947      * operands and narrowing the {@code double} result to a {@code float}
1948      * result.
1949 #end[intOrFloat]
1950      *
1951      * This method is also equivalent to the expression
1952      * {@link #lanewise(VectorOperators.Binary,Vector)
1953      *    lanewise}{@code (}{@link VectorOperators#POW
1954      *    POW}{@code , b)}.
1955      *
1956      * @param b a scalar exponent by which to raise this vector
1957      * @return the {@code b}-th power of this vector
1958      * @see #pow(Vector)
1959      * @see VectorOperators#POW
1960      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1961      */
1962     @ForceInline
1963     public final $abstractvectortype$ pow($type$ b) {
1964         return lanewise(POW, b);
1965     }
1966 #end[FP]
1967 
1968     /// UNARY METHODS
1969 
1970     /**
1971      * {@inheritDoc} <!--workaround-->
1972      */
1973     @Override
1974     @ForceInline
1975     public final
1976     $abstractvectortype$ neg() {
1977         return lanewise(NEG);
1978     }
1979 
1980     /**
1981      * {@inheritDoc} <!--workaround-->
1982      */
1983     @Override
1984     @ForceInline
1985     public final
1986     $abstractvectortype$ abs() {
1987         return lanewise(ABS);
1988     }
1989 
1990 #if[BITWISE]
1991     // not (~)
1992     /**
1993      * Computes the bitwise logical complement ({@code ~})
1994      * of this vector.
1995      *
1996      * This is a lane-wise binary operation which applies the
1997      * the primitive bitwise "not" operation ({@code ~})
1998      * to each lane value.
1999      *
2000      * This method is also equivalent to the expression
2001      * {@link #lanewise(VectorOperators.Unary)
2002      *    lanewise}{@code (}{@link VectorOperators#NOT
2003      *    NOT}{@code )}.
2004      *
2005      * <p>
2006      * This is not a full-service named operation like
2007      * {@link #add(Vector) add}.  A masked version of
2008      * this operation is not directly available
2009      * but may be obtained via the masked version of
2010      * {@code lanewise}.
2011      *
2012      * @return the bitwise complement {@code ~} of this vector
2013      * @see #and(Vector)
2014      * @see VectorOperators#NOT
2015      * @see #lanewise(VectorOperators.Unary,VectorMask)
2016      */
2017     @ForceInline
2018     public final $abstractvectortype$ not() {
2019         return lanewise(NOT);
2020     }
2021 #end[BITWISE]
2022 
2023 #if[FP]
2024     // sqrt
2025     /**
2026      * Computes the square root of this vector.
2027      *
2028      * This is a lane-wise unary operation which applies an operation
2029      * conforming to the specification of
2030      * {@link Math#sqrt Math.sqrt(a)}
2031      * to each lane value.
2032 #if[intOrFloat]
2033      * The operation is adapted to cast the operand and the result,
2034      * specifically widening the {@code float} operand to a {@code double}
2035      * operand and narrowing the {@code double} result to a {@code float}
2036      * result.
2037 #end[intOrFloat]
2038      *
2039      * This method is also equivalent to the expression
2040      * {@link #lanewise(VectorOperators.Unary)
2041      *    lanewise}{@code (}{@link VectorOperators#SQRT
2042      *    SQRT}{@code )}.
2043      *
2044      * @return the square root of this vector
2045      * @see VectorOperators#SQRT
2046      * @see #lanewise(VectorOperators.Unary,VectorMask)
2047      */
2048     @ForceInline
2049     public final $abstractvectortype$ sqrt() {
2050         return lanewise(SQRT);
2051     }
2052 #end[FP]
2053 
2054     /// COMPARISONS
2055 
2056     /**
2057      * {@inheritDoc} <!--workaround-->
2058      */
2059     @Override
2060     @ForceInline
2061     public final
2062     VectorMask<$Boxtype$> eq(Vector<$Boxtype$> v) {
2063         return compare(EQ, v);
2064     }
2065 
2066     /**
2067      * Tests if this vector is equal to an input scalar.
2068      *
2069      * This is a lane-wise binary test operation which applies
2070      * the primitive equals operation ({@code ==}) to each lane.
2071      * The result is the same as {@code compare(VectorOperators.Comparison.EQ, e)}.
2072      *
2073      * @param e the input scalar
2074      * @return the result mask of testing if this vector
2075      *         is equal to {@code e}
2076      * @see #compare(VectorOperators.Comparison,$type$)
2077      */
2078     @ForceInline
2079     public final
2080     VectorMask<$Boxtype$> eq($type$ e) {
2081         return compare(EQ, e);
2082     }
2083 
2084     /**
2085      * {@inheritDoc} <!--workaround-->
2086      */
2087     @Override
2088     @ForceInline
2089     public final
2090     VectorMask<$Boxtype$> lt(Vector<$Boxtype$> v) {
2091         return compare(LT, v);
2092     }
2093 
2094     /**
2095      * Tests if this vector is less than an input scalar.
2096      *
2097      * This is a lane-wise binary test operation which applies
2098      * the primitive less than operation ({@code <}) to each lane.
2099      * The result is the same as {@code compare(VectorOperators.LT, e)}.
2100      *
2101      * @param e the input scalar
2102      * @return the mask result of testing if this vector
2103      *         is less than the input scalar
2104      * @see #compare(VectorOperators.Comparison,$type$)
2105      */
2106     @ForceInline
2107     public final
2108     VectorMask<$Boxtype$> lt($type$ e) {
2109         return compare(LT, e);
2110     }
2111 
2112     /**
2113      * {@inheritDoc} <!--workaround-->
2114      */
2115     @Override
2116     public abstract
2117     VectorMask<$Boxtype$> test(VectorOperators.Test op);
2118 
2119     /*package-private*/
2120     @ForceInline
2121     final
2122     <M extends VectorMask<$Boxtype$>>
2123     M testTemplate(Class<M> maskType, Test op) {
2124         $Type$Species vsp = vspecies();
2125         if (opKind(op, VO_SPECIAL)) {
2126 #if[FP]
2127             $Bitstype$Vector bits = this.viewAsIntegralLanes();
2128 #end[FP]
2129             VectorMask<$Boxbitstype$> m;
2130             if (op == IS_DEFAULT) {
2131                 m = {#if[FP]?bits.}compare(EQ, ($bitstype$) 0);
2132             } else if (op == IS_NEGATIVE) {
2133                 m = {#if[FP]?bits.}compare(LT, ($bitstype$) 0);
2134             }
2135 #if[FP]
2136             else if (op == IS_FINITE ||
2137                      op == IS_NAN ||
2138                      op == IS_INFINITE) {
2139                 // first kill the sign:
2140                 bits = bits.and($Boxbitstype$.MAX_VALUE);
2141                 // next find the bit pattern for infinity:
2142                 $bitstype$ infbits = ($bitstype$) toBits($Boxtype$.POSITIVE_INFINITY);
2143                 // now compare:
2144                 if (op == IS_FINITE) {
2145                     m = bits.compare(LT, infbits);
2146                 } else if (op == IS_NAN) {
2147                     m = bits.compare(GT, infbits);
2148                 } else {
2149                     m = bits.compare(EQ, infbits);
2150                 }
2151             }
2152 #end[FP]
2153             else {
2154                 throw new AssertionError(op);
2155             }
2156             return maskType.cast(m{#if[FP]?.cast(vsp)});
2157         }
2158         int opc = opCode(op);
2159         throw new AssertionError(op);
2160     }
2161 
2162     /**
2163      * {@inheritDoc} <!--workaround-->
2164      */
2165     @Override
2166     public abstract
2167     VectorMask<$Boxtype$> test(VectorOperators.Test op,
2168                                   VectorMask<$Boxtype$> m);
2169 
2170     /*package-private*/
2171     @ForceInline
2172     final
2173     <M extends VectorMask<$Boxtype$>>
2174     M testTemplate(Class<M> maskType, Test op, M mask) {
2175         $Type$Species vsp = vspecies();
2176         mask.check(maskType, this);
2177         if (opKind(op, VO_SPECIAL)) {
2178 #if[FP]
2179             $Bitstype$Vector bits = this.viewAsIntegralLanes();
2180             VectorMask<$Boxbitstype$> m = mask.cast($Bitstype$Vector.species(shape()));
2181 #else[FP]
2182             VectorMask<$Boxbitstype$> m = mask;
2183 #end[FP]
2184             if (op == IS_DEFAULT) {
2185                 m = {#if[FP]?bits.}compare(EQ, ($bitstype$) 0, m);
2186             } else if (op == IS_NEGATIVE) {
2187                 m = {#if[FP]?bits.}compare(LT, ($bitstype$) 0, m);
2188             }
2189 #if[FP]
2190             else if (op == IS_FINITE ||
2191                      op == IS_NAN ||
2192                      op == IS_INFINITE) {
2193                 // first kill the sign:
2194                 bits = bits.and($Boxbitstype$.MAX_VALUE);
2195                 // next find the bit pattern for infinity:
2196                 $bitstype$ infbits = ($bitstype$) toBits($Boxtype$.POSITIVE_INFINITY);
2197                 // now compare:
2198                 if (op == IS_FINITE) {
2199                     m = bits.compare(LT, infbits, m);
2200                 } else if (op == IS_NAN) {
2201                     m = bits.compare(GT, infbits, m);
2202                 } else {
2203                     m = bits.compare(EQ, infbits, m);
2204                 }
2205             }
2206 #end[FP]
2207             else {
2208                 throw new AssertionError(op);
2209             }
2210             return maskType.cast(m{#if[FP]?.cast(vsp)});
2211         }
2212         int opc = opCode(op);
2213         throw new AssertionError(op);
2214     }
2215 
2216     /**
2217      * {@inheritDoc} <!--workaround-->
2218      */
2219     @Override
2220     public abstract
2221     VectorMask<$Boxtype$> compare(VectorOperators.Comparison op, Vector<$Boxtype$> v);
2222 
2223     /*package-private*/
2224     @ForceInline
2225     final
2226     <M extends VectorMask<$Boxtype$>>
2227     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v) {
2228         $abstractvectortype$ that = ($abstractvectortype$) v;
2229         that.check(this);
2230         int opc = opCode(op);
2231         return VectorSupport.compare(
2232             opc, getClass(), maskType, $type$.class, length(),
2233             this, that, null,
2234             (cond, v0, v1, m1) -> {
2235                 AbstractMask<$Boxtype$> m
2236                     = v0.bTest(cond, v1, (cond_, i, a, b)
2237                                -> compareWithOp(cond, a, b));
2238                 @SuppressWarnings("unchecked")
2239                 M m2 = (M) m;
2240                 return m2;
2241             });
2242     }
2243 
2244     /*package-private*/
2245     @ForceInline
2246     final
2247     <M extends VectorMask<$Boxtype$>>
2248     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v, M m) {
2249         $abstractvectortype$ that = ($abstractvectortype$) v;
2250         that.check(this);
2251         m.check(maskType, this);
2252         int opc = opCode(op);
2253         return VectorSupport.compare(
2254             opc, getClass(), maskType, $type$.class, length(),
2255             this, that, m,
2256             (cond, v0, v1, m1) -> {
2257                 AbstractMask<$Boxtype$> cmpM
2258                     = v0.bTest(cond, v1, (cond_, i, a, b)
2259                                -> compareWithOp(cond, a, b));
2260                 @SuppressWarnings("unchecked")
2261                 M m2 = (M) cmpM.and(m1);
2262                 return m2;
2263             });
2264     }
2265 
2266     @ForceInline
2267     private static boolean compareWithOp(int cond, $type$ a, $type$ b) {
2268         return switch (cond) {
2269             case BT_eq -> a == b;
2270             case BT_ne -> a != b;
2271             case BT_lt -> a < b;
2272             case BT_le -> a <= b;
2273             case BT_gt -> a > b;
2274             case BT_ge -> a >= b;
2275 #if[!FP]
2276             case BT_ult -> $Boxtype$.compareUnsigned(a, b) < 0;
2277             case BT_ule -> $Boxtype$.compareUnsigned(a, b) <= 0;
2278             case BT_ugt -> $Boxtype$.compareUnsigned(a, b) > 0;
2279             case BT_uge -> $Boxtype$.compareUnsigned(a, b) >= 0;
2280 #end[!FP]
2281             default -> throw new AssertionError();
2282         };
2283     }
2284 
2285     /**
2286      * Tests this vector by comparing it with an input scalar,
2287      * according to the given comparison operation.
2288      *
2289      * This is a lane-wise binary test operation which applies
2290      * the comparison operation to each lane.
2291      * <p>
2292      * The result is the same as
2293      * {@code compare(op, broadcast(species(), e))}.
2294      * That is, the scalar may be regarded as broadcast to
2295      * a vector of the same species, and then compared
2296      * against the original vector, using the selected
2297      * comparison operation.
2298      *
2299      * @param op the operation used to compare lane values
2300      * @param e the input scalar
2301      * @return the mask result of testing lane-wise if this vector
2302      *         compares to the input, according to the selected
2303      *         comparison operator
2304      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector)
2305      * @see #eq($type$)
2306      * @see #lt($type$)
2307      */
2308     public abstract
2309     VectorMask<$Boxtype$> compare(Comparison op, $type$ e);
2310 
2311     /*package-private*/
2312     @ForceInline
2313     final
2314     <M extends VectorMask<$Boxtype$>>
2315     M compareTemplate(Class<M> maskType, Comparison op, $type$ e) {
2316         return compareTemplate(maskType, op, broadcast(e));
2317     }
2318 
2319     /**
2320      * Tests this vector by comparing it with an input scalar,
2321      * according to the given comparison operation,
2322      * in lanes selected by a mask.
2323      *
2324      * This is a masked lane-wise binary test operation which applies
2325      * to each pair of corresponding lane values.
2326      *
2327      * The returned result is equal to the expression
2328      * {@code compare(op,s).and(m)}.
2329      *
2330      * @param op the operation used to compare lane values
2331      * @param e the input scalar
2332      * @param m the mask controlling lane selection
2333      * @return the mask result of testing lane-wise if this vector
2334      *         compares to the input, according to the selected
2335      *         comparison operator,
2336      *         and only in the lanes selected by the mask
2337      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector,VectorMask)
2338      */
2339     @ForceInline
2340     public final VectorMask<$Boxtype$> compare(VectorOperators.Comparison op,
2341                                                $type$ e,
2342                                                VectorMask<$Boxtype$> m) {
2343         return compare(op, broadcast(e), m);
2344     }
2345 
2346 #if[!long]
2347     /**
2348      * {@inheritDoc} <!--workaround-->
2349      */
2350     @Override
2351     public abstract
2352     VectorMask<$Boxtype$> compare(Comparison op, long e);
2353 
2354     /*package-private*/
2355     @ForceInline
2356     final
2357     <M extends VectorMask<$Boxtype$>>
2358     M compareTemplate(Class<M> maskType, Comparison op, long e) {
2359         return compareTemplate(maskType, op, broadcast(e));
2360     }
2361 
2362     /**
2363      * {@inheritDoc} <!--workaround-->
2364      */
2365     @Override
2366     @ForceInline
2367     public final
2368     VectorMask<$Boxtype$> compare(Comparison op, long e, VectorMask<$Boxtype$> m) {
2369         return compare(op, broadcast(e), m);
2370     }
2371 
2372 
2373 #end[!long]
2374 
2375     /**
2376      * {@inheritDoc} <!--workaround-->
2377      */
2378     @Override public abstract
2379     $abstractvectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m);
2380 
2381     /*package-private*/
2382     @ForceInline
2383     final
2384     <M extends VectorMask<$Boxtype$>>
2385     $abstractvectortype$
2386     blendTemplate(Class<M> maskType, $abstractvectortype$ v, M m) {
2387         v.check(this);
2388         return VectorSupport.blend(
2389             getClass(), maskType, $type$.class, length(),
2390             this, v, m,
2391             (v0, v1, m_) -> v0.bOp(v1, m_, (i, a, b) -> b));
2392     }
2393 
2394     /**
2395      * {@inheritDoc} <!--workaround-->
2396      */
2397     @Override public abstract $abstractvectortype$ addIndex(int scale);
2398 
2399     /*package-private*/
2400     @ForceInline
2401     final $abstractvectortype$ addIndexTemplate(int scale) {
2402         $Type$Species vsp = vspecies();
2403         // make sure VLENGTH*scale doesn't overflow:
2404         vsp.checkScale(scale);
2405         return VectorSupport.indexVector(
2406             getClass(), $type$.class, length(),
2407             this, scale, vsp,
2408             (v, scale_, s)
2409             -> {
2410                 // If the platform doesn't support an INDEX
2411                 // instruction directly, load IOTA from memory
2412                 // and multiply.
2413                 $abstractvectortype$ iota = s.iota();
2414                 $type$ sc = ($type$) scale_;
2415                 return v.add(sc == 1 ? iota : iota.mul(sc));
2416             });
2417     }
2418 
2419     /**
2420      * Replaces selected lanes of this vector with
2421      * a scalar value
2422      * under the control of a mask.
2423      *
2424      * This is a masked lane-wise binary operation which
2425      * selects each lane value from one or the other input.
2426      *
2427      * The returned result is equal to the expression
2428      * {@code blend(broadcast(e),m)}.
2429      *
2430      * @param e the input scalar, containing the replacement lane value
2431      * @param m the mask controlling lane selection of the scalar
2432      * @return the result of blending the lane elements of this vector with
2433      *         the scalar value
2434      */
2435     @ForceInline
2436     public final $abstractvectortype$ blend($type$ e,
2437                                             VectorMask<$Boxtype$> m) {
2438         return blend(broadcast(e), m);
2439     }
2440 
2441 #if[!long]
2442     /**
2443      * Replaces selected lanes of this vector with
2444      * a scalar value
2445      * under the control of a mask.
2446      *
2447      * This is a masked lane-wise binary operation which
2448      * selects each lane value from one or the other input.
2449      *
2450      * The returned result is equal to the expression
2451      * {@code blend(broadcast(e),m)}.
2452      *
2453      * @param e the input scalar, containing the replacement lane value
2454      * @param m the mask controlling lane selection of the scalar
2455      * @return the result of blending the lane elements of this vector with
2456      *         the scalar value
2457      */
2458     @ForceInline
2459     public final $abstractvectortype$ blend(long e,
2460                                             VectorMask<$Boxtype$> m) {
2461         return blend(broadcast(e), m);
2462     }
2463 #end[!long]
2464 
2465     /**
2466      * {@inheritDoc} <!--workaround-->
2467      */
2468     @Override
2469     public abstract
2470     $abstractvectortype$ slice(int origin, Vector<$Boxtype$> v1);
2471 
2472     /*package-private*/
2473     final
2474     @ForceInline
2475     $abstractvectortype$ sliceTemplate(int origin, Vector<$Boxtype$> v1) {
2476         $abstractvectortype$ that = ($abstractvectortype$) v1;
2477         that.check(this);
2478         Objects.checkIndex(origin, length() + 1);
2479         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2480         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin))));
2481         iota = iotaShuffle(origin, 1, true);
2482         return that.rearrange(iota).blend(this.rearrange(iota), blendMask);
2483     }
2484 
2485     /**
2486      * {@inheritDoc} <!--workaround-->
2487      */
2488     @Override
2489     @ForceInline
2490     public final
2491     $abstractvectortype$ slice(int origin,
2492                                Vector<$Boxtype$> w,
2493                                VectorMask<$Boxtype$> m) {
2494         return broadcast(0).blend(slice(origin, w), m);
2495     }
2496 
2497     /**
2498      * {@inheritDoc} <!--workaround-->
2499      */
2500     @Override
2501     public abstract
2502     $abstractvectortype$ slice(int origin);
2503 
2504     /*package-private*/
2505     final
2506     @ForceInline
2507     $abstractvectortype$ sliceTemplate(int origin) {
2508         Objects.checkIndex(origin, length() + 1);
2509         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2510         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin))));
2511         iota = iotaShuffle(origin, 1, true);
2512         return vspecies().zero().blend(this.rearrange(iota), blendMask);
2513     }
2514 
2515     /**
2516      * {@inheritDoc} <!--workaround-->
2517      */
2518     @Override
2519     public abstract
2520     $abstractvectortype$ unslice(int origin, Vector<$Boxtype$> w, int part);
2521 
2522     /*package-private*/
2523     final
2524     @ForceInline
2525     $abstractvectortype$
2526     unsliceTemplate(int origin, Vector<$Boxtype$> w, int part) {
2527         $abstractvectortype$ that = ($abstractvectortype$) w;
2528         that.check(this);
2529         Objects.checkIndex(origin, length() + 1);
2530         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2531         VectorMask<$Boxtype$> blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT,
2532                                                                   (broadcast(($type$)(origin))));
2533         iota = iotaShuffle(-origin, 1, true);
2534         return that.blend(this.rearrange(iota), blendMask);
2535     }
2536 
2537     /*package-private*/
2538     final
2539     @ForceInline
2540     <M extends VectorMask<$Boxtype$>>
2541     $abstractvectortype$
2542     unsliceTemplate(Class<M> maskType, int origin, Vector<$Boxtype$> w, int part, M m) {
2543         $abstractvectortype$ that = ($abstractvectortype$) w;
2544         that.check(this);
2545         $abstractvectortype$ slice = that.sliceTemplate(origin, that);
2546         slice = slice.blendTemplate(maskType, this, m);
2547         return slice.unsliceTemplate(origin, w, part);
2548     }
2549 
2550     /**
2551      * {@inheritDoc} <!--workaround-->
2552      */
2553     @Override
2554     public abstract
2555     $abstractvectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m);
2556 
2557     /**
2558      * {@inheritDoc} <!--workaround-->
2559      */
2560     @Override
2561     public abstract
2562     $abstractvectortype$ unslice(int origin);
2563 
2564     /*package-private*/
2565     final
2566     @ForceInline
2567     $abstractvectortype$
2568     unsliceTemplate(int origin) {
2569         Objects.checkIndex(origin, length() + 1);
2570         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2571         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.GE,
2572                                                                   (broadcast(($type$)(origin))));
2573         iota = iotaShuffle(-origin, 1, true);
2574         return vspecies().zero().blend(this.rearrange(iota), blendMask);
2575     }
2576 
2577     private ArrayIndexOutOfBoundsException
2578     wrongPartForSlice(int part) {
2579         String msg = String.format("bad part number %d for slice operation",
2580                                    part);
2581         return new ArrayIndexOutOfBoundsException(msg);
2582     }
2583 
2584     /**
2585      * {@inheritDoc} <!--workaround-->
2586      */
2587     @Override
2588     public abstract
2589     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> m);
2590 
2591     /*package-private*/
2592     @ForceInline
2593     final
2594     <S extends VectorShuffle<$Boxtype$>>
2595     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype, S shuffle) {
2596         shuffle.checkIndexes();
2597         return VectorSupport.rearrangeOp(
2598             getClass(), shuffletype, null, $type$.class, length(),
2599             this, shuffle, null,
2600             (v1, s_, m_) -> v1.uOp((i, a) -> {
2601                 int ei = s_.laneSource(i);
2602                 return v1.lane(ei);
2603             }));
2604     }
2605 
2606     /**
2607      * {@inheritDoc} <!--workaround-->
2608      */
2609     @Override
2610     public abstract
2611     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2612                                    VectorMask<$Boxtype$> m);
2613 
2614     /*package-private*/
2615     @ForceInline
2616     final
2617     <S extends VectorShuffle<$Boxtype$>, M extends VectorMask<$Boxtype$>>
2618     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2619                                            Class<M> masktype,
2620                                            S shuffle,
2621                                            M m) {
2622 
2623         m.check(masktype, this);
2624         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2625         if (m.andNot(valid).anyTrue()) {
2626             shuffle.checkIndexes();
2627             throw new AssertionError();
2628         }
2629         return VectorSupport.rearrangeOp(
2630                    getClass(), shuffletype, masktype, $type$.class, length(),
2631                    this, shuffle, m,
2632                    (v1, s_, m_) -> v1.uOp((i, a) -> {
2633                         int ei = s_.laneSource(i);
2634                         return ei < 0  || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
2635                    }));
2636     }
2637 
2638     /**
2639      * {@inheritDoc} <!--workaround-->
2640      */
2641     @Override
2642     public abstract
2643     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2644                                    Vector<$Boxtype$> v);
2645 
2646     /*package-private*/
2647     @ForceInline
2648     final
2649     <S extends VectorShuffle<$Boxtype$>>
2650     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2651                                            S shuffle,
2652                                            $abstractvectortype$ v) {
2653         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2654         @SuppressWarnings("unchecked")
2655         S ws = (S) shuffle.wrapIndexes();
2656         $abstractvectortype$ r0 =
2657             VectorSupport.rearrangeOp(
2658                 getClass(), shuffletype, null, $type$.class, length(),
2659                 this, ws, null,
2660                 (v0, s_, m_) -> v0.uOp((i, a) -> {
2661                     int ei = s_.laneSource(i);
2662                     return v0.lane(ei);
2663                 }));
2664         $abstractvectortype$ r1 =
2665             VectorSupport.rearrangeOp(
2666                 getClass(), shuffletype, null, $type$.class, length(),
2667                 v, ws, null,
2668                 (v1, s_, m_) -> v1.uOp((i, a) -> {
2669                     int ei = s_.laneSource(i);
2670                     return v1.lane(ei);
2671                 }));
2672         return r1.blend(r0, valid);
2673     }
2674 
2675     @ForceInline
2676     private final
2677     VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) {
2678         $type$[] a = toArray();
2679         int[] sa = new int[a.length];
2680         for (int i = 0; i < a.length; i++) {
2681             sa[i] = (int) a[i];
2682         }
2683         return VectorShuffle.fromArray(dsp, sa, 0);
2684     }
2685 
2686     /*package-private*/
2687     @ForceInline
2688     final
2689     VectorShuffle<$Boxtype$> toShuffleTemplate(Class<?> shuffleType) {
2690         $Type$Species vsp = vspecies();
2691         return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
2692                                      getClass(), $type$.class, length(),
2693                                      shuffleType, byte.class, length(),
2694                                      this, vsp,
2695                                      $Type$Vector::toShuffle0);
2696     }
2697 
2698     /**
2699      * {@inheritDoc} <!--workaround-->
2700      */
2701     @Override
2702     public abstract
2703     $abstractvectortype$ selectFrom(Vector<$Boxtype$> v);
2704 
2705     /*package-private*/
2706     @ForceInline
2707     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v) {
2708         return v.rearrange(this.toShuffle());
2709     }
2710 
2711     /**
2712      * {@inheritDoc} <!--workaround-->
2713      */
2714     @Override
2715     public abstract
2716     $abstractvectortype$ selectFrom(Vector<$Boxtype$> s, VectorMask<$Boxtype$> m);
2717 
2718     /*package-private*/
2719     @ForceInline
2720     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v,
2721                                                   AbstractMask<$Boxtype$> m) {
2722         return v.rearrange(this.toShuffle(), m);
2723     }
2724 
2725     /// Ternary operations
2726 
2727 #if[BITWISE]
2728     /**
2729      * Blends together the bits of two vectors under
2730      * the control of a third, which supplies mask bits.
2731      *
2732      * This is a lane-wise ternary operation which performs
2733      * a bitwise blending operation {@code (a&~c)|(b&c)}
2734      * to each lane.
2735      *
2736      * This method is also equivalent to the expression
2737      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2738      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2739      *    BITWISE_BLEND}{@code , bits, mask)}.
2740      *
2741      * @param bits input bits to blend into the current vector
2742      * @param mask a bitwise mask to enable blending of the input bits
2743      * @return the bitwise blend of the given bits into the current vector,
2744      *         under control of the bitwise mask
2745      * @see #bitwiseBlend($type$,$type$)
2746      * @see #bitwiseBlend($type$,Vector)
2747      * @see #bitwiseBlend(Vector,$type$)
2748      * @see VectorOperators#BITWISE_BLEND
2749      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
2750      */
2751     @ForceInline
2752     public final
2753     $abstractvectortype$ bitwiseBlend(Vector<$Boxtype$> bits, Vector<$Boxtype$> mask) {
2754         return lanewise(BITWISE_BLEND, bits, mask);
2755     }
2756 
2757     /**
2758      * Blends together the bits of a vector and a scalar under
2759      * the control of another scalar, which supplies mask bits.
2760      *
2761      * This is a lane-wise ternary operation which performs
2762      * a bitwise blending operation {@code (a&~c)|(b&c)}
2763      * to each lane.
2764      *
2765      * This method is also equivalent to the expression
2766      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2767      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2768      *    BITWISE_BLEND}{@code , bits, mask)}.
2769      *
2770      * @param bits input bits to blend into the current vector
2771      * @param mask a bitwise mask to enable blending of the input bits
2772      * @return the bitwise blend of the given bits into the current vector,
2773      *         under control of the bitwise mask
2774      * @see #bitwiseBlend(Vector,Vector)
2775      * @see VectorOperators#BITWISE_BLEND
2776      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
2777      */
2778     @ForceInline
2779     public final
2780     $abstractvectortype$ bitwiseBlend($type$ bits, $type$ mask) {
2781         return lanewise(BITWISE_BLEND, bits, mask);
2782     }
2783 
2784     /**
2785      * Blends together the bits of a vector and a scalar under
2786      * the control of another vector, which supplies mask bits.
2787      *
2788      * This is a lane-wise ternary operation which performs
2789      * a bitwise blending operation {@code (a&~c)|(b&c)}
2790      * to each lane.
2791      *
2792      * This method is also equivalent to the expression
2793      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2794      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2795      *    BITWISE_BLEND}{@code , bits, mask)}.
2796      *
2797      * @param bits input bits to blend into the current vector
2798      * @param mask a bitwise mask to enable blending of the input bits
2799      * @return the bitwise blend of the given bits into the current vector,
2800      *         under control of the bitwise mask
2801      * @see #bitwiseBlend(Vector,Vector)
2802      * @see VectorOperators#BITWISE_BLEND
2803      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
2804      */
2805     @ForceInline
2806     public final
2807     $abstractvectortype$ bitwiseBlend($type$ bits, Vector<$Boxtype$> mask) {
2808         return lanewise(BITWISE_BLEND, bits, mask);
2809     }
2810 
2811     /**
2812      * Blends together the bits of two vectors under
2813      * the control of a scalar, which supplies mask bits.
2814      *
2815      * This is a lane-wise ternary operation which performs
2816      * a bitwise blending operation {@code (a&~c)|(b&c)}
2817      * to each lane.
2818      *
2819      * This method is also equivalent to the expression
2820      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2821      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2822      *    BITWISE_BLEND}{@code , bits, mask)}.
2823      *
2824      * @param bits input bits to blend into the current vector
2825      * @param mask a bitwise mask to enable blending of the input bits
2826      * @return the bitwise blend of the given bits into the current vector,
2827      *         under control of the bitwise mask
2828      * @see #bitwiseBlend(Vector,Vector)
2829      * @see VectorOperators#BITWISE_BLEND
2830      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
2831      */
2832     @ForceInline
2833     public final
2834     $abstractvectortype$ bitwiseBlend(Vector<$Boxtype$> bits, $type$ mask) {
2835         return lanewise(BITWISE_BLEND, bits, mask);
2836     }
2837 #end[BITWISE]
2838 
2839 #if[FP]
2840     /**
2841      * Multiplies this vector by a second input vector, and sums
2842      * the result with a third.
2843      *
2844      * Extended precision is used for the intermediate result,
2845      * avoiding possible loss of precision from rounding once
2846      * for each of the two operations.
2847      * The result is numerically close to {@code this.mul(b).add(c)},
2848      * and is typically closer to the true mathematical result.
2849      *
2850      * This is a lane-wise ternary operation which applies an operation
2851      * conforming to the specification of
2852      * {@link Math#fma($type$,$type$,$type$) Math.fma(a,b,c)}
2853      * to each lane.
2854 #if[intOrFloat]
2855      * The operation is adapted to cast the operands and the result,
2856      * specifically widening {@code float} operands to {@code double}
2857      * operands and narrowing the {@code double} result to a {@code float}
2858      * result.
2859 #end[intOrFloat]
2860      *
2861      * This method is also equivalent to the expression
2862      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2863      *    lanewise}{@code (}{@link VectorOperators#FMA
2864      *    FMA}{@code , b, c)}.
2865      *
2866      * @param b the second input vector, supplying multiplier values
2867      * @param c the third input vector, supplying addend values
2868      * @return the product of this vector and the second input vector
2869      *         summed with the third input vector, using extended precision
2870      *         for the intermediate result
2871      * @see #fma($type$,$type$)
2872      * @see VectorOperators#FMA
2873      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
2874      */
2875     @ForceInline
2876     public final
2877     $abstractvectortype$ fma(Vector<$Boxtype$> b, Vector<$Boxtype$> c) {
2878         return lanewise(FMA, b, c);
2879     }
2880 
2881     /**
2882      * Multiplies this vector by a scalar multiplier, and sums
2883      * the result with a scalar addend.
2884      *
2885      * Extended precision is used for the intermediate result,
2886      * avoiding possible loss of precision from rounding once
2887      * for each of the two operations.
2888      * The result is numerically close to {@code this.mul(b).add(c)},
2889      * and is typically closer to the true mathematical result.
2890      *
2891      * This is a lane-wise ternary operation which applies an operation
2892      * conforming to the specification of
2893      * {@link Math#fma($type$,$type$,$type$) Math.fma(a,b,c)}
2894      * to each lane.
2895 #if[intOrFloat]
2896      * The operation is adapted to cast the operands and the result,
2897      * specifically widening {@code float} operands to {@code double}
2898      * operands and narrowing the {@code double} result to a {@code float}
2899      * result.
2900 #end[intOrFloat]
2901      *
2902      * This method is also equivalent to the expression
2903      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2904      *    lanewise}{@code (}{@link VectorOperators#FMA
2905      *    FMA}{@code , b, c)}.
2906      *
2907      * @param b the scalar multiplier
2908      * @param c the scalar addend
2909      * @return the product of this vector and the scalar multiplier
2910      *         summed with scalar addend, using extended precision
2911      *         for the intermediate result
2912      * @see #fma(Vector,Vector)
2913      * @see VectorOperators#FMA
2914      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
2915      */
2916     @ForceInline
2917     public final
2918     $abstractvectortype$ fma($type$ b, $type$ c) {
2919         return lanewise(FMA, b, c);
2920     }
2921 
2922     // Don't bother with (Vector,$type$) and ($type$,Vector) overloadings.
2923 #end[FP]
2924 
2925     // Type specific horizontal reductions
2926 
2927     /**
2928      * Returns a value accumulated from all the lanes of this vector.
2929      *
2930      * This is an associative cross-lane reduction operation which
2931      * applies the specified operation to all the lane elements.
2932      * <p>
2933      * A few reduction operations do not support arbitrary reordering
2934      * of their operands, yet are included here because of their
2935      * usefulness.
2936      * <ul>
2937      * <li>
2938      * In the case of {@code FIRST_NONZERO}, the reduction returns
2939      * the value from the lowest-numbered non-zero lane.
2940 #if[FP]
2941      * (As with {@code MAX} and {@code MIN}, floating point negative
2942      * zero {@code -0.0} is treated as a value distinct from
2943      * the default value, positive zero. So a first-nonzero lane reduction
2944      * might return {@code -0.0} even in the presence of non-zero
2945      * lane values.)
2946      * <li>
2947      * In the case of {@code ADD} and {@code MUL}, the
2948      * precise result will reflect the choice of an arbitrary order
2949      * of operations, which may even vary over time.
2950      * For further details see the section
2951      * <a href="VectorOperators.html#fp_assoc">Operations on floating point vectors</a>.
2952 #end[FP]
2953      * <li>
2954      * All other reduction operations are fully commutative and
2955      * associative.  The implementation can choose any order of
2956      * processing, yet it will always produce the same result.
2957      * </ul>
2958      *
2959      * @param op the operation used to combine lane values
2960      * @return the accumulated result
2961      * @throws UnsupportedOperationException if this vector does
2962      *         not support the requested operation
2963      * @see #reduceLanes(VectorOperators.Associative,VectorMask)
2964      * @see #add(Vector)
2965      * @see #mul(Vector)
2966      * @see #min(Vector)
2967      * @see #max(Vector)
2968 #if[BITWISE]
2969      * @see #and(Vector)
2970      * @see #or(Vector)
2971      * @see VectorOperators#XOR
2972 #end[BITWISE]
2973      * @see VectorOperators#FIRST_NONZERO
2974      */
2975     public abstract $type$ reduceLanes(VectorOperators.Associative op);
2976 
2977     /**
2978      * Returns a value accumulated from selected lanes of this vector,
2979      * controlled by a mask.
2980      *
2981      * This is an associative cross-lane reduction operation which
2982      * applies the specified operation to the selected lane elements.
2983      * <p>
2984      * If no elements are selected, an operation-specific identity
2985      * value is returned.
2986      * <ul>
2987      * <li>
2988      * If the operation is
2989 #if[BITWISE]
2990      *  {@code ADD}, {@code XOR}, {@code OR},
2991 #else[BITWISE]
2992      *  {@code ADD}
2993 #end[BITWISE]
2994      * or {@code FIRST_NONZERO},
2995      * then the identity value is {#if[FP]?positive }zero, the default {@code $type$} value.
2996      * <li>
2997      * If the operation is {@code MUL},
2998      * then the identity value is one.
2999 #if[BITWISE]
3000      * <li>
3001      * If the operation is {@code AND},
3002      * then the identity value is minus one (all bits set).
3003      * <li>
3004      * If the operation is {@code MAX},
3005      * then the identity value is {@code $Boxtype$.MIN_VALUE}.
3006      * <li>
3007      * If the operation is {@code MIN},
3008      * then the identity value is {@code $Boxtype$.MAX_VALUE}.
3009 #end[BITWISE]
3010 #if[FP]
3011      * <li>
3012      * If the operation is {@code MAX},
3013      * then the identity value is {@code $Boxtype$.NEGATIVE_INFINITY}.
3014      * <li>
3015      * If the operation is {@code MIN},
3016      * then the identity value is {@code $Boxtype$.POSITIVE_INFINITY}.
3017 #end[FP]
3018      * </ul>
3019      * <p>
3020      * A few reduction operations do not support arbitrary reordering
3021      * of their operands, yet are included here because of their
3022      * usefulness.
3023      * <ul>
3024      * <li>
3025      * In the case of {@code FIRST_NONZERO}, the reduction returns
3026      * the value from the lowest-numbered non-zero lane.
3027 #if[FP]
3028      * (As with {@code MAX} and {@code MIN}, floating point negative
3029      * zero {@code -0.0} is treated as a value distinct from
3030      * the default value, positive zero. So a first-nonzero lane reduction
3031      * might return {@code -0.0} even in the presence of non-zero
3032      * lane values.)
3033      * <li>
3034      * In the case of {@code ADD} and {@code MUL}, the
3035      * precise result will reflect the choice of an arbitrary order
3036      * of operations, which may even vary over time.
3037      * For further details see the section
3038      * <a href="VectorOperators.html#fp_assoc">Operations on floating point vectors</a>.
3039 #end[FP]
3040      * <li>
3041      * All other reduction operations are fully commutative and
3042      * associative.  The implementation can choose any order of
3043      * processing, yet it will always produce the same result.
3044      * </ul>
3045      *
3046      * @param op the operation used to combine lane values
3047      * @param m the mask controlling lane selection
3048      * @return the reduced result accumulated from the selected lane values
3049      * @throws UnsupportedOperationException if this vector does
3050      *         not support the requested operation
3051      * @see #reduceLanes(VectorOperators.Associative)
3052      */
3053     public abstract $type$ reduceLanes(VectorOperators.Associative op,
3054                                        VectorMask<$Boxtype$> m);
3055 
3056     /*package-private*/
3057     @ForceInline
3058     final
3059     $type$ reduceLanesTemplate(VectorOperators.Associative op,
3060                                Class<? extends VectorMask<$Boxtype$>> maskClass,
3061                                VectorMask<$Boxtype$> m) {
3062         m.check(maskClass, this);
3063         if (op == FIRST_NONZERO) {
3064             // FIXME:  The JIT should handle this.
3065             $abstractvectortype$ v = broadcast(($type$) 0).blend(this, m);
3066             return v.reduceLanesTemplate(op);
3067         }
3068         int opc = opCode(op);
3069         return fromBits(VectorSupport.reductionCoerced(
3070             opc, getClass(), maskClass, $type$.class, length(),
3071             this, m,
3072             REDUCE_IMPL.find(op, opc, $abstractvectortype$::reductionOperations)));
3073     }
3074 
3075     /*package-private*/
3076     @ForceInline
3077     final
3078     $type$ reduceLanesTemplate(VectorOperators.Associative op) {
3079         if (op == FIRST_NONZERO) {
3080             // FIXME:  The JIT should handle this.
3081             VectorMask<$Boxbitstype$> thisNZ
3082                 = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
3083             int ft = thisNZ.firstTrue();
3084             return ft < length() ? this.lane(ft) : ($type$) 0;
3085         }
3086         int opc = opCode(op);
3087         return fromBits(VectorSupport.reductionCoerced(
3088             opc, getClass(), null, $type$.class, length(),
3089             this, null,
3090             REDUCE_IMPL.find(op, opc, $abstractvectortype$::reductionOperations)));
3091     }
3092 
3093     private static final
3094     ImplCache<Associative, ReductionOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
3095         REDUCE_IMPL = new ImplCache<>(Associative.class, $Type$Vector.class);
3096 
3097     private static ReductionOperation<$abstractvectortype$, VectorMask<$Boxtype$>> reductionOperations(int opc_) {
3098         switch (opc_) {
3099             case VECTOR_OP_ADD: return (v, m) ->
3100                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a + b)));
3101             case VECTOR_OP_MUL: return (v, m) ->
3102                     toBits(v.rOp(($type$)1, m, (i, a, b) -> ($type$)(a * b)));
3103             case VECTOR_OP_MIN: return (v, m) ->
3104                     toBits(v.rOp(MAX_OR_INF, m, (i, a, b) -> ($type$) Math.min(a, b)));
3105             case VECTOR_OP_MAX: return (v, m) ->
3106                     toBits(v.rOp(MIN_OR_INF, m, (i, a, b) -> ($type$) Math.max(a, b)));
3107 #if[BITWISE]
3108             case VECTOR_OP_AND: return (v, m) ->
3109                     toBits(v.rOp(($type$)-1, m, (i, a, b) -> ($type$)(a & b)));
3110             case VECTOR_OP_OR: return (v, m) ->
3111                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a | b)));
3112             case VECTOR_OP_XOR: return (v, m) ->
3113                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a ^ b)));
3114 #end[BITWISE]
3115             default: return null;
3116         }
3117     }
3118 
3119 #if[FP]
3120     private static final $type$ MIN_OR_INF = $Boxtype$.NEGATIVE_INFINITY;
3121     private static final $type$ MAX_OR_INF = $Boxtype$.POSITIVE_INFINITY;
3122 #else[FP]
3123     private static final $type$ MIN_OR_INF = $Boxtype$.MIN_VALUE;
3124     private static final $type$ MAX_OR_INF = $Boxtype$.MAX_VALUE;
3125 #end[FP]
3126 
3127     public @Override abstract long reduceLanesToLong(VectorOperators.Associative op);
3128     public @Override abstract long reduceLanesToLong(VectorOperators.Associative op,
3129                                                      VectorMask<$Boxtype$> m);
3130 
3131     // Type specific accessors
3132 
3133     /**
3134      * Gets the lane element at lane index {@code i}
3135      *
3136      * @param i the lane index
3137      * @return the lane element at lane index {@code i}
3138      * @throws IllegalArgumentException if the index is is out of range
3139      * ({@code < 0 || >= length()})
3140      */
3141     public abstract $type$ lane(int i);
3142 
3143     /**
3144      * Replaces the lane element of this vector at lane index {@code i} with
3145      * value {@code e}.
3146      *
3147      * This is a cross-lane operation and behaves as if it returns the result
3148      * of blending this vector with an input vector that is the result of
3149      * broadcasting {@code e} and a mask that has only one lane set at lane
3150      * index {@code i}.
3151      *
3152      * @param i the lane index of the lane element to be replaced
3153      * @param e the value to be placed
3154      * @return the result of replacing the lane element of this vector at lane
3155      * index {@code i} with value {@code e}.
3156      * @throws IllegalArgumentException if the index is is out of range
3157      * ({@code < 0 || >= length()})
3158      */
3159     public abstract $abstractvectortype$ withLane(int i, $type$ e);
3160 
3161     // Memory load operations
3162 
3163     /**
3164      * Returns an array of type {@code $type$[]}
3165      * containing all the lane values.
3166      * The array length is the same as the vector length.
3167      * The array elements are stored in lane order.
3168      * <p>
3169      * This method behaves as if it stores
3170      * this vector into an allocated array
3171      * (using {@link #intoArray($type$[], int) intoArray})
3172      * and returns the array as follows:
3173      * <pre>{@code
3174      *   $type$[] a = new $type$[this.length()];
3175      *   this.intoArray(a, 0);
3176      *   return a;
3177      * }</pre>
3178      *
3179      * @return an array containing the lane values of this vector
3180      */
3181     @ForceInline
3182     @Override
3183     public final $type$[] toArray() {
3184         $type$[] a = new $type$[vspecies().laneCount()];
3185         intoArray(a, 0);
3186         return a;
3187     }
3188 
3189 #if[int]
3190     /**
3191      * {@inheritDoc} <!--workaround-->
3192      * This is an alias for {@link #toArray()}
3193      * When this method is used on used on vectors
3194      * of type {@code $abstractvectortype$},
3195      * there will be no loss of range or precision.
3196      */
3197     @ForceInline
3198     @Override
3199     public final int[] toIntArray() {
3200         return toArray();
3201     }
3202 #else[int]
3203     /** {@inheritDoc} <!--workaround-->
3204 #if[!FP]
3205 #if[!long]
3206      * @implNote
3207      * When this method is used on used on vectors
3208      * of type {@code $abstractvectortype$},
3209      * there will be no loss of precision or range,
3210      * and so no {@code UnsupportedOperationException} will
3211      * be thrown.
3212 #end[!long]
3213 #end[!FP]
3214      */
3215     @ForceInline
3216     @Override
3217     public final int[] toIntArray() {
3218         $type$[] a = toArray();
3219         int[] res = new int[a.length];
3220         for (int i = 0; i < a.length; i++) {
3221             $type$ e = a[i];
3222             res[i] = (int) $Type$Species.toIntegralChecked(e, true);
3223         }
3224         return res;
3225     }
3226 #end[int]
3227 
3228 #if[long]
3229     /**
3230      * {@inheritDoc} <!--workaround-->
3231      * This is an alias for {@link #toArray()}
3232      * When this method is used on used on vectors
3233      * of type {@code $abstractvectortype$},
3234      * there will be no loss of range or precision.
3235      */
3236     @ForceInline
3237     @Override
3238     public final long[] toLongArray() {
3239         return toArray();
3240     }
3241 #else[long]
3242     /** {@inheritDoc} <!--workaround-->
3243 #if[!FP]
3244      * @implNote
3245      * When this method is used on used on vectors
3246      * of type {@code $abstractvectortype$},
3247      * there will be no loss of precision or range,
3248      * and so no {@code UnsupportedOperationException} will
3249      * be thrown.
3250 #end[!FP]
3251      */
3252     @ForceInline
3253     @Override
3254     public final long[] toLongArray() {
3255         $type$[] a = toArray();
3256         long[] res = new long[a.length];
3257         for (int i = 0; i < a.length; i++) {
3258             $type$ e = a[i];
3259             res[i] = $Type$Species.toIntegralChecked(e, false);
3260         }
3261         return res;
3262     }
3263 #end[long]
3264 
3265 #if[double]
3266     /** {@inheritDoc} <!--workaround-->
3267      * @implNote
3268      * This is an alias for {@link #toArray()}
3269      * When this method is used on used on vectors
3270      * of type {@code $abstractvectortype$},
3271      * there will be no loss of precision.
3272      */
3273     @ForceInline
3274     @Override
3275     public final double[] toDoubleArray() {
3276         return toArray();
3277     }
3278 #else[double]
3279     /** {@inheritDoc} <!--workaround-->
3280 #if[long]
3281      * @implNote
3282      * When this method is used on used on vectors
3283      * of type {@code $abstractvectortype$},
3284      * up to nine bits of precision may be lost
3285      * for lane values of large magnitude.
3286 #else[long]
3287      * @implNote
3288      * When this method is used on used on vectors
3289      * of type {@code $abstractvectortype$},
3290      * there will be no loss of precision.
3291 #end[long]
3292      */
3293     @ForceInline
3294     @Override
3295     public final double[] toDoubleArray() {
3296         $type$[] a = toArray();
3297         double[] res = new double[a.length];
3298         for (int i = 0; i < a.length; i++) {
3299             res[i] = (double) a[i];
3300         }
3301         return res;
3302     }
3303 #end[double]
3304 
3305     /**
3306      * Loads a vector from a byte array starting at an offset.
3307      * Bytes are composed into primitive lane elements according
3308      * to the specified byte order.
3309      * The vector is arranged into lanes according to
3310      * <a href="Vector.html#lane-order">memory ordering</a>.
3311      * <p>
3312      * This method behaves as if it returns the result of calling
3313      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3314      * fromByteBuffer()} as follows:
3315      * <pre>{@code
3316      * var bb = ByteBuffer.wrap(a);
3317      * var m = species.maskAll(true);
3318      * return fromByteBuffer(species, bb, offset, bo, m);
3319      * }</pre>
3320      *
3321      * @param species species of desired vector
3322      * @param a the byte array
3323      * @param offset the offset into the array
3324      * @param bo the intended byte order
3325      * @return a vector loaded from a byte array
3326      * @throws IndexOutOfBoundsException
3327      *         if {@code offset+N*ESIZE < 0}
3328      *         or {@code offset+(N+1)*ESIZE > a.length}
3329      *         for any lane {@code N} in the vector
3330      */
3331     @ForceInline
3332     public static
3333     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3334                                        byte[] a, int offset,
3335                                        ByteOrder bo) {
3336         offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
3337         $Type$Species vsp = ($Type$Species) species;
3338         return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
3339     }
3340 
3341     /**
3342      * Loads a vector from a byte array starting at an offset
3343      * and using a mask.
3344      * Lanes where the mask is unset are filled with the default
3345      * value of {@code $type$} ({#if[FP]?positive }zero).
3346      * Bytes are composed into primitive lane elements according
3347      * to the specified byte order.
3348      * The vector is arranged into lanes according to
3349      * <a href="Vector.html#lane-order">memory ordering</a>.
3350      * <p>
3351      * This method behaves as if it returns the result of calling
3352      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3353      * fromByteBuffer()} as follows:
3354      * <pre>{@code
3355      * var bb = ByteBuffer.wrap(a);
3356      * return fromByteBuffer(species, bb, offset, bo, m);
3357      * }</pre>
3358      *
3359      * @param species species of desired vector
3360      * @param a the byte array
3361      * @param offset the offset into the array
3362      * @param bo the intended byte order
3363      * @param m the mask controlling lane selection
3364      * @return a vector loaded from a byte array
3365      * @throws IndexOutOfBoundsException
3366      *         if {@code offset+N*ESIZE < 0}
3367      *         or {@code offset+(N+1)*ESIZE > a.length}
3368      *         for any lane {@code N} in the vector
3369      *         where the mask is set
3370      */
3371     @ForceInline
3372     public static
3373     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3374                                        byte[] a, int offset,
3375                                        ByteOrder bo,
3376                                        VectorMask<$Boxtype$> m) {
3377         $Type$Species vsp = ($Type$Species) species;
3378         if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
3379             return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
3380         }
3381 
3382         // FIXME: optimize
3383         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
3384         ByteBuffer wb = wrapper(a, bo);
3385         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
3386                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
3387     }
3388 
3389     /**
3390      * Loads a vector from an array of type {@code $type$[]}
3391      * starting at an offset.
3392      * For each vector lane, where {@code N} is the vector lane index, the
3393      * array element at index {@code offset + N} is placed into the
3394      * resulting vector at lane index {@code N}.
3395      *
3396      * @param species species of desired vector
3397      * @param a the array
3398      * @param offset the offset into the array
3399      * @return the vector loaded from an array
3400      * @throws IndexOutOfBoundsException
3401      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3402      *         for any lane {@code N} in the vector
3403      */
3404     @ForceInline
3405     public static
3406     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3407                                    $type$[] a, int offset) {
3408         offset = checkFromIndexSize(offset, species.length(), a.length);
3409         $Type$Species vsp = ($Type$Species) species;
3410         return vsp.dummyVector().fromArray0(a, offset);
3411     }
3412 
3413     /**
3414      * Loads a vector from an array of type {@code $type$[]}
3415      * starting at an offset and using a mask.
3416      * Lanes where the mask is unset are filled with the default
3417      * value of {@code $type$} ({#if[FP]?positive }zero).
3418      * For each vector lane, where {@code N} is the vector lane index,
3419      * if the mask lane at index {@code N} is set then the array element at
3420      * index {@code offset + N} is placed into the resulting vector at lane index
3421      * {@code N}, otherwise the default element value is placed into the
3422      * resulting vector at lane index {@code N}.
3423      *
3424      * @param species species of desired vector
3425      * @param a the array
3426      * @param offset the offset into the array
3427      * @param m the mask controlling lane selection
3428      * @return the vector loaded from an array
3429      * @throws IndexOutOfBoundsException
3430      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3431      *         for any lane {@code N} in the vector
3432      *         where the mask is set
3433      */
3434     @ForceInline
3435     public static
3436     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3437                                    $type$[] a, int offset,
3438                                    VectorMask<$Boxtype$> m) {
3439         $Type$Species vsp = ($Type$Species) species;
3440         if (offset >= 0 && offset <= (a.length - species.length())) {
3441             return vsp.dummyVector().fromArray0(a, offset, m);
3442         }
3443 
3444         // FIXME: optimize
3445         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3446         return vsp.vOp(m, i -> a[offset + i]);
3447     }
3448 
3449     /**
3450      * Gathers a new vector composed of elements from an array of type
3451      * {@code $type$[]},
3452      * using indexes obtained by adding a fixed {@code offset} to a
3453      * series of secondary offsets from an <em>index map</em>.
3454      * The index map is a contiguous sequence of {@code VLENGTH}
3455      * elements in a second array of {@code int}s, starting at a given
3456      * {@code mapOffset}.
3457      * <p>
3458      * For each vector lane, where {@code N} is the vector lane index,
3459      * the lane is loaded from the array
3460      * element {@code a[f(N)]}, where {@code f(N)} is the
3461      * index mapping expression
3462      * {@code offset + indexMap[mapOffset + N]]}.
3463      *
3464      * @param species species of desired vector
3465      * @param a the array
3466      * @param offset the offset into the array, may be negative if relative
3467      * indexes in the index map compensate to produce a value within the
3468      * array bounds
3469      * @param indexMap the index map
3470      * @param mapOffset the offset into the index map
3471      * @return the vector loaded from the indexed elements of the array
3472      * @throws IndexOutOfBoundsException
3473      *         if {@code mapOffset+N < 0}
3474      *         or if {@code mapOffset+N >= indexMap.length},
3475      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3476      *         is an invalid index into {@code a},
3477      *         for any lane {@code N} in the vector
3478      * @see $abstractvectortype$#toIntArray()
3479      */
3480 #if[byteOrShort]
3481     @ForceInline
3482     public static
3483     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3484                                    $type$[] a, int offset,
3485                                    int[] indexMap, int mapOffset) {
3486         $Type$Species vsp = ($Type$Species) species;
3487         return vsp.vOp(n -> a[offset + indexMap[mapOffset + n]]);
3488     }
3489 #else[byteOrShort]
3490     @ForceInline
3491     public static
3492     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3493                                    $type$[] a, int offset,
3494                                    int[] indexMap, int mapOffset) {
3495         $Type$Species vsp = ($Type$Species) species;
3496         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3497         Objects.requireNonNull(a);
3498         Objects.requireNonNull(indexMap);
3499         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();
3500 
3501 #if[longOrDouble]
3502         if (vsp.laneCount() == 1) {
3503           return $abstractvectortype$.fromArray(vsp, a, offset + indexMap[mapOffset]);
3504         }
3505 
3506         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3507         IntVector vix;
3508         if (isp.laneCount() != vsp.laneCount()) {
3509             // For $Type$MaxVector,  if vector length is non-power-of-two or
3510             // 2048 bits, indexShape of $Type$ species is S_MAX_BIT.
3511             // Assume that vector length is 2048, then the lane count of $Type$
3512             // vector is 32. When converting $Type$ species to int species,
3513             // indexShape is still S_MAX_BIT, but the lane count of int vector
3514             // is 64. So when loading index vector (IntVector), only lower half
3515             // of index data is needed.
3516             vix = IntVector
3517                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3518                 .add(offset);
3519         } else {
3520             vix = IntVector
3521                 .fromArray(isp, indexMap, mapOffset)
3522                 .add(offset);
3523         }
3524 #else[longOrDouble]
3525         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3526         IntVector vix = IntVector
3527             .fromArray(isp, indexMap, mapOffset)
3528             .add(offset);
3529 #end[longOrDouble]
3530 
3531         vix = VectorIntrinsics.checkIndex(vix, a.length);
3532 
3533         return VectorSupport.loadWithMap(
3534             vectorType, null, $type$.class, vsp.laneCount(),
3535             isp.vectorType(),
3536             a, ARRAY_BASE, vix, null,
3537             a, offset, indexMap, mapOffset, vsp,
3538             (c, idx, iMap, idy, s, vm) ->
3539             s.vOp(n -> c[idx + iMap[idy+n]]));
3540     }
3541 #end[byteOrShort]
3542 
3543     /**
3544      * Gathers a new vector composed of elements from an array of type
3545      * {@code $type$[]},
3546      * under the control of a mask, and
3547      * using indexes obtained by adding a fixed {@code offset} to a
3548      * series of secondary offsets from an <em>index map</em>.
3549      * The index map is a contiguous sequence of {@code VLENGTH}
3550      * elements in a second array of {@code int}s, starting at a given
3551      * {@code mapOffset}.
3552      * <p>
3553      * For each vector lane, where {@code N} is the vector lane index,
3554      * if the lane is set in the mask,
3555      * the lane is loaded from the array
3556      * element {@code a[f(N)]}, where {@code f(N)} is the
3557      * index mapping expression
3558      * {@code offset + indexMap[mapOffset + N]]}.
3559      * Unset lanes in the resulting vector are set to zero.
3560      *
3561      * @param species species of desired vector
3562      * @param a the array
3563      * @param offset the offset into the array, may be negative if relative
3564      * indexes in the index map compensate to produce a value within the
3565      * array bounds
3566      * @param indexMap the index map
3567      * @param mapOffset the offset into the index map
3568      * @param m the mask controlling lane selection
3569      * @return the vector loaded from the indexed elements of the array
3570      * @throws IndexOutOfBoundsException
3571      *         if {@code mapOffset+N < 0}
3572      *         or if {@code mapOffset+N >= indexMap.length},
3573      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3574      *         is an invalid index into {@code a},
3575      *         for any lane {@code N} in the vector
3576      *         where the mask is set
3577      * @see $abstractvectortype$#toIntArray()
3578      */
3579 #if[byteOrShort]
3580     @ForceInline
3581     public static
3582     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3583                                    $type$[] a, int offset,
3584                                    int[] indexMap, int mapOffset,
3585                                    VectorMask<$Boxtype$> m) {
3586         $Type$Species vsp = ($Type$Species) species;
3587         return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
3588     }
3589 #else[byteOrShort]
3590     @ForceInline
3591     public static
3592     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3593                                    $type$[] a, int offset,
3594                                    int[] indexMap, int mapOffset,
3595                                    VectorMask<$Boxtype$> m) {
3596         if (m.allTrue()) {
3597             return fromArray(species, a, offset, indexMap, mapOffset);
3598         }
3599         else {
3600             $Type$Species vsp = ($Type$Species) species;
3601             return vsp.dummyVector().fromArray0(a, offset, indexMap, mapOffset, m);
3602         }
3603     }
3604 #end[byteOrShort]
3605 
3606 #if[short]
3607     /**
3608      * Loads a vector from an array of type {@code char[]}
3609      * starting at an offset.
3610      * For each vector lane, where {@code N} is the vector lane index, the
3611      * array element at index {@code offset + N}
3612      * is first cast to a {@code short} value and then
3613      * placed into the resulting vector at lane index {@code N}.
3614      *
3615      * @param species species of desired vector
3616      * @param a the array
3617      * @param offset the offset into the array
3618      * @return the vector loaded from an array
3619      * @throws IndexOutOfBoundsException
3620      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3621      *         for any lane {@code N} in the vector
3622      */
3623     @ForceInline
3624     public static
3625     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3626                                        char[] a, int offset) {
3627         offset = checkFromIndexSize(offset, species.length(), a.length);
3628         $Type$Species vsp = ($Type$Species) species;
3629         return vsp.dummyVector().fromCharArray0(a, offset);
3630     }
3631 
3632     /**
3633      * Loads a vector from an array of type {@code char[]}
3634      * starting at an offset and using a mask.
3635      * Lanes where the mask is unset are filled with the default
3636      * value of {@code $type$} ({#if[FP]?positive }zero).
3637      * For each vector lane, where {@code N} is the vector lane index,
3638      * if the mask lane at index {@code N} is set then the array element at
3639      * index {@code offset + N}
3640      * is first cast to a {@code short} value and then
3641      * placed into the resulting vector at lane index
3642      * {@code N}, otherwise the default element value is placed into the
3643      * resulting vector at lane index {@code N}.
3644      *
3645      * @param species species of desired vector
3646      * @param a the array
3647      * @param offset the offset into the array
3648      * @param m the mask controlling lane selection
3649      * @return the vector loaded from an array
3650      * @throws IndexOutOfBoundsException
3651      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3652      *         for any lane {@code N} in the vector
3653      *         where the mask is set
3654      */
3655     @ForceInline
3656     public static
3657     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3658                                        char[] a, int offset,
3659                                        VectorMask<$Boxtype$> m) {
3660         $Type$Species vsp = ($Type$Species) species;
3661         if (offset >= 0 && offset <= (a.length - species.length())) {
3662             return vsp.dummyVector().fromCharArray0(a, offset, m);
3663         }
3664 
3665         // FIXME: optimize
3666         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3667         return vsp.vOp(m, i -> (short) a[offset + i]);
3668     }
3669 
3670     /**
3671      * Gathers a new vector composed of elements from an array of type
3672      * {@code char[]},
3673      * using indexes obtained by adding a fixed {@code offset} to a
3674      * series of secondary offsets from an <em>index map</em>.
3675      * The index map is a contiguous sequence of {@code VLENGTH}
3676      * elements in a second array of {@code int}s, starting at a given
3677      * {@code mapOffset}.
3678      * <p>
3679      * For each vector lane, where {@code N} is the vector lane index,
3680      * the lane is loaded from the expression
3681      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3682      * index mapping expression
3683      * {@code offset + indexMap[mapOffset + N]]}.
3684      *
3685      * @param species species of desired vector
3686      * @param a the array
3687      * @param offset the offset into the array, may be negative if relative
3688      * indexes in the index map compensate to produce a value within the
3689      * array bounds
3690      * @param indexMap the index map
3691      * @param mapOffset the offset into the index map
3692      * @return the vector loaded from the indexed elements of the array
3693      * @throws IndexOutOfBoundsException
3694      *         if {@code mapOffset+N < 0}
3695      *         or if {@code mapOffset+N >= indexMap.length},
3696      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3697      *         is an invalid index into {@code a},
3698      *         for any lane {@code N} in the vector
3699      * @see $abstractvectortype$#toIntArray()
3700      */
3701     @ForceInline
3702     public static
3703     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3704                                        char[] a, int offset,
3705                                        int[] indexMap, int mapOffset) {
3706         // FIXME: optimize
3707         $Type$Species vsp = ($Type$Species) species;
3708         return vsp.vOp(n -> (short) a[offset + indexMap[mapOffset + n]]);
3709     }
3710 
3711     /**
3712      * Gathers a new vector composed of elements from an array of type
3713      * {@code char[]},
3714      * under the control of a mask, and
3715      * using indexes obtained by adding a fixed {@code offset} to a
3716      * series of secondary offsets from an <em>index map</em>.
3717      * The index map is a contiguous sequence of {@code VLENGTH}
3718      * elements in a second array of {@code int}s, starting at a given
3719      * {@code mapOffset}.
3720      * <p>
3721      * For each vector lane, where {@code N} is the vector lane index,
3722      * if the lane is set in the mask,
3723      * the lane is loaded from the expression
3724      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3725      * index mapping expression
3726      * {@code offset + indexMap[mapOffset + N]]}.
3727      * Unset lanes in the resulting vector are set to zero.
3728      *
3729      * @param species species of desired vector
3730      * @param a the array
3731      * @param offset the offset into the array, may be negative if relative
3732      * indexes in the index map compensate to produce a value within the
3733      * array bounds
3734      * @param indexMap the index map
3735      * @param mapOffset the offset into the index map
3736      * @param m the mask controlling lane selection
3737      * @return the vector loaded from the indexed elements of the array
3738      * @throws IndexOutOfBoundsException
3739      *         if {@code mapOffset+N < 0}
3740      *         or if {@code mapOffset+N >= indexMap.length},
3741      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3742      *         is an invalid index into {@code a},
3743      *         for any lane {@code N} in the vector
3744      *         where the mask is set
3745      * @see $abstractvectortype$#toIntArray()
3746      */
3747     @ForceInline
3748     public static
3749     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3750                                        char[] a, int offset,
3751                                        int[] indexMap, int mapOffset,
3752                                        VectorMask<$Boxtype$> m) {
3753         // FIXME: optimize
3754         $Type$Species vsp = ($Type$Species) species;
3755         return vsp.vOp(m, n -> (short) a[offset + indexMap[mapOffset + n]]);
3756     }
3757 #end[short]
3758 
3759 #if[byte]
3760     /**
3761      * Loads a vector from an array of type {@code boolean[]}
3762      * starting at an offset.
3763      * For each vector lane, where {@code N} is the vector lane index, the
3764      * array element at index {@code offset + N}
3765      * is first converted to a {@code byte} value and then
3766      * placed into the resulting vector at lane index {@code N}.
3767      * <p>
3768      * A {@code boolean} value is converted to a {@code byte} value by applying the
3769      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3770      *
3771      * @param species species of desired vector
3772      * @param a the array
3773      * @param offset the offset into the array
3774      * @return the vector loaded from an array
3775      * @throws IndexOutOfBoundsException
3776      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3777      *         for any lane {@code N} in the vector
3778      */
3779     @ForceInline
3780     public static
3781     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3782                                           boolean[] a, int offset) {
3783         offset = checkFromIndexSize(offset, species.length(), a.length);
3784         $Type$Species vsp = ($Type$Species) species;
3785         return vsp.dummyVector().fromBooleanArray0(a, offset);
3786     }
3787 
3788     /**
3789      * Loads a vector from an array of type {@code boolean[]}
3790      * starting at an offset and using a mask.
3791      * Lanes where the mask is unset are filled with the default
3792      * value of {@code $type$} ({#if[FP]?positive }zero).
3793      * For each vector lane, where {@code N} is the vector lane index,
3794      * if the mask lane at index {@code N} is set then the array element at
3795      * index {@code offset + N}
3796      * is first converted to a {@code byte} value and then
3797      * placed into the resulting vector at lane index
3798      * {@code N}, otherwise the default element value is placed into the
3799      * resulting vector at lane index {@code N}.
3800      * <p>
3801      * A {@code boolean} value is converted to a {@code byte} value by applying the
3802      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3803      *
3804      * @param species species of desired vector
3805      * @param a the array
3806      * @param offset the offset into the array
3807      * @param m the mask controlling lane selection
3808      * @return the vector loaded from an array
3809      * @throws IndexOutOfBoundsException
3810      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3811      *         for any lane {@code N} in the vector
3812      *         where the mask is set
3813      */
3814     @ForceInline
3815     public static
3816     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3817                                           boolean[] a, int offset,
3818                                           VectorMask<$Boxtype$> m) {
3819         $Type$Species vsp = ($Type$Species) species;
3820         if (offset >= 0 && offset <= (a.length - species.length())) {
3821             $abstractvectortype$ zero = vsp.zero();
3822             return vsp.dummyVector().fromBooleanArray0(a, offset, m);
3823         }
3824 
3825         // FIXME: optimize
3826         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3827         return vsp.vOp(m, i -> (byte) (a[offset + i] ? 1 : 0));
3828     }
3829 
3830     /**
3831      * Gathers a new vector composed of elements from an array of type
3832      * {@code boolean[]},
3833      * using indexes obtained by adding a fixed {@code offset} to a
3834      * series of secondary offsets from an <em>index map</em>.
3835      * The index map is a contiguous sequence of {@code VLENGTH}
3836      * elements in a second array of {@code int}s, starting at a given
3837      * {@code mapOffset}.
3838      * <p>
3839      * For each vector lane, where {@code N} is the vector lane index,
3840      * the lane is loaded from the expression
3841      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
3842      * index mapping expression
3843      * {@code offset + indexMap[mapOffset + N]]}.
3844      *
3845      * @param species species of desired vector
3846      * @param a the array
3847      * @param offset the offset into the array, may be negative if relative
3848      * indexes in the index map compensate to produce a value within the
3849      * array bounds
3850      * @param indexMap the index map
3851      * @param mapOffset the offset into the index map
3852      * @return the vector loaded from the indexed elements of the array
3853      * @throws IndexOutOfBoundsException
3854      *         if {@code mapOffset+N < 0}
3855      *         or if {@code mapOffset+N >= indexMap.length},
3856      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3857      *         is an invalid index into {@code a},
3858      *         for any lane {@code N} in the vector
3859      * @see $abstractvectortype$#toIntArray()
3860      */
3861     @ForceInline
3862     public static
3863     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3864                                           boolean[] a, int offset,
3865                                           int[] indexMap, int mapOffset) {
3866         // FIXME: optimize
3867         $Type$Species vsp = ($Type$Species) species;
3868         return vsp.vOp(n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
3869     }
3870 
3871     /**
3872      * Gathers a new vector composed of elements from an array of type
3873      * {@code boolean[]},
3874      * under the control of a mask, and
3875      * using indexes obtained by adding a fixed {@code offset} to a
3876      * series of secondary offsets from an <em>index map</em>.
3877      * The index map is a contiguous sequence of {@code VLENGTH}
3878      * elements in a second array of {@code int}s, starting at a given
3879      * {@code mapOffset}.
3880      * <p>
3881      * For each vector lane, where {@code N} is the vector lane index,
3882      * if the lane is set in the mask,
3883      * the lane is loaded from the expression
3884      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
3885      * index mapping expression
3886      * {@code offset + indexMap[mapOffset + N]]}.
3887      * Unset lanes in the resulting vector are set to zero.
3888      *
3889      * @param species species of desired vector
3890      * @param a the array
3891      * @param offset the offset into the array, may be negative if relative
3892      * indexes in the index map compensate to produce a value within the
3893      * array bounds
3894      * @param indexMap the index map
3895      * @param mapOffset the offset into the index map
3896      * @param m the mask controlling lane selection
3897      * @return the vector loaded from the indexed elements of the array
3898      * @throws IndexOutOfBoundsException
3899      *         if {@code mapOffset+N < 0}
3900      *         or if {@code mapOffset+N >= indexMap.length},
3901      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3902      *         is an invalid index into {@code a},
3903      *         for any lane {@code N} in the vector
3904      *         where the mask is set
3905      * @see $abstractvectortype$#toIntArray()
3906      */
3907     @ForceInline
3908     public static
3909     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3910                                           boolean[] a, int offset,
3911                                           int[] indexMap, int mapOffset,
3912                                           VectorMask<$Boxtype$> m) {
3913         // FIXME: optimize
3914         $Type$Species vsp = ($Type$Species) species;
3915         return vsp.vOp(m, n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
3916     }
3917 #end[byte]
3918 
3919     /**
3920      * Loads a vector from a {@linkplain ByteBuffer byte buffer}
3921      * starting at an offset into the byte buffer.
3922      * Bytes are composed into primitive lane elements according
3923      * to the specified byte order.
3924      * The vector is arranged into lanes according to
3925      * <a href="Vector.html#lane-order">memory ordering</a>.
3926      * <p>
3927      * This method behaves as if it returns the result of calling
3928      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3929      * fromByteBuffer()} as follows:
3930      * <pre>{@code
3931      * var m = species.maskAll(true);
3932      * return fromByteBuffer(species, bb, offset, bo, m);
3933      * }</pre>
3934      *
3935      * @param species species of desired vector
3936      * @param bb the byte buffer
3937      * @param offset the offset into the byte buffer
3938      * @param bo the intended byte order
3939      * @return a vector loaded from a byte buffer
3940      * @throws IndexOutOfBoundsException
3941      *         if {@code offset+N*$sizeInBytes$ < 0}
3942      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
3943      *         for any lane {@code N} in the vector
3944      */
3945     @ForceInline
3946     public static
3947     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
3948                                         ByteBuffer bb, int offset,
3949                                         ByteOrder bo) {
3950         offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
3951         $Type$Species vsp = ($Type$Species) species;
3952         return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
3953     }
3954 
3955     /**
3956      * Loads a vector from a {@linkplain ByteBuffer byte buffer}
3957      * starting at an offset into the byte buffer
3958      * and using a mask.
3959      * Lanes where the mask is unset are filled with the default
3960      * value of {@code $type$} ({#if[FP]?positive }zero).
3961      * Bytes are composed into primitive lane elements according
3962      * to the specified byte order.
3963      * The vector is arranged into lanes according to
3964      * <a href="Vector.html#lane-order">memory ordering</a>.
3965      * <p>
3966      * The following pseudocode illustrates the behavior:
3967      * <pre>{@code
3968      * $Type$Buffer eb = bb.duplicate()
3969      *     .position(offset){#if[byte]?;}
3970 #if[!byte]
3971      *     .order(bo).as$Type$Buffer();
3972 #end[!byte]
3973      * $type$[] ar = new $type$[species.length()];
3974      * for (int n = 0; n < ar.length; n++) {
3975      *     if (m.laneIsSet(n)) {
3976      *         ar[n] = eb.get(n);
3977      *     }
3978      * }
3979      * $abstractvectortype$ r = $abstractvectortype$.fromArray(species, ar, 0);
3980      * }</pre>
3981      * @implNote
3982 #if[!byte]
3983      * This operation is likely to be more efficient if
3984      * the specified byte order is the same as
3985      * {@linkplain ByteOrder#nativeOrder()
3986      * the platform native order},
3987      * since this method will not need to reorder
3988      * the bytes of lane values.
3989 #else[!byte]
3990      * The byte order argument is ignored.
3991 #end[!byte]
3992      *
3993      * @param species species of desired vector
3994      * @param bb the byte buffer
3995      * @param offset the offset into the byte buffer
3996      * @param bo the intended byte order
3997      * @param m the mask controlling lane selection
3998      * @return a vector loaded from a byte buffer
3999      * @throws IndexOutOfBoundsException
4000      *         if {@code offset+N*$sizeInBytes$ < 0}
4001      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
4002      *         for any lane {@code N} in the vector
4003      *         where the mask is set
4004      */
4005     @ForceInline
4006     public static
4007     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
4008                                         ByteBuffer bb, int offset,
4009                                         ByteOrder bo,
4010                                         VectorMask<$Boxtype$> m) {
4011         $Type$Species vsp = ($Type$Species) species;
4012         if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
4013             return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
4014         }
4015 
4016         // FIXME: optimize
4017         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4018         ByteBuffer wb = wrapper(bb, bo);
4019         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
4020                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4021     }
4022 
4023     // Memory store operations
4024 
4025     /**
4026      * Stores this vector into an array of type {@code $type$[]}
4027      * starting at an offset.
4028      * <p>
4029      * For each vector lane, where {@code N} is the vector lane index,
4030      * the lane element at index {@code N} is stored into the array
4031      * element {@code a[offset+N]}.
4032      *
4033      * @param a the array, of type {@code $type$[]}
4034      * @param offset the offset into the array
4035      * @throws IndexOutOfBoundsException
4036      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4037      *         for any lane {@code N} in the vector
4038      */
4039     @ForceInline
4040     public final
4041     void intoArray($type$[] a, int offset) {
4042         offset = checkFromIndexSize(offset, length(), a.length);
4043         $Type$Species vsp = vspecies();
4044         VectorSupport.store(
4045             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4046             a, arrayAddress(a, offset),
4047             this,
4048             a, offset,
4049             (arr, off, v)
4050             -> v.stOp(arr, off,
4051                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4052     }
4053 
4054     /**
4055      * Stores this vector into an array of type {@code $type$[]}
4056      * starting at offset and using a mask.
4057      * <p>
4058      * For each vector lane, where {@code N} is the vector lane index,
4059      * the lane element at index {@code N} is stored into the array
4060      * element {@code a[offset+N]}.
4061      * If the mask lane at {@code N} is unset then the corresponding
4062      * array element {@code a[offset+N]} is left unchanged.
4063      * <p>
4064      * Array range checking is done for lanes where the mask is set.
4065      * Lanes where the mask is unset are not stored and do not need
4066      * to correspond to legitimate elements of {@code a}.
4067      * That is, unset lanes may correspond to array indexes less than
4068      * zero or beyond the end of the array.
4069      *
4070      * @param a the array, of type {@code $type$[]}
4071      * @param offset the offset into the array
4072      * @param m the mask controlling lane storage
4073      * @throws IndexOutOfBoundsException
4074      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4075      *         for any lane {@code N} in the vector
4076      *         where the mask is set
4077      */
4078     @ForceInline
4079     public final
4080     void intoArray($type$[] a, int offset,
4081                    VectorMask<$Boxtype$> m) {
4082         if (m.allTrue()) {
4083             intoArray(a, offset);
4084         } else {
4085             $Type$Species vsp = vspecies();
4086             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4087             intoArray0(a, offset, m);
4088         }
4089     }
4090 
4091     /**
4092      * Scatters this vector into an array of type {@code $type$[]}
4093      * using indexes obtained by adding a fixed {@code offset} to a
4094      * series of secondary offsets from an <em>index map</em>.
4095      * The index map is a contiguous sequence of {@code VLENGTH}
4096      * elements in a second array of {@code int}s, starting at a given
4097      * {@code mapOffset}.
4098      * <p>
4099      * For each vector lane, where {@code N} is the vector lane index,
4100      * the lane element at index {@code N} is stored into the array
4101      * element {@code a[f(N)]}, where {@code f(N)} is the
4102      * index mapping expression
4103      * {@code offset + indexMap[mapOffset + N]]}.
4104      *
4105      * @param a the array
4106      * @param offset an offset to combine with the index map offsets
4107      * @param indexMap the index map
4108      * @param mapOffset the offset into the index map
4109      * @throws IndexOutOfBoundsException
4110      *         if {@code mapOffset+N < 0}
4111      *         or if {@code mapOffset+N >= indexMap.length},
4112      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4113      *         is an invalid index into {@code a},
4114      *         for any lane {@code N} in the vector
4115      * @see $abstractvectortype$#toIntArray()
4116      */
4117 #if[byteOrShort]
4118     @ForceInline
4119     public final
4120     void intoArray($type$[] a, int offset,
4121                    int[] indexMap, int mapOffset) {
4122         stOp(a, offset,
4123              (arr, off, i, e) -> {
4124                  int j = indexMap[mapOffset + i];
4125                  arr[off + j] = e;
4126              });
4127     }
4128 #else[byteOrShort]
4129     @ForceInline
4130     public final
4131     void intoArray($type$[] a, int offset,
4132                    int[] indexMap, int mapOffset) {
4133         $Type$Species vsp = vspecies();
4134         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4135 #if[longOrDouble]
4136         if (vsp.laneCount() == 1) {
4137             intoArray(a, offset + indexMap[mapOffset]);
4138             return;
4139         }
4140 
4141         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4142         IntVector vix;
4143         if (isp.laneCount() != vsp.laneCount()) {
4144             // For $Type$MaxVector,  if vector length  is 2048 bits, indexShape
4145             // of $Type$ species is S_MAX_BIT. and the lane count of $Type$
4146             // vector is 32. When converting $Type$ species to int species,
4147             // indexShape is still S_MAX_BIT, but the lane count of int vector
4148             // is 64. So when loading index vector (IntVector), only lower half
4149             // of index data is needed.
4150             vix = IntVector
4151                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4152                 .add(offset);
4153         } else {
4154             vix = IntVector
4155                 .fromArray(isp, indexMap, mapOffset)
4156                 .add(offset);
4157         }
4158 
4159 #else[longOrDouble]
4160         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4161         IntVector vix = IntVector
4162             .fromArray(isp, indexMap, mapOffset)
4163             .add(offset);
4164 #end[longOrDouble]
4165 
4166         vix = VectorIntrinsics.checkIndex(vix, a.length);
4167 
4168         VectorSupport.storeWithMap(
4169             vsp.vectorType(), null, vsp.elementType(), vsp.laneCount(),
4170             isp.vectorType(),
4171             a, arrayAddress(a, 0), vix,
4172             this, null,
4173             a, offset, indexMap, mapOffset,
4174             (arr, off, v, map, mo, vm)
4175             -> v.stOp(arr, off,
4176                       (arr_, off_, i, e) -> {
4177                           int j = map[mo + i];
4178                           arr[off + j] = e;
4179                       }));
4180     }
4181 #end[byteOrShort]
4182 
4183     /**
4184      * Scatters this vector into an array of type {@code $type$[]},
4185      * under the control of a mask, and
4186      * using indexes obtained by adding a fixed {@code offset} to a
4187      * series of secondary offsets from an <em>index map</em>.
4188      * The index map is a contiguous sequence of {@code VLENGTH}
4189      * elements in a second array of {@code int}s, starting at a given
4190      * {@code mapOffset}.
4191      * <p>
4192      * For each vector lane, where {@code N} is the vector lane index,
4193      * if the mask lane at index {@code N} is set then
4194      * the lane element at index {@code N} is stored into the array
4195      * element {@code a[f(N)]}, where {@code f(N)} is the
4196      * index mapping expression
4197      * {@code offset + indexMap[mapOffset + N]]}.
4198      *
4199      * @param a the array
4200      * @param offset an offset to combine with the index map offsets
4201      * @param indexMap the index map
4202      * @param mapOffset the offset into the index map
4203      * @param m the mask
4204      * @throws IndexOutOfBoundsException
4205      *         if {@code mapOffset+N < 0}
4206      *         or if {@code mapOffset+N >= indexMap.length},
4207      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4208      *         is an invalid index into {@code a},
4209      *         for any lane {@code N} in the vector
4210      *         where the mask is set
4211      * @see $abstractvectortype$#toIntArray()
4212      */
4213 #if[byteOrShort]
4214     @ForceInline
4215     public final
4216     void intoArray($type$[] a, int offset,
4217                    int[] indexMap, int mapOffset,
4218                    VectorMask<$Boxtype$> m) {
4219         stOp(a, offset, m,
4220              (arr, off, i, e) -> {
4221                  int j = indexMap[mapOffset + i];
4222                  arr[off + j] = e;
4223              });
4224     }
4225 #else[byteOrShort]
4226     @ForceInline
4227     public final
4228     void intoArray($type$[] a, int offset,
4229                    int[] indexMap, int mapOffset,
4230                    VectorMask<$Boxtype$> m) {
4231         if (m.allTrue()) {
4232             intoArray(a, offset, indexMap, mapOffset);
4233         }
4234         else {
4235             intoArray0(a, offset, indexMap, mapOffset, m);
4236         }
4237     }
4238 #end[byteOrShort]
4239 
4240 #if[short]
4241     /**
4242      * Stores this vector into an array of type {@code char[]}
4243      * starting at an offset.
4244      * <p>
4245      * For each vector lane, where {@code N} is the vector lane index,
4246      * the lane element at index {@code N}
4247      * is first cast to a {@code char} value and then
4248      * stored into the array element {@code a[offset+N]}.
4249      *
4250      * @param a the array, of type {@code char[]}
4251      * @param offset the offset into the array
4252      * @throws IndexOutOfBoundsException
4253      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4254      *         for any lane {@code N} in the vector
4255      */
4256     @ForceInline
4257     public final
4258     void intoCharArray(char[] a, int offset) {
4259         offset = checkFromIndexSize(offset, length(), a.length);
4260         $Type$Species vsp = vspecies();
4261         VectorSupport.store(
4262             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4263             a, charArrayAddress(a, offset),
4264             this,
4265             a, offset,
4266             (arr, off, v)
4267             -> v.stOp(arr, off,
4268                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
4269     }
4270 
4271     /**
4272      * Stores this vector into an array of type {@code char[]}
4273      * starting at offset and using a mask.
4274      * <p>
4275      * For each vector lane, where {@code N} is the vector lane index,
4276      * the lane element at index {@code N}
4277      * is first cast to a {@code char} value and then
4278      * stored into the array element {@code a[offset+N]}.
4279      * If the mask lane at {@code N} is unset then the corresponding
4280      * array element {@code a[offset+N]} is left unchanged.
4281      * <p>
4282      * Array range checking is done for lanes where the mask is set.
4283      * Lanes where the mask is unset are not stored and do not need
4284      * to correspond to legitimate elements of {@code a}.
4285      * That is, unset lanes may correspond to array indexes less than
4286      * zero or beyond the end of the array.
4287      *
4288      * @param a the array, of type {@code char[]}
4289      * @param offset the offset into the array
4290      * @param m the mask controlling lane storage
4291      * @throws IndexOutOfBoundsException
4292      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4293      *         for any lane {@code N} in the vector
4294      *         where the mask is set
4295      */
4296     @ForceInline
4297     public final
4298     void intoCharArray(char[] a, int offset,
4299                        VectorMask<$Boxtype$> m) {
4300         if (m.allTrue()) {
4301             intoCharArray(a, offset);
4302         } else {
4303             $Type$Species vsp = vspecies();
4304             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4305             intoCharArray0(a, offset, m);
4306         }
4307     }
4308 
4309     /**
4310      * Scatters this vector into an array of type {@code char[]}
4311      * using indexes obtained by adding a fixed {@code offset} to a
4312      * series of secondary offsets from an <em>index map</em>.
4313      * The index map is a contiguous sequence of {@code VLENGTH}
4314      * elements in a second array of {@code int}s, starting at a given
4315      * {@code mapOffset}.
4316      * <p>
4317      * For each vector lane, where {@code N} is the vector lane index,
4318      * the lane element at index {@code N}
4319      * is first cast to a {@code char} value and then
4320      * stored into the array
4321      * element {@code a[f(N)]}, where {@code f(N)} is the
4322      * index mapping expression
4323      * {@code offset + indexMap[mapOffset + N]]}.
4324      *
4325      * @param a the array
4326      * @param offset an offset to combine with the index map offsets
4327      * @param indexMap the index map
4328      * @param mapOffset the offset into the index map
4329      * @throws IndexOutOfBoundsException
4330      *         if {@code mapOffset+N < 0}
4331      *         or if {@code mapOffset+N >= indexMap.length},
4332      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4333      *         is an invalid index into {@code a},
4334      *         for any lane {@code N} in the vector
4335      * @see $abstractvectortype$#toIntArray()
4336      */
4337     @ForceInline
4338     public final
4339     void intoCharArray(char[] a, int offset,
4340                        int[] indexMap, int mapOffset) {
4341         // FIXME: optimize
4342         stOp(a, offset,
4343              (arr, off, i, e) -> {
4344                  int j = indexMap[mapOffset + i];
4345                  arr[off + j] = (char) e;
4346              });
4347     }
4348 
4349     /**
4350      * Scatters this vector into an array of type {@code char[]},
4351      * under the control of a mask, and
4352      * using indexes obtained by adding a fixed {@code offset} to a
4353      * series of secondary offsets from an <em>index map</em>.
4354      * The index map is a contiguous sequence of {@code VLENGTH}
4355      * elements in a second array of {@code int}s, starting at a given
4356      * {@code mapOffset}.
4357      * <p>
4358      * For each vector lane, where {@code N} is the vector lane index,
4359      * if the mask lane at index {@code N} is set then
4360      * the lane element at index {@code N}
4361      * is first cast to a {@code char} value and then
4362      * stored into the array
4363      * element {@code a[f(N)]}, where {@code f(N)} is the
4364      * index mapping expression
4365      * {@code offset + indexMap[mapOffset + N]]}.
4366      *
4367      * @param a the array
4368      * @param offset an offset to combine with the index map offsets
4369      * @param indexMap the index map
4370      * @param mapOffset the offset into the index map
4371      * @param m the mask
4372      * @throws IndexOutOfBoundsException
4373      *         if {@code mapOffset+N < 0}
4374      *         or if {@code mapOffset+N >= indexMap.length},
4375      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4376      *         is an invalid index into {@code a},
4377      *         for any lane {@code N} in the vector
4378      *         where the mask is set
4379      * @see $abstractvectortype$#toIntArray()
4380      */
4381     @ForceInline
4382     public final
4383     void intoCharArray(char[] a, int offset,
4384                        int[] indexMap, int mapOffset,
4385                        VectorMask<$Boxtype$> m) {
4386         // FIXME: optimize
4387         stOp(a, offset, m,
4388              (arr, off, i, e) -> {
4389                  int j = indexMap[mapOffset + i];
4390                  arr[off + j] = (char) e;
4391              });
4392     }
4393 #end[short]
4394 
4395 #if[byte]
4396     /**
4397      * Stores this vector into an array of type {@code boolean[]}
4398      * starting at an offset.
4399      * <p>
4400      * For each vector lane, where {@code N} is the vector lane index,
4401      * the lane element at index {@code N}
4402      * is first converted to a {@code boolean} value and then
4403      * stored into the array element {@code a[offset+N]}.
4404      * <p>
4405      * A {@code byte} value is converted to a {@code boolean} value by applying the
4406      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4407      *
4408      * @param a the array
4409      * @param offset the offset into the array
4410      * @throws IndexOutOfBoundsException
4411      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4412      *         for any lane {@code N} in the vector
4413      */
4414     @ForceInline
4415     public final
4416     void intoBooleanArray(boolean[] a, int offset) {
4417         offset = checkFromIndexSize(offset, length(), a.length);
4418         $Type$Species vsp = vspecies();
4419         ByteVector normalized = this.and((byte) 1);
4420         VectorSupport.store(
4421             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4422             a, booleanArrayAddress(a, offset),
4423             normalized,
4424             a, offset,
4425             (arr, off, v)
4426             -> v.stOp(arr, off,
4427                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4428     }
4429 
4430     /**
4431      * Stores this vector into an array of type {@code boolean[]}
4432      * starting at offset and using a mask.
4433      * <p>
4434      * For each vector lane, where {@code N} is the vector lane index,
4435      * the lane element at index {@code N}
4436      * is first converted to a {@code boolean} value and then
4437      * stored into the array element {@code a[offset+N]}.
4438      * If the mask lane at {@code N} is unset then the corresponding
4439      * array element {@code a[offset+N]} is left unchanged.
4440      * <p>
4441      * A {@code byte} value is converted to a {@code boolean} value by applying the
4442      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4443      * <p>
4444      * Array range checking is done for lanes where the mask is set.
4445      * Lanes where the mask is unset are not stored and do not need
4446      * to correspond to legitimate elements of {@code a}.
4447      * That is, unset lanes may correspond to array indexes less than
4448      * zero or beyond the end of the array.
4449      *
4450      * @param a the array
4451      * @param offset the offset into the array
4452      * @param m the mask controlling lane storage
4453      * @throws IndexOutOfBoundsException
4454      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4455      *         for any lane {@code N} in the vector
4456      *         where the mask is set
4457      */
4458     @ForceInline
4459     public final
4460     void intoBooleanArray(boolean[] a, int offset,
4461                           VectorMask<$Boxtype$> m) {
4462         if (m.allTrue()) {
4463             intoBooleanArray(a, offset);
4464         } else {
4465             $Type$Species vsp = vspecies();
4466             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4467             intoBooleanArray0(a, offset, m);
4468         }
4469     }
4470 
4471     /**
4472      * Scatters this vector into an array of type {@code boolean[]}
4473      * using indexes obtained by adding a fixed {@code offset} to a
4474      * series of secondary offsets from an <em>index map</em>.
4475      * The index map is a contiguous sequence of {@code VLENGTH}
4476      * elements in a second array of {@code int}s, starting at a given
4477      * {@code mapOffset}.
4478      * <p>
4479      * For each vector lane, where {@code N} is the vector lane index,
4480      * the lane element at index {@code N}
4481      * is first converted to a {@code boolean} value and then
4482      * stored into the array
4483      * element {@code a[f(N)]}, where {@code f(N)} is the
4484      * index mapping expression
4485      * {@code offset + indexMap[mapOffset + N]]}.
4486      * <p>
4487      * A {@code byte} value is converted to a {@code boolean} value by applying the
4488      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4489      *
4490      * @param a the array
4491      * @param offset an offset to combine with the index map offsets
4492      * @param indexMap the index map
4493      * @param mapOffset the offset into the index map
4494      * @throws IndexOutOfBoundsException
4495      *         if {@code mapOffset+N < 0}
4496      *         or if {@code mapOffset+N >= indexMap.length},
4497      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4498      *         is an invalid index into {@code a},
4499      *         for any lane {@code N} in the vector
4500      * @see $abstractvectortype$#toIntArray()
4501      */
4502     @ForceInline
4503     public final
4504     void intoBooleanArray(boolean[] a, int offset,
4505                           int[] indexMap, int mapOffset) {
4506         // FIXME: optimize
4507         stOp(a, offset,
4508              (arr, off, i, e) -> {
4509                  int j = indexMap[mapOffset + i];
4510                  arr[off + j] = (e & 1) != 0;
4511              });
4512     }
4513 
4514     /**
4515      * Scatters this vector into an array of type {@code boolean[]},
4516      * under the control of a mask, and
4517      * using indexes obtained by adding a fixed {@code offset} to a
4518      * series of secondary offsets from an <em>index map</em>.
4519      * The index map is a contiguous sequence of {@code VLENGTH}
4520      * elements in a second array of {@code int}s, starting at a given
4521      * {@code mapOffset}.
4522      * <p>
4523      * For each vector lane, where {@code N} is the vector lane index,
4524      * if the mask lane at index {@code N} is set then
4525      * the lane element at index {@code N}
4526      * is first converted to a {@code boolean} value and then
4527      * stored into the array
4528      * element {@code a[f(N)]}, where {@code f(N)} is the
4529      * index mapping expression
4530      * {@code offset + indexMap[mapOffset + N]]}.
4531      * <p>
4532      * A {@code byte} value is converted to a {@code boolean} value by applying the
4533      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4534      *
4535      * @param a the array
4536      * @param offset an offset to combine with the index map offsets
4537      * @param indexMap the index map
4538      * @param mapOffset the offset into the index map
4539      * @param m the mask
4540      * @throws IndexOutOfBoundsException
4541      *         if {@code mapOffset+N < 0}
4542      *         or if {@code mapOffset+N >= indexMap.length},
4543      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4544      *         is an invalid index into {@code a},
4545      *         for any lane {@code N} in the vector
4546      *         where the mask is set
4547      * @see $abstractvectortype$#toIntArray()
4548      */
4549     @ForceInline
4550     public final
4551     void intoBooleanArray(boolean[] a, int offset,
4552                           int[] indexMap, int mapOffset,
4553                           VectorMask<$Boxtype$> m) {
4554         // FIXME: optimize
4555         stOp(a, offset, m,
4556              (arr, off, i, e) -> {
4557                  int j = indexMap[mapOffset + i];
4558                  arr[off + j] = (e & 1) != 0;
4559              });
4560     }
4561 #end[byte]
4562 
4563     /**
4564      * {@inheritDoc} <!--workaround-->
4565      */
4566     @Override
4567     @ForceInline
4568     public final
4569     void intoByteArray(byte[] a, int offset,
4570                        ByteOrder bo) {
4571         offset = checkFromIndexSize(offset, byteSize(), a.length);
4572         maybeSwap(bo).intoByteArray0(a, offset);
4573     }
4574 
4575     /**
4576      * {@inheritDoc} <!--workaround-->
4577      */
4578     @Override
4579     @ForceInline
4580     public final
4581     void intoByteArray(byte[] a, int offset,
4582                        ByteOrder bo,
4583                        VectorMask<$Boxtype$> m) {
4584         if (m.allTrue()) {
4585             intoByteArray(a, offset, bo);
4586         } else {
4587             $Type$Species vsp = vspecies();
4588             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
4589             maybeSwap(bo).intoByteArray0(a, offset, m);
4590         }
4591     }
4592 
4593     /**
4594      * {@inheritDoc} <!--workaround-->
4595      */
4596     @Override
4597     @ForceInline
4598     public final
4599     void intoByteBuffer(ByteBuffer bb, int offset,
4600                         ByteOrder bo) {
4601         if (ScopedMemoryAccess.isReadOnly(bb)) {
4602             throw new ReadOnlyBufferException();
4603         }
4604         offset = checkFromIndexSize(offset, byteSize(), bb.limit());
4605         maybeSwap(bo).intoByteBuffer0(bb, offset);
4606     }
4607 
4608     /**
4609      * {@inheritDoc} <!--workaround-->
4610      */
4611     @Override
4612     @ForceInline
4613     public final
4614     void intoByteBuffer(ByteBuffer bb, int offset,
4615                         ByteOrder bo,
4616                         VectorMask<$Boxtype$> m) {
4617         if (m.allTrue()) {
4618             intoByteBuffer(bb, offset, bo);
4619         } else {
4620             if (bb.isReadOnly()) {
4621                 throw new ReadOnlyBufferException();
4622             }
4623             $Type$Species vsp = vspecies();
4624             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4625             maybeSwap(bo).intoByteBuffer0(bb, offset, m);
4626         }
4627     }
4628 
4629     // ================================================
4630 
4631     // Low-level memory operations.
4632     //
4633     // Note that all of these operations *must* inline into a context
4634     // where the exact species of the involved vector is a
4635     // compile-time constant.  Otherwise, the intrinsic generation
4636     // will fail and performance will suffer.
4637     //
4638     // In many cases this is achieved by re-deriving a version of the
4639     // method in each concrete subclass (per species).  The re-derived
4640     // method simply calls one of these generic methods, with exact
4641     // parameters for the controlling metadata, which is either a
4642     // typed vector or constant species instance.
4643 
4644     // Unchecked loading operations in native byte order.
4645     // Caller is responsible for applying index checks, masking, and
4646     // byte swapping.
4647 
4648     /*package-private*/
4649     abstract
4650     $abstractvectortype$ fromArray0($type$[] a, int offset);
4651     @ForceInline
4652     final
4653     $abstractvectortype$ fromArray0Template($type$[] a, int offset) {
4654         $Type$Species vsp = vspecies();
4655         return VectorSupport.load(
4656             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4657             a, arrayAddress(a, offset),
4658             a, offset, vsp,
4659             (arr, off, s) -> s.ldOp(arr, off,
4660                                     (arr_, off_, i) -> arr_[off_ + i]));
4661     }
4662 
4663     /*package-private*/
4664     abstract
4665     $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4666     @ForceInline
4667     final
4668     <M extends VectorMask<$Boxtype$>>
4669     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4670         m.check(species());
4671         $Type$Species vsp = vspecies();
4672         return VectorSupport.loadMasked(
4673             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4674             a, arrayAddress(a, offset), m,
4675             a, offset, vsp,
4676             (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4677                                         (arr_, off_, i) -> arr_[off_ + i]));
4678     }
4679 
4680 #if[!byteOrShort]
4681     /*package-private*/
4682     abstract
4683     $abstractvectortype$ fromArray0($type$[] a, int offset,
4684                                     int[] indexMap, int mapOffset,
4685                                     VectorMask<$Boxtype$> m);
4686     @ForceInline
4687     final
4688     <M extends VectorMask<$Boxtype$>>
4689     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset,
4690                                             int[] indexMap, int mapOffset, M m) {
4691         $Type$Species vsp = vspecies();
4692         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4693         Objects.requireNonNull(a);
4694         Objects.requireNonNull(indexMap);
4695         m.check(vsp);
4696         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();
4697 
4698 #if[longOrDouble]
4699         if (vsp.laneCount() == 1) {
4700           return $abstractvectortype$.fromArray(vsp, a, offset + indexMap[mapOffset], m);
4701         }
4702 
4703         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
4704         IntVector vix;
4705         if (isp.laneCount() != vsp.laneCount()) {
4706             // For $Type$MaxVector,  if vector length is non-power-of-two or
4707             // 2048 bits, indexShape of $Type$ species is S_MAX_BIT.
4708             // Assume that vector length is 2048, then the lane count of $Type$
4709             // vector is 32. When converting $Type$ species to int species,
4710             // indexShape is still S_MAX_BIT, but the lane count of int vector
4711             // is 64. So when loading index vector (IntVector), only lower half
4712             // of index data is needed.
4713             vix = IntVector
4714                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4715                 .add(offset);
4716         } else {
4717             vix = IntVector
4718                 .fromArray(isp, indexMap, mapOffset)
4719                 .add(offset);
4720         }
4721 #else[longOrDouble]
4722         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
4723         IntVector vix = IntVector
4724             .fromArray(isp, indexMap, mapOffset)
4725             .add(offset);
4726 #end[longOrDouble]
4727 
4728         // FIXME: Check index under mask controlling.
4729         vix = VectorIntrinsics.checkIndex(vix, a.length);
4730 
4731         return VectorSupport.loadWithMap(
4732             vectorType, maskClass, $type$.class, vsp.laneCount(),
4733             isp.vectorType(),
4734             a, ARRAY_BASE, vix, m,
4735             a, offset, indexMap, mapOffset, vsp,
4736             (c, idx, iMap, idy, s, vm) ->
4737             s.vOp(vm, n -> c[idx + iMap[idy+n]]));
4738     }
4739 #end[!byteOrShort]
4740 
4741 #if[short]
4742     /*package-private*/
4743     abstract
4744     $abstractvectortype$ fromCharArray0(char[] a, int offset);
4745     @ForceInline
4746     final
4747     $abstractvectortype$ fromCharArray0Template(char[] a, int offset) {
4748         $Type$Species vsp = vspecies();
4749         return VectorSupport.load(
4750             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4751             a, charArrayAddress(a, offset),
4752             a, offset, vsp,
4753             (arr, off, s) -> s.ldOp(arr, off,
4754                                     (arr_, off_, i) -> (short) arr_[off_ + i]));
4755     }
4756 
4757     /*package-private*/
4758     abstract
4759     $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
4760     @ForceInline
4761     final
4762     <M extends VectorMask<$Boxtype$>>
4763     $abstractvectortype$ fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
4764         m.check(species());
4765         $Type$Species vsp = vspecies();
4766         return VectorSupport.loadMasked(
4767                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4768                 a, charArrayAddress(a, offset), m,
4769                 a, offset, vsp,
4770                 (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4771                                             (arr_, off_, i) -> (short) arr_[off_ + i]));
4772     }
4773 #end[short]
4774 
4775 #if[byte]
4776     /*package-private*/
4777     abstract
4778     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset);
4779     @ForceInline
4780     final
4781     $abstractvectortype$ fromBooleanArray0Template(boolean[] a, int offset) {
4782         $Type$Species vsp = vspecies();
4783         return VectorSupport.load(
4784             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4785             a, booleanArrayAddress(a, offset),
4786             a, offset, vsp,
4787             (arr, off, s) -> s.ldOp(arr, off,
4788                                     (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4789     }
4790 
4791     /*package-private*/
4792     abstract
4793     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4794     @ForceInline
4795     final
4796     <M extends VectorMask<$Boxtype$>>
4797     $abstractvectortype$ fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4798         m.check(species());
4799         $Type$Species vsp = vspecies();
4800         return VectorSupport.loadMasked(
4801             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4802             a, booleanArrayAddress(a, offset), m,
4803             a, offset, vsp,
4804             (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4805                                         (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4806     }
4807 #end[byte]
4808 
4809     @Override
4810     abstract
4811     $abstractvectortype$ fromByteArray0(byte[] a, int offset);
4812     @ForceInline
4813     final
4814     $abstractvectortype$ fromByteArray0Template(byte[] a, int offset) {
4815         $Type$Species vsp = vspecies();
4816         return VectorSupport.load(
4817             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4818             a, byteArrayAddress(a, offset),
4819             a, offset, vsp,
4820             (arr, off, s) -> {
4821                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4822                 return s.ldOp(wb, off,
4823                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4824             });
4825     }
4826 
4827     abstract
4828     $abstractvectortype$ fromByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
4829     @ForceInline
4830     final
4831     <M extends VectorMask<$Boxtype$>>
4832     $abstractvectortype$ fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
4833         $Type$Species vsp = vspecies();
4834         m.check(vsp);
4835         return VectorSupport.loadMasked(
4836             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4837             a, byteArrayAddress(a, offset), m,
4838             a, offset, vsp,
4839             (arr, off, s, vm) -> {
4840                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4841                 return s.ldOp(wb, off, vm,
4842                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4843             });
4844     }
4845 
4846     abstract
4847     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset);
4848     @ForceInline
4849     final
4850     $abstractvectortype$ fromByteBuffer0Template(ByteBuffer bb, int offset) {
4851         $Type$Species vsp = vspecies();
4852         return ScopedMemoryAccess.loadFromByteBuffer(
4853                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4854                 bb, offset, vsp,
4855                 (buf, off, s) -> {
4856                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4857                     return s.ldOp(wb, off,
4858                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4859                 });
4860     }
4861 
4862     abstract
4863     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
4864     @ForceInline
4865     final
4866     <M extends VectorMask<$Boxtype$>>
4867     $abstractvectortype$ fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
4868         $Type$Species vsp = vspecies();
4869         m.check(vsp);
4870         return ScopedMemoryAccess.loadFromByteBufferMasked(
4871                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4872                 bb, offset, m, vsp,
4873                 (buf, off, s, vm) -> {
4874                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4875                     return s.ldOp(wb, off, vm,
4876                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4877                 });
4878     }
4879 
4880     // Unchecked storing operations in native byte order.
4881     // Caller is responsible for applying index checks, masking, and
4882     // byte swapping.
4883 
4884     abstract
4885     void intoArray0($type$[] a, int offset);
4886     @ForceInline
4887     final
4888     void intoArray0Template($type$[] a, int offset) {
4889         $Type$Species vsp = vspecies();
4890         VectorSupport.store(
4891             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4892             a, arrayAddress(a, offset),
4893             this, a, offset,
4894             (arr, off, v)
4895             -> v.stOp(arr, off,
4896                       (arr_, off_, i, e) -> arr_[off_+i] = e));
4897     }
4898 
4899     abstract
4900     void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4901     @ForceInline
4902     final
4903     <M extends VectorMask<$Boxtype$>>
4904     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4905         m.check(species());
4906         $Type$Species vsp = vspecies();
4907         VectorSupport.storeMasked(
4908             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4909             a, arrayAddress(a, offset),
4910             this, m, a, offset,
4911             (arr, off, v, vm)
4912             -> v.stOp(arr, off, vm,
4913                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4914     }
4915 
4916 #if[!byteOrShort]
4917     abstract
4918     void intoArray0($type$[] a, int offset,
4919                     int[] indexMap, int mapOffset,
4920                     VectorMask<$Boxtype$> m);
4921     @ForceInline
4922     final
4923     <M extends VectorMask<$Boxtype$>>
4924     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset,
4925                             int[] indexMap, int mapOffset, M m) {
4926         m.check(species());
4927         $Type$Species vsp = vspecies();
4928         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4929 #if[longOrDouble]
4930         if (vsp.laneCount() == 1) {
4931             intoArray(a, offset + indexMap[mapOffset], m);
4932             return;
4933         }
4934 
4935         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4936         IntVector vix;
4937         if (isp.laneCount() != vsp.laneCount()) {
4938             // For $Type$MaxVector,  if vector length  is 2048 bits, indexShape
4939             // of $Type$ species is S_MAX_BIT. and the lane count of $Type$
4940             // vector is 32. When converting $Type$ species to int species,
4941             // indexShape is still S_MAX_BIT, but the lane count of int vector
4942             // is 64. So when loading index vector (IntVector), only lower half
4943             // of index data is needed.
4944             vix = IntVector
4945                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4946                 .add(offset);
4947         } else {
4948             vix = IntVector
4949                 .fromArray(isp, indexMap, mapOffset)
4950                 .add(offset);
4951         }
4952 
4953 #else[longOrDouble]
4954         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4955         IntVector vix = IntVector
4956             .fromArray(isp, indexMap, mapOffset)
4957             .add(offset);
4958 #end[longOrDouble]
4959 
4960         // FIXME: Check index under mask controlling.
4961         vix = VectorIntrinsics.checkIndex(vix, a.length);
4962 
4963         VectorSupport.storeWithMap(
4964             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4965             isp.vectorType(),
4966             a, arrayAddress(a, 0), vix,
4967             this, m,
4968             a, offset, indexMap, mapOffset,
4969             (arr, off, v, map, mo, vm)
4970             -> v.stOp(arr, off, vm,
4971                       (arr_, off_, i, e) -> {
4972                           int j = map[mo + i];
4973                           arr[off + j] = e;
4974                       }));
4975     }
4976 #end[!byteOrShort]
4977 
4978 #if[byte]
4979     abstract
4980     void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4981     @ForceInline
4982     final
4983     <M extends VectorMask<$Boxtype$>>
4984     void intoBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4985         m.check(species());
4986         $Type$Species vsp = vspecies();
4987         ByteVector normalized = this.and((byte) 1);
4988         VectorSupport.storeMasked(
4989             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4990             a, booleanArrayAddress(a, offset),
4991             normalized, m, a, offset,
4992             (arr, off, v, vm)
4993             -> v.stOp(arr, off, vm,
4994                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4995     }
4996 #end[byte]
4997 
4998     abstract
4999     void intoByteArray0(byte[] a, int offset);
5000     @ForceInline
5001     final
5002     void intoByteArray0Template(byte[] a, int offset) {
5003         $Type$Species vsp = vspecies();
5004         VectorSupport.store(
5005             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
5006             a, byteArrayAddress(a, offset),
5007             this, a, offset,
5008             (arr, off, v) -> {
5009                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
5010                 v.stOp(wb, off,
5011                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5012             });
5013     }
5014 
5015     abstract
5016     void intoByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
5017     @ForceInline
5018     final
5019     <M extends VectorMask<$Boxtype$>>
5020     void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
5021         $Type$Species vsp = vspecies();
5022         m.check(vsp);
5023         VectorSupport.storeMasked(
5024             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5025             a, byteArrayAddress(a, offset),
5026             this, m, a, offset,
5027             (arr, off, v, vm) -> {
5028                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
5029                 v.stOp(wb, off, vm,
5030                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5031             });
5032     }
5033 
5034     @ForceInline
5035     final
5036     void intoByteBuffer0(ByteBuffer bb, int offset) {
5037         $Type$Species vsp = vspecies();
5038         ScopedMemoryAccess.storeIntoByteBuffer(
5039                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
5040                 this, bb, offset,
5041                 (buf, off, v) -> {
5042                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
5043                     v.stOp(wb, off,
5044                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5045                 });
5046     }
5047 
5048     abstract
5049     void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
5050     @ForceInline
5051     final
5052     <M extends VectorMask<$Boxtype$>>
5053     void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
5054         $Type$Species vsp = vspecies();
5055         m.check(vsp);
5056         ScopedMemoryAccess.storeIntoByteBufferMasked(
5057                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5058                 this, m, bb, offset,
5059                 (buf, off, v, vm) -> {
5060                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
5061                     v.stOp(wb, off, vm,
5062                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5063                 });
5064     }
5065 
5066 #if[short]
5067     /*package-private*/
5068     abstract
5069     void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
5070     @ForceInline
5071     final
5072     <M extends VectorMask<$Boxtype$>>
5073     void intoCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
5074         m.check(species());
5075         $Type$Species vsp = vspecies();
5076         VectorSupport.storeMasked(
5077             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5078             a, charArrayAddress(a, offset),
5079             this, m, a, offset,
5080             (arr, off, v, vm)
5081             -> v.stOp(arr, off, vm,
5082                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
5083     }
5084 #end[short]
5085 
5086     // End of low-level memory operations.
5087 
5088     private static
5089     void checkMaskFromIndexSize(int offset,
5090                                 $Type$Species vsp,
5091                                 VectorMask<$Boxtype$> m,
5092                                 int scale,
5093                                 int limit) {
5094         ((AbstractMask<$Boxtype$>)m)
5095             .checkIndexByLane(offset, limit, vsp.iota(), scale);
5096     }
5097 
5098     @ForceInline
5099     private void conditionalStoreNYI(int offset,
5100                                      $Type$Species vsp,
5101                                      VectorMask<$Boxtype$> m,
5102                                      int scale,
5103                                      int limit) {
5104         if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
5105             String msg =
5106                 String.format("unimplemented: store @%d in [0..%d), %s in %s",
5107                               offset, limit, m, vsp);
5108             throw new AssertionError(msg);
5109         }
5110     }
5111 
5112     /*package-private*/
5113     @Override
5114     @ForceInline
5115     final
5116     $abstractvectortype$ maybeSwap(ByteOrder bo) {
5117 #if[!byte]
5118         if (bo != NATIVE_ENDIAN) {
5119             return this.reinterpretAsBytes()
5120                 .rearrange(swapBytesShuffle())
5121                 .reinterpretAs$Type$s();
5122         }
5123 #end[!byte]
5124         return this;
5125     }
5126 
5127     static final int ARRAY_SHIFT =
5128         31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_$TYPE$_INDEX_SCALE);
5129     static final long ARRAY_BASE =
5130         Unsafe.ARRAY_$TYPE$_BASE_OFFSET;
5131 
5132     @ForceInline
5133     static long arrayAddress($type$[] a, int index) {
5134         return ARRAY_BASE + (((long)index) << ARRAY_SHIFT);
5135     }
5136 
5137 #if[short]
5138     static final int ARRAY_CHAR_SHIFT =
5139             31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_CHAR_INDEX_SCALE);
5140     static final long ARRAY_CHAR_BASE =
5141             Unsafe.ARRAY_CHAR_BASE_OFFSET;
5142 
5143     @ForceInline
5144     static long charArrayAddress(char[] a, int index) {
5145         return ARRAY_CHAR_BASE + (((long)index) << ARRAY_CHAR_SHIFT);
5146     }
5147 #end[short]
5148 
5149 #if[byte]
5150     static final int ARRAY_BOOLEAN_SHIFT =
5151             31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_BOOLEAN_INDEX_SCALE);
5152     static final long ARRAY_BOOLEAN_BASE =
5153             Unsafe.ARRAY_BOOLEAN_BASE_OFFSET;
5154 
5155     @ForceInline
5156     static long booleanArrayAddress(boolean[] a, int index) {
5157         return ARRAY_BOOLEAN_BASE + (((long)index) << ARRAY_BOOLEAN_SHIFT);
5158     }
5159 #end[byte]
5160 
5161     @ForceInline
5162     static long byteArrayAddress(byte[] a, int index) {
5163         return Unsafe.ARRAY_BYTE_BASE_OFFSET + index;
5164     }
5165 
5166     // ================================================
5167 
5168     /// Reinterpreting view methods:
5169     //   lanewise reinterpret: viewAsXVector()
5170     //   keep shape, redraw lanes: reinterpretAsEs()
5171 
5172     /**
5173      * {@inheritDoc} <!--workaround-->
5174      */
5175     @ForceInline
5176     @Override
5177     public final ByteVector reinterpretAsBytes() {
5178 #if[byte]
5179         return this;
5180 #else[byte]
5181          // Going to ByteVector, pay close attention to byte order.
5182          assert(REGISTER_ENDIAN == ByteOrder.LITTLE_ENDIAN);
5183          return asByteVectorRaw();
5184          //return asByteVectorRaw().rearrange(swapBytesShuffle());
5185 #end[byte]
5186     }
5187 
5188     /**
5189      * {@inheritDoc} <!--workaround-->
5190      */
5191     @ForceInline
5192     @Override
5193     public final $Bitstype$Vector viewAsIntegralLanes() {
5194 #if[BITWISE]
5195         return this;
5196 #else[BITWISE]
5197         LaneType ilt = LaneType.$TYPE$.asIntegral();
5198         return ($Bitstype$Vector) asVectorRaw(ilt);
5199 #end[BITWISE]
5200     }
5201 
5202     /**
5203      * {@inheritDoc} <!--workaround-->
5204 #if[byteOrShort]
5205      *
5206      * @implNote This method always throws
5207      * {@code UnsupportedOperationException}, because there is no floating
5208      * point type of the same size as {@code $type$}.  The return type
5209      * of this method is arbitrarily designated as
5210      * {@code Vector<?>}.  Future versions of this API may change the return
5211      * type if additional floating point types become available.
5212 #end[byteOrShort]
5213      */
5214     @ForceInline
5215     @Override
5216     public final
5217     {#if[byteOrShort]?Vector<?>:$Fptype$Vector}
5218     viewAsFloatingLanes() {
5219 #if[FP]
5220         return this;
5221 #else[FP]
5222         LaneType flt = LaneType.$TYPE$.asFloating();
5223 #if[!byteOrShort]
5224         return ($Fptype$Vector) asVectorRaw(flt);
5225 #else[!byteOrShort]
5226         // asFloating() will throw UnsupportedOperationException for the unsupported type $type$
5227         throw new AssertionError("Cannot reach here");
5228 #end[!byteOrShort]
5229 #end[FP]
5230     }
5231 
5232     // ================================================
5233 
5234     /// Object methods: toString, equals, hashCode
5235     //
5236     // Object methods are defined as if via Arrays.toString, etc.,
5237     // is applied to the array of elements.  Two equal vectors
5238     // are required to have equal species and equal lane values.
5239 
5240     /**
5241      * Returns a string representation of this vector, of the form
5242      * {@code "[0,1,2...]"}, reporting the lane values of this vector,
5243      * in lane order.
5244      *
5245      * The string is produced as if by a call to {@link
5246      * java.util.Arrays#toString($type$[]) Arrays.toString()},
5247      * as appropriate to the {@code $type$} array returned by
5248      * {@link #toArray this.toArray()}.
5249      *
5250      * @return a string of the form {@code "[0,1,2...]"}
5251      * reporting the lane values of this vector
5252      */
5253     @Override
5254     @ForceInline
5255     public final
5256     String toString() {
5257         // now that toArray is strongly typed, we can define this
5258         return Arrays.toString(toArray());
5259     }
5260 
5261     /**
5262      * {@inheritDoc} <!--workaround-->
5263      */
5264     @Override
5265     @ForceInline
5266     public final
5267     boolean equals(Object obj) {
5268         if (obj instanceof Vector) {
5269             Vector<?> that = (Vector<?>) obj;
5270             if (this.species().equals(that.species())) {
5271                 return this.eq(that.check(this.species())).allTrue();
5272             }
5273         }
5274         return false;
5275     }
5276 
5277     /**
5278      * {@inheritDoc} <!--workaround-->
5279      */
5280     @Override
5281     @ForceInline
5282     public final
5283     int hashCode() {
5284         // now that toArray is strongly typed, we can define this
5285         return Objects.hash(species(), Arrays.hashCode(toArray()));
5286     }
5287 
5288     // ================================================
5289 
5290     // Species
5291 
5292     /**
5293      * Class representing {@link $abstractvectortype$}'s of the same {@link VectorShape VectorShape}.
5294      */
5295     /*package-private*/
5296     static final class $Type$Species extends AbstractSpecies<$Boxtype$> {
5297         private $Type$Species(VectorShape shape,
5298                 Class<? extends $abstractvectortype$> vectorType,
5299                 Class<? extends AbstractMask<$Boxtype$>> maskType,
5300                 Function<Object, $abstractvectortype$> vectorFactory) {
5301             super(shape, LaneType.of($type$.class),
5302                   vectorType, maskType,
5303                   vectorFactory);
5304             assert(this.elementSize() == $Boxtype$.SIZE);
5305         }
5306 
5307         // Specializing overrides:
5308 
5309         @Override
5310         @ForceInline
5311         public final Class<$Boxtype$> elementType() {
5312             return $type$.class;
5313         }
5314 
5315         @Override
5316         @ForceInline
5317         final Class<$Boxtype$> genericElementType() {
5318             return $Boxtype$.class;
5319         }
5320 
5321         @SuppressWarnings("unchecked")
5322         @Override
5323         @ForceInline
5324         public final Class<? extends $Type$Vector> vectorType() {
5325             return (Class<? extends $Type$Vector>) vectorType;
5326         }
5327 
5328         @Override
5329         @ForceInline
5330         public final long checkValue(long e) {
5331             longToElementBits(e);  // only for exception
5332             return e;
5333         }
5334 
5335         /*package-private*/
5336         @Override
5337         @ForceInline
5338         final $abstractvectortype$ broadcastBits(long bits) {
5339             return ($abstractvectortype$)
5340                 VectorSupport.fromBitsCoerced(
5341                     vectorType, $type$.class, laneCount,
5342                     bits, MODE_BROADCAST, this,
5343                     (bits_, s_) -> s_.rvOp(i -> bits_));
5344         }
5345 
5346         /*package-private*/
5347         @ForceInline
5348         {#if[long]?public }final $abstractvectortype$ broadcast($type$ e) {
5349             return broadcastBits(toBits(e));
5350         }
5351 
5352 #if[!long]
5353         @Override
5354         @ForceInline
5355         public final $abstractvectortype$ broadcast(long e) {
5356             return broadcastBits(longToElementBits(e));
5357         }
5358 #end[!long]
5359 
5360         /*package-private*/
5361         final @Override
5362         @ForceInline
5363         long longToElementBits(long value) {
5364 #if[long]
5365             // In this case, the conversion can never fail.
5366             return value;
5367 #else[long]
5368             // Do the conversion, and then test it for failure.
5369             $type$ e = ($type$) value;
5370             if ((long) e != value) {
5371                 throw badElementBits(value, e);
5372             }
5373             return toBits(e);
5374 #end[long]
5375         }
5376 
5377         /*package-private*/
5378         @ForceInline
5379         static long toIntegralChecked($type$ e, boolean convertToInt) {
5380             long value = convertToInt ? (int) e : (long) e;
5381             if (($type$) value != e) {
5382                 throw badArrayBits(e, convertToInt, value);
5383             }
5384             return value;
5385         }
5386 
5387         /* this non-public one is for internal conversions */
5388         @Override
5389         @ForceInline
5390         final $abstractvectortype$ fromIntValues(int[] values) {
5391             VectorIntrinsics.requireLength(values.length, laneCount);
5392             $type$[] va = new $type$[laneCount()];
5393             for (int i = 0; i < va.length; i++) {
5394                 int lv = values[i];
5395                 $type$ v = ($type$) lv;
5396                 va[i] = v;
5397                 if ((int)v != lv) {
5398                     throw badElementBits(lv, v);
5399                 }
5400             }
5401             return dummyVector().fromArray0(va, 0);
5402         }
5403 
5404         // Virtual constructors
5405 
5406         @ForceInline
5407         @Override final
5408         public $abstractvectortype$ fromArray(Object a, int offset) {
5409             // User entry point:  Be careful with inputs.
5410             return $abstractvectortype$
5411                 .fromArray(this, ($type$[]) a, offset);
5412         }
5413 
5414         @ForceInline
5415         @Override final
5416         $abstractvectortype$ dummyVector() {
5417             return ($abstractvectortype$) super.dummyVector();
5418         }
5419 
5420         /*package-private*/
5421         final @Override
5422         @ForceInline
5423         $abstractvectortype$ rvOp(RVOp f) {
5424             $type$[] res = new $type$[laneCount()];
5425             for (int i = 0; i < res.length; i++) {
5426                 $bitstype$ bits = {#if[!long]?($bitstype$)} f.apply(i);
5427                 res[i] = fromBits(bits);
5428             }
5429             return dummyVector().vectorFactory(res);
5430         }
5431 
5432         $Type$Vector vOp(FVOp f) {
5433             $type$[] res = new $type$[laneCount()];
5434             for (int i = 0; i < res.length; i++) {
5435                 res[i] = f.apply(i);
5436             }
5437             return dummyVector().vectorFactory(res);
5438         }
5439 
5440         $Type$Vector vOp(VectorMask<$Boxtype$> m, FVOp f) {
5441             $type$[] res = new $type$[laneCount()];
5442             boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
5443             for (int i = 0; i < res.length; i++) {
5444                 if (mbits[i]) {
5445                     res[i] = f.apply(i);
5446                 }
5447             }
5448             return dummyVector().vectorFactory(res);
5449         }
5450 
5451         /*package-private*/
5452         @ForceInline
5453         <M> $abstractvectortype$ ldOp(M memory, int offset,
5454                                       FLdOp<M> f) {
5455             return dummyVector().ldOp(memory, offset, f);
5456         }
5457 
5458         /*package-private*/
5459         @ForceInline
5460         <M> $abstractvectortype$ ldOp(M memory, int offset,
5461                                       VectorMask<$Boxtype$> m,
5462                                       FLdOp<M> f) {
5463             return dummyVector().ldOp(memory, offset, m, f);
5464         }
5465 
5466         /*package-private*/
5467         @ForceInline
5468         <M> void stOp(M memory, int offset, FStOp<M> f) {
5469             dummyVector().stOp(memory, offset, f);
5470         }
5471 
5472         /*package-private*/
5473         @ForceInline
5474         <M> void stOp(M memory, int offset,
5475                       AbstractMask<$Boxtype$> m,
5476                       FStOp<M> f) {
5477             dummyVector().stOp(memory, offset, m, f);
5478         }
5479 
5480         // N.B. Make sure these constant vectors and
5481         // masks load up correctly into registers.
5482         //
5483         // Also, see if we can avoid all that switching.
5484         // Could we cache both vectors and both masks in
5485         // this species object?
5486 
5487         // Zero and iota vector access
5488         @Override
5489         @ForceInline
5490         public final $abstractvectortype$ zero() {
5491             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5492                 return $Type$MaxVector.ZERO;
5493             switch (vectorBitSize()) {
5494                 case 64: return $Type$64Vector.ZERO;
5495                 case 128: return $Type$128Vector.ZERO;
5496                 case 256: return $Type$256Vector.ZERO;
5497                 case 512: return $Type$512Vector.ZERO;
5498             }
5499             throw new AssertionError();
5500         }
5501 
5502         @Override
5503         @ForceInline
5504         public final $abstractvectortype$ iota() {
5505             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5506                 return $Type$MaxVector.IOTA;
5507             switch (vectorBitSize()) {
5508                 case 64: return $Type$64Vector.IOTA;
5509                 case 128: return $Type$128Vector.IOTA;
5510                 case 256: return $Type$256Vector.IOTA;
5511                 case 512: return $Type$512Vector.IOTA;
5512             }
5513             throw new AssertionError();
5514         }
5515 
5516         // Mask access
5517         @Override
5518         @ForceInline
5519         public final VectorMask<$Boxtype$> maskAll(boolean bit) {
5520             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5521                 return $Type$MaxVector.$Type$MaxMask.maskAll(bit);
5522             switch (vectorBitSize()) {
5523                 case 64: return $Type$64Vector.$Type$64Mask.maskAll(bit);
5524                 case 128: return $Type$128Vector.$Type$128Mask.maskAll(bit);
5525                 case 256: return $Type$256Vector.$Type$256Mask.maskAll(bit);
5526                 case 512: return $Type$512Vector.$Type$512Mask.maskAll(bit);
5527             }
5528             throw new AssertionError();
5529         }
5530     }
5531 
5532     /**
5533      * Finds a species for an element type of {@code $type$} and shape.
5534      *
5535      * @param s the shape
5536      * @return a species for an element type of {@code $type$} and shape
5537      * @throws IllegalArgumentException if no such species exists for the shape
5538      */
5539     static $Type$Species species(VectorShape s) {
5540         Objects.requireNonNull(s);
5541         switch (s.switchKey) {
5542             case VectorShape.SK_64_BIT: return ($Type$Species) SPECIES_64;
5543             case VectorShape.SK_128_BIT: return ($Type$Species) SPECIES_128;
5544             case VectorShape.SK_256_BIT: return ($Type$Species) SPECIES_256;
5545             case VectorShape.SK_512_BIT: return ($Type$Species) SPECIES_512;
5546             case VectorShape.SK_Max_BIT: return ($Type$Species) SPECIES_MAX;
5547             default: throw new IllegalArgumentException("Bad shape: " + s);
5548         }
5549     }
5550 
5551     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_64_BIT VectorShape.S_64_BIT}. */
5552     public static final VectorSpecies<$Boxtype$> SPECIES_64
5553         = new $Type$Species(VectorShape.S_64_BIT,
5554                             $Type$64Vector.class,
5555                             $Type$64Vector.$Type$64Mask.class,
5556                             $Type$64Vector::new);
5557 
5558     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
5559     public static final VectorSpecies<$Boxtype$> SPECIES_128
5560         = new $Type$Species(VectorShape.S_128_BIT,
5561                             $Type$128Vector.class,
5562                             $Type$128Vector.$Type$128Mask.class,
5563                             $Type$128Vector::new);
5564 
5565     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
5566     public static final VectorSpecies<$Boxtype$> SPECIES_256
5567         = new $Type$Species(VectorShape.S_256_BIT,
5568                             $Type$256Vector.class,
5569                             $Type$256Vector.$Type$256Mask.class,
5570                             $Type$256Vector::new);
5571 
5572     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
5573     public static final VectorSpecies<$Boxtype$> SPECIES_512
5574         = new $Type$Species(VectorShape.S_512_BIT,
5575                             $Type$512Vector.class,
5576                             $Type$512Vector.$Type$512Mask.class,
5577                             $Type$512Vector::new);
5578 
5579     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
5580     public static final VectorSpecies<$Boxtype$> SPECIES_MAX
5581         = new $Type$Species(VectorShape.S_Max_BIT,
5582                             $Type$MaxVector.class,
5583                             $Type$MaxVector.$Type$MaxMask.class,
5584                             $Type$MaxVector::new);
5585 
5586     /**
5587      * Preferred species for {@link $Type$Vector}s.
5588      * A preferred species is a species of maximal bit-size for the platform.
5589      */
5590     public static final VectorSpecies<$Boxtype$> SPECIES_PREFERRED
5591         = ($Type$Species) VectorSpecies.ofPreferred($type$.class);
5592 }