1 /*
   2  * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.nio.ByteOrder;
  29 import java.nio.ReadOnlyBufferException;
  30 import java.util.Arrays;
  31 import java.util.Objects;
  32 import java.util.function.Function;
  33 import java.util.function.UnaryOperator;
  34 
  35 import jdk.internal.misc.ScopedMemoryAccess;
  36 import jdk.internal.misc.Unsafe;
  37 import jdk.internal.vm.annotation.ForceInline;
  38 import jdk.internal.vm.vector.VectorSupport;
  39 
  40 import static jdk.internal.vm.vector.VectorSupport.*;
  41 import static jdk.incubator.vector.VectorIntrinsics.*;
  42 
  43 import static jdk.incubator.vector.VectorOperators.*;
  44 
  45 #warn This file is preprocessed before being compiled
  46 
  47 /**
  48  * A specialized {@link Vector} representing an ordered immutable sequence of
  49  * {@code $type$} values.
  50  */
  51 @SuppressWarnings("cast")  // warning: redundant cast
  52 public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
  53 
  54     $abstractvectortype$($type$[] vec) {
  55         super(vec);
  56     }
  57 
  58 #if[FP]
  59     static final int FORBID_OPCODE_KIND = VO_NOFP;
  60 #else[FP]
  61     static final int FORBID_OPCODE_KIND = VO_ONLYFP;
  62 #end[FP]
  63 
  64     @ForceInline
  65     static int opCode(Operator op) {
  66         return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
  67     }
  68     @ForceInline
  69     static int opCode(Operator op, int requireKind) {
  70         requireKind |= VO_OPCODE_VALID;
  71         return VectorOperators.opCode(op, requireKind, FORBID_OPCODE_KIND);
  72     }
  73     @ForceInline
  74     static boolean opKind(Operator op, int bit) {
  75         return VectorOperators.opKind(op, bit);
  76     }
  77 
  78     // Virtualized factories and operators,
  79     // coded with portable definitions.
  80     // These are all @ForceInline in case
  81     // they need to be used performantly.
  82     // The various shape-specific subclasses
  83     // also specialize them by wrapping
  84     // them in a call like this:
  85     //    return (Byte128Vector)
  86     //       super.bOp((Byte128Vector) o);
  87     // The purpose of that is to forcibly inline
  88     // the generic definition from this file
  89     // into a sharply type- and size-specific
  90     // wrapper in the subclass file, so that
  91     // the JIT can specialize the code.
  92     // The code is only inlined and expanded
  93     // if it gets hot.  Think of it as a cheap
  94     // and lazy version of C++ templates.
  95 
  96     // Virtualized getter
  97 
  98     /*package-private*/
  99     abstract $type$[] vec();
 100 
 101     // Virtualized constructors
 102 
 103     /**
 104      * Build a vector directly using my own constructor.
 105      * It is an error if the array is aliased elsewhere.
 106      */
 107     /*package-private*/
 108     abstract $abstractvectortype$ vectorFactory($type$[] vec);
 109 
 110     /**
 111      * Build a mask directly using my species.
 112      * It is an error if the array is aliased elsewhere.
 113      */
 114     /*package-private*/
 115     @ForceInline
 116     final
 117     AbstractMask<$Boxtype$> maskFactory(boolean[] bits) {
 118         return vspecies().maskFactory(bits);
 119     }
 120 
 121     // Constant loader (takes dummy as vector arg)
 122     interface FVOp {
 123         $type$ apply(int i);
 124     }
 125 
 126     /*package-private*/
 127     @ForceInline
 128     final
 129     $abstractvectortype$ vOp(FVOp f) {
 130         $type$[] res = new $type$[length()];
 131         for (int i = 0; i < res.length; i++) {
 132             res[i] = f.apply(i);
 133         }
 134         return vectorFactory(res);
 135     }
 136 
 137     @ForceInline
 138     final
 139     $abstractvectortype$ vOp(VectorMask<$Boxtype$> m, FVOp f) {
 140         $type$[] res = new $type$[length()];
 141         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 142         for (int i = 0; i < res.length; i++) {
 143             if (mbits[i]) {
 144                 res[i] = f.apply(i);
 145             }
 146         }
 147         return vectorFactory(res);
 148     }
 149 
 150     // Unary operator
 151 
 152     /*package-private*/
 153     interface FUnOp {
 154         $type$ apply(int i, $type$ a);
 155     }
 156 
 157     /*package-private*/
 158     abstract
 159     $abstractvectortype$ uOp(FUnOp f);
 160     @ForceInline
 161     final
 162     $abstractvectortype$ uOpTemplate(FUnOp f) {
 163         $type$[] vec = vec();
 164         $type$[] res = new $type$[length()];
 165         for (int i = 0; i < res.length; i++) {
 166             res[i] = f.apply(i, vec[i]);
 167         }
 168         return vectorFactory(res);
 169     }
 170 
 171     /*package-private*/
 172     abstract
 173     $abstractvectortype$ uOp(VectorMask<$Boxtype$> m,
 174                              FUnOp f);
 175     @ForceInline
 176     final
 177     $abstractvectortype$ uOpTemplate(VectorMask<$Boxtype$> m,
 178                                      FUnOp f) {
 179         if (m == null) {
 180             return uOpTemplate(f);
 181         }
 182         $type$[] vec = vec();
 183         $type$[] res = new $type$[length()];
 184         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 185         for (int i = 0; i < res.length; i++) {
 186             res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
 187         }
 188         return vectorFactory(res);
 189     }
 190 
 191     // Binary operator
 192 
 193     /*package-private*/
 194     interface FBinOp {
 195         $type$ apply(int i, $type$ a, $type$ b);
 196     }
 197 
 198     /*package-private*/
 199     abstract
 200     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 201                              FBinOp f);
 202     @ForceInline
 203     final
 204     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 205                                      FBinOp f) {
 206         $type$[] res = new $type$[length()];
 207         $type$[] vec1 = this.vec();
 208         $type$[] vec2 = (($abstractvectortype$)o).vec();
 209         for (int i = 0; i < res.length; i++) {
 210             res[i] = f.apply(i, vec1[i], vec2[i]);
 211         }
 212         return vectorFactory(res);
 213     }
 214 
 215     /*package-private*/
 216     abstract
 217     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 218                              VectorMask<$Boxtype$> m,
 219                              FBinOp f);
 220     @ForceInline
 221     final
 222     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 223                                      VectorMask<$Boxtype$> m,
 224                                      FBinOp f) {
 225         if (m == null) {
 226             return bOpTemplate(o, f);
 227         }
 228         $type$[] res = new $type$[length()];
 229         $type$[] vec1 = this.vec();
 230         $type$[] vec2 = (($abstractvectortype$)o).vec();
 231         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 232         for (int i = 0; i < res.length; i++) {
 233             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
 234         }
 235         return vectorFactory(res);
 236     }
 237 
 238     // Ternary operator
 239 
 240     /*package-private*/
 241     interface FTriOp {
 242         $type$ apply(int i, $type$ a, $type$ b, $type$ c);
 243     }
 244 
 245     /*package-private*/
 246     abstract
 247     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 248                              Vector<$Boxtype$> o2,
 249                              FTriOp f);
 250     @ForceInline
 251     final
 252     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 253                                      Vector<$Boxtype$> o2,
 254                                      FTriOp f) {
 255         $type$[] res = new $type$[length()];
 256         $type$[] vec1 = this.vec();
 257         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 258         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 259         for (int i = 0; i < res.length; i++) {
 260             res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
 261         }
 262         return vectorFactory(res);
 263     }
 264 
 265     /*package-private*/
 266     abstract
 267     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 268                              Vector<$Boxtype$> o2,
 269                              VectorMask<$Boxtype$> m,
 270                              FTriOp f);
 271     @ForceInline
 272     final
 273     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 274                                      Vector<$Boxtype$> o2,
 275                                      VectorMask<$Boxtype$> m,
 276                                      FTriOp f) {
 277         if (m == null) {
 278             return tOpTemplate(o1, o2, f);
 279         }
 280         $type$[] res = new $type$[length()];
 281         $type$[] vec1 = this.vec();
 282         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 283         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 284         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 285         for (int i = 0; i < res.length; i++) {
 286             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
 287         }
 288         return vectorFactory(res);
 289     }
 290 
 291     // Reduction operator
 292 
 293     /*package-private*/
 294     abstract
 295     $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f);
 296 
 297     @ForceInline
 298     final
 299     $type$ rOpTemplate($type$ v, VectorMask<$Boxtype$> m, FBinOp f) {
 300         if (m == null) {
 301             return rOpTemplate(v, f);
 302         }
 303         $type$[] vec = vec();
 304         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 305         for (int i = 0; i < vec.length; i++) {
 306             v = mbits[i] ? f.apply(i, v, vec[i]) : v;
 307         }
 308         return v;
 309     }
 310 
 311     @ForceInline
 312     final
 313     $type$ rOpTemplate($type$ v, FBinOp f) {
 314         $type$[] vec = vec();
 315         for (int i = 0; i < vec.length; i++) {
 316             v = f.apply(i, v, vec[i]);
 317         }
 318         return v;
 319     }
 320 
 321     // Memory reference
 322 
 323     /*package-private*/
 324     interface FLdOp<M> {
 325         $type$ apply(M memory, int offset, int i);
 326     }
 327 
 328     /*package-private*/
 329     @ForceInline
 330     final
 331     <M> $abstractvectortype$ ldOp(M memory, int offset,
 332                                   FLdOp<M> f) {
 333         //dummy; no vec = vec();
 334         $type$[] res = new $type$[length()];
 335         for (int i = 0; i < res.length; i++) {
 336             res[i] = f.apply(memory, offset, i);
 337         }
 338         return vectorFactory(res);
 339     }
 340 
 341     /*package-private*/
 342     @ForceInline
 343     final
 344     <M> $abstractvectortype$ ldOp(M memory, int offset,
 345                                   VectorMask<$Boxtype$> m,
 346                                   FLdOp<M> f) {
 347         //$type$[] vec = vec();
 348         $type$[] res = new $type$[length()];
 349         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 350         for (int i = 0; i < res.length; i++) {
 351             if (mbits[i]) {
 352                 res[i] = f.apply(memory, offset, i);
 353             }
 354         }
 355         return vectorFactory(res);
 356     }
 357 
 358     interface FStOp<M> {
 359         void apply(M memory, int offset, int i, $type$ a);
 360     }
 361 
 362     /*package-private*/
 363     @ForceInline
 364     final
 365     <M> void stOp(M memory, int offset,
 366                   FStOp<M> f) {
 367         $type$[] vec = vec();
 368         for (int i = 0; i < vec.length; i++) {
 369             f.apply(memory, offset, i, vec[i]);
 370         }
 371     }
 372 
 373     /*package-private*/
 374     @ForceInline
 375     final
 376     <M> void stOp(M memory, int offset,
 377                   VectorMask<$Boxtype$> m,
 378                   FStOp<M> f) {
 379         $type$[] vec = vec();
 380         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 381         for (int i = 0; i < vec.length; i++) {
 382             if (mbits[i]) {
 383                 f.apply(memory, offset, i, vec[i]);
 384             }
 385         }
 386     }
 387 
 388     // Binary test
 389 
 390     /*package-private*/
 391     interface FBinTest {
 392         boolean apply(int cond, int i, $type$ a, $type$ b);
 393     }
 394 
 395     /*package-private*/
 396     @ForceInline
 397     final
 398     AbstractMask<$Boxtype$> bTest(int cond,
 399                                   Vector<$Boxtype$> o,
 400                                   FBinTest f) {
 401         $type$[] vec1 = vec();
 402         $type$[] vec2 = (($abstractvectortype$)o).vec();
 403         boolean[] bits = new boolean[length()];
 404         for (int i = 0; i < length(); i++){
 405             bits[i] = f.apply(cond, i, vec1[i], vec2[i]);
 406         }
 407         return maskFactory(bits);
 408     }
 409 
 410 #if[BITWISE]
 411     /*package-private*/
 412     @ForceInline
 413     static $type$ rotateLeft($type$ a, int n) {
 414 #if[intOrLong]
 415         return $Boxtype$.rotateLeft(a, n);
 416 #else[intOrLong]
 417         return ($type$)((((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) << (n & $Boxtype$.SIZE-1)) | (((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) >>> ($Boxtype$.SIZE - (n & $Boxtype$.SIZE-1))));
 418 #end[intOrLong]
 419     }
 420 
 421     /*package-private*/
 422     @ForceInline
 423     static $type$ rotateRight($type$ a, int n) {
 424 #if[intOrLong]
 425         return $Boxtype$.rotateRight(a, n);
 426 #else[intOrLong]
 427         return ($type$)((((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) >>> (n & $Boxtype$.SIZE-1)) | (((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) << ($Boxtype$.SIZE - (n & $Boxtype$.SIZE-1))));
 428 #end[intOrLong]
 429     }
 430 #end[BITWISE]
 431 
 432     /*package-private*/
 433     @Override
 434     abstract $Type$Species vspecies();
 435 
 436     /*package-private*/
 437     @ForceInline
 438     static long toBits($type$ e) {
 439         return {#if[FP]? $Type$.$type$ToRaw$Bitstype$Bits(e): e};
 440     }
 441 
 442     /*package-private*/
 443     @ForceInline
 444     static $type$ fromBits(long bits) {
 445         return {#if[FP]?$Type$.$bitstype$BitsTo$Type$}(($bitstype$)bits);
 446     }
 447 
 448     // Static factories (other than memory operations)
 449 
 450     // Note: A surprising behavior in javadoc
 451     // sometimes makes a lone /** {@inheritDoc} */
 452     // comment drop the method altogether,
 453     // apparently if the method mentions an
 454     // parameter or return type of Vector<$Boxtype$>
 455     // instead of Vector<E> as originally specified.
 456     // Adding an empty HTML fragment appears to
 457     // nudge javadoc into providing the desired
 458     // inherited documentation.  We use the HTML
 459     // comment <!--workaround--> for this.
 460 
 461     /**
 462      * Returns a vector of the given species
 463      * where all lane elements are set to
 464      * zero, the default primitive value.
 465      *
 466      * @param species species of the desired zero vector
 467      * @return a zero vector
 468      */
 469     @ForceInline
 470     public static $abstractvectortype$ zero(VectorSpecies<$Boxtype$> species) {
 471         $Type$Species vsp = ($Type$Species) species;
 472 #if[FP]
 473         return VectorSupport.broadcastCoerced(vsp.vectorType(), $type$.class, species.length(),
 474                         toBits(0.0f), vsp,
 475                         ((bits_, s_) -> s_.rvOp(i -> bits_)));
 476 #else[FP]
 477         return VectorSupport.broadcastCoerced(vsp.vectorType(), $type$.class, species.length(),
 478                                 0, vsp,
 479                                 ((bits_, s_) -> s_.rvOp(i -> bits_)));
 480 #end[FP]
 481     }
 482 
 483     /**
 484      * Returns a vector of the same species as this one
 485      * where all lane elements are set to
 486      * the primitive value {@code e}.
 487      *
 488      * The contents of the current vector are discarded;
 489      * only the species is relevant to this operation.
 490      *
 491      * <p> This method returns the value of this expression:
 492      * {@code $abstractvectortype$.broadcast(this.species(), e)}.
 493      *
 494      * @apiNote
 495      * Unlike the similar method named {@code broadcast()}
 496      * in the supertype {@code Vector}, this method does not
 497      * need to validate its argument, and cannot throw
 498      * {@code IllegalArgumentException}.  This method is
 499      * therefore preferable to the supertype method.
 500      *
 501      * @param e the value to broadcast
 502      * @return a vector where all lane elements are set to
 503      *         the primitive value {@code e}
 504      * @see #broadcast(VectorSpecies,long)
 505      * @see Vector#broadcast(long)
 506      * @see VectorSpecies#broadcast(long)
 507      */
 508     public abstract $abstractvectortype$ broadcast($type$ e);
 509 
 510     /**
 511      * Returns a vector of the given species
 512      * where all lane elements are set to
 513      * the primitive value {@code e}.
 514      *
 515      * @param species species of the desired vector
 516      * @param e the value to broadcast
 517      * @return a vector where all lane elements are set to
 518      *         the primitive value {@code e}
 519      * @see #broadcast(long)
 520      * @see Vector#broadcast(long)
 521      * @see VectorSpecies#broadcast(long)
 522      */
 523     @ForceInline
 524     public static $abstractvectortype$ broadcast(VectorSpecies<$Boxtype$> species, $type$ e) {
 525         $Type$Species vsp = ($Type$Species) species;
 526         return vsp.broadcast(e);
 527     }
 528 
 529     /*package-private*/
 530     @ForceInline
 531     final $abstractvectortype$ broadcastTemplate($type$ e) {
 532         $Type$Species vsp = vspecies();
 533         return vsp.broadcast(e);
 534     }
 535 
 536 #if[!long]
 537     /**
 538      * {@inheritDoc} <!--workaround-->
 539      * @apiNote
 540      * When working with vector subtypes like {@code $abstractvectortype$},
 541      * {@linkplain #broadcast($type$) the more strongly typed method}
 542      * is typically selected.  It can be explicitly selected
 543      * using a cast: {@code v.broadcast(($type$)e)}.
 544      * The two expressions will produce numerically identical results.
 545      */
 546     @Override
 547     public abstract $abstractvectortype$ broadcast(long e);
 548 
 549     /**
 550      * Returns a vector of the given species
 551      * where all lane elements are set to
 552      * the primitive value {@code e}.
 553      *
 554      * The {@code long} value must be accurately representable
 555      * by the {@code ETYPE} of the vector species, so that
 556      * {@code e==(long)(ETYPE)e}.
 557      *
 558      * @param species species of the desired vector
 559      * @param e the value to broadcast
 560      * @return a vector where all lane elements are set to
 561      *         the primitive value {@code e}
 562      * @throws IllegalArgumentException
 563      *         if the given {@code long} value cannot
 564      *         be represented by the vector's {@code ETYPE}
 565      * @see #broadcast(VectorSpecies,$type$)
 566      * @see VectorSpecies#checkValue(long)
 567      */
 568     @ForceInline
 569     public static $abstractvectortype$ broadcast(VectorSpecies<$Boxtype$> species, long e) {
 570         $Type$Species vsp = ($Type$Species) species;
 571         return vsp.broadcast(e);
 572     }
 573 
 574     /*package-private*/
 575     @ForceInline
 576     final $abstractvectortype$ broadcastTemplate(long e) {
 577         return vspecies().broadcast(e);
 578     }
 579 #end[!long]
 580 
 581     // Unary lanewise support
 582 
 583     /**
 584      * {@inheritDoc} <!--workaround-->
 585      */
 586     public abstract
 587     $abstractvectortype$ lanewise(VectorOperators.Unary op);
 588 
 589     @ForceInline
 590     final
 591     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op) {
 592         if (opKind(op, VO_SPECIAL)) {
 593             if (op == ZOMO) {
 594                 return blend(broadcast(-1), compare(NE, 0));
 595             }
 596 #if[BITWISE]
 597             if (op == NOT) {
 598                 return broadcast(-1).lanewise(XOR, this);
 599             } else if (op == NEG) {
 600                 // FIXME: Support this in the JIT.
 601                 return broadcast(0).lanewise(SUB, this);
 602             }
 603 #end[BITWISE]
 604         }
 605         int opc = opCode(op);
 606         return VectorSupport.unaryOp(
 607             opc, getClass(), null, $type$.class, length(),
 608             this, null,
 609             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));
 610     }
 611 
 612     /**
 613      * {@inheritDoc} <!--workaround-->
 614      */
 615     @Override
 616     public abstract
 617     $abstractvectortype$ lanewise(VectorOperators.Unary op,
 618                                   VectorMask<$Boxtype$> m);
 619     @ForceInline
 620     final
 621     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op,
 622                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
 623                                           VectorMask<$Boxtype$> m) {
 624         m.check(maskClass, this);
 625         if (opKind(op, VO_SPECIAL)) {
 626             if (op == ZOMO) {
 627                 return blend(broadcast(-1), compare(NE, 0, m));
 628             }
 629 #if[BITWISE]
 630             if (op == NOT) {
 631                 return lanewise(XOR, broadcast(-1), m);
 632             } else if (op == NEG) {
 633                 return lanewise(NOT, m).lanewise(ADD, broadcast(1), m);
 634             }
 635 #end[BITWISE]
 636         }
 637         int opc = opCode(op);
 638         return VectorSupport.unaryOp(
 639             opc, getClass(), maskClass, $type$.class, length(),
 640             this, m,
 641             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));
 642     }
 643 
 644     private static final
 645     ImplCache<Unary, UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 646         UN_IMPL = new ImplCache<>(Unary.class, $Type$Vector.class);
 647 
 648     private static UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> unaryOperations(int opc_) {
 649         switch (opc_) {
 650             case VECTOR_OP_NEG: return (v0, m) ->
 651                     v0.uOp(m, (i, a) -> ($type$) -a);
 652             case VECTOR_OP_ABS: return (v0, m) ->
 653                     v0.uOp(m, (i, a) -> ($type$) Math.abs(a));
 654 #if[FP]
 655             case VECTOR_OP_SIN: return (v0, m) ->
 656                     v0.uOp(m, (i, a) -> ($type$) Math.sin(a));
 657             case VECTOR_OP_COS: return (v0, m) ->
 658                     v0.uOp(m, (i, a) -> ($type$) Math.cos(a));
 659             case VECTOR_OP_TAN: return (v0, m) ->
 660                     v0.uOp(m, (i, a) -> ($type$) Math.tan(a));
 661             case VECTOR_OP_ASIN: return (v0, m) ->
 662                     v0.uOp(m, (i, a) -> ($type$) Math.asin(a));
 663             case VECTOR_OP_ACOS: return (v0, m) ->
 664                     v0.uOp(m, (i, a) -> ($type$) Math.acos(a));
 665             case VECTOR_OP_ATAN: return (v0, m) ->
 666                     v0.uOp(m, (i, a) -> ($type$) Math.atan(a));
 667             case VECTOR_OP_EXP: return (v0, m) ->
 668                     v0.uOp(m, (i, a) -> ($type$) Math.exp(a));
 669             case VECTOR_OP_LOG: return (v0, m) ->
 670                     v0.uOp(m, (i, a) -> ($type$) Math.log(a));
 671             case VECTOR_OP_LOG10: return (v0, m) ->
 672                     v0.uOp(m, (i, a) -> ($type$) Math.log10(a));
 673             case VECTOR_OP_SQRT: return (v0, m) ->
 674                     v0.uOp(m, (i, a) -> ($type$) Math.sqrt(a));
 675             case VECTOR_OP_CBRT: return (v0, m) ->
 676                     v0.uOp(m, (i, a) -> ($type$) Math.cbrt(a));
 677             case VECTOR_OP_SINH: return (v0, m) ->
 678                     v0.uOp(m, (i, a) -> ($type$) Math.sinh(a));
 679             case VECTOR_OP_COSH: return (v0, m) ->
 680                     v0.uOp(m, (i, a) -> ($type$) Math.cosh(a));
 681             case VECTOR_OP_TANH: return (v0, m) ->
 682                     v0.uOp(m, (i, a) -> ($type$) Math.tanh(a));
 683             case VECTOR_OP_EXPM1: return (v0, m) ->
 684                     v0.uOp(m, (i, a) -> ($type$) Math.expm1(a));
 685             case VECTOR_OP_LOG1P: return (v0, m) ->
 686                     v0.uOp(m, (i, a) -> ($type$) Math.log1p(a));
 687 #end[FP]
 688             default: return null;
 689         }
 690     }
 691 
 692     // Binary lanewise support
 693 
 694     /**
 695      * {@inheritDoc} <!--workaround-->
 696      * @see #lanewise(VectorOperators.Binary,$type$)
 697      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 698      */
 699     @Override
 700     public abstract
 701     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 702                                   Vector<$Boxtype$> v);
 703     @ForceInline
 704     final
 705     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 706                                           Vector<$Boxtype$> v) {
 707         $abstractvectortype$ that = ($abstractvectortype$) v;
 708         that.check(this);
 709 
 710         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 711             if (op == FIRST_NONZERO) {
 712                 // FIXME: Support this in the JIT.
 713                 VectorMask<$Boxbitstype$> thisNZ
 714                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 715                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 716                 op = OR_UNCHECKED;
 717 #if[FP]
 718                 // FIXME: Support OR_UNCHECKED on float/double also!
 719                 return this.viewAsIntegralLanes()
 720                     .lanewise(op, that.viewAsIntegralLanes())
 721                     .viewAsFloatingLanes();
 722 #end[FP]
 723             }
 724 #if[BITWISE]
 725 #if[!FP]
 726             if (opKind(op, VO_SHIFT)) {
 727                 // As per shift specification for Java, mask the shift count.
 728                 // This allows the JIT to ignore some ISA details.
 729                 that = that.lanewise(AND, SHIFT_MASK);
 730             }
 731 #end[!FP]
 732             if (op == AND_NOT) {
 733                 // FIXME: Support this in the JIT.
 734                 that = that.lanewise(NOT);
 735                 op = AND;
 736             } else if (op == DIV) {
 737                 VectorMask<$Boxtype$> eqz = that.eq(($type$) 0);
 738                 if (eqz.anyTrue()) {
 739                     throw that.divZeroException();
 740                 }
 741             }
 742 #end[BITWISE]
 743         }
 744 
 745         int opc = opCode(op);
 746         return VectorSupport.binaryOp(
 747             opc, getClass(), null, $type$.class, length(),
 748             this, that, null,
 749             BIN_IMPL.find(op, opc, $abstractvectortype$::binaryOperations));
 750     }
 751 
 752     /**
 753      * {@inheritDoc} <!--workaround-->
 754      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 755      */
 756     @Override
 757     public abstract
 758     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 759                                   Vector<$Boxtype$> v,
 760                                   VectorMask<$Boxtype$> m);
 761     @ForceInline
 762     final
 763     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 764                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
 765                                           Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 766         $abstractvectortype$ that = ($abstractvectortype$) v;
 767         that.check(this);
 768         m.check(maskClass, this);
 769 
 770         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 771             if (op == FIRST_NONZERO) {
 772 #if[FP]
 773                 return blend(lanewise(op, v), m);
 774 #else[FP]
 775                 // FIXME: Support this in the JIT.
 776                 VectorMask<$Boxbitstype$> thisNZ
 777                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 778                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 779                 op = OR_UNCHECKED;
 780 #end[FP]
 781             }
 782 #if[BITWISE]
 783 #if[!FP]
 784             if (opKind(op, VO_SHIFT)) {
 785                 // As per shift specification for Java, mask the shift count.
 786                 // This allows the JIT to ignore some ISA details.
 787                 that = that.lanewise(AND, SHIFT_MASK);
 788             }
 789 #end[!FP]
 790             if (op == AND_NOT) {
 791                 // FIXME: Support this in the JIT.
 792                 that = that.lanewise(NOT);
 793                 op = AND;
 794             } else if (op == DIV) {
 795                 VectorMask<$Boxtype$> eqz = that.eq(($type$)0);
 796                 if (eqz.and(m).anyTrue()) {
 797                     throw that.divZeroException();
 798                 }
 799                 // suppress div/0 exceptions in unset lanes
 800                 that = that.lanewise(NOT, eqz);
 801             }
 802 #end[BITWISE]
 803         }
 804 
 805         int opc = opCode(op);
 806         return VectorSupport.binaryOp(
 807             opc, getClass(), maskClass, $type$.class, length(),
 808             this, that, m,
 809             BIN_IMPL.find(op, opc, $abstractvectortype$::binaryOperations));
 810     }
 811 
 812     private static final
 813     ImplCache<Binary, BinaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 814         BIN_IMPL = new ImplCache<>(Binary.class, $Type$Vector.class);
 815 
 816     private static BinaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> binaryOperations(int opc_) {
 817         switch (opc_) {
 818             case VECTOR_OP_ADD: return (v0, v1, vm) ->
 819                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a + b));
 820             case VECTOR_OP_SUB: return (v0, v1, vm) ->
 821                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a - b));
 822             case VECTOR_OP_MUL: return (v0, v1, vm) ->
 823                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a * b));
 824             case VECTOR_OP_DIV: return (v0, v1, vm) ->
 825                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a / b));
 826             case VECTOR_OP_MAX: return (v0, v1, vm) ->
 827                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.max(a, b));
 828             case VECTOR_OP_MIN: return (v0, v1, vm) ->
 829                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.min(a, b));
 830 #if[BITWISE]
 831             case VECTOR_OP_AND: return (v0, v1, vm) ->
 832                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a & b));
 833             case VECTOR_OP_OR: return (v0, v1, vm) ->
 834                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a | b));
 835             case VECTOR_OP_XOR: return (v0, v1, vm) ->
 836                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a ^ b));
 837             case VECTOR_OP_LSHIFT: return (v0, v1, vm) ->
 838                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a << n));
 839             case VECTOR_OP_RSHIFT: return (v0, v1, vm) ->
 840                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a >> n));
 841             case VECTOR_OP_URSHIFT: return (v0, v1, vm) ->
 842                     v0.bOp(v1, vm, (i, a, n) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 843             case VECTOR_OP_LROTATE: return (v0, v1, vm) ->
 844                     v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
 845             case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
 846                     v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
 847 #end[BITWISE]
 848 #if[FP]
 849             case VECTOR_OP_OR: return (v0, v1, vm) ->
 850                     v0.bOp(v1, vm, (i, a, b) -> fromBits(toBits(a) | toBits(b)));
 851             case VECTOR_OP_ATAN2: return (v0, v1, vm) ->
 852                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.atan2(a, b));
 853             case VECTOR_OP_POW: return (v0, v1, vm) ->
 854                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.pow(a, b));
 855             case VECTOR_OP_HYPOT: return (v0, v1, vm) ->
 856                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.hypot(a, b));
 857 #end[FP]
 858             default: return null;
 859         }
 860     }
 861 
 862     // FIXME: Maybe all of the public final methods in this file (the
 863     // simple ones that just call lanewise) should be pushed down to
 864     // the X-VectorBits template.  They can't optimize properly at
 865     // this level, and must rely on inlining.  Does it work?
 866     // (If it works, of course keep the code here.)
 867 
 868     /**
 869      * Combines the lane values of this vector
 870      * with the value of a broadcast scalar.
 871      *
 872      * This is a lane-wise binary operation which applies
 873      * the selected operation to each lane.
 874      * The return value will be equal to this expression:
 875      * {@code this.lanewise(op, this.broadcast(e))}.
 876      *
 877      * @param op the operation used to process lane values
 878      * @param e the input scalar
 879      * @return the result of applying the operation lane-wise
 880      *         to the two input vectors
 881      * @throws UnsupportedOperationException if this vector does
 882      *         not support the requested operation
 883      * @see #lanewise(VectorOperators.Binary,Vector)
 884      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 885      */
 886     @ForceInline
 887     public final
 888     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 889                                   $type$ e) {
 890 #if[BITWISE]
 891         if (opKind(op, VO_SHIFT) && ($type$)(int)e == e) {
 892             return lanewiseShift(op, (int) e);
 893         }
 894         if (op == AND_NOT) {
 895             op = AND; e = ($type$) ~e;
 896         }
 897 #end[BITWISE]
 898         return lanewise(op, broadcast(e));
 899     }
 900 
 901     /**
 902      * Combines the lane values of this vector
 903      * with the value of a broadcast scalar,
 904      * with selection of lane elements controlled by a mask.
 905      *
 906      * This is a masked lane-wise binary operation which applies
 907      * the selected operation to each lane.
 908      * The return value will be equal to this expression:
 909      * {@code this.lanewise(op, this.broadcast(e), m)}.
 910      *
 911      * @param op the operation used to process lane values
 912      * @param e the input scalar
 913      * @param m the mask controlling lane selection
 914      * @return the result of applying the operation lane-wise
 915      *         to the input vector and the scalar
 916      * @throws UnsupportedOperationException if this vector does
 917      *         not support the requested operation
 918      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
 919      * @see #lanewise(VectorOperators.Binary,$type$)
 920      */
 921     @ForceInline
 922     public final
 923     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 924                                   $type$ e,
 925                                   VectorMask<$Boxtype$> m) {
 926 #if[BITWISE]
 927         if (opKind(op, VO_SHIFT) && ($type$)(int)e == e) {
 928             return lanewiseShift(op, (int) e, m);
 929         }
 930         if (op == AND_NOT) {
 931             op = AND; e = ($type$) ~e;
 932         }
 933 #end[BITWISE]
 934         return lanewise(op, broadcast(e), m);
 935     }
 936 
 937 #if[!long]
 938     /**
 939      * {@inheritDoc} <!--workaround-->
 940      * @apiNote
 941      * When working with vector subtypes like {@code $abstractvectortype$},
 942      * {@linkplain #lanewise(VectorOperators.Binary,$type$)
 943      * the more strongly typed method}
 944      * is typically selected.  It can be explicitly selected
 945      * using a cast: {@code v.lanewise(op,($type$)e)}.
 946      * The two expressions will produce numerically identical results.
 947      */
 948     @ForceInline
 949     public final
 950     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 951                                   long e) {
 952         $type$ e1 = ($type$) e;
 953 #if[BITWISE]
 954         if ((long)e1 != e
 955             // allow shift ops to clip down their int parameters
 956             && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
 957 #else[BITWISE]
 958         if ((long)e1 != e) {
 959 #end[BITWISE]
 960             vspecies().checkValue(e);  // for exception
 961         }
 962         return lanewise(op, e1);
 963     }
 964 
 965     /**
 966      * {@inheritDoc} <!--workaround-->
 967      * @apiNote
 968      * When working with vector subtypes like {@code $abstractvectortype$},
 969      * {@linkplain #lanewise(VectorOperators.Binary,$type$,VectorMask)
 970      * the more strongly typed method}
 971      * is typically selected.  It can be explicitly selected
 972      * using a cast: {@code v.lanewise(op,($type$)e,m)}.
 973      * The two expressions will produce numerically identical results.
 974      */
 975     @ForceInline
 976     public final
 977     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 978                                   long e, VectorMask<$Boxtype$> m) {
 979         $type$ e1 = ($type$) e;
 980 #if[BITWISE]
 981         if ((long)e1 != e
 982             // allow shift ops to clip down their int parameters
 983             && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
 984 #else[BITWISE]
 985         if ((long)e1 != e) {
 986 #end[BITWISE]
 987             vspecies().checkValue(e);  // for exception
 988         }
 989         return lanewise(op, e1, m);
 990     }
 991 #end[!long]
 992 
 993 #if[BITWISE]
 994     /*package-private*/
 995     abstract $abstractvectortype$
 996     lanewiseShift(VectorOperators.Binary op, int e);
 997 
 998     /*package-private*/
 999     @ForceInline
1000     final $abstractvectortype$
1001     lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
1002         // Special handling for these.  FIXME: Refactor?
1003         assert(opKind(op, VO_SHIFT));
1004         // As per shift specification for Java, mask the shift count.
1005         e &= SHIFT_MASK;
1006         int opc = opCode(op);
1007         return VectorSupport.broadcastInt(
1008             opc, getClass(), null, $type$.class, length(),
1009             this, e, null,
1010             BIN_INT_IMPL.find(op, opc, $abstractvectortype$::broadcastIntOperations));
1011     }
1012 
1013     /*package-private*/
1014     abstract $abstractvectortype$
1015     lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m);
1016 
1017     /*package-private*/
1018     @ForceInline
1019     final $abstractvectortype$
1020     lanewiseShiftTemplate(VectorOperators.Binary op,
1021                           Class<? extends VectorMask<$Boxtype$>> maskClass,
1022                           int e, VectorMask<$Boxtype$> m) {
1023         m.check(maskClass, this);
1024         assert(opKind(op, VO_SHIFT));
1025         // As per shift specification for Java, mask the shift count.
1026         e &= SHIFT_MASK;
1027         int opc = opCode(op);
1028         return VectorSupport.broadcastInt(
1029             opc, getClass(), maskClass, $type$.class, length(),
1030             this, e, m,
1031             BIN_INT_IMPL.find(op, opc, $abstractvectortype$::broadcastIntOperations));
1032     }
1033 
1034     private static final
1035     ImplCache<Binary,VectorBroadcastIntOp<$abstractvectortype$, VectorMask<$Boxtype$>>> BIN_INT_IMPL
1036         = new ImplCache<>(Binary.class, $Type$Vector.class);
1037 
1038     private static VectorBroadcastIntOp<$abstractvectortype$, VectorMask<$Boxtype$>> broadcastIntOperations(int opc_) {
1039         switch (opc_) {
1040             case VECTOR_OP_LSHIFT: return (v, n, m) ->
1041                     v.uOp(m, (i, a) -> ($type$)(a << n));
1042             case VECTOR_OP_RSHIFT: return (v, n, m) ->
1043                     v.uOp(m, (i, a) -> ($type$)(a >> n));
1044             case VECTOR_OP_URSHIFT: return (v, n, m) ->
1045                     v.uOp(m, (i, a) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
1046             case VECTOR_OP_LROTATE: return (v, n, m) ->
1047                     v.uOp(m, (i, a) -> rotateLeft(a, (int)n));
1048             case VECTOR_OP_RROTATE: return (v, n, m) ->
1049                     v.uOp(m, (i, a) -> rotateRight(a, (int)n));
1050             default: return null;
1051         }
1052     }
1053 
1054     // As per shift specification for Java, mask the shift count.
1055     // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
1056     // The latter two maskings go beyond the JLS, but seem reasonable
1057     // since our lane types are first-class types, not just dressed
1058     // up ints.
1059     private static final int SHIFT_MASK = ($Boxtype$.SIZE - 1);
1060 #if[byteOrShort]
1061     // Also simulate >>> on sub-word variables with a mask.
1062     private static final int LSHR_SETUP_MASK = ((1 << $Boxtype$.SIZE) - 1);
1063 #else[byteOrShort]
1064     private static final $type$ LSHR_SETUP_MASK = -1;
1065 #end[byteOrShort]
1066 #end[BITWISE]
1067 
1068     // Ternary lanewise support
1069 
1070     // Ternary operators come in eight variations:
1071     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
1072     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
1073 
1074     // It is annoying to support all of these variations of masking
1075     // and broadcast, but it would be more surprising not to continue
1076     // the obvious pattern started by unary and binary.
1077 
1078    /**
1079      * {@inheritDoc} <!--workaround-->
1080      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1081      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1082      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1083      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1084      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
1085      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
1086      */
1087     @Override
1088     public abstract
1089     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
1090                                                   Vector<$Boxtype$> v1,
1091                                                   Vector<$Boxtype$> v2);
1092     @ForceInline
1093     final
1094     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
1095                                           Vector<$Boxtype$> v1,
1096                                           Vector<$Boxtype$> v2) {
1097         $abstractvectortype$ that = ($abstractvectortype$) v1;
1098         $abstractvectortype$ tother = ($abstractvectortype$) v2;
1099         // It's a word: https://www.dictionary.com/browse/tother
1100         // See also Chapter 11 of Dickens, Our Mutual Friend:
1101         // "Totherest Governor," replied Mr Riderhood...
1102         that.check(this);
1103         tother.check(this);
1104 #if[BITWISE]
1105         if (op == BITWISE_BLEND) {
1106             // FIXME: Support this in the JIT.
1107             that = this.lanewise(XOR, that).lanewise(AND, tother);
1108             return this.lanewise(XOR, that);
1109         }
1110 #end[BITWISE]
1111         int opc = opCode(op);
1112         return VectorSupport.ternaryOp(
1113             opc, getClass(), null, $type$.class, length(),
1114             this, that, tother, null,
1115             TERN_IMPL.find(op, opc, $abstractvectortype$::ternaryOperations));
1116     }
1117 
1118     /**
1119      * {@inheritDoc} <!--workaround-->
1120      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1121      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1122      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1123      */
1124     @Override
1125     public abstract
1126     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
1127                                   Vector<$Boxtype$> v1,
1128                                   Vector<$Boxtype$> v2,
1129                                   VectorMask<$Boxtype$> m);
1130     @ForceInline
1131     final
1132     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
1133                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
1134                                           Vector<$Boxtype$> v1,
1135                                           Vector<$Boxtype$> v2,
1136                                           VectorMask<$Boxtype$> m) {
1137         $abstractvectortype$ that = ($abstractvectortype$) v1;
1138         $abstractvectortype$ tother = ($abstractvectortype$) v2;
1139         // It's a word: https://www.dictionary.com/browse/tother
1140         // See also Chapter 11 of Dickens, Our Mutual Friend:
1141         // "Totherest Governor," replied Mr Riderhood...
1142         that.check(this);
1143         tother.check(this);
1144         m.check(maskClass, this);
1145 
1146 #if[BITWISE]
1147         if (op == BITWISE_BLEND) {
1148             // FIXME: Support this in the JIT.
1149             that = this.lanewise(XOR, that).lanewise(AND, tother);
1150             return this.lanewise(XOR, that, m);
1151         }
1152 #end[BITWISE]
1153         int opc = opCode(op);
1154         return VectorSupport.ternaryOp(
1155             opc, getClass(), maskClass, $type$.class, length(),
1156             this, that, tother, m,
1157             TERN_IMPL.find(op, opc, $abstractvectortype$::ternaryOperations));
1158     }
1159 
1160     private static final
1161     ImplCache<Ternary, TernaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
1162         TERN_IMPL = new ImplCache<>(Ternary.class, $Type$Vector.class);
1163 
1164     private static TernaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> ternaryOperations(int opc_) {
1165         switch (opc_) {
1166 #if[FP]
1167             case VECTOR_OP_FMA: return (v0, v1_, v2_, m) ->
1168                     v0.tOp(v1_, v2_, m, (i, a, b, c) -> Math.fma(a, b, c));
1169 #end[FP]
1170             default: return null;
1171         }
1172     }
1173 
1174     /**
1175      * Combines the lane values of this vector
1176      * with the values of two broadcast scalars.
1177      *
1178      * This is a lane-wise ternary operation which applies
1179      * the selected operation to each lane.
1180      * The return value will be equal to this expression:
1181      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
1182      *
1183      * @param op the operation used to combine lane values
1184      * @param e1 the first input scalar
1185      * @param e2 the second input scalar
1186      * @return the result of applying the operation lane-wise
1187      *         to the input vector and the scalars
1188      * @throws UnsupportedOperationException if this vector does
1189      *         not support the requested operation
1190      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1191      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1192      */
1193     @ForceInline
1194     public final
1195     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2)
1196                                   $type$ e1,
1197                                   $type$ e2) {
1198         return lanewise(op, broadcast(e1), broadcast(e2));
1199     }
1200 
1201     /**
1202      * Combines the lane values of this vector
1203      * with the values of two broadcast scalars,
1204      * with selection of lane elements controlled by a mask.
1205      *
1206      * This is a masked lane-wise ternary operation which applies
1207      * the selected operation to each lane.
1208      * The return value will be equal to this expression:
1209      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
1210      *
1211      * @param op the operation used to combine lane values
1212      * @param e1 the first input scalar
1213      * @param e2 the second input scalar
1214      * @param m the mask controlling lane selection
1215      * @return the result of applying the operation lane-wise
1216      *         to the input vector and the scalars
1217      * @throws UnsupportedOperationException if this vector does
1218      *         not support the requested operation
1219      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1220      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1221      */
1222     @ForceInline
1223     public final
1224     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
1225                                   $type$ e1,
1226                                   $type$ e2,
1227                                   VectorMask<$Boxtype$> m) {
1228         return lanewise(op, broadcast(e1), broadcast(e2), m);
1229     }
1230 
1231     /**
1232      * Combines the lane values of this vector
1233      * with the values of another vector and a broadcast scalar.
1234      *
1235      * This is a lane-wise ternary operation which applies
1236      * the selected operation to each lane.
1237      * The return value will be equal to this expression:
1238      * {@code this.lanewise(op, v1, this.broadcast(e2))}.
1239      *
1240      * @param op the operation used to combine lane values
1241      * @param v1 the other input vector
1242      * @param e2 the input scalar
1243      * @return the result of applying the operation lane-wise
1244      *         to the input vectors and the scalar
1245      * @throws UnsupportedOperationException if this vector does
1246      *         not support the requested operation
1247      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1248      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1249      */
1250     @ForceInline
1251     public final
1252     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2)
1253                                   Vector<$Boxtype$> v1,
1254                                   $type$ e2) {
1255         return lanewise(op, v1, broadcast(e2));
1256     }
1257 
1258     /**
1259      * Combines the lane values of this vector
1260      * with the values of another vector and a broadcast scalar,
1261      * with selection of lane elements controlled by a mask.
1262      *
1263      * This is a masked lane-wise ternary operation which applies
1264      * the selected operation to each lane.
1265      * The return value will be equal to this expression:
1266      * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
1267      *
1268      * @param op the operation used to combine lane values
1269      * @param v1 the other input vector
1270      * @param e2 the input scalar
1271      * @param m the mask controlling lane selection
1272      * @return the result of applying the operation lane-wise
1273      *         to the input vectors and the scalar
1274      * @throws UnsupportedOperationException if this vector does
1275      *         not support the requested operation
1276      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1277      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1278      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
1279      */
1280     @ForceInline
1281     public final
1282     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
1283                                   Vector<$Boxtype$> v1,
1284                                   $type$ e2,
1285                                   VectorMask<$Boxtype$> m) {
1286         return lanewise(op, v1, broadcast(e2), m);
1287     }
1288 
1289     /**
1290      * Combines the lane values of this vector
1291      * with the values of another vector and a broadcast scalar.
1292      *
1293      * This is a lane-wise ternary operation which applies
1294      * the selected operation to each lane.
1295      * The return value will be equal to this expression:
1296      * {@code this.lanewise(op, this.broadcast(e1), v2)}.
1297      *
1298      * @param op the operation used to combine lane values
1299      * @param e1 the input scalar
1300      * @param v2 the other input vector
1301      * @return the result of applying the operation lane-wise
1302      *         to the input vectors and the scalar
1303      * @throws UnsupportedOperationException if this vector does
1304      *         not support the requested operation
1305      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1306      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1307      */
1308     @ForceInline
1309     public final
1310     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2)
1311                                   $type$ e1,
1312                                   Vector<$Boxtype$> v2) {
1313         return lanewise(op, broadcast(e1), v2);
1314     }
1315 
1316     /**
1317      * Combines the lane values of this vector
1318      * with the values of another vector and a broadcast scalar,
1319      * with selection of lane elements controlled by a mask.
1320      *
1321      * This is a masked lane-wise ternary operation which applies
1322      * the selected operation to each lane.
1323      * The return value will be equal to this expression:
1324      * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
1325      *
1326      * @param op the operation used to combine lane values
1327      * @param e1 the input scalar
1328      * @param v2 the other input vector
1329      * @param m the mask controlling lane selection
1330      * @return the result of applying the operation lane-wise
1331      *         to the input vectors and the scalar
1332      * @throws UnsupportedOperationException if this vector does
1333      *         not support the requested operation
1334      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1335      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
1336      */
1337     @ForceInline
1338     public final
1339     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
1340                                   $type$ e1,
1341                                   Vector<$Boxtype$> v2,
1342                                   VectorMask<$Boxtype$> m) {
1343         return lanewise(op, broadcast(e1), v2, m);
1344     }
1345 
1346     // (Thus endeth the Great and Mighty Ternary Ogdoad.)
1347     // https://en.wikipedia.org/wiki/Ogdoad
1348 
1349     /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1350     //
1351     // These include masked and non-masked versions.
1352     // This subclass adds broadcast (masked or not).
1353 
1354     /**
1355      * {@inheritDoc} <!--workaround-->
1356      * @see #add($type$)
1357      */
1358     @Override
1359     @ForceInline
1360     public final $abstractvectortype$ add(Vector<$Boxtype$> v) {
1361         return lanewise(ADD, v);
1362     }
1363 
1364     /**
1365      * Adds this vector to the broadcast of an input scalar.
1366      *
1367      * This is a lane-wise binary operation which applies
1368      * the primitive addition operation ({@code +}) to each lane.
1369      *
1370      * This method is also equivalent to the expression
1371      * {@link #lanewise(VectorOperators.Binary,$type$)
1372      *    lanewise}{@code (}{@link VectorOperators#ADD
1373      *    ADD}{@code , e)}.
1374      *
1375      * @param e the input scalar
1376      * @return the result of adding each lane of this vector to the scalar
1377      * @see #add(Vector)
1378      * @see #broadcast($type$)
1379      * @see #add($type$,VectorMask)
1380      * @see VectorOperators#ADD
1381      * @see #lanewise(VectorOperators.Binary,Vector)
1382      * @see #lanewise(VectorOperators.Binary,$type$)
1383      */
1384     @ForceInline
1385     public final
1386     $abstractvectortype$ add($type$ e) {
1387         return lanewise(ADD, e);
1388     }
1389 
1390     /**
1391      * {@inheritDoc} <!--workaround-->
1392      * @see #add($type$,VectorMask)
1393      */
1394     @Override
1395     @ForceInline
1396     public final $abstractvectortype$ add(Vector<$Boxtype$> v,
1397                                           VectorMask<$Boxtype$> m) {
1398         return lanewise(ADD, v, m);
1399     }
1400 
1401     /**
1402      * Adds this vector to the broadcast of an input scalar,
1403      * selecting lane elements controlled by a mask.
1404      *
1405      * This is a masked lane-wise binary operation which applies
1406      * the primitive addition operation ({@code +}) to each lane.
1407      *
1408      * This method is also equivalent to the expression
1409      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1410      *    lanewise}{@code (}{@link VectorOperators#ADD
1411      *    ADD}{@code , s, m)}.
1412      *
1413      * @param e the input scalar
1414      * @param m the mask controlling lane selection
1415      * @return the result of adding each lane of this vector to the scalar
1416      * @see #add(Vector,VectorMask)
1417      * @see #broadcast($type$)
1418      * @see #add($type$)
1419      * @see VectorOperators#ADD
1420      * @see #lanewise(VectorOperators.Binary,Vector)
1421      * @see #lanewise(VectorOperators.Binary,$type$)
1422      */
1423     @ForceInline
1424     public final $abstractvectortype$ add($type$ e,
1425                                           VectorMask<$Boxtype$> m) {
1426         return lanewise(ADD, e, m);
1427     }
1428 
1429     /**
1430      * {@inheritDoc} <!--workaround-->
1431      * @see #sub($type$)
1432      */
1433     @Override
1434     @ForceInline
1435     public final $abstractvectortype$ sub(Vector<$Boxtype$> v) {
1436         return lanewise(SUB, v);
1437     }
1438 
1439     /**
1440      * Subtracts an input scalar from this vector.
1441      *
1442      * This is a masked lane-wise binary operation which applies
1443      * the primitive subtraction operation ({@code -}) to each lane.
1444      *
1445      * This method is also equivalent to the expression
1446      * {@link #lanewise(VectorOperators.Binary,$type$)
1447      *    lanewise}{@code (}{@link VectorOperators#SUB
1448      *    SUB}{@code , e)}.
1449      *
1450      * @param e the input scalar
1451      * @return the result of subtracting the scalar from each lane of this vector
1452      * @see #sub(Vector)
1453      * @see #broadcast($type$)
1454      * @see #sub($type$,VectorMask)
1455      * @see VectorOperators#SUB
1456      * @see #lanewise(VectorOperators.Binary,Vector)
1457      * @see #lanewise(VectorOperators.Binary,$type$)
1458      */
1459     @ForceInline
1460     public final $abstractvectortype$ sub($type$ e) {
1461         return lanewise(SUB, e);
1462     }
1463 
1464     /**
1465      * {@inheritDoc} <!--workaround-->
1466      * @see #sub($type$,VectorMask)
1467      */
1468     @Override
1469     @ForceInline
1470     public final $abstractvectortype$ sub(Vector<$Boxtype$> v,
1471                                           VectorMask<$Boxtype$> m) {
1472         return lanewise(SUB, v, m);
1473     }
1474 
1475     /**
1476      * Subtracts an input scalar from this vector
1477      * under the control of a mask.
1478      *
1479      * This is a masked lane-wise binary operation which applies
1480      * the primitive subtraction operation ({@code -}) to each lane.
1481      *
1482      * This method is also equivalent to the expression
1483      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1484      *    lanewise}{@code (}{@link VectorOperators#SUB
1485      *    SUB}{@code , s, m)}.
1486      *
1487      * @param e the input scalar
1488      * @param m the mask controlling lane selection
1489      * @return the result of subtracting the scalar from each lane of this vector
1490      * @see #sub(Vector,VectorMask)
1491      * @see #broadcast($type$)
1492      * @see #sub($type$)
1493      * @see VectorOperators#SUB
1494      * @see #lanewise(VectorOperators.Binary,Vector)
1495      * @see #lanewise(VectorOperators.Binary,$type$)
1496      */
1497     @ForceInline
1498     public final $abstractvectortype$ sub($type$ e,
1499                                           VectorMask<$Boxtype$> m) {
1500         return lanewise(SUB, e, m);
1501     }
1502 
1503     /**
1504      * {@inheritDoc} <!--workaround-->
1505      * @see #mul($type$)
1506      */
1507     @Override
1508     @ForceInline
1509     public final $abstractvectortype$ mul(Vector<$Boxtype$> v) {
1510         return lanewise(MUL, v);
1511     }
1512 
1513     /**
1514      * Multiplies this vector by the broadcast of an input scalar.
1515      *
1516      * This is a lane-wise binary operation which applies
1517      * the primitive multiplication operation ({@code *}) to each lane.
1518      *
1519      * This method is also equivalent to the expression
1520      * {@link #lanewise(VectorOperators.Binary,$type$)
1521      *    lanewise}{@code (}{@link VectorOperators#MUL
1522      *    MUL}{@code , e)}.
1523      *
1524      * @param e the input scalar
1525      * @return the result of multiplying this vector by the given scalar
1526      * @see #mul(Vector)
1527      * @see #broadcast($type$)
1528      * @see #mul($type$,VectorMask)
1529      * @see VectorOperators#MUL
1530      * @see #lanewise(VectorOperators.Binary,Vector)
1531      * @see #lanewise(VectorOperators.Binary,$type$)
1532      */
1533     @ForceInline
1534     public final $abstractvectortype$ mul($type$ e) {
1535         return lanewise(MUL, e);
1536     }
1537 
1538     /**
1539      * {@inheritDoc} <!--workaround-->
1540      * @see #mul($type$,VectorMask)
1541      */
1542     @Override
1543     @ForceInline
1544     public final $abstractvectortype$ mul(Vector<$Boxtype$> v,
1545                                           VectorMask<$Boxtype$> m) {
1546         return lanewise(MUL, v, m);
1547     }
1548 
1549     /**
1550      * Multiplies this vector by the broadcast of an input scalar,
1551      * selecting lane elements controlled by a mask.
1552      *
1553      * This is a masked lane-wise binary operation which applies
1554      * the primitive multiplication operation ({@code *}) to each lane.
1555      *
1556      * This method is also equivalent to the expression
1557      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1558      *    lanewise}{@code (}{@link VectorOperators#MUL
1559      *    MUL}{@code , s, m)}.
1560      *
1561      * @param e the input scalar
1562      * @param m the mask controlling lane selection
1563      * @return the result of muling each lane of this vector to the scalar
1564      * @see #mul(Vector,VectorMask)
1565      * @see #broadcast($type$)
1566      * @see #mul($type$)
1567      * @see VectorOperators#MUL
1568      * @see #lanewise(VectorOperators.Binary,Vector)
1569      * @see #lanewise(VectorOperators.Binary,$type$)
1570      */
1571     @ForceInline
1572     public final $abstractvectortype$ mul($type$ e,
1573                                           VectorMask<$Boxtype$> m) {
1574         return lanewise(MUL, e, m);
1575     }
1576 
1577     /**
1578      * {@inheritDoc} <!--workaround-->
1579 #if[FP]
1580      * @apiNote Because the underlying scalar operator is an IEEE
1581      * floating point number, division by zero in fact will
1582      * not throw an exception, but will yield a signed
1583      * infinity or NaN.
1584 #else[FP]
1585      * @apiNote If there is a zero divisor, {@code
1586      * ArithmeticException} will be thrown.
1587 #end[FP]
1588      */
1589     @Override
1590     @ForceInline
1591     public final $abstractvectortype$ div(Vector<$Boxtype$> v) {
1592         return lanewise(DIV, v);
1593     }
1594 
1595     /**
1596      * Divides this vector by the broadcast of an input scalar.
1597      *
1598      * This is a lane-wise binary operation which applies
1599      * the primitive division operation ({@code /}) to each lane.
1600      *
1601      * This method is also equivalent to the expression
1602      * {@link #lanewise(VectorOperators.Binary,$type$)
1603      *    lanewise}{@code (}{@link VectorOperators#DIV
1604      *    DIV}{@code , e)}.
1605      *
1606 #if[FP]
1607      * @apiNote Because the underlying scalar operator is an IEEE
1608      * floating point number, division by zero in fact will
1609      * not throw an exception, but will yield a signed
1610      * infinity or NaN.
1611 #else[FP]
1612      * @apiNote If there is a zero divisor, {@code
1613      * ArithmeticException} will be thrown.
1614 #end[FP]
1615      *
1616      * @param e the input scalar
1617      * @return the result of dividing each lane of this vector by the scalar
1618      * @see #div(Vector)
1619      * @see #broadcast($type$)
1620      * @see #div($type$,VectorMask)
1621      * @see VectorOperators#DIV
1622      * @see #lanewise(VectorOperators.Binary,Vector)
1623      * @see #lanewise(VectorOperators.Binary,$type$)
1624      */
1625     @ForceInline
1626     public final $abstractvectortype$ div($type$ e) {
1627         return lanewise(DIV, e);
1628     }
1629 
1630     /**
1631      * {@inheritDoc} <!--workaround-->
1632      * @see #div($type$,VectorMask)
1633 #if[FP]
1634      * @apiNote Because the underlying scalar operator is an IEEE
1635      * floating point number, division by zero in fact will
1636      * not throw an exception, but will yield a signed
1637      * infinity or NaN.
1638 #else[FP]
1639      * @apiNote If there is a zero divisor, {@code
1640      * ArithmeticException} will be thrown.
1641 #end[FP]
1642      */
1643     @Override
1644     @ForceInline
1645     public final $abstractvectortype$ div(Vector<$Boxtype$> v,
1646                                           VectorMask<$Boxtype$> m) {
1647         return lanewise(DIV, v, m);
1648     }
1649 
1650     /**
1651      * Divides this vector by the broadcast of an input scalar,
1652      * selecting lane elements controlled by a mask.
1653      *
1654      * This is a masked lane-wise binary operation which applies
1655      * the primitive division operation ({@code /}) to each lane.
1656      *
1657      * This method is also equivalent to the expression
1658      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1659      *    lanewise}{@code (}{@link VectorOperators#DIV
1660      *    DIV}{@code , s, m)}.
1661      *
1662 #if[FP]
1663      * @apiNote Because the underlying scalar operator is an IEEE
1664      * floating point number, division by zero in fact will
1665      * not throw an exception, but will yield a signed
1666      * infinity or NaN.
1667 #else[FP]
1668      * @apiNote If there is a zero divisor, {@code
1669      * ArithmeticException} will be thrown.
1670 #end[FP]
1671      *
1672      * @param e the input scalar
1673      * @param m the mask controlling lane selection
1674      * @return the result of dividing each lane of this vector by the scalar
1675      * @see #div(Vector,VectorMask)
1676      * @see #broadcast($type$)
1677      * @see #div($type$)
1678      * @see VectorOperators#DIV
1679      * @see #lanewise(VectorOperators.Binary,Vector)
1680      * @see #lanewise(VectorOperators.Binary,$type$)
1681      */
1682     @ForceInline
1683     public final $abstractvectortype$ div($type$ e,
1684                                           VectorMask<$Boxtype$> m) {
1685         return lanewise(DIV, e, m);
1686     }
1687 
1688     /// END OF FULL-SERVICE BINARY METHODS
1689 
1690     /// SECOND-TIER BINARY METHODS
1691     //
1692     // There are no masked versions.
1693 
1694     /**
1695      * {@inheritDoc} <!--workaround-->
1696 #if[FP]
1697      * @apiNote
1698      * For this method, floating point negative
1699      * zero {@code -0.0} is treated as a value distinct from, and less
1700      * than the default value (positive zero).
1701 #end[FP]
1702      */
1703     @Override
1704     @ForceInline
1705     public final $abstractvectortype$ min(Vector<$Boxtype$> v) {
1706         return lanewise(MIN, v);
1707     }
1708 
1709     // FIXME:  "broadcast of an input scalar" is really wordy.  Reduce?
1710     /**
1711      * Computes the smaller of this vector and the broadcast of an input scalar.
1712      *
1713      * This is a lane-wise binary operation which applies the
1714      * operation {@code Math.min()} to each pair of
1715      * corresponding lane values.
1716      *
1717      * This method is also equivalent to the expression
1718      * {@link #lanewise(VectorOperators.Binary,$type$)
1719      *    lanewise}{@code (}{@link VectorOperators#MIN
1720      *    MIN}{@code , e)}.
1721      *
1722      * @param e the input scalar
1723      * @return the result of multiplying this vector by the given scalar
1724      * @see #min(Vector)
1725      * @see #broadcast($type$)
1726      * @see VectorOperators#MIN
1727      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1728 #if[FP]
1729      * @apiNote
1730      * For this method, floating point negative
1731      * zero {@code -0.0} is treated as a value distinct from, and less
1732      * than the default value (positive zero).
1733 #end[FP]
1734      */
1735     @ForceInline
1736     public final $abstractvectortype$ min($type$ e) {
1737         return lanewise(MIN, e);
1738     }
1739 
1740     /**
1741      * {@inheritDoc} <!--workaround-->
1742 #if[FP]
1743      * @apiNote
1744      * For this method, floating point negative
1745      * zero {@code -0.0} is treated as a value distinct from, and less
1746      * than the default value (positive zero).
1747 #end[FP]
1748      */
1749     @Override
1750     @ForceInline
1751     public final $abstractvectortype$ max(Vector<$Boxtype$> v) {
1752         return lanewise(MAX, v);
1753     }
1754 
1755     /**
1756      * Computes the larger of this vector and the broadcast of an input scalar.
1757      *
1758      * This is a lane-wise binary operation which applies the
1759      * operation {@code Math.max()} to each pair of
1760      * corresponding lane values.
1761      *
1762      * This method is also equivalent to the expression
1763      * {@link #lanewise(VectorOperators.Binary,$type$)
1764      *    lanewise}{@code (}{@link VectorOperators#MAX
1765      *    MAX}{@code , e)}.
1766      *
1767      * @param e the input scalar
1768      * @return the result of multiplying this vector by the given scalar
1769      * @see #max(Vector)
1770      * @see #broadcast($type$)
1771      * @see VectorOperators#MAX
1772      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1773 #if[FP]
1774      * @apiNote
1775      * For this method, floating point negative
1776      * zero {@code -0.0} is treated as a value distinct from, and less
1777      * than the default value (positive zero).
1778 #end[FP]
1779      */
1780     @ForceInline
1781     public final $abstractvectortype$ max($type$ e) {
1782         return lanewise(MAX, e);
1783     }
1784 
1785 #if[BITWISE]
1786     // common bitwise operators: and, or, not (with scalar versions)
1787     /**
1788      * Computes the bitwise logical conjunction ({@code &})
1789      * of this vector and a second input vector.
1790      *
1791      * This is a lane-wise binary operation which applies the
1792      * the primitive bitwise "and" operation ({@code &})
1793      * to each pair of corresponding lane values.
1794      *
1795      * This method is also equivalent to the expression
1796      * {@link #lanewise(VectorOperators.Binary,Vector)
1797      *    lanewise}{@code (}{@link VectorOperators#AND
1798      *    AND}{@code , v)}.
1799      *
1800      * <p>
1801      * This is not a full-service named operation like
1802      * {@link #add(Vector) add}.  A masked version of
1803      * this operation is not directly available
1804      * but may be obtained via the masked version of
1805      * {@code lanewise}.
1806      *
1807      * @param v a second input vector
1808      * @return the bitwise {@code &} of this vector and the second input vector
1809      * @see #and($type$)
1810      * @see #or(Vector)
1811      * @see #not()
1812      * @see VectorOperators#AND
1813      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1814      */
1815     @ForceInline
1816     public final $abstractvectortype$ and(Vector<$Boxtype$> v) {
1817         return lanewise(AND, v);
1818     }
1819 
1820     /**
1821      * Computes the bitwise logical conjunction ({@code &})
1822      * of this vector and a scalar.
1823      *
1824      * This is a lane-wise binary operation which applies the
1825      * the primitive bitwise "and" operation ({@code &})
1826      * to each pair of corresponding lane values.
1827      *
1828      * This method is also equivalent to the expression
1829      * {@link #lanewise(VectorOperators.Binary,Vector)
1830      *    lanewise}{@code (}{@link VectorOperators#AND
1831      *    AND}{@code , e)}.
1832      *
1833      * @param e an input scalar
1834      * @return the bitwise {@code &} of this vector and scalar
1835      * @see #and(Vector)
1836      * @see VectorOperators#AND
1837      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1838      */
1839     @ForceInline
1840     public final $abstractvectortype$ and($type$ e) {
1841         return lanewise(AND, e);
1842     }
1843 
1844     /**
1845      * Computes the bitwise logical disjunction ({@code |})
1846      * of this vector and a second input vector.
1847      *
1848      * This is a lane-wise binary operation which applies the
1849      * the primitive bitwise "or" operation ({@code |})
1850      * to each pair of corresponding lane values.
1851      *
1852      * This method is also equivalent to the expression
1853      * {@link #lanewise(VectorOperators.Binary,Vector)
1854      *    lanewise}{@code (}{@link VectorOperators#OR
1855      *    AND}{@code , v)}.
1856      *
1857      * <p>
1858      * This is not a full-service named operation like
1859      * {@link #add(Vector) add}.  A masked version of
1860      * this operation is not directly available
1861      * but may be obtained via the masked version of
1862      * {@code lanewise}.
1863      *
1864      * @param v a second input vector
1865      * @return the bitwise {@code |} of this vector and the second input vector
1866      * @see #or($type$)
1867      * @see #and(Vector)
1868      * @see #not()
1869      * @see VectorOperators#OR
1870      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1871      */
1872     @ForceInline
1873     public final $abstractvectortype$ or(Vector<$Boxtype$> v) {
1874         return lanewise(OR, v);
1875     }
1876 
1877     /**
1878      * Computes the bitwise logical disjunction ({@code |})
1879      * of this vector and a scalar.
1880      *
1881      * This is a lane-wise binary operation which applies the
1882      * the primitive bitwise "or" operation ({@code |})
1883      * to each pair of corresponding lane values.
1884      *
1885      * This method is also equivalent to the expression
1886      * {@link #lanewise(VectorOperators.Binary,Vector)
1887      *    lanewise}{@code (}{@link VectorOperators#OR
1888      *    OR}{@code , e)}.
1889      *
1890      * @param e an input scalar
1891      * @return the bitwise {@code |} of this vector and scalar
1892      * @see #or(Vector)
1893      * @see VectorOperators#OR
1894      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1895      */
1896     @ForceInline
1897     public final $abstractvectortype$ or($type$ e) {
1898         return lanewise(OR, e);
1899     }
1900 
1901 #end[BITWISE]
1902 
1903 #if[FP]
1904     // common FP operator: pow
1905     /**
1906      * Raises this vector to the power of a second input vector.
1907      *
1908      * This is a lane-wise binary operation which applies an operation
1909      * conforming to the specification of
1910      * {@link Math#pow Math.pow(a,b)}
1911      * to each pair of corresponding lane values.
1912 #if[intOrFloat]
1913      * The operation is adapted to cast the operands and the result,
1914      * specifically widening {@code float} operands to {@code double}
1915      * operands and narrowing the {@code double} result to a {@code float}
1916      * result.
1917 #end[intOrFloat]
1918      *
1919      * This method is also equivalent to the expression
1920      * {@link #lanewise(VectorOperators.Binary,Vector)
1921      *    lanewise}{@code (}{@link VectorOperators#POW
1922      *    POW}{@code , b)}.
1923      *
1924      * <p>
1925      * This is not a full-service named operation like
1926      * {@link #add(Vector) add}.  A masked version of
1927      * this operation is not directly available
1928      * but may be obtained via the masked version of
1929      * {@code lanewise}.
1930      *
1931      * @param b a vector exponent by which to raise this vector
1932      * @return the {@code b}-th power of this vector
1933      * @see #pow($type$)
1934      * @see VectorOperators#POW
1935      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1936      */
1937     @ForceInline
1938     public final $abstractvectortype$ pow(Vector<$Boxtype$> b) {
1939         return lanewise(POW, b);
1940     }
1941 
1942     /**
1943      * Raises this vector to a scalar power.
1944      *
1945      * This is a lane-wise binary operation which applies an operation
1946      * conforming to the specification of
1947      * {@link Math#pow Math.pow(a,b)}
1948      * to each pair of corresponding lane values.
1949 #if[intOrFloat]
1950      * The operation is adapted to cast the operands and the result,
1951      * specifically widening {@code float} operands to {@code double}
1952      * operands and narrowing the {@code double} result to a {@code float}
1953      * result.
1954 #end[intOrFloat]
1955      *
1956      * This method is also equivalent to the expression
1957      * {@link #lanewise(VectorOperators.Binary,Vector)
1958      *    lanewise}{@code (}{@link VectorOperators#POW
1959      *    POW}{@code , b)}.
1960      *
1961      * @param b a scalar exponent by which to raise this vector
1962      * @return the {@code b}-th power of this vector
1963      * @see #pow(Vector)
1964      * @see VectorOperators#POW
1965      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1966      */
1967     @ForceInline
1968     public final $abstractvectortype$ pow($type$ b) {
1969         return lanewise(POW, b);
1970     }
1971 #end[FP]
1972 
1973     /// UNARY METHODS
1974 
1975     /**
1976      * {@inheritDoc} <!--workaround-->
1977      */
1978     @Override
1979     @ForceInline
1980     public final
1981     $abstractvectortype$ neg() {
1982         return lanewise(NEG);
1983     }
1984 
1985     /**
1986      * {@inheritDoc} <!--workaround-->
1987      */
1988     @Override
1989     @ForceInline
1990     public final
1991     $abstractvectortype$ abs() {
1992         return lanewise(ABS);
1993     }
1994 
1995 #if[BITWISE]
1996     // not (~)
1997     /**
1998      * Computes the bitwise logical complement ({@code ~})
1999      * of this vector.
2000      *
2001      * This is a lane-wise binary operation which applies the
2002      * the primitive bitwise "not" operation ({@code ~})
2003      * to each lane value.
2004      *
2005      * This method is also equivalent to the expression
2006      * {@link #lanewise(VectorOperators.Unary)
2007      *    lanewise}{@code (}{@link VectorOperators#NOT
2008      *    NOT}{@code )}.
2009      *
2010      * <p>
2011      * This is not a full-service named operation like
2012      * {@link #add(Vector) add}.  A masked version of
2013      * this operation is not directly available
2014      * but may be obtained via the masked version of
2015      * {@code lanewise}.
2016      *
2017      * @return the bitwise complement {@code ~} of this vector
2018      * @see #and(Vector)
2019      * @see VectorOperators#NOT
2020      * @see #lanewise(VectorOperators.Unary,VectorMask)
2021      */
2022     @ForceInline
2023     public final $abstractvectortype$ not() {
2024         return lanewise(NOT);
2025     }
2026 #end[BITWISE]
2027 
2028 #if[FP]
2029     // sqrt
2030     /**
2031      * Computes the square root of this vector.
2032      *
2033      * This is a lane-wise unary operation which applies an operation
2034      * conforming to the specification of
2035      * {@link Math#sqrt Math.sqrt(a)}
2036      * to each lane value.
2037 #if[intOrFloat]
2038      * The operation is adapted to cast the operand and the result,
2039      * specifically widening the {@code float} operand to a {@code double}
2040      * operand and narrowing the {@code double} result to a {@code float}
2041      * result.
2042 #end[intOrFloat]
2043      *
2044      * This method is also equivalent to the expression
2045      * {@link #lanewise(VectorOperators.Unary)
2046      *    lanewise}{@code (}{@link VectorOperators#SQRT
2047      *    SQRT}{@code )}.
2048      *
2049      * @return the square root of this vector
2050      * @see VectorOperators#SQRT
2051      * @see #lanewise(VectorOperators.Unary,VectorMask)
2052      */
2053     @ForceInline
2054     public final $abstractvectortype$ sqrt() {
2055         return lanewise(SQRT);
2056     }
2057 #end[FP]
2058 
2059     /// COMPARISONS
2060 
2061     /**
2062      * {@inheritDoc} <!--workaround-->
2063      */
2064     @Override
2065     @ForceInline
2066     public final
2067     VectorMask<$Boxtype$> eq(Vector<$Boxtype$> v) {
2068         return compare(EQ, v);
2069     }
2070 
2071     /**
2072      * Tests if this vector is equal to an input scalar.
2073      *
2074      * This is a lane-wise binary test operation which applies
2075      * the primitive equals operation ({@code ==}) to each lane.
2076      * The result is the same as {@code compare(VectorOperators.Comparison.EQ, e)}.
2077      *
2078      * @param e the input scalar
2079      * @return the result mask of testing if this vector
2080      *         is equal to {@code e}
2081      * @see #compare(VectorOperators.Comparison,$type$)
2082      */
2083     @ForceInline
2084     public final
2085     VectorMask<$Boxtype$> eq($type$ e) {
2086         return compare(EQ, e);
2087     }
2088 
2089     /**
2090      * {@inheritDoc} <!--workaround-->
2091      */
2092     @Override
2093     @ForceInline
2094     public final
2095     VectorMask<$Boxtype$> lt(Vector<$Boxtype$> v) {
2096         return compare(LT, v);
2097     }
2098 
2099     /**
2100      * Tests if this vector is less than an input scalar.
2101      *
2102      * This is a lane-wise binary test operation which applies
2103      * the primitive less than operation ({@code <}) to each lane.
2104      * The result is the same as {@code compare(VectorOperators.LT, e)}.
2105      *
2106      * @param e the input scalar
2107      * @return the mask result of testing if this vector
2108      *         is less than the input scalar
2109      * @see #compare(VectorOperators.Comparison,$type$)
2110      */
2111     @ForceInline
2112     public final
2113     VectorMask<$Boxtype$> lt($type$ e) {
2114         return compare(LT, e);
2115     }
2116 
2117     /**
2118      * {@inheritDoc} <!--workaround-->
2119      */
2120     @Override
2121     public abstract
2122     VectorMask<$Boxtype$> test(VectorOperators.Test op);
2123 
2124     /*package-private*/
2125     @ForceInline
2126     final
2127     <M extends VectorMask<$Boxtype$>>
2128     M testTemplate(Class<M> maskType, Test op) {
2129         $Type$Species vsp = vspecies();
2130         if (opKind(op, VO_SPECIAL)) {
2131             $Bitstype$Vector bits = this.viewAsIntegralLanes();
2132             VectorMask<$Boxbitstype$> m;
2133             if (op == IS_DEFAULT) {
2134                 m = bits.compare(EQ, ($bitstype$) 0);
2135             } else if (op == IS_NEGATIVE) {
2136                 m = bits.compare(LT, ($bitstype$) 0);
2137             }
2138 #if[FP]
2139             else if (op == IS_FINITE ||
2140                      op == IS_NAN ||
2141                      op == IS_INFINITE) {
2142                 // first kill the sign:
2143                 bits = bits.and($Boxbitstype$.MAX_VALUE);
2144                 // next find the bit pattern for infinity:
2145                 $bitstype$ infbits = ($bitstype$) toBits($Boxtype$.POSITIVE_INFINITY);
2146                 // now compare:
2147                 if (op == IS_FINITE) {
2148                     m = bits.compare(LT, infbits);
2149                 } else if (op == IS_NAN) {
2150                     m = bits.compare(GT, infbits);
2151                 } else {
2152                     m = bits.compare(EQ, infbits);
2153                 }
2154             }
2155 #end[FP]
2156             else {
2157                 throw new AssertionError(op);
2158             }
2159             return maskType.cast(m{#if[FP]?.cast(this.vspecies())});
2160         }
2161         int opc = opCode(op);
2162         throw new AssertionError(op);
2163     }
2164 
2165     /**
2166      * {@inheritDoc} <!--workaround-->
2167      */
2168     @Override
2169     @ForceInline
2170     public final
2171     VectorMask<$Boxtype$> test(VectorOperators.Test op,
2172                                   VectorMask<$Boxtype$> m) {
2173         return test(op).and(m);
2174     }
2175 
2176     /**
2177      * {@inheritDoc} <!--workaround-->
2178      */
2179     @Override
2180     public abstract
2181     VectorMask<$Boxtype$> compare(VectorOperators.Comparison op, Vector<$Boxtype$> v);
2182 
2183     /*package-private*/
2184     @ForceInline
2185     final
2186     <M extends VectorMask<$Boxtype$>>
2187     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v) {
2188         $abstractvectortype$ that = ($abstractvectortype$) v;
2189         that.check(this);
2190         int opc = opCode(op);
2191         return VectorSupport.compare(
2192             opc, getClass(), maskType, $type$.class, length(),
2193             this, that, null,
2194             (cond, v0, v1, m1) -> {
2195                 AbstractMask<$Boxtype$> m
2196                     = v0.bTest(cond, v1, (cond_, i, a, b)
2197                                -> compareWithOp(cond, a, b));
2198                 @SuppressWarnings("unchecked")
2199                 M m2 = (M) m;
2200                 return m2;
2201             });
2202     }
2203 
2204     /*package-private*/
2205     @ForceInline
2206     final
2207     <M extends VectorMask<$Boxtype$>>
2208     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v, M m) {
2209         $abstractvectortype$ that = ($abstractvectortype$) v;
2210         that.check(this);
2211         m.check(maskType, this);
2212         int opc = opCode(op);
2213         return VectorSupport.compare(
2214             opc, getClass(), maskType, $type$.class, length(),
2215             this, that, m,
2216             (cond, v0, v1, m1) -> {
2217                 AbstractMask<$Boxtype$> cmpM
2218                     = v0.bTest(cond, v1, (cond_, i, a, b)
2219                                -> compareWithOp(cond, a, b));
2220                 @SuppressWarnings("unchecked")
2221                 M m2 = (M) cmpM.and(m1);
2222                 return m2;
2223             });
2224     }
2225 
2226     @ForceInline
2227     private static boolean compareWithOp(int cond, $type$ a, $type$ b) {
2228         return switch (cond) {
2229             case BT_eq -> a == b;
2230             case BT_ne -> a != b;
2231             case BT_lt -> a < b;
2232             case BT_le -> a <= b;
2233             case BT_gt -> a > b;
2234             case BT_ge -> a >= b;
2235 #if[!FP]
2236             case BT_ult -> $Boxtype$.compareUnsigned(a, b) < 0;
2237             case BT_ule -> $Boxtype$.compareUnsigned(a, b) <= 0;
2238             case BT_ugt -> $Boxtype$.compareUnsigned(a, b) > 0;
2239             case BT_uge -> $Boxtype$.compareUnsigned(a, b) >= 0;
2240 #end[!FP]
2241             default -> throw new AssertionError();
2242         };
2243     }
2244 
2245     /**
2246      * Tests this vector by comparing it with an input scalar,
2247      * according to the given comparison operation.
2248      *
2249      * This is a lane-wise binary test operation which applies
2250      * the comparison operation to each lane.
2251      * <p>
2252      * The result is the same as
2253      * {@code compare(op, broadcast(species(), e))}.
2254      * That is, the scalar may be regarded as broadcast to
2255      * a vector of the same species, and then compared
2256      * against the original vector, using the selected
2257      * comparison operation.
2258      *
2259      * @param op the operation used to compare lane values
2260      * @param e the input scalar
2261      * @return the mask result of testing lane-wise if this vector
2262      *         compares to the input, according to the selected
2263      *         comparison operator
2264      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector)
2265      * @see #eq($type$)
2266      * @see #lt($type$)
2267      */
2268     public abstract
2269     VectorMask<$Boxtype$> compare(Comparison op, $type$ e);
2270 
2271     /*package-private*/
2272     @ForceInline
2273     final
2274     <M extends VectorMask<$Boxtype$>>
2275     M compareTemplate(Class<M> maskType, Comparison op, $type$ e) {
2276         return compareTemplate(maskType, op, broadcast(e));
2277     }
2278 
2279     /**
2280      * Tests this vector by comparing it with an input scalar,
2281      * according to the given comparison operation,
2282      * in lanes selected by a mask.
2283      *
2284      * This is a masked lane-wise binary test operation which applies
2285      * to each pair of corresponding lane values.
2286      *
2287      * The returned result is equal to the expression
2288      * {@code compare(op,s).and(m)}.
2289      *
2290      * @param op the operation used to compare lane values
2291      * @param e the input scalar
2292      * @param m the mask controlling lane selection
2293      * @return the mask result of testing lane-wise if this vector
2294      *         compares to the input, according to the selected
2295      *         comparison operator,
2296      *         and only in the lanes selected by the mask
2297      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector,VectorMask)
2298      */
2299     @ForceInline
2300     public final VectorMask<$Boxtype$> compare(VectorOperators.Comparison op,
2301                                                $type$ e,
2302                                                VectorMask<$Boxtype$> m) {
2303         return compare(op, broadcast(e), m);
2304     }
2305 
2306 #if[!long]
2307     /**
2308      * {@inheritDoc} <!--workaround-->
2309      */
2310     @Override
2311     public abstract
2312     VectorMask<$Boxtype$> compare(Comparison op, long e);
2313 
2314     /*package-private*/
2315     @ForceInline
2316     final
2317     <M extends VectorMask<$Boxtype$>>
2318     M compareTemplate(Class<M> maskType, Comparison op, long e) {
2319         return compareTemplate(maskType, op, broadcast(e));
2320     }
2321 
2322     /**
2323      * {@inheritDoc} <!--workaround-->
2324      */
2325     @Override
2326     @ForceInline
2327     public final
2328     VectorMask<$Boxtype$> compare(Comparison op, long e, VectorMask<$Boxtype$> m) {
2329         return compare(op, broadcast(e), m);
2330     }
2331 
2332 
2333 #end[!long]
2334 
2335     /**
2336      * {@inheritDoc} <!--workaround-->
2337      */
2338     @Override public abstract
2339     $abstractvectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m);
2340 
2341     /*package-private*/
2342     @ForceInline
2343     final
2344     <M extends VectorMask<$Boxtype$>>
2345     $abstractvectortype$
2346     blendTemplate(Class<M> maskType, $abstractvectortype$ v, M m) {
2347         v.check(this);
2348         return VectorSupport.blend(
2349             getClass(), maskType, $type$.class, length(),
2350             this, v, m,
2351             (v0, v1, m_) -> v0.bOp(v1, m_, (i, a, b) -> b));
2352     }
2353 
2354     /**
2355      * {@inheritDoc} <!--workaround-->
2356      */
2357     @Override public abstract $abstractvectortype$ addIndex(int scale);
2358 
2359     /*package-private*/
2360     @ForceInline
2361     final $abstractvectortype$ addIndexTemplate(int scale) {
2362         $Type$Species vsp = vspecies();
2363         // make sure VLENGTH*scale doesn't overflow:
2364         vsp.checkScale(scale);
2365         return VectorSupport.indexVector(
2366             getClass(), $type$.class, length(),
2367             this, scale, vsp,
2368             (v, scale_, s)
2369             -> {
2370                 // If the platform doesn't support an INDEX
2371                 // instruction directly, load IOTA from memory
2372                 // and multiply.
2373                 $abstractvectortype$ iota = s.iota();
2374                 $type$ sc = ($type$) scale_;
2375                 return v.add(sc == 1 ? iota : iota.mul(sc));
2376             });
2377     }
2378 
2379     /**
2380      * Replaces selected lanes of this vector with
2381      * a scalar value
2382      * under the control of a mask.
2383      *
2384      * This is a masked lane-wise binary operation which
2385      * selects each lane value from one or the other input.
2386      *
2387      * The returned result is equal to the expression
2388      * {@code blend(broadcast(e),m)}.
2389      *
2390      * @param e the input scalar, containing the replacement lane value
2391      * @param m the mask controlling lane selection of the scalar
2392      * @return the result of blending the lane elements of this vector with
2393      *         the scalar value
2394      */
2395     @ForceInline
2396     public final $abstractvectortype$ blend($type$ e,
2397                                             VectorMask<$Boxtype$> m) {
2398         return blend(broadcast(e), m);
2399     }
2400 
2401 #if[!long]
2402     /**
2403      * Replaces selected lanes of this vector with
2404      * a scalar value
2405      * under the control of a mask.
2406      *
2407      * This is a masked lane-wise binary operation which
2408      * selects each lane value from one or the other input.
2409      *
2410      * The returned result is equal to the expression
2411      * {@code blend(broadcast(e),m)}.
2412      *
2413      * @param e the input scalar, containing the replacement lane value
2414      * @param m the mask controlling lane selection of the scalar
2415      * @return the result of blending the lane elements of this vector with
2416      *         the scalar value
2417      */
2418     @ForceInline
2419     public final $abstractvectortype$ blend(long e,
2420                                             VectorMask<$Boxtype$> m) {
2421         return blend(broadcast(e), m);
2422     }
2423 #end[!long]
2424 
2425     /**
2426      * {@inheritDoc} <!--workaround-->
2427      */
2428     @Override
2429     public abstract
2430     $abstractvectortype$ slice(int origin, Vector<$Boxtype$> v1);
2431 
2432     /*package-private*/
2433     final
2434     @ForceInline
2435     $abstractvectortype$ sliceTemplate(int origin, Vector<$Boxtype$> v1) {
2436         $abstractvectortype$ that = ($abstractvectortype$) v1;
2437         that.check(this);
2438         Objects.checkIndex(origin, length() + 1);
2439         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2440         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin))));
2441         iota = iotaShuffle(origin, 1, true);
2442         return that.rearrange(iota).blend(this.rearrange(iota), blendMask);
2443     }
2444 
2445     /**
2446      * {@inheritDoc} <!--workaround-->
2447      */
2448     @Override
2449     @ForceInline
2450     public final
2451     $abstractvectortype$ slice(int origin,
2452                                Vector<$Boxtype$> w,
2453                                VectorMask<$Boxtype$> m) {
2454         return broadcast(0).blend(slice(origin, w), m);
2455     }
2456 
2457     /**
2458      * {@inheritDoc} <!--workaround-->
2459      */
2460     @Override
2461     public abstract
2462     $abstractvectortype$ slice(int origin);
2463 
2464     /*package-private*/
2465     final
2466     @ForceInline
2467     $abstractvectortype$ sliceTemplate(int origin) {
2468         Objects.checkIndex(origin, length() + 1);
2469         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2470         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin))));
2471         iota = iotaShuffle(origin, 1, true);
2472         return vspecies().zero().blend(this.rearrange(iota), blendMask);
2473     }
2474 
2475     /**
2476      * {@inheritDoc} <!--workaround-->
2477      */
2478     @Override
2479     public abstract
2480     $abstractvectortype$ unslice(int origin, Vector<$Boxtype$> w, int part);
2481 
2482     /*package-private*/
2483     final
2484     @ForceInline
2485     $abstractvectortype$
2486     unsliceTemplate(int origin, Vector<$Boxtype$> w, int part) {
2487         $abstractvectortype$ that = ($abstractvectortype$) w;
2488         that.check(this);
2489         Objects.checkIndex(origin, length() + 1);
2490         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2491         VectorMask<$Boxtype$> blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT,
2492                                                                   (broadcast(($type$)(origin))));
2493         iota = iotaShuffle(-origin, 1, true);
2494         return that.blend(this.rearrange(iota), blendMask);
2495     }
2496 
2497     /*package-private*/
2498     final
2499     @ForceInline
2500     <M extends VectorMask<$Boxtype$>>
2501     $abstractvectortype$
2502     unsliceTemplate(Class<M> maskType, int origin, Vector<$Boxtype$> w, int part, M m) {
2503         $abstractvectortype$ that = ($abstractvectortype$) w;
2504         that.check(this);
2505         $abstractvectortype$ slice = that.sliceTemplate(origin, that);
2506         slice = slice.blendTemplate(maskType, this, m);
2507         return slice.unsliceTemplate(origin, w, part);
2508     }
2509 
2510     /**
2511      * {@inheritDoc} <!--workaround-->
2512      */
2513     @Override
2514     public abstract
2515     $abstractvectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m);
2516 
2517     /**
2518      * {@inheritDoc} <!--workaround-->
2519      */
2520     @Override
2521     public abstract
2522     $abstractvectortype$ unslice(int origin);
2523 
2524     /*package-private*/
2525     final
2526     @ForceInline
2527     $abstractvectortype$
2528     unsliceTemplate(int origin) {
2529         Objects.checkIndex(origin, length() + 1);
2530         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2531         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.GE,
2532                                                                   (broadcast(($type$)(origin))));
2533         iota = iotaShuffle(-origin, 1, true);
2534         return vspecies().zero().blend(this.rearrange(iota), blendMask);
2535     }
2536 
2537     private ArrayIndexOutOfBoundsException
2538     wrongPartForSlice(int part) {
2539         String msg = String.format("bad part number %d for slice operation",
2540                                    part);
2541         return new ArrayIndexOutOfBoundsException(msg);
2542     }
2543 
2544     /**
2545      * {@inheritDoc} <!--workaround-->
2546      */
2547     @Override
2548     public abstract
2549     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> m);
2550 
2551     /*package-private*/
2552     @ForceInline
2553     final
2554     <S extends VectorShuffle<$Boxtype$>>
2555     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype, S shuffle) {
2556         shuffle.checkIndexes();
2557         return VectorSupport.rearrangeOp(
2558             getClass(), shuffletype, null, $type$.class, length(),
2559             this, shuffle, null,
2560             (v1, s_, m_) -> v1.uOp((i, a) -> {
2561                 int ei = s_.laneSource(i);
2562                 return v1.lane(ei);
2563             }));
2564     }
2565 
2566     /**
2567      * {@inheritDoc} <!--workaround-->
2568      */
2569     @Override
2570     public abstract
2571     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2572                                    VectorMask<$Boxtype$> m);
2573 
2574     /*package-private*/
2575     @ForceInline
2576     final
2577     <S extends VectorShuffle<$Boxtype$>, M extends VectorMask<$Boxtype$>>
2578     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2579                                            Class<M> masktype,
2580                                            S shuffle,
2581                                            M m) {
2582 
2583         m.check(masktype, this);
2584         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2585         if (m.andNot(valid).anyTrue()) {
2586             shuffle.checkIndexes();
2587             throw new AssertionError();
2588         }
2589         return VectorSupport.rearrangeOp(
2590                    getClass(), shuffletype, masktype, $type$.class, length(),
2591                    this, shuffle, m,
2592                    (v1, s_, m_) -> v1.uOp((i, a) -> {
2593                         int ei = s_.laneSource(i);
2594                         return ei < 0  || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
2595                    }));
2596     }
2597 
2598     /**
2599      * {@inheritDoc} <!--workaround-->
2600      */
2601     @Override
2602     public abstract
2603     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2604                                    Vector<$Boxtype$> v);
2605 
2606     /*package-private*/
2607     @ForceInline
2608     final
2609     <S extends VectorShuffle<$Boxtype$>>
2610     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2611                                            S shuffle,
2612                                            $abstractvectortype$ v) {
2613         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2614         @SuppressWarnings("unchecked")
2615         S ws = (S) shuffle.wrapIndexes();
2616         $abstractvectortype$ r0 =
2617             VectorSupport.rearrangeOp(
2618                 getClass(), shuffletype, null, $type$.class, length(),
2619                 this, ws, null,
2620                 (v0, s_, m_) -> v0.uOp((i, a) -> {
2621                     int ei = s_.laneSource(i);
2622                     return v0.lane(ei);
2623                 }));
2624         $abstractvectortype$ r1 =
2625             VectorSupport.rearrangeOp(
2626                 getClass(), shuffletype, null, $type$.class, length(),
2627                 v, ws, null,
2628                 (v1, s_, m_) -> v1.uOp((i, a) -> {
2629                     int ei = s_.laneSource(i);
2630                     return v1.lane(ei);
2631                 }));
2632         return r1.blend(r0, valid);
2633     }
2634 
2635     @ForceInline
2636     private final
2637     VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) {
2638         $type$[] a = toArray();
2639         int[] sa = new int[a.length];
2640         for (int i = 0; i < a.length; i++) {
2641             sa[i] = (int) a[i];
2642         }
2643         return VectorShuffle.fromArray(dsp, sa, 0);
2644     }
2645 
2646     /*package-private*/
2647     @ForceInline
2648     final
2649     VectorShuffle<$Boxtype$> toShuffleTemplate(Class<?> shuffleType) {
2650         $Type$Species vsp = vspecies();
2651         return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
2652                                      getClass(), $type$.class, length(),
2653                                      shuffleType, byte.class, length(),
2654                                      this, vsp,
2655                                      $Type$Vector::toShuffle0);
2656     }
2657 
2658     /**
2659      * {@inheritDoc} <!--workaround-->
2660      */
2661     @Override
2662     public abstract
2663     $abstractvectortype$ selectFrom(Vector<$Boxtype$> v);
2664 
2665     /*package-private*/
2666     @ForceInline
2667     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v) {
2668         return v.rearrange(this.toShuffle());
2669     }
2670 
2671     /**
2672      * {@inheritDoc} <!--workaround-->
2673      */
2674     @Override
2675     public abstract
2676     $abstractvectortype$ selectFrom(Vector<$Boxtype$> s, VectorMask<$Boxtype$> m);
2677 
2678     /*package-private*/
2679     @ForceInline
2680     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v,
2681                                                   AbstractMask<$Boxtype$> m) {
2682         return v.rearrange(this.toShuffle(), m);
2683     }
2684 
2685     /// Ternary operations
2686 
2687 #if[BITWISE]
2688     /**
2689      * Blends together the bits of two vectors under
2690      * the control of a third, which supplies mask bits.
2691      *
2692      * This is a lane-wise ternary operation which performs
2693      * a bitwise blending operation {@code (a&~c)|(b&c)}
2694      * to each lane.
2695      *
2696      * This method is also equivalent to the expression
2697      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2698      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2699      *    BITWISE_BLEND}{@code , bits, mask)}.
2700      *
2701      * @param bits input bits to blend into the current vector
2702      * @param mask a bitwise mask to enable blending of the input bits
2703      * @return the bitwise blend of the given bits into the current vector,
2704      *         under control of the bitwise mask
2705      * @see #bitwiseBlend($type$,$type$)
2706      * @see #bitwiseBlend($type$,Vector)
2707      * @see #bitwiseBlend(Vector,$type$)
2708      * @see VectorOperators#BITWISE_BLEND
2709      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
2710      */
2711     @ForceInline
2712     public final
2713     $abstractvectortype$ bitwiseBlend(Vector<$Boxtype$> bits, Vector<$Boxtype$> mask) {
2714         return lanewise(BITWISE_BLEND, bits, mask);
2715     }
2716 
2717     /**
2718      * Blends together the bits of a vector and a scalar under
2719      * the control of another scalar, which supplies mask bits.
2720      *
2721      * This is a lane-wise ternary operation which performs
2722      * a bitwise blending operation {@code (a&~c)|(b&c)}
2723      * to each lane.
2724      *
2725      * This method is also equivalent to the expression
2726      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2727      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2728      *    BITWISE_BLEND}{@code , bits, mask)}.
2729      *
2730      * @param bits input bits to blend into the current vector
2731      * @param mask a bitwise mask to enable blending of the input bits
2732      * @return the bitwise blend of the given bits into the current vector,
2733      *         under control of the bitwise mask
2734      * @see #bitwiseBlend(Vector,Vector)
2735      * @see VectorOperators#BITWISE_BLEND
2736      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
2737      */
2738     @ForceInline
2739     public final
2740     $abstractvectortype$ bitwiseBlend($type$ bits, $type$ mask) {
2741         return lanewise(BITWISE_BLEND, bits, mask);
2742     }
2743 
2744     /**
2745      * Blends together the bits of a vector and a scalar under
2746      * the control of another vector, which supplies mask bits.
2747      *
2748      * This is a lane-wise ternary operation which performs
2749      * a bitwise blending operation {@code (a&~c)|(b&c)}
2750      * to each lane.
2751      *
2752      * This method is also equivalent to the expression
2753      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2754      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2755      *    BITWISE_BLEND}{@code , bits, mask)}.
2756      *
2757      * @param bits input bits to blend into the current vector
2758      * @param mask a bitwise mask to enable blending of the input bits
2759      * @return the bitwise blend of the given bits into the current vector,
2760      *         under control of the bitwise mask
2761      * @see #bitwiseBlend(Vector,Vector)
2762      * @see VectorOperators#BITWISE_BLEND
2763      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
2764      */
2765     @ForceInline
2766     public final
2767     $abstractvectortype$ bitwiseBlend($type$ bits, Vector<$Boxtype$> mask) {
2768         return lanewise(BITWISE_BLEND, bits, mask);
2769     }
2770 
2771     /**
2772      * Blends together the bits of two vectors under
2773      * the control of a scalar, which supplies mask bits.
2774      *
2775      * This is a lane-wise ternary operation which performs
2776      * a bitwise blending operation {@code (a&~c)|(b&c)}
2777      * to each lane.
2778      *
2779      * This method is also equivalent to the expression
2780      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2781      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2782      *    BITWISE_BLEND}{@code , bits, mask)}.
2783      *
2784      * @param bits input bits to blend into the current vector
2785      * @param mask a bitwise mask to enable blending of the input bits
2786      * @return the bitwise blend of the given bits into the current vector,
2787      *         under control of the bitwise mask
2788      * @see #bitwiseBlend(Vector,Vector)
2789      * @see VectorOperators#BITWISE_BLEND
2790      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
2791      */
2792     @ForceInline
2793     public final
2794     $abstractvectortype$ bitwiseBlend(Vector<$Boxtype$> bits, $type$ mask) {
2795         return lanewise(BITWISE_BLEND, bits, mask);
2796     }
2797 #end[BITWISE]
2798 
2799 #if[FP]
2800     /**
2801      * Multiplies this vector by a second input vector, and sums
2802      * the result with a third.
2803      *
2804      * Extended precision is used for the intermediate result,
2805      * avoiding possible loss of precision from rounding once
2806      * for each of the two operations.
2807      * The result is numerically close to {@code this.mul(b).add(c)},
2808      * and is typically closer to the true mathematical result.
2809      *
2810      * This is a lane-wise ternary operation which applies an operation
2811      * conforming to the specification of
2812      * {@link Math#fma($type$,$type$,$type$) Math.fma(a,b,c)}
2813      * to each lane.
2814 #if[intOrFloat]
2815      * The operation is adapted to cast the operands and the result,
2816      * specifically widening {@code float} operands to {@code double}
2817      * operands and narrowing the {@code double} result to a {@code float}
2818      * result.
2819 #end[intOrFloat]
2820      *
2821      * This method is also equivalent to the expression
2822      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2823      *    lanewise}{@code (}{@link VectorOperators#FMA
2824      *    FMA}{@code , b, c)}.
2825      *
2826      * @param b the second input vector, supplying multiplier values
2827      * @param c the third input vector, supplying addend values
2828      * @return the product of this vector and the second input vector
2829      *         summed with the third input vector, using extended precision
2830      *         for the intermediate result
2831      * @see #fma($type$,$type$)
2832      * @see VectorOperators#FMA
2833      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
2834      */
2835     @ForceInline
2836     public final
2837     $abstractvectortype$ fma(Vector<$Boxtype$> b, Vector<$Boxtype$> c) {
2838         return lanewise(FMA, b, c);
2839     }
2840 
2841     /**
2842      * Multiplies this vector by a scalar multiplier, and sums
2843      * the result with a scalar addend.
2844      *
2845      * Extended precision is used for the intermediate result,
2846      * avoiding possible loss of precision from rounding once
2847      * for each of the two operations.
2848      * The result is numerically close to {@code this.mul(b).add(c)},
2849      * and is typically closer to the true mathematical result.
2850      *
2851      * This is a lane-wise ternary operation which applies an operation
2852      * conforming to the specification of
2853      * {@link Math#fma($type$,$type$,$type$) Math.fma(a,b,c)}
2854      * to each lane.
2855 #if[intOrFloat]
2856      * The operation is adapted to cast the operands and the result,
2857      * specifically widening {@code float} operands to {@code double}
2858      * operands and narrowing the {@code double} result to a {@code float}
2859      * result.
2860 #end[intOrFloat]
2861      *
2862      * This method is also equivalent to the expression
2863      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2864      *    lanewise}{@code (}{@link VectorOperators#FMA
2865      *    FMA}{@code , b, c)}.
2866      *
2867      * @param b the scalar multiplier
2868      * @param c the scalar addend
2869      * @return the product of this vector and the scalar multiplier
2870      *         summed with scalar addend, using extended precision
2871      *         for the intermediate result
2872      * @see #fma(Vector,Vector)
2873      * @see VectorOperators#FMA
2874      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
2875      */
2876     @ForceInline
2877     public final
2878     $abstractvectortype$ fma($type$ b, $type$ c) {
2879         return lanewise(FMA, b, c);
2880     }
2881 
2882     // Don't bother with (Vector,$type$) and ($type$,Vector) overloadings.
2883 #end[FP]
2884 
2885     // Type specific horizontal reductions
2886 
2887     /**
2888      * Returns a value accumulated from all the lanes of this vector.
2889      *
2890      * This is an associative cross-lane reduction operation which
2891      * applies the specified operation to all the lane elements.
2892      * <p>
2893      * A few reduction operations do not support arbitrary reordering
2894      * of their operands, yet are included here because of their
2895      * usefulness.
2896      * <ul>
2897      * <li>
2898      * In the case of {@code FIRST_NONZERO}, the reduction returns
2899      * the value from the lowest-numbered non-zero lane.
2900 #if[FP]
2901      * (As with {@code MAX} and {@code MIN}, floating point negative
2902      * zero {@code -0.0} is treated as a value distinct from
2903      * the default value, positive zero. So a first-nonzero lane reduction
2904      * might return {@code -0.0} even in the presence of non-zero
2905      * lane values.)
2906      * <li>
2907      * In the case of {@code ADD} and {@code MUL}, the
2908      * precise result will reflect the choice of an arbitrary order
2909      * of operations, which may even vary over time.
2910      * For further details see the section
2911      * <a href="VectorOperators.html#fp_assoc">Operations on floating point vectors</a>.
2912 #end[FP]
2913      * <li>
2914      * All other reduction operations are fully commutative and
2915      * associative.  The implementation can choose any order of
2916      * processing, yet it will always produce the same result.
2917      * </ul>
2918      *
2919      * @param op the operation used to combine lane values
2920      * @return the accumulated result
2921      * @throws UnsupportedOperationException if this vector does
2922      *         not support the requested operation
2923      * @see #reduceLanes(VectorOperators.Associative,VectorMask)
2924      * @see #add(Vector)
2925      * @see #mul(Vector)
2926      * @see #min(Vector)
2927      * @see #max(Vector)
2928 #if[BITWISE]
2929      * @see #and(Vector)
2930      * @see #or(Vector)
2931      * @see VectorOperators#XOR
2932 #end[BITWISE]
2933      * @see VectorOperators#FIRST_NONZERO
2934      */
2935     public abstract $type$ reduceLanes(VectorOperators.Associative op);
2936 
2937     /**
2938      * Returns a value accumulated from selected lanes of this vector,
2939      * controlled by a mask.
2940      *
2941      * This is an associative cross-lane reduction operation which
2942      * applies the specified operation to the selected lane elements.
2943      * <p>
2944      * If no elements are selected, an operation-specific identity
2945      * value is returned.
2946      * <ul>
2947      * <li>
2948      * If the operation is
2949 #if[BITWISE]
2950      *  {@code ADD}, {@code XOR}, {@code OR},
2951 #else[BITWISE]
2952      *  {@code ADD}
2953 #end[BITWISE]
2954      * or {@code FIRST_NONZERO},
2955      * then the identity value is {#if[FP]?positive }zero, the default {@code $type$} value.
2956      * <li>
2957      * If the operation is {@code MUL},
2958      * then the identity value is one.
2959 #if[BITWISE]
2960      * <li>
2961      * If the operation is {@code AND},
2962      * then the identity value is minus one (all bits set).
2963      * <li>
2964      * If the operation is {@code MAX},
2965      * then the identity value is {@code $Boxtype$.MIN_VALUE}.
2966      * <li>
2967      * If the operation is {@code MIN},
2968      * then the identity value is {@code $Boxtype$.MAX_VALUE}.
2969 #end[BITWISE]
2970 #if[FP]
2971      * <li>
2972      * If the operation is {@code MAX},
2973      * then the identity value is {@code $Boxtype$.NEGATIVE_INFINITY}.
2974      * <li>
2975      * If the operation is {@code MIN},
2976      * then the identity value is {@code $Boxtype$.POSITIVE_INFINITY}.
2977 #end[FP]
2978      * </ul>
2979      * <p>
2980      * A few reduction operations do not support arbitrary reordering
2981      * of their operands, yet are included here because of their
2982      * usefulness.
2983      * <ul>
2984      * <li>
2985      * In the case of {@code FIRST_NONZERO}, the reduction returns
2986      * the value from the lowest-numbered non-zero lane.
2987 #if[FP]
2988      * (As with {@code MAX} and {@code MIN}, floating point negative
2989      * zero {@code -0.0} is treated as a value distinct from
2990      * the default value, positive zero. So a first-nonzero lane reduction
2991      * might return {@code -0.0} even in the presence of non-zero
2992      * lane values.)
2993      * <li>
2994      * In the case of {@code ADD} and {@code MUL}, the
2995      * precise result will reflect the choice of an arbitrary order
2996      * of operations, which may even vary over time.
2997      * For further details see the section
2998      * <a href="VectorOperators.html#fp_assoc">Operations on floating point vectors</a>.
2999 #end[FP]
3000      * <li>
3001      * All other reduction operations are fully commutative and
3002      * associative.  The implementation can choose any order of
3003      * processing, yet it will always produce the same result.
3004      * </ul>
3005      *
3006      * @param op the operation used to combine lane values
3007      * @param m the mask controlling lane selection
3008      * @return the reduced result accumulated from the selected lane values
3009      * @throws UnsupportedOperationException if this vector does
3010      *         not support the requested operation
3011      * @see #reduceLanes(VectorOperators.Associative)
3012      */
3013     public abstract $type$ reduceLanes(VectorOperators.Associative op,
3014                                        VectorMask<$Boxtype$> m);
3015 
3016     /*package-private*/
3017     @ForceInline
3018     final
3019     $type$ reduceLanesTemplate(VectorOperators.Associative op,
3020                                Class<? extends VectorMask<$Boxtype$>> maskClass,
3021                                VectorMask<$Boxtype$> m) {
3022         m.check(maskClass, this);
3023         if (op == FIRST_NONZERO) {
3024             $abstractvectortype$ v = reduceIdentityVector(op).blend(this, m);
3025             return v.reduceLanesTemplate(op);
3026         }
3027         int opc = opCode(op);
3028         return fromBits(VectorSupport.reductionCoerced(
3029             opc, getClass(), maskClass, $type$.class, length(),
3030             this, m,
3031             REDUCE_IMPL.find(op, opc, $abstractvectortype$::reductionOperations)));
3032     }
3033 
3034     /*package-private*/
3035     @ForceInline
3036     final
3037     $type$ reduceLanesTemplate(VectorOperators.Associative op) {
3038         if (op == FIRST_NONZERO) {
3039             // FIXME:  The JIT should handle this, and other scan ops alos.
3040             VectorMask<$Boxbitstype$> thisNZ
3041                 = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
3042             return this.lane(thisNZ.firstTrue());
3043         }
3044         int opc = opCode(op);
3045         return fromBits(VectorSupport.reductionCoerced(
3046             opc, getClass(), null, $type$.class, length(),
3047             this, null,
3048             REDUCE_IMPL.find(op, opc, $abstractvectortype$::reductionOperations)));
3049     }
3050 
3051     private static final
3052     ImplCache<Associative, ReductionOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
3053         REDUCE_IMPL = new ImplCache<>(Associative.class, $Type$Vector.class);
3054 
3055     private static ReductionOperation<$abstractvectortype$, VectorMask<$Boxtype$>> reductionOperations(int opc_) {
3056         switch (opc_) {
3057             case VECTOR_OP_ADD: return (v, m) ->
3058                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a + b)));
3059             case VECTOR_OP_MUL: return (v, m) ->
3060                     toBits(v.rOp(($type$)1, m, (i, a, b) -> ($type$)(a * b)));
3061             case VECTOR_OP_MIN: return (v, m) ->
3062                     toBits(v.rOp(MAX_OR_INF, m, (i, a, b) -> ($type$) Math.min(a, b)));
3063             case VECTOR_OP_MAX: return (v, m) ->
3064                     toBits(v.rOp(MIN_OR_INF, m, (i, a, b) -> ($type$) Math.max(a, b)));
3065 #if[BITWISE]
3066             case VECTOR_OP_AND: return (v, m) ->
3067                     toBits(v.rOp(($type$)-1, m, (i, a, b) -> ($type$)(a & b)));
3068             case VECTOR_OP_OR: return (v, m) ->
3069                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a | b)));
3070             case VECTOR_OP_XOR: return (v, m) ->
3071                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a ^ b)));
3072 #end[BITWISE]
3073             default: return null;
3074         }
3075     }
3076 
3077     private
3078     @ForceInline
3079     $abstractvectortype$ reduceIdentityVector(VectorOperators.Associative op) {
3080         int opc = opCode(op);
3081         UnaryOperator<$abstractvectortype$> fn
3082             = REDUCE_ID_IMPL.find(op, opc, (opc_) -> {
3083                 switch (opc_) {
3084                 case VECTOR_OP_ADD:
3085 #if[BITWISE]
3086                 case VECTOR_OP_OR:
3087                 case VECTOR_OP_XOR:
3088 #end[BITWISE]
3089                     return v -> v.broadcast(0);
3090                 case VECTOR_OP_MUL:
3091                     return v -> v.broadcast(1);
3092 #if[BITWISE]
3093                 case VECTOR_OP_AND:
3094                     return v -> v.broadcast(-1);
3095 #end[BITWISE]
3096                 case VECTOR_OP_MIN:
3097                     return v -> v.broadcast(MAX_OR_INF);
3098                 case VECTOR_OP_MAX:
3099                     return v -> v.broadcast(MIN_OR_INF);
3100                 default: return null;
3101                 }
3102             });
3103         return fn.apply(this);
3104     }
3105     private static final
3106     ImplCache<Associative,UnaryOperator<$abstractvectortype$>> REDUCE_ID_IMPL
3107         = new ImplCache<>(Associative.class, $Type$Vector.class);
3108 
3109 #if[FP]
3110     private static final $type$ MIN_OR_INF = $Boxtype$.NEGATIVE_INFINITY;
3111     private static final $type$ MAX_OR_INF = $Boxtype$.POSITIVE_INFINITY;
3112 #else[FP]
3113     private static final $type$ MIN_OR_INF = $Boxtype$.MIN_VALUE;
3114     private static final $type$ MAX_OR_INF = $Boxtype$.MAX_VALUE;
3115 #end[FP]
3116 
3117     public @Override abstract long reduceLanesToLong(VectorOperators.Associative op);
3118     public @Override abstract long reduceLanesToLong(VectorOperators.Associative op,
3119                                                      VectorMask<$Boxtype$> m);
3120 
3121     // Type specific accessors
3122 
3123     /**
3124      * Gets the lane element at lane index {@code i}
3125      *
3126      * @param i the lane index
3127      * @return the lane element at lane index {@code i}
3128      * @throws IllegalArgumentException if the index is is out of range
3129      * ({@code < 0 || >= length()})
3130      */
3131     public abstract $type$ lane(int i);
3132 
3133     /**
3134      * Replaces the lane element of this vector at lane index {@code i} with
3135      * value {@code e}.
3136      *
3137      * This is a cross-lane operation and behaves as if it returns the result
3138      * of blending this vector with an input vector that is the result of
3139      * broadcasting {@code e} and a mask that has only one lane set at lane
3140      * index {@code i}.
3141      *
3142      * @param i the lane index of the lane element to be replaced
3143      * @param e the value to be placed
3144      * @return the result of replacing the lane element of this vector at lane
3145      * index {@code i} with value {@code e}.
3146      * @throws IllegalArgumentException if the index is is out of range
3147      * ({@code < 0 || >= length()})
3148      */
3149     public abstract $abstractvectortype$ withLane(int i, $type$ e);
3150 
3151     // Memory load operations
3152 
3153     /**
3154      * Returns an array of type {@code $type$[]}
3155      * containing all the lane values.
3156      * The array length is the same as the vector length.
3157      * The array elements are stored in lane order.
3158      * <p>
3159      * This method behaves as if it stores
3160      * this vector into an allocated array
3161      * (using {@link #intoArray($type$[], int) intoArray})
3162      * and returns the array as follows:
3163      * <pre>{@code
3164      *   $type$[] a = new $type$[this.length()];
3165      *   this.intoArray(a, 0);
3166      *   return a;
3167      * }</pre>
3168      *
3169      * @return an array containing the lane values of this vector
3170      */
3171     @ForceInline
3172     @Override
3173     public final $type$[] toArray() {
3174         $type$[] a = new $type$[vspecies().laneCount()];
3175         intoArray(a, 0);
3176         return a;
3177     }
3178 
3179 #if[int]
3180     /**
3181      * {@inheritDoc} <!--workaround-->
3182      * This is an alias for {@link #toArray()}
3183      * When this method is used on used on vectors
3184      * of type {@code $abstractvectortype$},
3185      * there will be no loss of range or precision.
3186      */
3187     @ForceInline
3188     @Override
3189     public final int[] toIntArray() {
3190         return toArray();
3191     }
3192 #else[int]
3193     /** {@inheritDoc} <!--workaround-->
3194 #if[!FP]
3195 #if[!long]
3196      * @implNote
3197      * When this method is used on used on vectors
3198      * of type {@code $abstractvectortype$},
3199      * there will be no loss of precision or range,
3200      * and so no {@code UnsupportedOperationException} will
3201      * be thrown.
3202 #end[!long]
3203 #end[!FP]
3204      */
3205     @ForceInline
3206     @Override
3207     public final int[] toIntArray() {
3208         $type$[] a = toArray();
3209         int[] res = new int[a.length];
3210         for (int i = 0; i < a.length; i++) {
3211             $type$ e = a[i];
3212             res[i] = (int) $Type$Species.toIntegralChecked(e, true);
3213         }
3214         return res;
3215     }
3216 #end[int]
3217 
3218 #if[long]
3219     /**
3220      * {@inheritDoc} <!--workaround-->
3221      * This is an alias for {@link #toArray()}
3222      * When this method is used on used on vectors
3223      * of type {@code $abstractvectortype$},
3224      * there will be no loss of range or precision.
3225      */
3226     @ForceInline
3227     @Override
3228     public final long[] toLongArray() {
3229         return toArray();
3230     }
3231 #else[long]
3232     /** {@inheritDoc} <!--workaround-->
3233 #if[!FP]
3234      * @implNote
3235      * When this method is used on used on vectors
3236      * of type {@code $abstractvectortype$},
3237      * there will be no loss of precision or range,
3238      * and so no {@code UnsupportedOperationException} will
3239      * be thrown.
3240 #end[!FP]
3241      */
3242     @ForceInline
3243     @Override
3244     public final long[] toLongArray() {
3245         $type$[] a = toArray();
3246         long[] res = new long[a.length];
3247         for (int i = 0; i < a.length; i++) {
3248             $type$ e = a[i];
3249             res[i] = $Type$Species.toIntegralChecked(e, false);
3250         }
3251         return res;
3252     }
3253 #end[long]
3254 
3255 #if[double]
3256     /** {@inheritDoc} <!--workaround-->
3257      * @implNote
3258      * This is an alias for {@link #toArray()}
3259      * When this method is used on used on vectors
3260      * of type {@code $abstractvectortype$},
3261      * there will be no loss of precision.
3262      */
3263     @ForceInline
3264     @Override
3265     public final double[] toDoubleArray() {
3266         return toArray();
3267     }
3268 #else[double]
3269     /** {@inheritDoc} <!--workaround-->
3270 #if[long]
3271      * @implNote
3272      * When this method is used on used on vectors
3273      * of type {@code $abstractvectortype$},
3274      * up to nine bits of precision may be lost
3275      * for lane values of large magnitude.
3276 #else[long]
3277      * @implNote
3278      * When this method is used on used on vectors
3279      * of type {@code $abstractvectortype$},
3280      * there will be no loss of precision.
3281 #end[long]
3282      */
3283     @ForceInline
3284     @Override
3285     public final double[] toDoubleArray() {
3286         $type$[] a = toArray();
3287         double[] res = new double[a.length];
3288         for (int i = 0; i < a.length; i++) {
3289             res[i] = (double) a[i];
3290         }
3291         return res;
3292     }
3293 #end[double]
3294 
3295     /**
3296      * Loads a vector from a byte array starting at an offset.
3297      * Bytes are composed into primitive lane elements according
3298      * to the specified byte order.
3299      * The vector is arranged into lanes according to
3300      * <a href="Vector.html#lane-order">memory ordering</a>.
3301      * <p>
3302      * This method behaves as if it returns the result of calling
3303      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3304      * fromByteBuffer()} as follows:
3305      * <pre>{@code
3306      * var bb = ByteBuffer.wrap(a);
3307      * var m = species.maskAll(true);
3308      * return fromByteBuffer(species, bb, offset, bo, m);
3309      * }</pre>
3310      *
3311      * @param species species of desired vector
3312      * @param a the byte array
3313      * @param offset the offset into the array
3314      * @param bo the intended byte order
3315      * @return a vector loaded from a byte array
3316      * @throws IndexOutOfBoundsException
3317      *         if {@code offset+N*ESIZE < 0}
3318      *         or {@code offset+(N+1)*ESIZE > a.length}
3319      *         for any lane {@code N} in the vector
3320      */
3321     @ForceInline
3322     public static
3323     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3324                                        byte[] a, int offset,
3325                                        ByteOrder bo) {
3326         offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
3327         $Type$Species vsp = ($Type$Species) species;
3328         return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
3329     }
3330 
3331     /**
3332      * Loads a vector from a byte array starting at an offset
3333      * and using a mask.
3334      * Lanes where the mask is unset are filled with the default
3335      * value of {@code $type$} ({#if[FP]?positive }zero).
3336      * Bytes are composed into primitive lane elements according
3337      * to the specified byte order.
3338      * The vector is arranged into lanes according to
3339      * <a href="Vector.html#lane-order">memory ordering</a>.
3340      * <p>
3341      * This method behaves as if it returns the result of calling
3342      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3343      * fromByteBuffer()} as follows:
3344      * <pre>{@code
3345      * var bb = ByteBuffer.wrap(a);
3346      * return fromByteBuffer(species, bb, offset, bo, m);
3347      * }</pre>
3348      *
3349      * @param species species of desired vector
3350      * @param a the byte array
3351      * @param offset the offset into the array
3352      * @param bo the intended byte order
3353      * @param m the mask controlling lane selection
3354      * @return a vector loaded from a byte array
3355      * @throws IndexOutOfBoundsException
3356      *         if {@code offset+N*ESIZE < 0}
3357      *         or {@code offset+(N+1)*ESIZE > a.length}
3358      *         for any lane {@code N} in the vector
3359      *         where the mask is set
3360      */
3361     @ForceInline
3362     public static
3363     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3364                                        byte[] a, int offset,
3365                                        ByteOrder bo,
3366                                        VectorMask<$Boxtype$> m) {
3367         $Type$Species vsp = ($Type$Species) species;
3368         if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
3369             return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
3370         }
3371 
3372         // FIXME: optimize
3373         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
3374         ByteBuffer wb = wrapper(a, bo);
3375         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
3376                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
3377     }
3378 
3379     /**
3380      * Loads a vector from an array of type {@code $type$[]}
3381      * starting at an offset.
3382      * For each vector lane, where {@code N} is the vector lane index, the
3383      * array element at index {@code offset + N} is placed into the
3384      * resulting vector at lane index {@code N}.
3385      *
3386      * @param species species of desired vector
3387      * @param a the array
3388      * @param offset the offset into the array
3389      * @return the vector loaded from an array
3390      * @throws IndexOutOfBoundsException
3391      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3392      *         for any lane {@code N} in the vector
3393      */
3394     @ForceInline
3395     public static
3396     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3397                                    $type$[] a, int offset) {
3398         offset = checkFromIndexSize(offset, species.length(), a.length);
3399         $Type$Species vsp = ($Type$Species) species;
3400         return vsp.dummyVector().fromArray0(a, offset);
3401     }
3402 
3403     /**
3404      * Loads a vector from an array of type {@code $type$[]}
3405      * starting at an offset and using a mask.
3406      * Lanes where the mask is unset are filled with the default
3407      * value of {@code $type$} ({#if[FP]?positive }zero).
3408      * For each vector lane, where {@code N} is the vector lane index,
3409      * if the mask lane at index {@code N} is set then the array element at
3410      * index {@code offset + N} is placed into the resulting vector at lane index
3411      * {@code N}, otherwise the default element value is placed into the
3412      * resulting vector at lane index {@code N}.
3413      *
3414      * @param species species of desired vector
3415      * @param a the array
3416      * @param offset the offset into the array
3417      * @param m the mask controlling lane selection
3418      * @return the vector loaded from an array
3419      * @throws IndexOutOfBoundsException
3420      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3421      *         for any lane {@code N} in the vector
3422      *         where the mask is set
3423      */
3424     @ForceInline
3425     public static
3426     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3427                                    $type$[] a, int offset,
3428                                    VectorMask<$Boxtype$> m) {
3429         $Type$Species vsp = ($Type$Species) species;
3430         if (offset >= 0 && offset <= (a.length - species.length())) {
3431             return vsp.dummyVector().fromArray0(a, offset, m);
3432         }
3433 
3434         // FIXME: optimize
3435         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3436         return vsp.vOp(m, i -> a[offset + i]);
3437     }
3438 
3439     /**
3440      * Gathers a new vector composed of elements from an array of type
3441      * {@code $type$[]},
3442      * using indexes obtained by adding a fixed {@code offset} to a
3443      * series of secondary offsets from an <em>index map</em>.
3444      * The index map is a contiguous sequence of {@code VLENGTH}
3445      * elements in a second array of {@code int}s, starting at a given
3446      * {@code mapOffset}.
3447      * <p>
3448      * For each vector lane, where {@code N} is the vector lane index,
3449      * the lane is loaded from the array
3450      * element {@code a[f(N)]}, where {@code f(N)} is the
3451      * index mapping expression
3452      * {@code offset + indexMap[mapOffset + N]]}.
3453      *
3454      * @param species species of desired vector
3455      * @param a the array
3456      * @param offset the offset into the array, may be negative if relative
3457      * indexes in the index map compensate to produce a value within the
3458      * array bounds
3459      * @param indexMap the index map
3460      * @param mapOffset the offset into the index map
3461      * @return the vector loaded from the indexed elements of the array
3462      * @throws IndexOutOfBoundsException
3463      *         if {@code mapOffset+N < 0}
3464      *         or if {@code mapOffset+N >= indexMap.length},
3465      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3466      *         is an invalid index into {@code a},
3467      *         for any lane {@code N} in the vector
3468      * @see $abstractvectortype$#toIntArray()
3469      */
3470 #if[byteOrShort]
3471     @ForceInline
3472     public static
3473     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3474                                    $type$[] a, int offset,
3475                                    int[] indexMap, int mapOffset) {
3476         $Type$Species vsp = ($Type$Species) species;
3477         return vsp.vOp(n -> a[offset + indexMap[mapOffset + n]]);
3478     }
3479 #else[byteOrShort]
3480     @ForceInline
3481     public static
3482     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3483                                    $type$[] a, int offset,
3484                                    int[] indexMap, int mapOffset) {
3485         $Type$Species vsp = ($Type$Species) species;
3486         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3487         Objects.requireNonNull(a);
3488         Objects.requireNonNull(indexMap);
3489         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();
3490 
3491 #if[longOrDouble]
3492         if (vsp.laneCount() == 1) {
3493           return $abstractvectortype$.fromArray(vsp, a, offset + indexMap[mapOffset]);
3494         }
3495 
3496         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3497         IntVector vix;
3498         if (isp.laneCount() != vsp.laneCount()) {
3499             // For $Type$MaxVector,  if vector length is non-power-of-two or
3500             // 2048 bits, indexShape of $Type$ species is S_MAX_BIT.
3501             // Assume that vector length is 2048, then the lane count of $Type$
3502             // vector is 32. When converting $Type$ species to int species,
3503             // indexShape is still S_MAX_BIT, but the lane count of int vector
3504             // is 64. So when loading index vector (IntVector), only lower half
3505             // of index data is needed.
3506             vix = IntVector
3507                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3508                 .add(offset);
3509         } else {
3510             vix = IntVector
3511                 .fromArray(isp, indexMap, mapOffset)
3512                 .add(offset);
3513         }
3514 #else[longOrDouble]
3515         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3516         IntVector vix = IntVector
3517             .fromArray(isp, indexMap, mapOffset)
3518             .add(offset);
3519 #end[longOrDouble]
3520 
3521         vix = VectorIntrinsics.checkIndex(vix, a.length);
3522 
3523         return VectorSupport.loadWithMap(
3524             vectorType, null, $type$.class, vsp.laneCount(),
3525             isp.vectorType(),
3526             a, ARRAY_BASE, vix, null,
3527             a, offset, indexMap, mapOffset, vsp,
3528             (c, idx, iMap, idy, s, vm) ->
3529             s.vOp(n -> c[idx + iMap[idy+n]]));
3530     }
3531 #end[byteOrShort]
3532 
3533     /**
3534      * Gathers a new vector composed of elements from an array of type
3535      * {@code $type$[]},
3536      * under the control of a mask, and
3537      * using indexes obtained by adding a fixed {@code offset} to a
3538      * series of secondary offsets from an <em>index map</em>.
3539      * The index map is a contiguous sequence of {@code VLENGTH}
3540      * elements in a second array of {@code int}s, starting at a given
3541      * {@code mapOffset}.
3542      * <p>
3543      * For each vector lane, where {@code N} is the vector lane index,
3544      * if the lane is set in the mask,
3545      * the lane is loaded from the array
3546      * element {@code a[f(N)]}, where {@code f(N)} is the
3547      * index mapping expression
3548      * {@code offset + indexMap[mapOffset + N]]}.
3549      * Unset lanes in the resulting vector are set to zero.
3550      *
3551      * @param species species of desired vector
3552      * @param a the array
3553      * @param offset the offset into the array, may be negative if relative
3554      * indexes in the index map compensate to produce a value within the
3555      * array bounds
3556      * @param indexMap the index map
3557      * @param mapOffset the offset into the index map
3558      * @param m the mask controlling lane selection
3559      * @return the vector loaded from the indexed elements of the array
3560      * @throws IndexOutOfBoundsException
3561      *         if {@code mapOffset+N < 0}
3562      *         or if {@code mapOffset+N >= indexMap.length},
3563      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3564      *         is an invalid index into {@code a},
3565      *         for any lane {@code N} in the vector
3566      *         where the mask is set
3567      * @see $abstractvectortype$#toIntArray()
3568      */
3569 #if[byteOrShort]
3570     @ForceInline
3571     public static
3572     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3573                                    $type$[] a, int offset,
3574                                    int[] indexMap, int mapOffset,
3575                                    VectorMask<$Boxtype$> m) {
3576         $Type$Species vsp = ($Type$Species) species;
3577         return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
3578     }
3579 #else[byteOrShort]
3580     @ForceInline
3581     public static
3582     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3583                                    $type$[] a, int offset,
3584                                    int[] indexMap, int mapOffset,
3585                                    VectorMask<$Boxtype$> m) {
3586         if (m.allTrue()) {
3587             return fromArray(species, a, offset, indexMap, mapOffset);
3588         }
3589         else {
3590             $Type$Species vsp = ($Type$Species) species;
3591             return vsp.dummyVector().fromArray0(a, offset, indexMap, mapOffset, m);
3592         }
3593     }
3594 #end[byteOrShort]
3595 
3596 #if[short]
3597     /**
3598      * Loads a vector from an array of type {@code char[]}
3599      * starting at an offset.
3600      * For each vector lane, where {@code N} is the vector lane index, the
3601      * array element at index {@code offset + N}
3602      * is first cast to a {@code short} value and then
3603      * placed into the resulting vector at lane index {@code N}.
3604      *
3605      * @param species species of desired vector
3606      * @param a the array
3607      * @param offset the offset into the array
3608      * @return the vector loaded from an array
3609      * @throws IndexOutOfBoundsException
3610      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3611      *         for any lane {@code N} in the vector
3612      */
3613     @ForceInline
3614     public static
3615     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3616                                        char[] a, int offset) {
3617         offset = checkFromIndexSize(offset, species.length(), a.length);
3618         $Type$Species vsp = ($Type$Species) species;
3619         return vsp.dummyVector().fromCharArray0(a, offset);
3620     }
3621 
3622     /**
3623      * Loads a vector from an array of type {@code char[]}
3624      * starting at an offset and using a mask.
3625      * Lanes where the mask is unset are filled with the default
3626      * value of {@code $type$} ({#if[FP]?positive }zero).
3627      * For each vector lane, where {@code N} is the vector lane index,
3628      * if the mask lane at index {@code N} is set then the array element at
3629      * index {@code offset + N}
3630      * is first cast to a {@code short} value and then
3631      * placed into the resulting vector at lane index
3632      * {@code N}, otherwise the default element value is placed into the
3633      * resulting vector at lane index {@code N}.
3634      *
3635      * @param species species of desired vector
3636      * @param a the array
3637      * @param offset the offset into the array
3638      * @param m the mask controlling lane selection
3639      * @return the vector loaded from an array
3640      * @throws IndexOutOfBoundsException
3641      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3642      *         for any lane {@code N} in the vector
3643      *         where the mask is set
3644      */
3645     @ForceInline
3646     public static
3647     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3648                                        char[] a, int offset,
3649                                        VectorMask<$Boxtype$> m) {
3650         $Type$Species vsp = ($Type$Species) species;
3651         if (offset >= 0 && offset <= (a.length - species.length())) {
3652             return vsp.dummyVector().fromCharArray0(a, offset, m);
3653         }
3654 
3655         // FIXME: optimize
3656         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3657         return vsp.vOp(m, i -> (short) a[offset + i]);
3658     }
3659 
3660     /**
3661      * Gathers a new vector composed of elements from an array of type
3662      * {@code char[]},
3663      * using indexes obtained by adding a fixed {@code offset} to a
3664      * series of secondary offsets from an <em>index map</em>.
3665      * The index map is a contiguous sequence of {@code VLENGTH}
3666      * elements in a second array of {@code int}s, starting at a given
3667      * {@code mapOffset}.
3668      * <p>
3669      * For each vector lane, where {@code N} is the vector lane index,
3670      * the lane is loaded from the expression
3671      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3672      * index mapping expression
3673      * {@code offset + indexMap[mapOffset + N]]}.
3674      *
3675      * @param species species of desired vector
3676      * @param a the array
3677      * @param offset the offset into the array, may be negative if relative
3678      * indexes in the index map compensate to produce a value within the
3679      * array bounds
3680      * @param indexMap the index map
3681      * @param mapOffset the offset into the index map
3682      * @return the vector loaded from the indexed elements of the array
3683      * @throws IndexOutOfBoundsException
3684      *         if {@code mapOffset+N < 0}
3685      *         or if {@code mapOffset+N >= indexMap.length},
3686      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3687      *         is an invalid index into {@code a},
3688      *         for any lane {@code N} in the vector
3689      * @see $abstractvectortype$#toIntArray()
3690      */
3691     @ForceInline
3692     public static
3693     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3694                                        char[] a, int offset,
3695                                        int[] indexMap, int mapOffset) {
3696         // FIXME: optimize
3697         $Type$Species vsp = ($Type$Species) species;
3698         return vsp.vOp(n -> (short) a[offset + indexMap[mapOffset + n]]);
3699     }
3700 
3701     /**
3702      * Gathers a new vector composed of elements from an array of type
3703      * {@code char[]},
3704      * under the control of a mask, and
3705      * using indexes obtained by adding a fixed {@code offset} to a
3706      * series of secondary offsets from an <em>index map</em>.
3707      * The index map is a contiguous sequence of {@code VLENGTH}
3708      * elements in a second array of {@code int}s, starting at a given
3709      * {@code mapOffset}.
3710      * <p>
3711      * For each vector lane, where {@code N} is the vector lane index,
3712      * if the lane is set in the mask,
3713      * the lane is loaded from the expression
3714      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3715      * index mapping expression
3716      * {@code offset + indexMap[mapOffset + N]]}.
3717      * Unset lanes in the resulting vector are set to zero.
3718      *
3719      * @param species species of desired vector
3720      * @param a the array
3721      * @param offset the offset into the array, may be negative if relative
3722      * indexes in the index map compensate to produce a value within the
3723      * array bounds
3724      * @param indexMap the index map
3725      * @param mapOffset the offset into the index map
3726      * @param m the mask controlling lane selection
3727      * @return the vector loaded from the indexed elements of the array
3728      * @throws IndexOutOfBoundsException
3729      *         if {@code mapOffset+N < 0}
3730      *         or if {@code mapOffset+N >= indexMap.length},
3731      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3732      *         is an invalid index into {@code a},
3733      *         for any lane {@code N} in the vector
3734      *         where the mask is set
3735      * @see $abstractvectortype$#toIntArray()
3736      */
3737     @ForceInline
3738     public static
3739     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3740                                        char[] a, int offset,
3741                                        int[] indexMap, int mapOffset,
3742                                        VectorMask<$Boxtype$> m) {
3743         // FIXME: optimize
3744         $Type$Species vsp = ($Type$Species) species;
3745         return vsp.vOp(m, n -> (short) a[offset + indexMap[mapOffset + n]]);
3746     }
3747 #end[short]
3748 
3749 #if[byte]
3750     /**
3751      * Loads a vector from an array of type {@code boolean[]}
3752      * starting at an offset.
3753      * For each vector lane, where {@code N} is the vector lane index, the
3754      * array element at index {@code offset + N}
3755      * is first converted to a {@code byte} value and then
3756      * placed into the resulting vector at lane index {@code N}.
3757      * <p>
3758      * A {@code boolean} value is converted to a {@code byte} value by applying the
3759      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3760      *
3761      * @param species species of desired vector
3762      * @param a the array
3763      * @param offset the offset into the array
3764      * @return the vector loaded from an array
3765      * @throws IndexOutOfBoundsException
3766      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3767      *         for any lane {@code N} in the vector
3768      */
3769     @ForceInline
3770     public static
3771     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3772                                           boolean[] a, int offset) {
3773         offset = checkFromIndexSize(offset, species.length(), a.length);
3774         $Type$Species vsp = ($Type$Species) species;
3775         return vsp.dummyVector().fromBooleanArray0(a, offset);
3776     }
3777 
3778     /**
3779      * Loads a vector from an array of type {@code boolean[]}
3780      * starting at an offset and using a mask.
3781      * Lanes where the mask is unset are filled with the default
3782      * value of {@code $type$} ({#if[FP]?positive }zero).
3783      * For each vector lane, where {@code N} is the vector lane index,
3784      * if the mask lane at index {@code N} is set then the array element at
3785      * index {@code offset + N}
3786      * is first converted to a {@code byte} value and then
3787      * placed into the resulting vector at lane index
3788      * {@code N}, otherwise the default element value is placed into the
3789      * resulting vector at lane index {@code N}.
3790      * <p>
3791      * A {@code boolean} value is converted to a {@code byte} value by applying the
3792      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3793      *
3794      * @param species species of desired vector
3795      * @param a the array
3796      * @param offset the offset into the array
3797      * @param m the mask controlling lane selection
3798      * @return the vector loaded from an array
3799      * @throws IndexOutOfBoundsException
3800      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3801      *         for any lane {@code N} in the vector
3802      *         where the mask is set
3803      */
3804     @ForceInline
3805     public static
3806     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3807                                           boolean[] a, int offset,
3808                                           VectorMask<$Boxtype$> m) {
3809         $Type$Species vsp = ($Type$Species) species;
3810         if (offset >= 0 && offset <= (a.length - species.length())) {
3811             $abstractvectortype$ zero = vsp.zero();
3812             return vsp.dummyVector().fromBooleanArray0(a, offset, m);
3813         }
3814 
3815         // FIXME: optimize
3816         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3817         return vsp.vOp(m, i -> (byte) (a[offset + i] ? 1 : 0));
3818     }
3819 
3820     /**
3821      * Gathers a new vector composed of elements from an array of type
3822      * {@code boolean[]},
3823      * using indexes obtained by adding a fixed {@code offset} to a
3824      * series of secondary offsets from an <em>index map</em>.
3825      * The index map is a contiguous sequence of {@code VLENGTH}
3826      * elements in a second array of {@code int}s, starting at a given
3827      * {@code mapOffset}.
3828      * <p>
3829      * For each vector lane, where {@code N} is the vector lane index,
3830      * the lane is loaded from the expression
3831      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
3832      * index mapping expression
3833      * {@code offset + indexMap[mapOffset + N]]}.
3834      *
3835      * @param species species of desired vector
3836      * @param a the array
3837      * @param offset the offset into the array, may be negative if relative
3838      * indexes in the index map compensate to produce a value within the
3839      * array bounds
3840      * @param indexMap the index map
3841      * @param mapOffset the offset into the index map
3842      * @return the vector loaded from the indexed elements of the array
3843      * @throws IndexOutOfBoundsException
3844      *         if {@code mapOffset+N < 0}
3845      *         or if {@code mapOffset+N >= indexMap.length},
3846      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3847      *         is an invalid index into {@code a},
3848      *         for any lane {@code N} in the vector
3849      * @see $abstractvectortype$#toIntArray()
3850      */
3851     @ForceInline
3852     public static
3853     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3854                                           boolean[] a, int offset,
3855                                           int[] indexMap, int mapOffset) {
3856         // FIXME: optimize
3857         $Type$Species vsp = ($Type$Species) species;
3858         return vsp.vOp(n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
3859     }
3860 
3861     /**
3862      * Gathers a new vector composed of elements from an array of type
3863      * {@code boolean[]},
3864      * under the control of a mask, and
3865      * using indexes obtained by adding a fixed {@code offset} to a
3866      * series of secondary offsets from an <em>index map</em>.
3867      * The index map is a contiguous sequence of {@code VLENGTH}
3868      * elements in a second array of {@code int}s, starting at a given
3869      * {@code mapOffset}.
3870      * <p>
3871      * For each vector lane, where {@code N} is the vector lane index,
3872      * if the lane is set in the mask,
3873      * the lane is loaded from the expression
3874      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
3875      * index mapping expression
3876      * {@code offset + indexMap[mapOffset + N]]}.
3877      * Unset lanes in the resulting vector are set to zero.
3878      *
3879      * @param species species of desired vector
3880      * @param a the array
3881      * @param offset the offset into the array, may be negative if relative
3882      * indexes in the index map compensate to produce a value within the
3883      * array bounds
3884      * @param indexMap the index map
3885      * @param mapOffset the offset into the index map
3886      * @param m the mask controlling lane selection
3887      * @return the vector loaded from the indexed elements of the array
3888      * @throws IndexOutOfBoundsException
3889      *         if {@code mapOffset+N < 0}
3890      *         or if {@code mapOffset+N >= indexMap.length},
3891      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3892      *         is an invalid index into {@code a},
3893      *         for any lane {@code N} in the vector
3894      *         where the mask is set
3895      * @see $abstractvectortype$#toIntArray()
3896      */
3897     @ForceInline
3898     public static
3899     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3900                                           boolean[] a, int offset,
3901                                           int[] indexMap, int mapOffset,
3902                                           VectorMask<$Boxtype$> m) {
3903         // FIXME: optimize
3904         $Type$Species vsp = ($Type$Species) species;
3905         return vsp.vOp(m, n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
3906     }
3907 #end[byte]
3908 
3909     /**
3910      * Loads a vector from a {@linkplain ByteBuffer byte buffer}
3911      * starting at an offset into the byte buffer.
3912      * Bytes are composed into primitive lane elements according
3913      * to the specified byte order.
3914      * The vector is arranged into lanes according to
3915      * <a href="Vector.html#lane-order">memory ordering</a>.
3916      * <p>
3917      * This method behaves as if it returns the result of calling
3918      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3919      * fromByteBuffer()} as follows:
3920      * <pre>{@code
3921      * var m = species.maskAll(true);
3922      * return fromByteBuffer(species, bb, offset, bo, m);
3923      * }</pre>
3924      *
3925      * @param species species of desired vector
3926      * @param bb the byte buffer
3927      * @param offset the offset into the byte buffer
3928      * @param bo the intended byte order
3929      * @return a vector loaded from a byte buffer
3930      * @throws IndexOutOfBoundsException
3931      *         if {@code offset+N*$sizeInBytes$ < 0}
3932      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
3933      *         for any lane {@code N} in the vector
3934      */
3935     @ForceInline
3936     public static
3937     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
3938                                         ByteBuffer bb, int offset,
3939                                         ByteOrder bo) {
3940         offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
3941         $Type$Species vsp = ($Type$Species) species;
3942         return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
3943     }
3944 
3945     /**
3946      * Loads a vector from a {@linkplain ByteBuffer byte buffer}
3947      * starting at an offset into the byte buffer
3948      * and using a mask.
3949      * Lanes where the mask is unset are filled with the default
3950      * value of {@code $type$} ({#if[FP]?positive }zero).
3951      * Bytes are composed into primitive lane elements according
3952      * to the specified byte order.
3953      * The vector is arranged into lanes according to
3954      * <a href="Vector.html#lane-order">memory ordering</a>.
3955      * <p>
3956      * The following pseudocode illustrates the behavior:
3957      * <pre>{@code
3958      * $Type$Buffer eb = bb.duplicate()
3959      *     .position(offset){#if[byte]?;}
3960 #if[!byte]
3961      *     .order(bo).as$Type$Buffer();
3962 #end[!byte]
3963      * $type$[] ar = new $type$[species.length()];
3964      * for (int n = 0; n < ar.length; n++) {
3965      *     if (m.laneIsSet(n)) {
3966      *         ar[n] = eb.get(n);
3967      *     }
3968      * }
3969      * $abstractvectortype$ r = $abstractvectortype$.fromArray(species, ar, 0);
3970      * }</pre>
3971      * @implNote
3972 #if[!byte]
3973      * This operation is likely to be more efficient if
3974      * the specified byte order is the same as
3975      * {@linkplain ByteOrder#nativeOrder()
3976      * the platform native order},
3977      * since this method will not need to reorder
3978      * the bytes of lane values.
3979 #else[!byte]
3980      * The byte order argument is ignored.
3981 #end[!byte]
3982      *
3983      * @param species species of desired vector
3984      * @param bb the byte buffer
3985      * @param offset the offset into the byte buffer
3986      * @param bo the intended byte order
3987      * @param m the mask controlling lane selection
3988      * @return a vector loaded from a byte buffer
3989      * @throws IndexOutOfBoundsException
3990      *         if {@code offset+N*$sizeInBytes$ < 0}
3991      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
3992      *         for any lane {@code N} in the vector
3993      *         where the mask is set
3994      */
3995     @ForceInline
3996     public static
3997     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
3998                                         ByteBuffer bb, int offset,
3999                                         ByteOrder bo,
4000                                         VectorMask<$Boxtype$> m) {
4001         $Type$Species vsp = ($Type$Species) species;
4002         if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
4003             return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
4004         }
4005 
4006         // FIXME: optimize
4007         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4008         ByteBuffer wb = wrapper(bb, bo);
4009         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
4010                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4011     }
4012 
4013     // Memory store operations
4014 
4015     /**
4016      * Stores this vector into an array of type {@code $type$[]}
4017      * starting at an offset.
4018      * <p>
4019      * For each vector lane, where {@code N} is the vector lane index,
4020      * the lane element at index {@code N} is stored into the array
4021      * element {@code a[offset+N]}.
4022      *
4023      * @param a the array, of type {@code $type$[]}
4024      * @param offset the offset into the array
4025      * @throws IndexOutOfBoundsException
4026      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4027      *         for any lane {@code N} in the vector
4028      */
4029     @ForceInline
4030     public final
4031     void intoArray($type$[] a, int offset) {
4032         offset = checkFromIndexSize(offset, length(), a.length);
4033         $Type$Species vsp = vspecies();
4034         VectorSupport.store(
4035             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4036             a, arrayAddress(a, offset),
4037             this,
4038             a, offset,
4039             (arr, off, v)
4040             -> v.stOp(arr, off,
4041                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4042     }
4043 
4044     /**
4045      * Stores this vector into an array of type {@code $type$[]}
4046      * starting at offset and using a mask.
4047      * <p>
4048      * For each vector lane, where {@code N} is the vector lane index,
4049      * the lane element at index {@code N} is stored into the array
4050      * element {@code a[offset+N]}.
4051      * If the mask lane at {@code N} is unset then the corresponding
4052      * array element {@code a[offset+N]} is left unchanged.
4053      * <p>
4054      * Array range checking is done for lanes where the mask is set.
4055      * Lanes where the mask is unset are not stored and do not need
4056      * to correspond to legitimate elements of {@code a}.
4057      * That is, unset lanes may correspond to array indexes less than
4058      * zero or beyond the end of the array.
4059      *
4060      * @param a the array, of type {@code $type$[]}
4061      * @param offset the offset into the array
4062      * @param m the mask controlling lane storage
4063      * @throws IndexOutOfBoundsException
4064      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4065      *         for any lane {@code N} in the vector
4066      *         where the mask is set
4067      */
4068     @ForceInline
4069     public final
4070     void intoArray($type$[] a, int offset,
4071                    VectorMask<$Boxtype$> m) {
4072         if (m.allTrue()) {
4073             intoArray(a, offset);
4074         } else {
4075             $Type$Species vsp = vspecies();
4076             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4077             intoArray0(a, offset, m);
4078         }
4079     }
4080 
4081     /**
4082      * Scatters this vector into an array of type {@code $type$[]}
4083      * using indexes obtained by adding a fixed {@code offset} to a
4084      * series of secondary offsets from an <em>index map</em>.
4085      * The index map is a contiguous sequence of {@code VLENGTH}
4086      * elements in a second array of {@code int}s, starting at a given
4087      * {@code mapOffset}.
4088      * <p>
4089      * For each vector lane, where {@code N} is the vector lane index,
4090      * the lane element at index {@code N} is stored into the array
4091      * element {@code a[f(N)]}, where {@code f(N)} is the
4092      * index mapping expression
4093      * {@code offset + indexMap[mapOffset + N]]}.
4094      *
4095      * @param a the array
4096      * @param offset an offset to combine with the index map offsets
4097      * @param indexMap the index map
4098      * @param mapOffset the offset into the index map
4099      * @throws IndexOutOfBoundsException
4100      *         if {@code mapOffset+N < 0}
4101      *         or if {@code mapOffset+N >= indexMap.length},
4102      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4103      *         is an invalid index into {@code a},
4104      *         for any lane {@code N} in the vector
4105      * @see $abstractvectortype$#toIntArray()
4106      */
4107 #if[byteOrShort]
4108     @ForceInline
4109     public final
4110     void intoArray($type$[] a, int offset,
4111                    int[] indexMap, int mapOffset) {
4112         stOp(a, offset,
4113              (arr, off, i, e) -> {
4114                  int j = indexMap[mapOffset + i];
4115                  arr[off + j] = e;
4116              });
4117     }
4118 #else[byteOrShort]
4119     @ForceInline
4120     public final
4121     void intoArray($type$[] a, int offset,
4122                    int[] indexMap, int mapOffset) {
4123         $Type$Species vsp = vspecies();
4124         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4125 #if[longOrDouble]
4126         if (vsp.laneCount() == 1) {
4127             intoArray(a, offset + indexMap[mapOffset]);
4128             return;
4129         }
4130 
4131         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4132         IntVector vix;
4133         if (isp.laneCount() != vsp.laneCount()) {
4134             // For $Type$MaxVector,  if vector length  is 2048 bits, indexShape
4135             // of $Type$ species is S_MAX_BIT. and the lane count of $Type$
4136             // vector is 32. When converting $Type$ species to int species,
4137             // indexShape is still S_MAX_BIT, but the lane count of int vector
4138             // is 64. So when loading index vector (IntVector), only lower half
4139             // of index data is needed.
4140             vix = IntVector
4141                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4142                 .add(offset);
4143         } else {
4144             vix = IntVector
4145                 .fromArray(isp, indexMap, mapOffset)
4146                 .add(offset);
4147         }
4148 
4149 #else[longOrDouble]
4150         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4151         IntVector vix = IntVector
4152             .fromArray(isp, indexMap, mapOffset)
4153             .add(offset);
4154 #end[longOrDouble]
4155 
4156         vix = VectorIntrinsics.checkIndex(vix, a.length);
4157 
4158         VectorSupport.storeWithMap(
4159             vsp.vectorType(), null, vsp.elementType(), vsp.laneCount(),
4160             isp.vectorType(),
4161             a, arrayAddress(a, 0), vix,
4162             this, null,
4163             a, offset, indexMap, mapOffset,
4164             (arr, off, v, map, mo, vm)
4165             -> v.stOp(arr, off,
4166                       (arr_, off_, i, e) -> {
4167                           int j = map[mo + i];
4168                           arr[off + j] = e;
4169                       }));
4170     }
4171 #end[byteOrShort]
4172 
4173     /**
4174      * Scatters this vector into an array of type {@code $type$[]},
4175      * under the control of a mask, and
4176      * using indexes obtained by adding a fixed {@code offset} to a
4177      * series of secondary offsets from an <em>index map</em>.
4178      * The index map is a contiguous sequence of {@code VLENGTH}
4179      * elements in a second array of {@code int}s, starting at a given
4180      * {@code mapOffset}.
4181      * <p>
4182      * For each vector lane, where {@code N} is the vector lane index,
4183      * if the mask lane at index {@code N} is set then
4184      * the lane element at index {@code N} is stored into the array
4185      * element {@code a[f(N)]}, where {@code f(N)} is the
4186      * index mapping expression
4187      * {@code offset + indexMap[mapOffset + N]]}.
4188      *
4189      * @param a the array
4190      * @param offset an offset to combine with the index map offsets
4191      * @param indexMap the index map
4192      * @param mapOffset the offset into the index map
4193      * @param m the mask
4194      * @throws IndexOutOfBoundsException
4195      *         if {@code mapOffset+N < 0}
4196      *         or if {@code mapOffset+N >= indexMap.length},
4197      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4198      *         is an invalid index into {@code a},
4199      *         for any lane {@code N} in the vector
4200      *         where the mask is set
4201      * @see $abstractvectortype$#toIntArray()
4202      */
4203 #if[byteOrShort]
4204     @ForceInline
4205     public final
4206     void intoArray($type$[] a, int offset,
4207                    int[] indexMap, int mapOffset,
4208                    VectorMask<$Boxtype$> m) {
4209         stOp(a, offset, m,
4210              (arr, off, i, e) -> {
4211                  int j = indexMap[mapOffset + i];
4212                  arr[off + j] = e;
4213              });
4214     }
4215 #else[byteOrShort]
4216     @ForceInline
4217     public final
4218     void intoArray($type$[] a, int offset,
4219                    int[] indexMap, int mapOffset,
4220                    VectorMask<$Boxtype$> m) {
4221         if (m.allTrue()) {
4222             intoArray(a, offset, indexMap, mapOffset);
4223         }
4224         else {
4225             intoArray0(a, offset, indexMap, mapOffset, m);
4226         }
4227     }
4228 #end[byteOrShort]
4229 
4230 #if[short]
4231     /**
4232      * Stores this vector into an array of type {@code char[]}
4233      * starting at an offset.
4234      * <p>
4235      * For each vector lane, where {@code N} is the vector lane index,
4236      * the lane element at index {@code N}
4237      * is first cast to a {@code char} value and then
4238      * stored into the array element {@code a[offset+N]}.
4239      *
4240      * @param a the array, of type {@code char[]}
4241      * @param offset the offset into the array
4242      * @throws IndexOutOfBoundsException
4243      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4244      *         for any lane {@code N} in the vector
4245      */
4246     @ForceInline
4247     public final
4248     void intoCharArray(char[] a, int offset) {
4249         offset = checkFromIndexSize(offset, length(), a.length);
4250         $Type$Species vsp = vspecies();
4251         VectorSupport.store(
4252             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4253             a, charArrayAddress(a, offset),
4254             this,
4255             a, offset,
4256             (arr, off, v)
4257             -> v.stOp(arr, off,
4258                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
4259     }
4260 
4261     /**
4262      * Stores this vector into an array of type {@code char[]}
4263      * starting at offset and using a mask.
4264      * <p>
4265      * For each vector lane, where {@code N} is the vector lane index,
4266      * the lane element at index {@code N}
4267      * is first cast to a {@code char} value and then
4268      * stored into the array element {@code a[offset+N]}.
4269      * If the mask lane at {@code N} is unset then the corresponding
4270      * array element {@code a[offset+N]} is left unchanged.
4271      * <p>
4272      * Array range checking is done for lanes where the mask is set.
4273      * Lanes where the mask is unset are not stored and do not need
4274      * to correspond to legitimate elements of {@code a}.
4275      * That is, unset lanes may correspond to array indexes less than
4276      * zero or beyond the end of the array.
4277      *
4278      * @param a the array, of type {@code char[]}
4279      * @param offset the offset into the array
4280      * @param m the mask controlling lane storage
4281      * @throws IndexOutOfBoundsException
4282      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4283      *         for any lane {@code N} in the vector
4284      *         where the mask is set
4285      */
4286     @ForceInline
4287     public final
4288     void intoCharArray(char[] a, int offset,
4289                        VectorMask<$Boxtype$> m) {
4290         if (m.allTrue()) {
4291             intoCharArray(a, offset);
4292         } else {
4293             $Type$Species vsp = vspecies();
4294             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4295             intoCharArray0(a, offset, m);
4296         }
4297     }
4298 
4299     /**
4300      * Scatters this vector into an array of type {@code char[]}
4301      * using indexes obtained by adding a fixed {@code offset} to a
4302      * series of secondary offsets from an <em>index map</em>.
4303      * The index map is a contiguous sequence of {@code VLENGTH}
4304      * elements in a second array of {@code int}s, starting at a given
4305      * {@code mapOffset}.
4306      * <p>
4307      * For each vector lane, where {@code N} is the vector lane index,
4308      * the lane element at index {@code N}
4309      * is first cast to a {@code char} value and then
4310      * stored into the array
4311      * element {@code a[f(N)]}, where {@code f(N)} is the
4312      * index mapping expression
4313      * {@code offset + indexMap[mapOffset + N]]}.
4314      *
4315      * @param a the array
4316      * @param offset an offset to combine with the index map offsets
4317      * @param indexMap the index map
4318      * @param mapOffset the offset into the index map
4319      * @throws IndexOutOfBoundsException
4320      *         if {@code mapOffset+N < 0}
4321      *         or if {@code mapOffset+N >= indexMap.length},
4322      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4323      *         is an invalid index into {@code a},
4324      *         for any lane {@code N} in the vector
4325      * @see $abstractvectortype$#toIntArray()
4326      */
4327     @ForceInline
4328     public final
4329     void intoCharArray(char[] a, int offset,
4330                        int[] indexMap, int mapOffset) {
4331         // FIXME: optimize
4332         stOp(a, offset,
4333              (arr, off, i, e) -> {
4334                  int j = indexMap[mapOffset + i];
4335                  arr[off + j] = (char) e;
4336              });
4337     }
4338 
4339     /**
4340      * Scatters this vector into an array of type {@code char[]},
4341      * under the control of a mask, and
4342      * using indexes obtained by adding a fixed {@code offset} to a
4343      * series of secondary offsets from an <em>index map</em>.
4344      * The index map is a contiguous sequence of {@code VLENGTH}
4345      * elements in a second array of {@code int}s, starting at a given
4346      * {@code mapOffset}.
4347      * <p>
4348      * For each vector lane, where {@code N} is the vector lane index,
4349      * if the mask lane at index {@code N} is set then
4350      * the lane element at index {@code N}
4351      * is first cast to a {@code char} value and then
4352      * stored into the array
4353      * element {@code a[f(N)]}, where {@code f(N)} is the
4354      * index mapping expression
4355      * {@code offset + indexMap[mapOffset + N]]}.
4356      *
4357      * @param a the array
4358      * @param offset an offset to combine with the index map offsets
4359      * @param indexMap the index map
4360      * @param mapOffset the offset into the index map
4361      * @param m the mask
4362      * @throws IndexOutOfBoundsException
4363      *         if {@code mapOffset+N < 0}
4364      *         or if {@code mapOffset+N >= indexMap.length},
4365      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4366      *         is an invalid index into {@code a},
4367      *         for any lane {@code N} in the vector
4368      *         where the mask is set
4369      * @see $abstractvectortype$#toIntArray()
4370      */
4371     @ForceInline
4372     public final
4373     void intoCharArray(char[] a, int offset,
4374                        int[] indexMap, int mapOffset,
4375                        VectorMask<$Boxtype$> m) {
4376         // FIXME: optimize
4377         stOp(a, offset, m,
4378              (arr, off, i, e) -> {
4379                  int j = indexMap[mapOffset + i];
4380                  arr[off + j] = (char) e;
4381              });
4382     }
4383 #end[short]
4384 
4385 #if[byte]
4386     /**
4387      * Stores this vector into an array of type {@code boolean[]}
4388      * starting at an offset.
4389      * <p>
4390      * For each vector lane, where {@code N} is the vector lane index,
4391      * the lane element at index {@code N}
4392      * is first converted to a {@code boolean} value and then
4393      * stored into the array element {@code a[offset+N]}.
4394      * <p>
4395      * A {@code byte} value is converted to a {@code boolean} value by applying the
4396      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4397      *
4398      * @param a the array
4399      * @param offset the offset into the array
4400      * @throws IndexOutOfBoundsException
4401      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4402      *         for any lane {@code N} in the vector
4403      */
4404     @ForceInline
4405     public final
4406     void intoBooleanArray(boolean[] a, int offset) {
4407         offset = checkFromIndexSize(offset, length(), a.length);
4408         $Type$Species vsp = vspecies();
4409         ByteVector normalized = this.and((byte) 1);
4410         VectorSupport.store(
4411             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4412             a, booleanArrayAddress(a, offset),
4413             normalized,
4414             a, offset,
4415             (arr, off, v)
4416             -> v.stOp(arr, off,
4417                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4418     }
4419 
4420     /**
4421      * Stores this vector into an array of type {@code boolean[]}
4422      * starting at offset and using a mask.
4423      * <p>
4424      * For each vector lane, where {@code N} is the vector lane index,
4425      * the lane element at index {@code N}
4426      * is first converted to a {@code boolean} value and then
4427      * stored into the array element {@code a[offset+N]}.
4428      * If the mask lane at {@code N} is unset then the corresponding
4429      * array element {@code a[offset+N]} is left unchanged.
4430      * <p>
4431      * A {@code byte} value is converted to a {@code boolean} value by applying the
4432      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4433      * <p>
4434      * Array range checking is done for lanes where the mask is set.
4435      * Lanes where the mask is unset are not stored and do not need
4436      * to correspond to legitimate elements of {@code a}.
4437      * That is, unset lanes may correspond to array indexes less than
4438      * zero or beyond the end of the array.
4439      *
4440      * @param a the array
4441      * @param offset the offset into the array
4442      * @param m the mask controlling lane storage
4443      * @throws IndexOutOfBoundsException
4444      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4445      *         for any lane {@code N} in the vector
4446      *         where the mask is set
4447      */
4448     @ForceInline
4449     public final
4450     void intoBooleanArray(boolean[] a, int offset,
4451                           VectorMask<$Boxtype$> m) {
4452         if (m.allTrue()) {
4453             intoBooleanArray(a, offset);
4454         } else {
4455             $Type$Species vsp = vspecies();
4456             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4457             intoBooleanArray0(a, offset, m);
4458         }
4459     }
4460 
4461     /**
4462      * Scatters this vector into an array of type {@code boolean[]}
4463      * using indexes obtained by adding a fixed {@code offset} to a
4464      * series of secondary offsets from an <em>index map</em>.
4465      * The index map is a contiguous sequence of {@code VLENGTH}
4466      * elements in a second array of {@code int}s, starting at a given
4467      * {@code mapOffset}.
4468      * <p>
4469      * For each vector lane, where {@code N} is the vector lane index,
4470      * the lane element at index {@code N}
4471      * is first converted to a {@code boolean} value and then
4472      * stored into the array
4473      * element {@code a[f(N)]}, where {@code f(N)} is the
4474      * index mapping expression
4475      * {@code offset + indexMap[mapOffset + N]]}.
4476      * <p>
4477      * A {@code byte} value is converted to a {@code boolean} value by applying the
4478      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4479      *
4480      * @param a the array
4481      * @param offset an offset to combine with the index map offsets
4482      * @param indexMap the index map
4483      * @param mapOffset the offset into the index map
4484      * @throws IndexOutOfBoundsException
4485      *         if {@code mapOffset+N < 0}
4486      *         or if {@code mapOffset+N >= indexMap.length},
4487      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4488      *         is an invalid index into {@code a},
4489      *         for any lane {@code N} in the vector
4490      * @see $abstractvectortype$#toIntArray()
4491      */
4492     @ForceInline
4493     public final
4494     void intoBooleanArray(boolean[] a, int offset,
4495                           int[] indexMap, int mapOffset) {
4496         // FIXME: optimize
4497         stOp(a, offset,
4498              (arr, off, i, e) -> {
4499                  int j = indexMap[mapOffset + i];
4500                  arr[off + j] = (e & 1) != 0;
4501              });
4502     }
4503 
4504     /**
4505      * Scatters this vector into an array of type {@code boolean[]},
4506      * under the control of a mask, and
4507      * using indexes obtained by adding a fixed {@code offset} to a
4508      * series of secondary offsets from an <em>index map</em>.
4509      * The index map is a contiguous sequence of {@code VLENGTH}
4510      * elements in a second array of {@code int}s, starting at a given
4511      * {@code mapOffset}.
4512      * <p>
4513      * For each vector lane, where {@code N} is the vector lane index,
4514      * if the mask lane at index {@code N} is set then
4515      * the lane element at index {@code N}
4516      * is first converted to a {@code boolean} value and then
4517      * stored into the array
4518      * element {@code a[f(N)]}, where {@code f(N)} is the
4519      * index mapping expression
4520      * {@code offset + indexMap[mapOffset + N]]}.
4521      * <p>
4522      * A {@code byte} value is converted to a {@code boolean} value by applying the
4523      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4524      *
4525      * @param a the array
4526      * @param offset an offset to combine with the index map offsets
4527      * @param indexMap the index map
4528      * @param mapOffset the offset into the index map
4529      * @param m the mask
4530      * @throws IndexOutOfBoundsException
4531      *         if {@code mapOffset+N < 0}
4532      *         or if {@code mapOffset+N >= indexMap.length},
4533      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4534      *         is an invalid index into {@code a},
4535      *         for any lane {@code N} in the vector
4536      *         where the mask is set
4537      * @see $abstractvectortype$#toIntArray()
4538      */
4539     @ForceInline
4540     public final
4541     void intoBooleanArray(boolean[] a, int offset,
4542                           int[] indexMap, int mapOffset,
4543                           VectorMask<$Boxtype$> m) {
4544         // FIXME: optimize
4545         stOp(a, offset, m,
4546              (arr, off, i, e) -> {
4547                  int j = indexMap[mapOffset + i];
4548                  arr[off + j] = (e & 1) != 0;
4549              });
4550     }
4551 #end[byte]
4552 
4553     /**
4554      * {@inheritDoc} <!--workaround-->
4555      */
4556     @Override
4557     @ForceInline
4558     public final
4559     void intoByteArray(byte[] a, int offset,
4560                        ByteOrder bo) {
4561         offset = checkFromIndexSize(offset, byteSize(), a.length);
4562         maybeSwap(bo).intoByteArray0(a, offset);
4563     }
4564 
4565     /**
4566      * {@inheritDoc} <!--workaround-->
4567      */
4568     @Override
4569     @ForceInline
4570     public final
4571     void intoByteArray(byte[] a, int offset,
4572                        ByteOrder bo,
4573                        VectorMask<$Boxtype$> m) {
4574         if (m.allTrue()) {
4575             intoByteArray(a, offset, bo);
4576         } else {
4577             $Type$Species vsp = vspecies();
4578             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
4579             maybeSwap(bo).intoByteArray0(a, offset, m);
4580         }
4581     }
4582 
4583     /**
4584      * {@inheritDoc} <!--workaround-->
4585      */
4586     @Override
4587     @ForceInline
4588     public final
4589     void intoByteBuffer(ByteBuffer bb, int offset,
4590                         ByteOrder bo) {
4591         if (ScopedMemoryAccess.isReadOnly(bb)) {
4592             throw new ReadOnlyBufferException();
4593         }
4594         offset = checkFromIndexSize(offset, byteSize(), bb.limit());
4595         maybeSwap(bo).intoByteBuffer0(bb, offset);
4596     }
4597 
4598     /**
4599      * {@inheritDoc} <!--workaround-->
4600      */
4601     @Override
4602     @ForceInline
4603     public final
4604     void intoByteBuffer(ByteBuffer bb, int offset,
4605                         ByteOrder bo,
4606                         VectorMask<$Boxtype$> m) {
4607         if (m.allTrue()) {
4608             intoByteBuffer(bb, offset, bo);
4609         } else {
4610             if (bb.isReadOnly()) {
4611                 throw new ReadOnlyBufferException();
4612             }
4613             $Type$Species vsp = vspecies();
4614             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4615             maybeSwap(bo).intoByteBuffer0(bb, offset, m);
4616         }
4617     }
4618 
4619     // ================================================
4620 
4621     // Low-level memory operations.
4622     //
4623     // Note that all of these operations *must* inline into a context
4624     // where the exact species of the involved vector is a
4625     // compile-time constant.  Otherwise, the intrinsic generation
4626     // will fail and performance will suffer.
4627     //
4628     // In many cases this is achieved by re-deriving a version of the
4629     // method in each concrete subclass (per species).  The re-derived
4630     // method simply calls one of these generic methods, with exact
4631     // parameters for the controlling metadata, which is either a
4632     // typed vector or constant species instance.
4633 
4634     // Unchecked loading operations in native byte order.
4635     // Caller is responsible for applying index checks, masking, and
4636     // byte swapping.
4637 
4638     /*package-private*/
4639     abstract
4640     $abstractvectortype$ fromArray0($type$[] a, int offset);
4641     @ForceInline
4642     final
4643     $abstractvectortype$ fromArray0Template($type$[] a, int offset) {
4644         $Type$Species vsp = vspecies();
4645         return VectorSupport.load(
4646             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4647             a, arrayAddress(a, offset),
4648             a, offset, vsp,
4649             (arr, off, s) -> s.ldOp(arr, off,
4650                                     (arr_, off_, i) -> arr_[off_ + i]));
4651     }
4652 
4653     /*package-private*/
4654     abstract
4655     $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4656     @ForceInline
4657     final
4658     <M extends VectorMask<$Boxtype$>>
4659     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4660         m.check(species());
4661         $Type$Species vsp = vspecies();
4662         return VectorSupport.loadMasked(
4663             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4664             a, arrayAddress(a, offset), m,
4665             a, offset, vsp,
4666             (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4667                                         (arr_, off_, i) -> arr_[off_ + i]));
4668     }
4669 
4670 #if[!byteOrShort]
4671     /*package-private*/
4672     abstract
4673     $abstractvectortype$ fromArray0($type$[] a, int offset,
4674                                     int[] indexMap, int mapOffset,
4675                                     VectorMask<$Boxtype$> m);
4676     @ForceInline
4677     final
4678     <M extends VectorMask<$Boxtype$>>
4679     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset,
4680                                             int[] indexMap, int mapOffset, M m) {
4681         $Type$Species vsp = vspecies();
4682         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4683         Objects.requireNonNull(a);
4684         Objects.requireNonNull(indexMap);
4685         m.check(vsp);
4686         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();
4687 
4688 #if[longOrDouble]
4689         if (vsp.laneCount() == 1) {
4690           return $abstractvectortype$.fromArray(vsp, a, offset + indexMap[mapOffset], m);
4691         }
4692 
4693         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
4694         IntVector vix;
4695         if (isp.laneCount() != vsp.laneCount()) {
4696             // For $Type$MaxVector,  if vector length is non-power-of-two or
4697             // 2048 bits, indexShape of $Type$ species is S_MAX_BIT.
4698             // Assume that vector length is 2048, then the lane count of $Type$
4699             // vector is 32. When converting $Type$ species to int species,
4700             // indexShape is still S_MAX_BIT, but the lane count of int vector
4701             // is 64. So when loading index vector (IntVector), only lower half
4702             // of index data is needed.
4703             vix = IntVector
4704                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4705                 .add(offset);
4706         } else {
4707             vix = IntVector
4708                 .fromArray(isp, indexMap, mapOffset)
4709                 .add(offset);
4710         }
4711 #else[longOrDouble]
4712         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
4713         IntVector vix = IntVector
4714             .fromArray(isp, indexMap, mapOffset)
4715             .add(offset);
4716 #end[longOrDouble]
4717 
4718         // FIXME: Check index under mask controlling.
4719         vix = VectorIntrinsics.checkIndex(vix, a.length);
4720 
4721         return VectorSupport.loadWithMap(
4722             vectorType, maskClass, $type$.class, vsp.laneCount(),
4723             isp.vectorType(),
4724             a, ARRAY_BASE, vix, m,
4725             a, offset, indexMap, mapOffset, vsp,
4726             (c, idx, iMap, idy, s, vm) ->
4727             s.vOp(vm, n -> c[idx + iMap[idy+n]]));
4728     }
4729 #end[!byteOrShort]
4730 
4731 #if[short]
4732     /*package-private*/
4733     abstract
4734     $abstractvectortype$ fromCharArray0(char[] a, int offset);
4735     @ForceInline
4736     final
4737     $abstractvectortype$ fromCharArray0Template(char[] a, int offset) {
4738         $Type$Species vsp = vspecies();
4739         return VectorSupport.load(
4740             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4741             a, charArrayAddress(a, offset),
4742             a, offset, vsp,
4743             (arr, off, s) -> s.ldOp(arr, off,
4744                                     (arr_, off_, i) -> (short) arr_[off_ + i]));
4745     }
4746 
4747     /*package-private*/
4748     abstract
4749     $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
4750     @ForceInline
4751     final
4752     <M extends VectorMask<$Boxtype$>>
4753     $abstractvectortype$ fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
4754         m.check(species());
4755         $Type$Species vsp = vspecies();
4756         return VectorSupport.loadMasked(
4757                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4758                 a, charArrayAddress(a, offset), m,
4759                 a, offset, vsp,
4760                 (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4761                                             (arr_, off_, i) -> (short) arr_[off_ + i]));
4762     }
4763 #end[short]
4764 
4765 #if[byte]
4766     /*package-private*/
4767     abstract
4768     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset);
4769     @ForceInline
4770     final
4771     $abstractvectortype$ fromBooleanArray0Template(boolean[] a, int offset) {
4772         $Type$Species vsp = vspecies();
4773         return VectorSupport.load(
4774             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4775             a, booleanArrayAddress(a, offset),
4776             a, offset, vsp,
4777             (arr, off, s) -> s.ldOp(arr, off,
4778                                     (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4779     }
4780 
4781     /*package-private*/
4782     abstract
4783     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4784     @ForceInline
4785     final
4786     <M extends VectorMask<$Boxtype$>>
4787     $abstractvectortype$ fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4788         m.check(species());
4789         $Type$Species vsp = vspecies();
4790         return VectorSupport.loadMasked(
4791             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4792             a, booleanArrayAddress(a, offset), m,
4793             a, offset, vsp,
4794             (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4795                                         (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4796     }
4797 #end[byte]
4798 
4799     @Override
4800     abstract
4801     $abstractvectortype$ fromByteArray0(byte[] a, int offset);
4802     @ForceInline
4803     final
4804     $abstractvectortype$ fromByteArray0Template(byte[] a, int offset) {
4805         $Type$Species vsp = vspecies();
4806         return VectorSupport.load(
4807             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4808             a, byteArrayAddress(a, offset),
4809             a, offset, vsp,
4810             (arr, off, s) -> {
4811                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4812                 return s.ldOp(wb, off,
4813                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4814             });
4815     }
4816 
4817     abstract
4818     $abstractvectortype$ fromByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
4819     @ForceInline
4820     final
4821     <M extends VectorMask<$Boxtype$>>
4822     $abstractvectortype$ fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
4823         $Type$Species vsp = vspecies();
4824         m.check(vsp);
4825         return VectorSupport.loadMasked(
4826             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4827             a, byteArrayAddress(a, offset), m,
4828             a, offset, vsp,
4829             (arr, off, s, vm) -> {
4830                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4831                 return s.ldOp(wb, off, vm,
4832                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4833             });
4834     }
4835 
4836     abstract
4837     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset);
4838     @ForceInline
4839     final
4840     $abstractvectortype$ fromByteBuffer0Template(ByteBuffer bb, int offset) {
4841         $Type$Species vsp = vspecies();
4842         return ScopedMemoryAccess.loadFromByteBuffer(
4843                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4844                 bb, offset, vsp,
4845                 (buf, off, s) -> {
4846                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4847                     return s.ldOp(wb, off,
4848                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4849                 });
4850     }
4851 
4852     abstract
4853     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
4854     @ForceInline
4855     final
4856     <M extends VectorMask<$Boxtype$>>
4857     $abstractvectortype$ fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
4858         $Type$Species vsp = vspecies();
4859         m.check(vsp);
4860         return ScopedMemoryAccess.loadFromByteBufferMasked(
4861                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4862                 bb, offset, m, vsp,
4863                 (buf, off, s, vm) -> {
4864                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4865                     return s.ldOp(wb, off, vm,
4866                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4867                 });
4868     }
4869 
4870     // Unchecked storing operations in native byte order.
4871     // Caller is responsible for applying index checks, masking, and
4872     // byte swapping.
4873 
4874     abstract
4875     void intoArray0($type$[] a, int offset);
4876     @ForceInline
4877     final
4878     void intoArray0Template($type$[] a, int offset) {
4879         $Type$Species vsp = vspecies();
4880         VectorSupport.store(
4881             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4882             a, arrayAddress(a, offset),
4883             this, a, offset,
4884             (arr, off, v)
4885             -> v.stOp(arr, off,
4886                       (arr_, off_, i, e) -> arr_[off_+i] = e));
4887     }
4888 
4889     abstract
4890     void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4891     @ForceInline
4892     final
4893     <M extends VectorMask<$Boxtype$>>
4894     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4895         m.check(species());
4896         $Type$Species vsp = vspecies();
4897         VectorSupport.storeMasked(
4898             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4899             a, arrayAddress(a, offset),
4900             this, m, a, offset,
4901             (arr, off, v, vm)
4902             -> v.stOp(arr, off, vm,
4903                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4904     }
4905 
4906 #if[!byteOrShort]
4907     abstract
4908     void intoArray0($type$[] a, int offset,
4909                     int[] indexMap, int mapOffset,
4910                     VectorMask<$Boxtype$> m);
4911     @ForceInline
4912     final
4913     <M extends VectorMask<$Boxtype$>>
4914     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset,
4915                             int[] indexMap, int mapOffset, M m) {
4916         m.check(species());
4917         $Type$Species vsp = vspecies();
4918         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4919 #if[longOrDouble]
4920         if (vsp.laneCount() == 1) {
4921             intoArray(a, offset + indexMap[mapOffset], m);
4922             return;
4923         }
4924 
4925         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4926         IntVector vix;
4927         if (isp.laneCount() != vsp.laneCount()) {
4928             // For $Type$MaxVector,  if vector length  is 2048 bits, indexShape
4929             // of $Type$ species is S_MAX_BIT. and the lane count of $Type$
4930             // vector is 32. When converting $Type$ species to int species,
4931             // indexShape is still S_MAX_BIT, but the lane count of int vector
4932             // is 64. So when loading index vector (IntVector), only lower half
4933             // of index data is needed.
4934             vix = IntVector
4935                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4936                 .add(offset);
4937         } else {
4938             vix = IntVector
4939                 .fromArray(isp, indexMap, mapOffset)
4940                 .add(offset);
4941         }
4942 
4943 #else[longOrDouble]
4944         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4945         IntVector vix = IntVector
4946             .fromArray(isp, indexMap, mapOffset)
4947             .add(offset);
4948 #end[longOrDouble]
4949 
4950         // FIXME: Check index under mask controlling.
4951         vix = VectorIntrinsics.checkIndex(vix, a.length);
4952 
4953         VectorSupport.storeWithMap(
4954             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4955             isp.vectorType(),
4956             a, arrayAddress(a, 0), vix,
4957             this, m,
4958             a, offset, indexMap, mapOffset,
4959             (arr, off, v, map, mo, vm)
4960             -> v.stOp(arr, off, vm,
4961                       (arr_, off_, i, e) -> {
4962                           int j = map[mo + i];
4963                           arr[off + j] = e;
4964                       }));
4965     }
4966 #end[!byteOrShort]
4967 
4968 #if[byte]
4969     abstract
4970     void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4971     @ForceInline
4972     final
4973     <M extends VectorMask<$Boxtype$>>
4974     void intoBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4975         m.check(species());
4976         $Type$Species vsp = vspecies();
4977         ByteVector normalized = this.and((byte) 1);
4978         VectorSupport.storeMasked(
4979             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4980             a, booleanArrayAddress(a, offset),
4981             normalized, m, a, offset,
4982             (arr, off, v, vm)
4983             -> v.stOp(arr, off, vm,
4984                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4985     }
4986 #end[byte]
4987 
4988     abstract
4989     void intoByteArray0(byte[] a, int offset);
4990     @ForceInline
4991     final
4992     void intoByteArray0Template(byte[] a, int offset) {
4993         $Type$Species vsp = vspecies();
4994         VectorSupport.store(
4995             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4996             a, byteArrayAddress(a, offset),
4997             this, a, offset,
4998             (arr, off, v) -> {
4999                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
5000                 v.stOp(wb, off,
5001                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5002             });
5003     }
5004 
5005     abstract
5006     void intoByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
5007     @ForceInline
5008     final
5009     <M extends VectorMask<$Boxtype$>>
5010     void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
5011         $Type$Species vsp = vspecies();
5012         m.check(vsp);
5013         VectorSupport.storeMasked(
5014             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5015             a, byteArrayAddress(a, offset),
5016             this, m, a, offset,
5017             (arr, off, v, vm) -> {
5018                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
5019                 v.stOp(wb, off, vm,
5020                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5021             });
5022     }
5023 
5024     @ForceInline
5025     final
5026     void intoByteBuffer0(ByteBuffer bb, int offset) {
5027         $Type$Species vsp = vspecies();
5028         ScopedMemoryAccess.storeIntoByteBuffer(
5029                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
5030                 this, bb, offset,
5031                 (buf, off, v) -> {
5032                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
5033                     v.stOp(wb, off,
5034                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5035                 });
5036     }
5037 
5038     abstract
5039     void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
5040     @ForceInline
5041     final
5042     <M extends VectorMask<$Boxtype$>>
5043     void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
5044         $Type$Species vsp = vspecies();
5045         m.check(vsp);
5046         ScopedMemoryAccess.storeIntoByteBufferMasked(
5047                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5048                 this, m, bb, offset,
5049                 (buf, off, v, vm) -> {
5050                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
5051                     v.stOp(wb, off, vm,
5052                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5053                 });
5054     }
5055 
5056 #if[short]
5057     /*package-private*/
5058     abstract
5059     void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
5060     @ForceInline
5061     final
5062     <M extends VectorMask<$Boxtype$>>
5063     void intoCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
5064         m.check(species());
5065         $Type$Species vsp = vspecies();
5066         VectorSupport.storeMasked(
5067             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5068             a, charArrayAddress(a, offset),
5069             this, m, a, offset,
5070             (arr, off, v, vm)
5071             -> v.stOp(arr, off, vm,
5072                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
5073     }
5074 #end[short]
5075 
5076     // End of low-level memory operations.
5077 
5078     private static
5079     void checkMaskFromIndexSize(int offset,
5080                                 $Type$Species vsp,
5081                                 VectorMask<$Boxtype$> m,
5082                                 int scale,
5083                                 int limit) {
5084         ((AbstractMask<$Boxtype$>)m)
5085             .checkIndexByLane(offset, limit, vsp.iota(), scale);
5086     }
5087 
5088     @ForceInline
5089     private void conditionalStoreNYI(int offset,
5090                                      $Type$Species vsp,
5091                                      VectorMask<$Boxtype$> m,
5092                                      int scale,
5093                                      int limit) {
5094         if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
5095             String msg =
5096                 String.format("unimplemented: store @%d in [0..%d), %s in %s",
5097                               offset, limit, m, vsp);
5098             throw new AssertionError(msg);
5099         }
5100     }
5101 
5102     /*package-private*/
5103     @Override
5104     @ForceInline
5105     final
5106     $abstractvectortype$ maybeSwap(ByteOrder bo) {
5107 #if[!byte]
5108         if (bo != NATIVE_ENDIAN) {
5109             return this.reinterpretAsBytes()
5110                 .rearrange(swapBytesShuffle())
5111                 .reinterpretAs$Type$s();
5112         }
5113 #end[!byte]
5114         return this;
5115     }
5116 
5117     static final int ARRAY_SHIFT =
5118         31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_$TYPE$_INDEX_SCALE);
5119     static final long ARRAY_BASE =
5120         Unsafe.ARRAY_$TYPE$_BASE_OFFSET;
5121 
5122     @ForceInline
5123     static long arrayAddress($type$[] a, int index) {
5124         return ARRAY_BASE + (((long)index) << ARRAY_SHIFT);
5125     }
5126 
5127 #if[short]
5128     static final int ARRAY_CHAR_SHIFT =
5129             31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_CHAR_INDEX_SCALE);
5130     static final long ARRAY_CHAR_BASE =
5131             Unsafe.ARRAY_CHAR_BASE_OFFSET;
5132 
5133     @ForceInline
5134     static long charArrayAddress(char[] a, int index) {
5135         return ARRAY_CHAR_BASE + (((long)index) << ARRAY_CHAR_SHIFT);
5136     }
5137 #end[short]
5138 
5139 #if[byte]
5140     static final int ARRAY_BOOLEAN_SHIFT =
5141             31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_BOOLEAN_INDEX_SCALE);
5142     static final long ARRAY_BOOLEAN_BASE =
5143             Unsafe.ARRAY_BOOLEAN_BASE_OFFSET;
5144 
5145     @ForceInline
5146     static long booleanArrayAddress(boolean[] a, int index) {
5147         return ARRAY_BOOLEAN_BASE + (((long)index) << ARRAY_BOOLEAN_SHIFT);
5148     }
5149 #end[byte]
5150 
5151     @ForceInline
5152     static long byteArrayAddress(byte[] a, int index) {
5153         return Unsafe.ARRAY_BYTE_BASE_OFFSET + index;
5154     }
5155 
5156     // ================================================
5157 
5158     /// Reinterpreting view methods:
5159     //   lanewise reinterpret: viewAsXVector()
5160     //   keep shape, redraw lanes: reinterpretAsEs()
5161 
5162     /**
5163      * {@inheritDoc} <!--workaround-->
5164      */
5165     @ForceInline
5166     @Override
5167     public final ByteVector reinterpretAsBytes() {
5168 #if[byte]
5169         return this;
5170 #else[byte]
5171          // Going to ByteVector, pay close attention to byte order.
5172          assert(REGISTER_ENDIAN == ByteOrder.LITTLE_ENDIAN);
5173          return asByteVectorRaw();
5174          //return asByteVectorRaw().rearrange(swapBytesShuffle());
5175 #end[byte]
5176     }
5177 
5178     /**
5179      * {@inheritDoc} <!--workaround-->
5180      */
5181     @ForceInline
5182     @Override
5183     public final $Bitstype$Vector viewAsIntegralLanes() {
5184 #if[BITWISE]
5185         return this;
5186 #else[BITWISE]
5187         LaneType ilt = LaneType.$TYPE$.asIntegral();
5188         return ($Bitstype$Vector) asVectorRaw(ilt);
5189 #end[BITWISE]
5190     }
5191 
5192     /**
5193      * {@inheritDoc} <!--workaround-->
5194 #if[byteOrShort]
5195      *
5196      * @implNote This method always throws
5197      * {@code UnsupportedOperationException}, because there is no floating
5198      * point type of the same size as {@code $type$}.  The return type
5199      * of this method is arbitrarily designated as
5200      * {@code Vector<?>}.  Future versions of this API may change the return
5201      * type if additional floating point types become available.
5202 #end[byteOrShort]
5203      */
5204     @ForceInline
5205     @Override
5206     public final
5207     {#if[byteOrShort]?Vector<?>:$Fptype$Vector}
5208     viewAsFloatingLanes() {
5209 #if[FP]
5210         return this;
5211 #else[FP]
5212         LaneType flt = LaneType.$TYPE$.asFloating();
5213 #if[!byteOrShort]
5214         return ($Fptype$Vector) asVectorRaw(flt);
5215 #else[!byteOrShort]
5216         // asFloating() will throw UnsupportedOperationException for the unsupported type $type$
5217         throw new AssertionError("Cannot reach here");
5218 #end[!byteOrShort]
5219 #end[FP]
5220     }
5221 
5222     // ================================================
5223 
5224     /// Object methods: toString, equals, hashCode
5225     //
5226     // Object methods are defined as if via Arrays.toString, etc.,
5227     // is applied to the array of elements.  Two equal vectors
5228     // are required to have equal species and equal lane values.
5229 
5230     /**
5231      * Returns a string representation of this vector, of the form
5232      * {@code "[0,1,2...]"}, reporting the lane values of this vector,
5233      * in lane order.
5234      *
5235      * The string is produced as if by a call to {@link
5236      * java.util.Arrays#toString($type$[]) Arrays.toString()},
5237      * as appropriate to the {@code $type$} array returned by
5238      * {@link #toArray this.toArray()}.
5239      *
5240      * @return a string of the form {@code "[0,1,2...]"}
5241      * reporting the lane values of this vector
5242      */
5243     @Override
5244     @ForceInline
5245     public final
5246     String toString() {
5247         // now that toArray is strongly typed, we can define this
5248         return Arrays.toString(toArray());
5249     }
5250 
5251     /**
5252      * {@inheritDoc} <!--workaround-->
5253      */
5254     @Override
5255     @ForceInline
5256     public final
5257     boolean equals(Object obj) {
5258         if (obj instanceof Vector) {
5259             Vector<?> that = (Vector<?>) obj;
5260             if (this.species().equals(that.species())) {
5261                 return this.eq(that.check(this.species())).allTrue();
5262             }
5263         }
5264         return false;
5265     }
5266 
5267     /**
5268      * {@inheritDoc} <!--workaround-->
5269      */
5270     @Override
5271     @ForceInline
5272     public final
5273     int hashCode() {
5274         // now that toArray is strongly typed, we can define this
5275         return Objects.hash(species(), Arrays.hashCode(toArray()));
5276     }
5277 
5278     // ================================================
5279 
5280     // Species
5281 
5282     /**
5283      * Class representing {@link $abstractvectortype$}'s of the same {@link VectorShape VectorShape}.
5284      */
5285     /*package-private*/
5286     static final class $Type$Species extends AbstractSpecies<$Boxtype$> {
5287         private $Type$Species(VectorShape shape,
5288                 Class<? extends $abstractvectortype$> vectorType,
5289                 Class<? extends AbstractMask<$Boxtype$>> maskType,
5290                 Function<Object, $abstractvectortype$> vectorFactory) {
5291             super(shape, LaneType.of($type$.class),
5292                   vectorType, maskType,
5293                   vectorFactory);
5294             assert(this.elementSize() == $Boxtype$.SIZE);
5295         }
5296 
5297         // Specializing overrides:
5298 
5299         @Override
5300         @ForceInline
5301         public final Class<$Boxtype$> elementType() {
5302             return $type$.class;
5303         }
5304 
5305         @Override
5306         @ForceInline
5307         final Class<$Boxtype$> genericElementType() {
5308             return $Boxtype$.class;
5309         }
5310 
5311         @SuppressWarnings("unchecked")
5312         @Override
5313         @ForceInline
5314         public final Class<? extends $Type$Vector> vectorType() {
5315             return (Class<? extends $Type$Vector>) vectorType;
5316         }
5317 
5318         @Override
5319         @ForceInline
5320         public final long checkValue(long e) {
5321             longToElementBits(e);  // only for exception
5322             return e;
5323         }
5324 
5325         /*package-private*/
5326         @Override
5327         @ForceInline
5328         final $abstractvectortype$ broadcastBits(long bits) {
5329             return ($abstractvectortype$)
5330                 VectorSupport.broadcastCoerced(
5331                     vectorType, $type$.class, laneCount,
5332                     bits, this,
5333                     (bits_, s_) -> s_.rvOp(i -> bits_));
5334         }
5335 
5336         /*package-private*/
5337         @ForceInline
5338         {#if[long]?public }final $abstractvectortype$ broadcast($type$ e) {
5339             return broadcastBits(toBits(e));
5340         }
5341 
5342 #if[!long]
5343         @Override
5344         @ForceInline
5345         public final $abstractvectortype$ broadcast(long e) {
5346             return broadcastBits(longToElementBits(e));
5347         }
5348 #end[!long]
5349 
5350         /*package-private*/
5351         final @Override
5352         @ForceInline
5353         long longToElementBits(long value) {
5354 #if[long]
5355             // In this case, the conversion can never fail.
5356             return value;
5357 #else[long]
5358             // Do the conversion, and then test it for failure.
5359             $type$ e = ($type$) value;
5360             if ((long) e != value) {
5361                 throw badElementBits(value, e);
5362             }
5363             return toBits(e);
5364 #end[long]
5365         }
5366 
5367         /*package-private*/
5368         @ForceInline
5369         static long toIntegralChecked($type$ e, boolean convertToInt) {
5370             long value = convertToInt ? (int) e : (long) e;
5371             if (($type$) value != e) {
5372                 throw badArrayBits(e, convertToInt, value);
5373             }
5374             return value;
5375         }
5376 
5377         /* this non-public one is for internal conversions */
5378         @Override
5379         @ForceInline
5380         final $abstractvectortype$ fromIntValues(int[] values) {
5381             VectorIntrinsics.requireLength(values.length, laneCount);
5382             $type$[] va = new $type$[laneCount()];
5383             for (int i = 0; i < va.length; i++) {
5384                 int lv = values[i];
5385                 $type$ v = ($type$) lv;
5386                 va[i] = v;
5387                 if ((int)v != lv) {
5388                     throw badElementBits(lv, v);
5389                 }
5390             }
5391             return dummyVector().fromArray0(va, 0);
5392         }
5393 
5394         // Virtual constructors
5395 
5396         @ForceInline
5397         @Override final
5398         public $abstractvectortype$ fromArray(Object a, int offset) {
5399             // User entry point:  Be careful with inputs.
5400             return $abstractvectortype$
5401                 .fromArray(this, ($type$[]) a, offset);
5402         }
5403 
5404         @ForceInline
5405         @Override final
5406         $abstractvectortype$ dummyVector() {
5407             return ($abstractvectortype$) super.dummyVector();
5408         }
5409 
5410         /*package-private*/
5411         final @Override
5412         @ForceInline
5413         $abstractvectortype$ rvOp(RVOp f) {
5414             $type$[] res = new $type$[laneCount()];
5415             for (int i = 0; i < res.length; i++) {
5416                 $bitstype$ bits = {#if[!long]?($bitstype$)} f.apply(i);
5417                 res[i] = fromBits(bits);
5418             }
5419             return dummyVector().vectorFactory(res);
5420         }
5421 
5422         $Type$Vector vOp(FVOp f) {
5423             $type$[] res = new $type$[laneCount()];
5424             for (int i = 0; i < res.length; i++) {
5425                 res[i] = f.apply(i);
5426             }
5427             return dummyVector().vectorFactory(res);
5428         }
5429 
5430         $Type$Vector vOp(VectorMask<$Boxtype$> m, FVOp f) {
5431             $type$[] res = new $type$[laneCount()];
5432             boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
5433             for (int i = 0; i < res.length; i++) {
5434                 if (mbits[i]) {
5435                     res[i] = f.apply(i);
5436                 }
5437             }
5438             return dummyVector().vectorFactory(res);
5439         }
5440 
5441         /*package-private*/
5442         @ForceInline
5443         <M> $abstractvectortype$ ldOp(M memory, int offset,
5444                                       FLdOp<M> f) {
5445             return dummyVector().ldOp(memory, offset, f);
5446         }
5447 
5448         /*package-private*/
5449         @ForceInline
5450         <M> $abstractvectortype$ ldOp(M memory, int offset,
5451                                       VectorMask<$Boxtype$> m,
5452                                       FLdOp<M> f) {
5453             return dummyVector().ldOp(memory, offset, m, f);
5454         }
5455 
5456         /*package-private*/
5457         @ForceInline
5458         <M> void stOp(M memory, int offset, FStOp<M> f) {
5459             dummyVector().stOp(memory, offset, f);
5460         }
5461 
5462         /*package-private*/
5463         @ForceInline
5464         <M> void stOp(M memory, int offset,
5465                       AbstractMask<$Boxtype$> m,
5466                       FStOp<M> f) {
5467             dummyVector().stOp(memory, offset, m, f);
5468         }
5469 
5470         // N.B. Make sure these constant vectors and
5471         // masks load up correctly into registers.
5472         //
5473         // Also, see if we can avoid all that switching.
5474         // Could we cache both vectors and both masks in
5475         // this species object?
5476 
5477         // Zero and iota vector access
5478         @Override
5479         @ForceInline
5480         public final $abstractvectortype$ zero() {
5481             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5482                 return $Type$MaxVector.ZERO;
5483             switch (vectorBitSize()) {
5484                 case 64: return $Type$64Vector.ZERO;
5485                 case 128: return $Type$128Vector.ZERO;
5486                 case 256: return $Type$256Vector.ZERO;
5487                 case 512: return $Type$512Vector.ZERO;
5488             }
5489             throw new AssertionError();
5490         }
5491 
5492         @Override
5493         @ForceInline
5494         public final $abstractvectortype$ iota() {
5495             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5496                 return $Type$MaxVector.IOTA;
5497             switch (vectorBitSize()) {
5498                 case 64: return $Type$64Vector.IOTA;
5499                 case 128: return $Type$128Vector.IOTA;
5500                 case 256: return $Type$256Vector.IOTA;
5501                 case 512: return $Type$512Vector.IOTA;
5502             }
5503             throw new AssertionError();
5504         }
5505 
5506         // Mask access
5507         @Override
5508         @ForceInline
5509         public final VectorMask<$Boxtype$> maskAll(boolean bit) {
5510             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5511                 return $Type$MaxVector.$Type$MaxMask.maskAll(bit);
5512             switch (vectorBitSize()) {
5513                 case 64: return $Type$64Vector.$Type$64Mask.maskAll(bit);
5514                 case 128: return $Type$128Vector.$Type$128Mask.maskAll(bit);
5515                 case 256: return $Type$256Vector.$Type$256Mask.maskAll(bit);
5516                 case 512: return $Type$512Vector.$Type$512Mask.maskAll(bit);
5517             }
5518             throw new AssertionError();
5519         }
5520     }
5521 
5522     /**
5523      * Finds a species for an element type of {@code $type$} and shape.
5524      *
5525      * @param s the shape
5526      * @return a species for an element type of {@code $type$} and shape
5527      * @throws IllegalArgumentException if no such species exists for the shape
5528      */
5529     static $Type$Species species(VectorShape s) {
5530         Objects.requireNonNull(s);
5531         switch (s) {
5532             case S_64_BIT: return ($Type$Species) SPECIES_64;
5533             case S_128_BIT: return ($Type$Species) SPECIES_128;
5534             case S_256_BIT: return ($Type$Species) SPECIES_256;
5535             case S_512_BIT: return ($Type$Species) SPECIES_512;
5536             case S_Max_BIT: return ($Type$Species) SPECIES_MAX;
5537             default: throw new IllegalArgumentException("Bad shape: " + s);
5538         }
5539     }
5540 
5541     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_64_BIT VectorShape.S_64_BIT}. */
5542     public static final VectorSpecies<$Boxtype$> SPECIES_64
5543         = new $Type$Species(VectorShape.S_64_BIT,
5544                             $Type$64Vector.class,
5545                             $Type$64Vector.$Type$64Mask.class,
5546                             $Type$64Vector::new);
5547 
5548     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
5549     public static final VectorSpecies<$Boxtype$> SPECIES_128
5550         = new $Type$Species(VectorShape.S_128_BIT,
5551                             $Type$128Vector.class,
5552                             $Type$128Vector.$Type$128Mask.class,
5553                             $Type$128Vector::new);
5554 
5555     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
5556     public static final VectorSpecies<$Boxtype$> SPECIES_256
5557         = new $Type$Species(VectorShape.S_256_BIT,
5558                             $Type$256Vector.class,
5559                             $Type$256Vector.$Type$256Mask.class,
5560                             $Type$256Vector::new);
5561 
5562     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
5563     public static final VectorSpecies<$Boxtype$> SPECIES_512
5564         = new $Type$Species(VectorShape.S_512_BIT,
5565                             $Type$512Vector.class,
5566                             $Type$512Vector.$Type$512Mask.class,
5567                             $Type$512Vector::new);
5568 
5569     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
5570     public static final VectorSpecies<$Boxtype$> SPECIES_MAX
5571         = new $Type$Species(VectorShape.S_Max_BIT,
5572                             $Type$MaxVector.class,
5573                             $Type$MaxVector.$Type$MaxMask.class,
5574                             $Type$MaxVector::new);
5575 
5576     /**
5577      * Preferred species for {@link $Type$Vector}s.
5578      * A preferred species is a species of maximal bit-size for the platform.
5579      */
5580     public static final VectorSpecies<$Boxtype$> SPECIES_PREFERRED
5581         = ($Type$Species) VectorSpecies.ofPreferred($type$.class);
5582 }