1 /*
   2  * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteOrder;
  28 import java.util.Arrays;
  29 import java.util.Objects;
  30 import java.util.function.Function;
  31 
  32 import jdk.incubator.foreign.MemorySegment;
  33 import jdk.incubator.foreign.ValueLayout;
  34 import jdk.internal.access.foreign.MemorySegmentProxy;
  35 import jdk.internal.misc.ScopedMemoryAccess;
  36 import jdk.internal.misc.Unsafe;
  37 import jdk.internal.vm.annotation.ForceInline;
  38 import jdk.internal.vm.vector.VectorSupport;
  39 
  40 import static jdk.internal.vm.vector.VectorSupport.*;
  41 import static jdk.incubator.vector.VectorIntrinsics.*;
  42 
  43 import static jdk.incubator.vector.VectorOperators.*;
  44 
  45 #warn This file is preprocessed before being compiled
  46 
  47 /**
  48  * A specialized {@link Vector} representing an ordered immutable sequence of
  49  * {@code $type$} values.
  50  */
  51 @SuppressWarnings("cast")  // warning: redundant cast
  52 public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
  53 
  54     $abstractvectortype$($type$[] vec) {
  55         super(vec);
  56     }
  57 
  58 #if[FP]
  59     static final int FORBID_OPCODE_KIND = VO_NOFP;
  60 #else[FP]
  61     static final int FORBID_OPCODE_KIND = VO_ONLYFP;
  62 #end[FP]
  63 
  64     static final ValueLayout.Of$Type$ ELEMENT_LAYOUT = ValueLayout.JAVA_$TYPE$.withBitAlignment(8);
  65 
  66     @ForceInline
  67     static int opCode(Operator op) {
  68         return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
  69     }
  70     @ForceInline
  71     static int opCode(Operator op, int requireKind) {
  72         requireKind |= VO_OPCODE_VALID;
  73         return VectorOperators.opCode(op, requireKind, FORBID_OPCODE_KIND);
  74     }
  75     @ForceInline
  76     static boolean opKind(Operator op, int bit) {
  77         return VectorOperators.opKind(op, bit);
  78     }
  79 
  80     // Virtualized factories and operators,
  81     // coded with portable definitions.
  82     // These are all @ForceInline in case
  83     // they need to be used performantly.
  84     // The various shape-specific subclasses
  85     // also specialize them by wrapping
  86     // them in a call like this:
  87     //    return (Byte128Vector)
  88     //       super.bOp((Byte128Vector) o);
  89     // The purpose of that is to forcibly inline
  90     // the generic definition from this file
  91     // into a sharply type- and size-specific
  92     // wrapper in the subclass file, so that
  93     // the JIT can specialize the code.
  94     // The code is only inlined and expanded
  95     // if it gets hot.  Think of it as a cheap
  96     // and lazy version of C++ templates.
  97 
  98     // Virtualized getter
  99 
 100     /*package-private*/
 101     abstract $type$[] vec();
 102 
 103     // Virtualized constructors
 104 
 105     /**
 106      * Build a vector directly using my own constructor.
 107      * It is an error if the array is aliased elsewhere.
 108      */
 109     /*package-private*/
 110     abstract $abstractvectortype$ vectorFactory($type$[] vec);
 111 
 112     /**
 113      * Build a mask directly using my species.
 114      * It is an error if the array is aliased elsewhere.
 115      */
 116     /*package-private*/
 117     @ForceInline
 118     final
 119     AbstractMask<$Boxtype$> maskFactory(boolean[] bits) {
 120         return vspecies().maskFactory(bits);
 121     }
 122 
 123     // Constant loader (takes dummy as vector arg)
 124     interface FVOp {
 125         $type$ apply(int i);
 126     }
 127 
 128     /*package-private*/
 129     @ForceInline
 130     final
 131     $abstractvectortype$ vOp(FVOp f) {
 132         $type$[] res = new $type$[length()];
 133         for (int i = 0; i < res.length; i++) {
 134             res[i] = f.apply(i);
 135         }
 136         return vectorFactory(res);
 137     }
 138 
 139     @ForceInline
 140     final
 141     $abstractvectortype$ vOp(VectorMask<$Boxtype$> m, FVOp f) {
 142         $type$[] res = new $type$[length()];
 143         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 144         for (int i = 0; i < res.length; i++) {
 145             if (mbits[i]) {
 146                 res[i] = f.apply(i);
 147             }
 148         }
 149         return vectorFactory(res);
 150     }
 151 
 152     // Unary operator
 153 
 154     /*package-private*/
 155     interface FUnOp {
 156         $type$ apply(int i, $type$ a);
 157     }
 158 
 159     /*package-private*/
 160     abstract
 161     $abstractvectortype$ uOp(FUnOp f);
 162     @ForceInline
 163     final
 164     $abstractvectortype$ uOpTemplate(FUnOp f) {
 165         $type$[] vec = vec();
 166         $type$[] res = new $type$[length()];
 167         for (int i = 0; i < res.length; i++) {
 168             res[i] = f.apply(i, vec[i]);
 169         }
 170         return vectorFactory(res);
 171     }
 172 
 173     /*package-private*/
 174     abstract
 175     $abstractvectortype$ uOp(VectorMask<$Boxtype$> m,
 176                              FUnOp f);
 177     @ForceInline
 178     final
 179     $abstractvectortype$ uOpTemplate(VectorMask<$Boxtype$> m,
 180                                      FUnOp f) {
 181         if (m == null) {
 182             return uOpTemplate(f);
 183         }
 184         $type$[] vec = vec();
 185         $type$[] res = new $type$[length()];
 186         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 187         for (int i = 0; i < res.length; i++) {
 188             res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
 189         }
 190         return vectorFactory(res);
 191     }
 192 
 193     // Binary operator
 194 
 195     /*package-private*/
 196     interface FBinOp {
 197         $type$ apply(int i, $type$ a, $type$ b);
 198     }
 199 
 200     /*package-private*/
 201     abstract
 202     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 203                              FBinOp f);
 204     @ForceInline
 205     final
 206     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 207                                      FBinOp f) {
 208         $type$[] res = new $type$[length()];
 209         $type$[] vec1 = this.vec();
 210         $type$[] vec2 = (($abstractvectortype$)o).vec();
 211         for (int i = 0; i < res.length; i++) {
 212             res[i] = f.apply(i, vec1[i], vec2[i]);
 213         }
 214         return vectorFactory(res);
 215     }
 216 
 217     /*package-private*/
 218     abstract
 219     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 220                              VectorMask<$Boxtype$> m,
 221                              FBinOp f);
 222     @ForceInline
 223     final
 224     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 225                                      VectorMask<$Boxtype$> m,
 226                                      FBinOp f) {
 227         if (m == null) {
 228             return bOpTemplate(o, f);
 229         }
 230         $type$[] res = new $type$[length()];
 231         $type$[] vec1 = this.vec();
 232         $type$[] vec2 = (($abstractvectortype$)o).vec();
 233         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 234         for (int i = 0; i < res.length; i++) {
 235             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
 236         }
 237         return vectorFactory(res);
 238     }
 239 
 240     // Ternary operator
 241 
 242     /*package-private*/
 243     interface FTriOp {
 244         $type$ apply(int i, $type$ a, $type$ b, $type$ c);
 245     }
 246 
 247     /*package-private*/
 248     abstract
 249     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 250                              Vector<$Boxtype$> o2,
 251                              FTriOp f);
 252     @ForceInline
 253     final
 254     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 255                                      Vector<$Boxtype$> o2,
 256                                      FTriOp f) {
 257         $type$[] res = new $type$[length()];
 258         $type$[] vec1 = this.vec();
 259         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 260         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 261         for (int i = 0; i < res.length; i++) {
 262             res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
 263         }
 264         return vectorFactory(res);
 265     }
 266 
 267     /*package-private*/
 268     abstract
 269     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 270                              Vector<$Boxtype$> o2,
 271                              VectorMask<$Boxtype$> m,
 272                              FTriOp f);
 273     @ForceInline
 274     final
 275     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 276                                      Vector<$Boxtype$> o2,
 277                                      VectorMask<$Boxtype$> m,
 278                                      FTriOp f) {
 279         if (m == null) {
 280             return tOpTemplate(o1, o2, f);
 281         }
 282         $type$[] res = new $type$[length()];
 283         $type$[] vec1 = this.vec();
 284         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 285         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 286         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 287         for (int i = 0; i < res.length; i++) {
 288             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
 289         }
 290         return vectorFactory(res);
 291     }
 292 
 293     // Reduction operator
 294 
 295     /*package-private*/
 296     abstract
 297     $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f);
 298 
 299     @ForceInline
 300     final
 301     $type$ rOpTemplate($type$ v, VectorMask<$Boxtype$> m, FBinOp f) {
 302         if (m == null) {
 303             return rOpTemplate(v, f);
 304         }
 305         $type$[] vec = vec();
 306         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 307         for (int i = 0; i < vec.length; i++) {
 308             v = mbits[i] ? f.apply(i, v, vec[i]) : v;
 309         }
 310         return v;
 311     }
 312 
 313     @ForceInline
 314     final
 315     $type$ rOpTemplate($type$ v, FBinOp f) {
 316         $type$[] vec = vec();
 317         for (int i = 0; i < vec.length; i++) {
 318             v = f.apply(i, v, vec[i]);
 319         }
 320         return v;
 321     }
 322 
 323     // Memory reference
 324 
 325     /*package-private*/
 326     interface FLdOp<M> {
 327         $type$ apply(M memory, int offset, int i);
 328     }
 329 
 330     /*package-private*/
 331     @ForceInline
 332     final
 333     <M> $abstractvectortype$ ldOp(M memory, int offset,
 334                                   FLdOp<M> f) {
 335         //dummy; no vec = vec();
 336         $type$[] res = new $type$[length()];
 337         for (int i = 0; i < res.length; i++) {
 338             res[i] = f.apply(memory, offset, i);
 339         }
 340         return vectorFactory(res);
 341     }
 342 
 343     /*package-private*/
 344     @ForceInline
 345     final
 346     <M> $abstractvectortype$ ldOp(M memory, int offset,
 347                                   VectorMask<$Boxtype$> m,
 348                                   FLdOp<M> f) {
 349         //$type$[] vec = vec();
 350         $type$[] res = new $type$[length()];
 351         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 352         for (int i = 0; i < res.length; i++) {
 353             if (mbits[i]) {
 354                 res[i] = f.apply(memory, offset, i);
 355             }
 356         }
 357         return vectorFactory(res);
 358     }
 359 
 360     /*package-private*/
 361     interface FLdLongOp {
 362         $type$ apply(MemorySegment memory, long offset, int i);
 363     }
 364 
 365     /*package-private*/
 366     @ForceInline
 367     final
 368     $abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
 369                                   FLdLongOp f) {
 370         //dummy; no vec = vec();
 371         $type$[] res = new $type$[length()];
 372         for (int i = 0; i < res.length; i++) {
 373             res[i] = f.apply(memory, offset, i);
 374         }
 375         return vectorFactory(res);
 376     }
 377 
 378     /*package-private*/
 379     @ForceInline
 380     final
 381     $abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
 382                                   VectorMask<$Boxtype$> m,
 383                                   FLdLongOp f) {
 384         //$type$[] vec = vec();
 385         $type$[] res = new $type$[length()];
 386         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 387         for (int i = 0; i < res.length; i++) {
 388             if (mbits[i]) {
 389                 res[i] = f.apply(memory, offset, i);
 390             }
 391         }
 392         return vectorFactory(res);
 393     }
 394 
 395     static $type$ memorySegmentGet(MemorySegment ms, long o, int i) {
 396         return ms.get(ELEMENT_LAYOUT, o + i * $sizeInBytes$L);
 397     }
 398 
 399     interface FStOp<M> {
 400         void apply(M memory, int offset, int i, $type$ a);
 401     }
 402 
 403     /*package-private*/
 404     @ForceInline
 405     final
 406     <M> void stOp(M memory, int offset,
 407                   FStOp<M> f) {
 408         $type$[] vec = vec();
 409         for (int i = 0; i < vec.length; i++) {
 410             f.apply(memory, offset, i, vec[i]);
 411         }
 412     }
 413 
 414     /*package-private*/
 415     @ForceInline
 416     final
 417     <M> void stOp(M memory, int offset,
 418                   VectorMask<$Boxtype$> m,
 419                   FStOp<M> f) {
 420         $type$[] vec = vec();
 421         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 422         for (int i = 0; i < vec.length; i++) {
 423             if (mbits[i]) {
 424                 f.apply(memory, offset, i, vec[i]);
 425             }
 426         }
 427     }
 428 
 429     interface FStLongOp {
 430         void apply(MemorySegment memory, long offset, int i, $type$ a);
 431     }
 432 
 433     /*package-private*/
 434     @ForceInline
 435     final
 436     void stLongOp(MemorySegment memory, long offset,
 437                   FStLongOp f) {
 438         $type$[] vec = vec();
 439         for (int i = 0; i < vec.length; i++) {
 440             f.apply(memory, offset, i, vec[i]);
 441         }
 442     }
 443 
 444     /*package-private*/
 445     @ForceInline
 446     final
 447     void stLongOp(MemorySegment memory, long offset,
 448                   VectorMask<$Boxtype$> m,
 449                   FStLongOp f) {
 450         $type$[] vec = vec();
 451         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 452         for (int i = 0; i < vec.length; i++) {
 453             if (mbits[i]) {
 454                 f.apply(memory, offset, i, vec[i]);
 455             }
 456         }
 457     }
 458 
 459     static void memorySegmentSet(MemorySegment ms, long o, int i, $type$ e) {
 460         ms.set(ELEMENT_LAYOUT, o + i * $sizeInBytes$L, e);
 461     }
 462 
 463     // Binary test
 464 
 465     /*package-private*/
 466     interface FBinTest {
 467         boolean apply(int cond, int i, $type$ a, $type$ b);
 468     }
 469 
 470     /*package-private*/
 471     @ForceInline
 472     final
 473     AbstractMask<$Boxtype$> bTest(int cond,
 474                                   Vector<$Boxtype$> o,
 475                                   FBinTest f) {
 476         $type$[] vec1 = vec();
 477         $type$[] vec2 = (($abstractvectortype$)o).vec();
 478         boolean[] bits = new boolean[length()];
 479         for (int i = 0; i < length(); i++){
 480             bits[i] = f.apply(cond, i, vec1[i], vec2[i]);
 481         }
 482         return maskFactory(bits);
 483     }
 484 
 485 #if[BITWISE]
 486     /*package-private*/
 487     @ForceInline
 488     static $type$ rotateLeft($type$ a, int n) {
 489 #if[intOrLong]
 490         return $Boxtype$.rotateLeft(a, n);
 491 #else[intOrLong]
 492         return ($type$)((((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) << (n & $Boxtype$.SIZE-1)) | (((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) >>> ($Boxtype$.SIZE - (n & $Boxtype$.SIZE-1))));
 493 #end[intOrLong]
 494     }
 495 
 496     /*package-private*/
 497     @ForceInline
 498     static $type$ rotateRight($type$ a, int n) {
 499 #if[intOrLong]
 500         return $Boxtype$.rotateRight(a, n);
 501 #else[intOrLong]
 502         return ($type$)((((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) >>> (n & $Boxtype$.SIZE-1)) | (((($type$)a) & $Boxtype$.toUnsignedInt(($type$)-1)) << ($Boxtype$.SIZE - (n & $Boxtype$.SIZE-1))));
 503 #end[intOrLong]
 504     }
 505 #end[BITWISE]
 506 
 507     /*package-private*/
 508     @Override
 509     abstract $Type$Species vspecies();
 510 
 511     /*package-private*/
 512     @ForceInline
 513     static long toBits($type$ e) {
 514         return {#if[FP]? $Type$.$type$ToRaw$Bitstype$Bits(e): e};
 515     }
 516 
 517     /*package-private*/
 518     @ForceInline
 519     static $type$ fromBits(long bits) {
 520         return {#if[FP]?$Type$.$bitstype$BitsTo$Type$}(($bitstype$)bits);
 521     }
 522 
 523     static $abstractvectortype$ expandHelper(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 524         VectorSpecies<$Boxtype$> vsp = m.vectorSpecies();
 525         $abstractvectortype$ r  = ($abstractvectortype$) vsp.zero();
 526         $abstractvectortype$ vi = ($abstractvectortype$) v;
 527         if (m.allTrue()) {
 528             return vi;
 529         }
 530         for (int i = 0, j = 0; i < vsp.length(); i++) {
 531             if (m.laneIsSet(i)) {
 532                 r = r.withLane(i, vi.lane(j++));
 533             }
 534         }
 535         return r;
 536     }
 537 
 538     static $abstractvectortype$ compressHelper(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 539         VectorSpecies<$Boxtype$> vsp = m.vectorSpecies();
 540         $abstractvectortype$ r  = ($abstractvectortype$) vsp.zero();
 541         $abstractvectortype$ vi = ($abstractvectortype$) v;
 542         if (m.allTrue()) {
 543             return vi;
 544         }
 545         for (int i = 0, j = 0; i < vsp.length(); i++) {
 546             if (m.laneIsSet(i)) {
 547                 r = r.withLane(j++, vi.lane(i));
 548             }
 549         }
 550         return r;
 551     }
 552 
 553     // Static factories (other than memory operations)
 554 
 555     // Note: A surprising behavior in javadoc
 556     // sometimes makes a lone /** {@inheritDoc} */
 557     // comment drop the method altogether,
 558     // apparently if the method mentions an
 559     // parameter or return type of Vector<$Boxtype$>
 560     // instead of Vector<E> as originally specified.
 561     // Adding an empty HTML fragment appears to
 562     // nudge javadoc into providing the desired
 563     // inherited documentation.  We use the HTML
 564     // comment <!--workaround--> for this.
 565 
 566     /**
 567      * Returns a vector of the given species
 568      * where all lane elements are set to
 569      * zero, the default primitive value.
 570      *
 571      * @param species species of the desired zero vector
 572      * @return a zero vector
 573      */
 574     @ForceInline
 575     public static $abstractvectortype$ zero(VectorSpecies<$Boxtype$> species) {
 576         $Type$Species vsp = ($Type$Species) species;
 577 #if[FP]
 578         return VectorSupport.fromBitsCoerced(vsp.vectorType(), $type$.class, species.length(),
 579                         toBits(0.0f), MODE_BROADCAST, vsp,
 580                         ((bits_, s_) -> s_.rvOp(i -> bits_)));
 581 #else[FP]
 582         return VectorSupport.fromBitsCoerced(vsp.vectorType(), $type$.class, species.length(),
 583                                 0, MODE_BROADCAST, vsp,
 584                                 ((bits_, s_) -> s_.rvOp(i -> bits_)));
 585 #end[FP]
 586     }
 587 
 588     /**
 589      * Returns a vector of the same species as this one
 590      * where all lane elements are set to
 591      * the primitive value {@code e}.
 592      *
 593      * The contents of the current vector are discarded;
 594      * only the species is relevant to this operation.
 595      *
 596      * <p> This method returns the value of this expression:
 597      * {@code $abstractvectortype$.broadcast(this.species(), e)}.
 598      *
 599      * @apiNote
 600      * Unlike the similar method named {@code broadcast()}
 601      * in the supertype {@code Vector}, this method does not
 602      * need to validate its argument, and cannot throw
 603      * {@code IllegalArgumentException}.  This method is
 604      * therefore preferable to the supertype method.
 605      *
 606      * @param e the value to broadcast
 607      * @return a vector where all lane elements are set to
 608      *         the primitive value {@code e}
 609      * @see #broadcast(VectorSpecies,long)
 610      * @see Vector#broadcast(long)
 611      * @see VectorSpecies#broadcast(long)
 612      */
 613     public abstract $abstractvectortype$ broadcast($type$ e);
 614 
 615     /**
 616      * Returns a vector of the given species
 617      * where all lane elements are set to
 618      * the primitive value {@code e}.
 619      *
 620      * @param species species of the desired vector
 621      * @param e the value to broadcast
 622      * @return a vector where all lane elements are set to
 623      *         the primitive value {@code e}
 624      * @see #broadcast(long)
 625      * @see Vector#broadcast(long)
 626      * @see VectorSpecies#broadcast(long)
 627      */
 628     @ForceInline
 629     public static $abstractvectortype$ broadcast(VectorSpecies<$Boxtype$> species, $type$ e) {
 630         $Type$Species vsp = ($Type$Species) species;
 631         return vsp.broadcast(e);
 632     }
 633 
 634     /*package-private*/
 635     @ForceInline
 636     final $abstractvectortype$ broadcastTemplate($type$ e) {
 637         $Type$Species vsp = vspecies();
 638         return vsp.broadcast(e);
 639     }
 640 
 641 #if[!long]
 642     /**
 643      * {@inheritDoc} <!--workaround-->
 644      * @apiNote
 645      * When working with vector subtypes like {@code $abstractvectortype$},
 646      * {@linkplain #broadcast($type$) the more strongly typed method}
 647      * is typically selected.  It can be explicitly selected
 648      * using a cast: {@code v.broadcast(($type$)e)}.
 649      * The two expressions will produce numerically identical results.
 650      */
 651     @Override
 652     public abstract $abstractvectortype$ broadcast(long e);
 653 
 654     /**
 655      * Returns a vector of the given species
 656      * where all lane elements are set to
 657      * the primitive value {@code e}.
 658      *
 659      * The {@code long} value must be accurately representable
 660      * by the {@code ETYPE} of the vector species, so that
 661      * {@code e==(long)(ETYPE)e}.
 662      *
 663      * @param species species of the desired vector
 664      * @param e the value to broadcast
 665      * @return a vector where all lane elements are set to
 666      *         the primitive value {@code e}
 667      * @throws IllegalArgumentException
 668      *         if the given {@code long} value cannot
 669      *         be represented by the vector's {@code ETYPE}
 670      * @see #broadcast(VectorSpecies,$type$)
 671      * @see VectorSpecies#checkValue(long)
 672      */
 673     @ForceInline
 674     public static $abstractvectortype$ broadcast(VectorSpecies<$Boxtype$> species, long e) {
 675         $Type$Species vsp = ($Type$Species) species;
 676         return vsp.broadcast(e);
 677     }
 678 
 679     /*package-private*/
 680     @ForceInline
 681     final $abstractvectortype$ broadcastTemplate(long e) {
 682         return vspecies().broadcast(e);
 683     }
 684 #end[!long]
 685 
 686     // Unary lanewise support
 687 
 688     /**
 689      * {@inheritDoc} <!--workaround-->
 690      */
 691     public abstract
 692     $abstractvectortype$ lanewise(VectorOperators.Unary op);
 693 
 694     @ForceInline
 695     final
 696     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op) {
 697         if (opKind(op, VO_SPECIAL)) {
 698             if (op == ZOMO) {
 699                 return blend(broadcast(-1), compare(NE, 0));
 700             }
 701 #if[BITWISE]
 702             if (op == NOT) {
 703                 return broadcast(-1).lanewise(XOR, this);
 704             }
 705 #end[BITWISE]
 706         }
 707         int opc = opCode(op);
 708         return VectorSupport.unaryOp(
 709             opc, getClass(), null, $type$.class, length(),
 710             this, null,
 711             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));
 712     }
 713 
 714     /**
 715      * {@inheritDoc} <!--workaround-->
 716      */
 717     @Override
 718     public abstract
 719     $abstractvectortype$ lanewise(VectorOperators.Unary op,
 720                                   VectorMask<$Boxtype$> m);
 721     @ForceInline
 722     final
 723     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op,
 724                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
 725                                           VectorMask<$Boxtype$> m) {
 726         m.check(maskClass, this);
 727         if (opKind(op, VO_SPECIAL)) {
 728             if (op == ZOMO) {
 729                 return blend(broadcast(-1), compare(NE, 0, m));
 730             }
 731 #if[BITWISE]
 732             if (op == NOT) {
 733                 return lanewise(XOR, broadcast(-1), m);
 734             }
 735 #end[BITWISE]
 736         }
 737         int opc = opCode(op);
 738         return VectorSupport.unaryOp(
 739             opc, getClass(), maskClass, $type$.class, length(),
 740             this, m,
 741             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));
 742     }
 743 
 744     private static final
 745     ImplCache<Unary, UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 746         UN_IMPL = new ImplCache<>(Unary.class, $Type$Vector.class);
 747 
 748     private static UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> unaryOperations(int opc_) {
 749         switch (opc_) {
 750             case VECTOR_OP_NEG: return (v0, m) ->
 751                     v0.uOp(m, (i, a) -> ($type$) -a);
 752             case VECTOR_OP_ABS: return (v0, m) ->
 753                     v0.uOp(m, (i, a) -> ($type$) Math.abs(a));
 754 #if[!FP]
 755 #if[intOrLong]
 756             case VECTOR_OP_BIT_COUNT: return (v0, m) ->
 757                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.bitCount(a));
 758             case VECTOR_OP_TZ_COUNT: return (v0, m) ->
 759                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.numberOfTrailingZeros(a));
 760             case VECTOR_OP_LZ_COUNT: return (v0, m) ->
 761                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.numberOfLeadingZeros(a));
 762             case VECTOR_OP_REVERSE: return (v0, m) ->
 763                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.reverse(a));
 764 #else[intOrLong]
 765             case VECTOR_OP_BIT_COUNT: return (v0, m) ->
 766                     v0.uOp(m, (i, a) -> ($type$) bitCount(a));
 767             case VECTOR_OP_TZ_COUNT: return (v0, m) ->
 768                     v0.uOp(m, (i, a) -> ($type$) numberOfTrailingZeros(a));
 769             case VECTOR_OP_LZ_COUNT: return (v0, m) ->
 770                     v0.uOp(m, (i, a) -> ($type$) numberOfLeadingZeros(a));
 771             case VECTOR_OP_REVERSE: return (v0, m) ->
 772                     v0.uOp(m, (i, a) -> reverse(a));
 773 #end[intOrLong]
 774 #if[BITWISE]
 775 #if[byte]
 776             case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
 777                     v0.uOp(m, (i, a) -> a);
 778 #else[byte]
 779             case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
 780                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.reverseBytes(a));
 781 #end[byte]
 782 #end[BITWISE]
 783 #end[!FP]
 784 #if[FP]
 785             case VECTOR_OP_SIN: return (v0, m) ->
 786                     v0.uOp(m, (i, a) -> ($type$) Math.sin(a));
 787             case VECTOR_OP_COS: return (v0, m) ->
 788                     v0.uOp(m, (i, a) -> ($type$) Math.cos(a));
 789             case VECTOR_OP_TAN: return (v0, m) ->
 790                     v0.uOp(m, (i, a) -> ($type$) Math.tan(a));
 791             case VECTOR_OP_ASIN: return (v0, m) ->
 792                     v0.uOp(m, (i, a) -> ($type$) Math.asin(a));
 793             case VECTOR_OP_ACOS: return (v0, m) ->
 794                     v0.uOp(m, (i, a) -> ($type$) Math.acos(a));
 795             case VECTOR_OP_ATAN: return (v0, m) ->
 796                     v0.uOp(m, (i, a) -> ($type$) Math.atan(a));
 797             case VECTOR_OP_EXP: return (v0, m) ->
 798                     v0.uOp(m, (i, a) -> ($type$) Math.exp(a));
 799             case VECTOR_OP_LOG: return (v0, m) ->
 800                     v0.uOp(m, (i, a) -> ($type$) Math.log(a));
 801             case VECTOR_OP_LOG10: return (v0, m) ->
 802                     v0.uOp(m, (i, a) -> ($type$) Math.log10(a));
 803             case VECTOR_OP_SQRT: return (v0, m) ->
 804                     v0.uOp(m, (i, a) -> ($type$) Math.sqrt(a));
 805             case VECTOR_OP_CBRT: return (v0, m) ->
 806                     v0.uOp(m, (i, a) -> ($type$) Math.cbrt(a));
 807             case VECTOR_OP_SINH: return (v0, m) ->
 808                     v0.uOp(m, (i, a) -> ($type$) Math.sinh(a));
 809             case VECTOR_OP_COSH: return (v0, m) ->
 810                     v0.uOp(m, (i, a) -> ($type$) Math.cosh(a));
 811             case VECTOR_OP_TANH: return (v0, m) ->
 812                     v0.uOp(m, (i, a) -> ($type$) Math.tanh(a));
 813             case VECTOR_OP_EXPM1: return (v0, m) ->
 814                     v0.uOp(m, (i, a) -> ($type$) Math.expm1(a));
 815             case VECTOR_OP_LOG1P: return (v0, m) ->
 816                     v0.uOp(m, (i, a) -> ($type$) Math.log1p(a));
 817 #end[FP]
 818             default: return null;
 819         }
 820     }
 821 
 822     // Binary lanewise support
 823 
 824     /**
 825      * {@inheritDoc} <!--workaround-->
 826      * @see #lanewise(VectorOperators.Binary,$type$)
 827      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 828      */
 829     @Override
 830     public abstract
 831     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 832                                   Vector<$Boxtype$> v);
 833     @ForceInline
 834     final
 835     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 836                                           Vector<$Boxtype$> v) {
 837         $abstractvectortype$ that = ($abstractvectortype$) v;
 838         that.check(this);
 839 
 840         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 841             if (op == FIRST_NONZERO) {
 842                 // FIXME: Support this in the JIT.
 843                 VectorMask<$Boxbitstype$> thisNZ
 844                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 845                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 846                 op = OR_UNCHECKED;
 847 #if[FP]
 848                 // FIXME: Support OR_UNCHECKED on float/double also!
 849                 return this.viewAsIntegralLanes()
 850                     .lanewise(op, that.viewAsIntegralLanes())
 851                     .viewAsFloatingLanes();
 852 #end[FP]
 853             }
 854 #if[BITWISE]
 855 #if[!FP]
 856             if (opKind(op, VO_SHIFT)) {
 857                 // As per shift specification for Java, mask the shift count.
 858                 // This allows the JIT to ignore some ISA details.
 859                 that = that.lanewise(AND, SHIFT_MASK);
 860             }
 861 #end[!FP]
 862             if (op == AND_NOT) {
 863                 // FIXME: Support this in the JIT.
 864                 that = that.lanewise(NOT);
 865                 op = AND;
 866             } else if (op == DIV) {
 867                 VectorMask<$Boxtype$> eqz = that.eq(($type$) 0);
 868                 if (eqz.anyTrue()) {
 869                     throw that.divZeroException();
 870                 }
 871             }
 872 #end[BITWISE]
 873         }
 874 
 875         int opc = opCode(op);
 876         return VectorSupport.binaryOp(
 877             opc, getClass(), null, $type$.class, length(),
 878             this, that, null,
 879             BIN_IMPL.find(op, opc, $abstractvectortype$::binaryOperations));
 880     }
 881 
 882     /**
 883      * {@inheritDoc} <!--workaround-->
 884      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 885      */
 886     @Override
 887     public abstract
 888     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 889                                   Vector<$Boxtype$> v,
 890                                   VectorMask<$Boxtype$> m);
 891     @ForceInline
 892     final
 893     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 894                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
 895                                           Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 896         $abstractvectortype$ that = ($abstractvectortype$) v;
 897         that.check(this);
 898         m.check(maskClass, this);
 899 
 900         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 901             if (op == FIRST_NONZERO) {
 902 #if[FP]
 903                 return blend(lanewise(op, v), m);
 904 #else[FP]
 905                 // FIXME: Support this in the JIT.
 906                 VectorMask<$Boxbitstype$> thisNZ
 907                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 908                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 909                 op = OR_UNCHECKED;
 910 #end[FP]
 911             }
 912 #if[BITWISE]
 913 #if[!FP]
 914             if (opKind(op, VO_SHIFT)) {
 915                 // As per shift specification for Java, mask the shift count.
 916                 // This allows the JIT to ignore some ISA details.
 917                 that = that.lanewise(AND, SHIFT_MASK);
 918             }
 919 #end[!FP]
 920             if (op == AND_NOT) {
 921                 // FIXME: Support this in the JIT.
 922                 that = that.lanewise(NOT);
 923                 op = AND;
 924             } else if (op == DIV) {
 925                 VectorMask<$Boxtype$> eqz = that.eq(($type$)0);
 926                 if (eqz.and(m).anyTrue()) {
 927                     throw that.divZeroException();
 928                 }
 929                 // suppress div/0 exceptions in unset lanes
 930                 that = that.lanewise(NOT, eqz);
 931             }
 932 #end[BITWISE]
 933         }
 934 
 935         int opc = opCode(op);
 936         return VectorSupport.binaryOp(
 937             opc, getClass(), maskClass, $type$.class, length(),
 938             this, that, m,
 939             BIN_IMPL.find(op, opc, $abstractvectortype$::binaryOperations));
 940     }
 941 
 942     private static final
 943     ImplCache<Binary, BinaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 944         BIN_IMPL = new ImplCache<>(Binary.class, $Type$Vector.class);
 945 
 946     private static BinaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> binaryOperations(int opc_) {
 947         switch (opc_) {
 948             case VECTOR_OP_ADD: return (v0, v1, vm) ->
 949                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a + b));
 950             case VECTOR_OP_SUB: return (v0, v1, vm) ->
 951                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a - b));
 952             case VECTOR_OP_MUL: return (v0, v1, vm) ->
 953                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a * b));
 954             case VECTOR_OP_DIV: return (v0, v1, vm) ->
 955                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a / b));
 956             case VECTOR_OP_MAX: return (v0, v1, vm) ->
 957                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.max(a, b));
 958             case VECTOR_OP_MIN: return (v0, v1, vm) ->
 959                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.min(a, b));
 960 #if[BITWISE]
 961             case VECTOR_OP_AND: return (v0, v1, vm) ->
 962                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a & b));
 963             case VECTOR_OP_OR: return (v0, v1, vm) ->
 964                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a | b));
 965             case VECTOR_OP_XOR: return (v0, v1, vm) ->
 966                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a ^ b));
 967             case VECTOR_OP_LSHIFT: return (v0, v1, vm) ->
 968                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a << n));
 969             case VECTOR_OP_RSHIFT: return (v0, v1, vm) ->
 970                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a >> n));
 971             case VECTOR_OP_URSHIFT: return (v0, v1, vm) ->
 972                     v0.bOp(v1, vm, (i, a, n) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 973             case VECTOR_OP_LROTATE: return (v0, v1, vm) ->
 974                     v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
 975             case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
 976                     v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
 977 #if[intOrLong]
 978             case VECTOR_OP_COMPRESS_BITS: return (v0, v1, vm) ->
 979                     v0.bOp(v1, vm, (i, a, n) -> $Boxtype$.compress(a, n));
 980             case VECTOR_OP_EXPAND_BITS: return (v0, v1, vm) ->
 981                     v0.bOp(v1, vm, (i, a, n) -> $Boxtype$.expand(a, n));
 982 #end[intOrLong]
 983 #end[BITWISE]
 984 #if[FP]
 985             case VECTOR_OP_OR: return (v0, v1, vm) ->
 986                     v0.bOp(v1, vm, (i, a, b) -> fromBits(toBits(a) | toBits(b)));
 987             case VECTOR_OP_ATAN2: return (v0, v1, vm) ->
 988                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.atan2(a, b));
 989             case VECTOR_OP_POW: return (v0, v1, vm) ->
 990                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.pow(a, b));
 991             case VECTOR_OP_HYPOT: return (v0, v1, vm) ->
 992                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.hypot(a, b));
 993 #end[FP]
 994             default: return null;
 995         }
 996     }
 997 
 998     // FIXME: Maybe all of the public final methods in this file (the
 999     // simple ones that just call lanewise) should be pushed down to
1000     // the X-VectorBits template.  They can't optimize properly at
1001     // this level, and must rely on inlining.  Does it work?
1002     // (If it works, of course keep the code here.)
1003 
1004     /**
1005      * Combines the lane values of this vector
1006      * with the value of a broadcast scalar.
1007      *
1008      * This is a lane-wise binary operation which applies
1009      * the selected operation to each lane.
1010      * The return value will be equal to this expression:
1011      * {@code this.lanewise(op, this.broadcast(e))}.
1012      *
1013      * @param op the operation used to process lane values
1014      * @param e the input scalar
1015      * @return the result of applying the operation lane-wise
1016      *         to the two input vectors
1017      * @throws UnsupportedOperationException if this vector does
1018      *         not support the requested operation
1019      * @see #lanewise(VectorOperators.Binary,Vector)
1020      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1021      */
1022     @ForceInline
1023     public final
1024     $abstractvectortype$ lanewise(VectorOperators.Binary op,
1025                                   $type$ e) {
1026 #if[BITWISE]
1027         if (opKind(op, VO_SHIFT) && ($type$)(int)e == e) {
1028             return lanewiseShift(op, (int) e);
1029         }
1030         if (op == AND_NOT) {
1031             op = AND; e = ($type$) ~e;
1032         }
1033 #end[BITWISE]
1034         return lanewise(op, broadcast(e));
1035     }
1036 
1037     /**
1038      * Combines the lane values of this vector
1039      * with the value of a broadcast scalar,
1040      * with selection of lane elements controlled by a mask.
1041      *
1042      * This is a masked lane-wise binary operation which applies
1043      * the selected operation to each lane.
1044      * The return value will be equal to this expression:
1045      * {@code this.lanewise(op, this.broadcast(e), m)}.
1046      *
1047      * @param op the operation used to process lane values
1048      * @param e the input scalar
1049      * @param m the mask controlling lane selection
1050      * @return the result of applying the operation lane-wise
1051      *         to the input vector and the scalar
1052      * @throws UnsupportedOperationException if this vector does
1053      *         not support the requested operation
1054      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1055      * @see #lanewise(VectorOperators.Binary,$type$)
1056      */
1057     @ForceInline
1058     public final
1059     $abstractvectortype$ lanewise(VectorOperators.Binary op,
1060                                   $type$ e,
1061                                   VectorMask<$Boxtype$> m) {
1062 #if[BITWISE]
1063         if (opKind(op, VO_SHIFT) && ($type$)(int)e == e) {
1064             return lanewiseShift(op, (int) e, m);
1065         }
1066         if (op == AND_NOT) {
1067             op = AND; e = ($type$) ~e;
1068         }
1069 #end[BITWISE]
1070         return lanewise(op, broadcast(e), m);
1071     }
1072 
1073 #if[!long]
1074     /**
1075      * {@inheritDoc} <!--workaround-->
1076      * @apiNote
1077      * When working with vector subtypes like {@code $abstractvectortype$},
1078      * {@linkplain #lanewise(VectorOperators.Binary,$type$)
1079      * the more strongly typed method}
1080      * is typically selected.  It can be explicitly selected
1081      * using a cast: {@code v.lanewise(op,($type$)e)}.
1082      * The two expressions will produce numerically identical results.
1083      */
1084     @ForceInline
1085     public final
1086     $abstractvectortype$ lanewise(VectorOperators.Binary op,
1087                                   long e) {
1088         $type$ e1 = ($type$) e;
1089 #if[BITWISE]
1090         if ((long)e1 != e
1091             // allow shift ops to clip down their int parameters
1092             && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
1093 #else[BITWISE]
1094         if ((long)e1 != e) {
1095 #end[BITWISE]
1096             vspecies().checkValue(e);  // for exception
1097         }
1098         return lanewise(op, e1);
1099     }
1100 
1101     /**
1102      * {@inheritDoc} <!--workaround-->
1103      * @apiNote
1104      * When working with vector subtypes like {@code $abstractvectortype$},
1105      * {@linkplain #lanewise(VectorOperators.Binary,$type$,VectorMask)
1106      * the more strongly typed method}
1107      * is typically selected.  It can be explicitly selected
1108      * using a cast: {@code v.lanewise(op,($type$)e,m)}.
1109      * The two expressions will produce numerically identical results.
1110      */
1111     @ForceInline
1112     public final
1113     $abstractvectortype$ lanewise(VectorOperators.Binary op,
1114                                   long e, VectorMask<$Boxtype$> m) {
1115         $type$ e1 = ($type$) e;
1116 #if[BITWISE]
1117         if ((long)e1 != e
1118             // allow shift ops to clip down their int parameters
1119             && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
1120 #else[BITWISE]
1121         if ((long)e1 != e) {
1122 #end[BITWISE]
1123             vspecies().checkValue(e);  // for exception
1124         }
1125         return lanewise(op, e1, m);
1126     }
1127 #end[!long]
1128 
1129 #if[BITWISE]
1130     /*package-private*/
1131     abstract $abstractvectortype$
1132     lanewiseShift(VectorOperators.Binary op, int e);
1133 
1134     /*package-private*/
1135     @ForceInline
1136     final $abstractvectortype$
1137     lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
1138         // Special handling for these.  FIXME: Refactor?
1139         assert(opKind(op, VO_SHIFT));
1140         // As per shift specification for Java, mask the shift count.
1141         e &= SHIFT_MASK;
1142         int opc = opCode(op);
1143         return VectorSupport.broadcastInt(
1144             opc, getClass(), null, $type$.class, length(),
1145             this, e, null,
1146             BIN_INT_IMPL.find(op, opc, $abstractvectortype$::broadcastIntOperations));
1147     }
1148 
1149     /*package-private*/
1150     abstract $abstractvectortype$
1151     lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m);
1152 
1153     /*package-private*/
1154     @ForceInline
1155     final $abstractvectortype$
1156     lanewiseShiftTemplate(VectorOperators.Binary op,
1157                           Class<? extends VectorMask<$Boxtype$>> maskClass,
1158                           int e, VectorMask<$Boxtype$> m) {
1159         m.check(maskClass, this);
1160         assert(opKind(op, VO_SHIFT));
1161         // As per shift specification for Java, mask the shift count.
1162         e &= SHIFT_MASK;
1163         int opc = opCode(op);
1164         return VectorSupport.broadcastInt(
1165             opc, getClass(), maskClass, $type$.class, length(),
1166             this, e, m,
1167             BIN_INT_IMPL.find(op, opc, $abstractvectortype$::broadcastIntOperations));
1168     }
1169 
1170     private static final
1171     ImplCache<Binary,VectorBroadcastIntOp<$abstractvectortype$, VectorMask<$Boxtype$>>> BIN_INT_IMPL
1172         = new ImplCache<>(Binary.class, $Type$Vector.class);
1173 
1174     private static VectorBroadcastIntOp<$abstractvectortype$, VectorMask<$Boxtype$>> broadcastIntOperations(int opc_) {
1175         switch (opc_) {
1176             case VECTOR_OP_LSHIFT: return (v, n, m) ->
1177                     v.uOp(m, (i, a) -> ($type$)(a << n));
1178             case VECTOR_OP_RSHIFT: return (v, n, m) ->
1179                     v.uOp(m, (i, a) -> ($type$)(a >> n));
1180             case VECTOR_OP_URSHIFT: return (v, n, m) ->
1181                     v.uOp(m, (i, a) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
1182             case VECTOR_OP_LROTATE: return (v, n, m) ->
1183                     v.uOp(m, (i, a) -> rotateLeft(a, (int)n));
1184             case VECTOR_OP_RROTATE: return (v, n, m) ->
1185                     v.uOp(m, (i, a) -> rotateRight(a, (int)n));
1186             default: return null;
1187         }
1188     }
1189 
1190     // As per shift specification for Java, mask the shift count.
1191     // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
1192     // The latter two maskings go beyond the JLS, but seem reasonable
1193     // since our lane types are first-class types, not just dressed
1194     // up ints.
1195     private static final int SHIFT_MASK = ($Boxtype$.SIZE - 1);
1196 #if[byteOrShort]
1197     // Also simulate >>> on sub-word variables with a mask.
1198     private static final int LSHR_SETUP_MASK = ((1 << $Boxtype$.SIZE) - 1);
1199 #else[byteOrShort]
1200     private static final $type$ LSHR_SETUP_MASK = -1;
1201 #end[byteOrShort]
1202 #end[BITWISE]
1203 
1204     // Ternary lanewise support
1205 
1206     // Ternary operators come in eight variations:
1207     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
1208     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
1209 
1210     // It is annoying to support all of these variations of masking
1211     // and broadcast, but it would be more surprising not to continue
1212     // the obvious pattern started by unary and binary.
1213 
1214    /**
1215      * {@inheritDoc} <!--workaround-->
1216      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1217      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1218      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1219      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1220      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
1221      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
1222      */
1223     @Override
1224     public abstract
1225     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
1226                                                   Vector<$Boxtype$> v1,
1227                                                   Vector<$Boxtype$> v2);
1228     @ForceInline
1229     final
1230     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
1231                                           Vector<$Boxtype$> v1,
1232                                           Vector<$Boxtype$> v2) {
1233         $abstractvectortype$ that = ($abstractvectortype$) v1;
1234         $abstractvectortype$ tother = ($abstractvectortype$) v2;
1235         // It's a word: https://www.dictionary.com/browse/tother
1236         // See also Chapter 11 of Dickens, Our Mutual Friend:
1237         // "Totherest Governor," replied Mr Riderhood...
1238         that.check(this);
1239         tother.check(this);
1240 #if[BITWISE]
1241         if (op == BITWISE_BLEND) {
1242             // FIXME: Support this in the JIT.
1243             that = this.lanewise(XOR, that).lanewise(AND, tother);
1244             return this.lanewise(XOR, that);
1245         }
1246 #end[BITWISE]
1247         int opc = opCode(op);
1248         return VectorSupport.ternaryOp(
1249             opc, getClass(), null, $type$.class, length(),
1250             this, that, tother, null,
1251             TERN_IMPL.find(op, opc, $abstractvectortype$::ternaryOperations));
1252     }
1253 
1254     /**
1255      * {@inheritDoc} <!--workaround-->
1256      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1257      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1258      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1259      */
1260     @Override
1261     public abstract
1262     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
1263                                   Vector<$Boxtype$> v1,
1264                                   Vector<$Boxtype$> v2,
1265                                   VectorMask<$Boxtype$> m);
1266     @ForceInline
1267     final
1268     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
1269                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
1270                                           Vector<$Boxtype$> v1,
1271                                           Vector<$Boxtype$> v2,
1272                                           VectorMask<$Boxtype$> m) {
1273         $abstractvectortype$ that = ($abstractvectortype$) v1;
1274         $abstractvectortype$ tother = ($abstractvectortype$) v2;
1275         // It's a word: https://www.dictionary.com/browse/tother
1276         // See also Chapter 11 of Dickens, Our Mutual Friend:
1277         // "Totherest Governor," replied Mr Riderhood...
1278         that.check(this);
1279         tother.check(this);
1280         m.check(maskClass, this);
1281 
1282 #if[BITWISE]
1283         if (op == BITWISE_BLEND) {
1284             // FIXME: Support this in the JIT.
1285             that = this.lanewise(XOR, that).lanewise(AND, tother);
1286             return this.lanewise(XOR, that, m);
1287         }
1288 #end[BITWISE]
1289         int opc = opCode(op);
1290         return VectorSupport.ternaryOp(
1291             opc, getClass(), maskClass, $type$.class, length(),
1292             this, that, tother, m,
1293             TERN_IMPL.find(op, opc, $abstractvectortype$::ternaryOperations));
1294     }
1295 
1296     private static final
1297     ImplCache<Ternary, TernaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
1298         TERN_IMPL = new ImplCache<>(Ternary.class, $Type$Vector.class);
1299 
1300     private static TernaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> ternaryOperations(int opc_) {
1301         switch (opc_) {
1302 #if[FP]
1303             case VECTOR_OP_FMA: return (v0, v1_, v2_, m) ->
1304                     v0.tOp(v1_, v2_, m, (i, a, b, c) -> Math.fma(a, b, c));
1305 #end[FP]
1306             default: return null;
1307         }
1308     }
1309 
1310     /**
1311      * Combines the lane values of this vector
1312      * with the values of two broadcast scalars.
1313      *
1314      * This is a lane-wise ternary operation which applies
1315      * the selected operation to each lane.
1316      * The return value will be equal to this expression:
1317      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
1318      *
1319      * @param op the operation used to combine lane values
1320      * @param e1 the first input scalar
1321      * @param e2 the second input scalar
1322      * @return the result of applying the operation lane-wise
1323      *         to the input vector and the scalars
1324      * @throws UnsupportedOperationException if this vector does
1325      *         not support the requested operation
1326      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1327      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1328      */
1329     @ForceInline
1330     public final
1331     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2)
1332                                   $type$ e1,
1333                                   $type$ e2) {
1334         return lanewise(op, broadcast(e1), broadcast(e2));
1335     }
1336 
1337     /**
1338      * Combines the lane values of this vector
1339      * with the values of two broadcast scalars,
1340      * with selection of lane elements controlled by a mask.
1341      *
1342      * This is a masked lane-wise ternary operation which applies
1343      * the selected operation to each lane.
1344      * The return value will be equal to this expression:
1345      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
1346      *
1347      * @param op the operation used to combine lane values
1348      * @param e1 the first input scalar
1349      * @param e2 the second input scalar
1350      * @param m the mask controlling lane selection
1351      * @return the result of applying the operation lane-wise
1352      *         to the input vector and the scalars
1353      * @throws UnsupportedOperationException if this vector does
1354      *         not support the requested operation
1355      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1356      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1357      */
1358     @ForceInline
1359     public final
1360     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
1361                                   $type$ e1,
1362                                   $type$ e2,
1363                                   VectorMask<$Boxtype$> m) {
1364         return lanewise(op, broadcast(e1), broadcast(e2), m);
1365     }
1366 
1367     /**
1368      * Combines the lane values of this vector
1369      * with the values of another vector and a broadcast scalar.
1370      *
1371      * This is a lane-wise ternary operation which applies
1372      * the selected operation to each lane.
1373      * The return value will be equal to this expression:
1374      * {@code this.lanewise(op, v1, this.broadcast(e2))}.
1375      *
1376      * @param op the operation used to combine lane values
1377      * @param v1 the other input vector
1378      * @param e2 the input scalar
1379      * @return the result of applying the operation lane-wise
1380      *         to the input vectors and the scalar
1381      * @throws UnsupportedOperationException if this vector does
1382      *         not support the requested operation
1383      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1384      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1385      */
1386     @ForceInline
1387     public final
1388     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2)
1389                                   Vector<$Boxtype$> v1,
1390                                   $type$ e2) {
1391         return lanewise(op, v1, broadcast(e2));
1392     }
1393 
1394     /**
1395      * Combines the lane values of this vector
1396      * with the values of another vector and a broadcast scalar,
1397      * with selection of lane elements controlled by a mask.
1398      *
1399      * This is a masked lane-wise ternary operation which applies
1400      * the selected operation to each lane.
1401      * The return value will be equal to this expression:
1402      * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
1403      *
1404      * @param op the operation used to combine lane values
1405      * @param v1 the other input vector
1406      * @param e2 the input scalar
1407      * @param m the mask controlling lane selection
1408      * @return the result of applying the operation lane-wise
1409      *         to the input vectors and the scalar
1410      * @throws UnsupportedOperationException if this vector does
1411      *         not support the requested operation
1412      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1413      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1414      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
1415      */
1416     @ForceInline
1417     public final
1418     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
1419                                   Vector<$Boxtype$> v1,
1420                                   $type$ e2,
1421                                   VectorMask<$Boxtype$> m) {
1422         return lanewise(op, v1, broadcast(e2), m);
1423     }
1424 
1425     /**
1426      * Combines the lane values of this vector
1427      * with the values of another vector and a broadcast scalar.
1428      *
1429      * This is a lane-wise ternary operation which applies
1430      * the selected operation to each lane.
1431      * The return value will be equal to this expression:
1432      * {@code this.lanewise(op, this.broadcast(e1), v2)}.
1433      *
1434      * @param op the operation used to combine lane values
1435      * @param e1 the input scalar
1436      * @param v2 the other input vector
1437      * @return the result of applying the operation lane-wise
1438      *         to the input vectors and the scalar
1439      * @throws UnsupportedOperationException if this vector does
1440      *         not support the requested operation
1441      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1442      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1443      */
1444     @ForceInline
1445     public final
1446     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2)
1447                                   $type$ e1,
1448                                   Vector<$Boxtype$> v2) {
1449         return lanewise(op, broadcast(e1), v2);
1450     }
1451 
1452     /**
1453      * Combines the lane values of this vector
1454      * with the values of another vector and a broadcast scalar,
1455      * with selection of lane elements controlled by a mask.
1456      *
1457      * This is a masked lane-wise ternary operation which applies
1458      * the selected operation to each lane.
1459      * The return value will be equal to this expression:
1460      * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
1461      *
1462      * @param op the operation used to combine lane values
1463      * @param e1 the input scalar
1464      * @param v2 the other input vector
1465      * @param m the mask controlling lane selection
1466      * @return the result of applying the operation lane-wise
1467      *         to the input vectors and the scalar
1468      * @throws UnsupportedOperationException if this vector does
1469      *         not support the requested operation
1470      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1471      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
1472      */
1473     @ForceInline
1474     public final
1475     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
1476                                   $type$ e1,
1477                                   Vector<$Boxtype$> v2,
1478                                   VectorMask<$Boxtype$> m) {
1479         return lanewise(op, broadcast(e1), v2, m);
1480     }
1481 
1482     // (Thus endeth the Great and Mighty Ternary Ogdoad.)
1483     // https://en.wikipedia.org/wiki/Ogdoad
1484 
1485     /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1486     //
1487     // These include masked and non-masked versions.
1488     // This subclass adds broadcast (masked or not).
1489 
1490     /**
1491      * {@inheritDoc} <!--workaround-->
1492      * @see #add($type$)
1493      */
1494     @Override
1495     @ForceInline
1496     public final $abstractvectortype$ add(Vector<$Boxtype$> v) {
1497         return lanewise(ADD, v);
1498     }
1499 
1500     /**
1501      * Adds this vector to the broadcast of an input scalar.
1502      *
1503      * This is a lane-wise binary operation which applies
1504      * the primitive addition operation ({@code +}) to each lane.
1505      *
1506      * This method is also equivalent to the expression
1507      * {@link #lanewise(VectorOperators.Binary,$type$)
1508      *    lanewise}{@code (}{@link VectorOperators#ADD
1509      *    ADD}{@code , e)}.
1510      *
1511      * @param e the input scalar
1512      * @return the result of adding each lane of this vector to the scalar
1513      * @see #add(Vector)
1514      * @see #broadcast($type$)
1515      * @see #add($type$,VectorMask)
1516      * @see VectorOperators#ADD
1517      * @see #lanewise(VectorOperators.Binary,Vector)
1518      * @see #lanewise(VectorOperators.Binary,$type$)
1519      */
1520     @ForceInline
1521     public final
1522     $abstractvectortype$ add($type$ e) {
1523         return lanewise(ADD, e);
1524     }
1525 
1526     /**
1527      * {@inheritDoc} <!--workaround-->
1528      * @see #add($type$,VectorMask)
1529      */
1530     @Override
1531     @ForceInline
1532     public final $abstractvectortype$ add(Vector<$Boxtype$> v,
1533                                           VectorMask<$Boxtype$> m) {
1534         return lanewise(ADD, v, m);
1535     }
1536 
1537     /**
1538      * Adds this vector to the broadcast of an input scalar,
1539      * selecting lane elements controlled by a mask.
1540      *
1541      * This is a masked lane-wise binary operation which applies
1542      * the primitive addition operation ({@code +}) to each lane.
1543      *
1544      * This method is also equivalent to the expression
1545      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1546      *    lanewise}{@code (}{@link VectorOperators#ADD
1547      *    ADD}{@code , s, m)}.
1548      *
1549      * @param e the input scalar
1550      * @param m the mask controlling lane selection
1551      * @return the result of adding each lane of this vector to the scalar
1552      * @see #add(Vector,VectorMask)
1553      * @see #broadcast($type$)
1554      * @see #add($type$)
1555      * @see VectorOperators#ADD
1556      * @see #lanewise(VectorOperators.Binary,Vector)
1557      * @see #lanewise(VectorOperators.Binary,$type$)
1558      */
1559     @ForceInline
1560     public final $abstractvectortype$ add($type$ e,
1561                                           VectorMask<$Boxtype$> m) {
1562         return lanewise(ADD, e, m);
1563     }
1564 
1565     /**
1566      * {@inheritDoc} <!--workaround-->
1567      * @see #sub($type$)
1568      */
1569     @Override
1570     @ForceInline
1571     public final $abstractvectortype$ sub(Vector<$Boxtype$> v) {
1572         return lanewise(SUB, v);
1573     }
1574 
1575     /**
1576      * Subtracts an input scalar from this vector.
1577      *
1578      * This is a masked lane-wise binary operation which applies
1579      * the primitive subtraction operation ({@code -}) to each lane.
1580      *
1581      * This method is also equivalent to the expression
1582      * {@link #lanewise(VectorOperators.Binary,$type$)
1583      *    lanewise}{@code (}{@link VectorOperators#SUB
1584      *    SUB}{@code , e)}.
1585      *
1586      * @param e the input scalar
1587      * @return the result of subtracting the scalar from each lane of this vector
1588      * @see #sub(Vector)
1589      * @see #broadcast($type$)
1590      * @see #sub($type$,VectorMask)
1591      * @see VectorOperators#SUB
1592      * @see #lanewise(VectorOperators.Binary,Vector)
1593      * @see #lanewise(VectorOperators.Binary,$type$)
1594      */
1595     @ForceInline
1596     public final $abstractvectortype$ sub($type$ e) {
1597         return lanewise(SUB, e);
1598     }
1599 
1600     /**
1601      * {@inheritDoc} <!--workaround-->
1602      * @see #sub($type$,VectorMask)
1603      */
1604     @Override
1605     @ForceInline
1606     public final $abstractvectortype$ sub(Vector<$Boxtype$> v,
1607                                           VectorMask<$Boxtype$> m) {
1608         return lanewise(SUB, v, m);
1609     }
1610 
1611     /**
1612      * Subtracts an input scalar from this vector
1613      * under the control of a mask.
1614      *
1615      * This is a masked lane-wise binary operation which applies
1616      * the primitive subtraction operation ({@code -}) to each lane.
1617      *
1618      * This method is also equivalent to the expression
1619      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1620      *    lanewise}{@code (}{@link VectorOperators#SUB
1621      *    SUB}{@code , s, m)}.
1622      *
1623      * @param e the input scalar
1624      * @param m the mask controlling lane selection
1625      * @return the result of subtracting the scalar from each lane of this vector
1626      * @see #sub(Vector,VectorMask)
1627      * @see #broadcast($type$)
1628      * @see #sub($type$)
1629      * @see VectorOperators#SUB
1630      * @see #lanewise(VectorOperators.Binary,Vector)
1631      * @see #lanewise(VectorOperators.Binary,$type$)
1632      */
1633     @ForceInline
1634     public final $abstractvectortype$ sub($type$ e,
1635                                           VectorMask<$Boxtype$> m) {
1636         return lanewise(SUB, e, m);
1637     }
1638 
1639     /**
1640      * {@inheritDoc} <!--workaround-->
1641      * @see #mul($type$)
1642      */
1643     @Override
1644     @ForceInline
1645     public final $abstractvectortype$ mul(Vector<$Boxtype$> v) {
1646         return lanewise(MUL, v);
1647     }
1648 
1649     /**
1650      * Multiplies this vector by the broadcast of an input scalar.
1651      *
1652      * This is a lane-wise binary operation which applies
1653      * the primitive multiplication operation ({@code *}) to each lane.
1654      *
1655      * This method is also equivalent to the expression
1656      * {@link #lanewise(VectorOperators.Binary,$type$)
1657      *    lanewise}{@code (}{@link VectorOperators#MUL
1658      *    MUL}{@code , e)}.
1659      *
1660      * @param e the input scalar
1661      * @return the result of multiplying this vector by the given scalar
1662      * @see #mul(Vector)
1663      * @see #broadcast($type$)
1664      * @see #mul($type$,VectorMask)
1665      * @see VectorOperators#MUL
1666      * @see #lanewise(VectorOperators.Binary,Vector)
1667      * @see #lanewise(VectorOperators.Binary,$type$)
1668      */
1669     @ForceInline
1670     public final $abstractvectortype$ mul($type$ e) {
1671         return lanewise(MUL, e);
1672     }
1673 
1674     /**
1675      * {@inheritDoc} <!--workaround-->
1676      * @see #mul($type$,VectorMask)
1677      */
1678     @Override
1679     @ForceInline
1680     public final $abstractvectortype$ mul(Vector<$Boxtype$> v,
1681                                           VectorMask<$Boxtype$> m) {
1682         return lanewise(MUL, v, m);
1683     }
1684 
1685     /**
1686      * Multiplies this vector by the broadcast of an input scalar,
1687      * selecting lane elements controlled by a mask.
1688      *
1689      * This is a masked lane-wise binary operation which applies
1690      * the primitive multiplication operation ({@code *}) to each lane.
1691      *
1692      * This method is also equivalent to the expression
1693      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1694      *    lanewise}{@code (}{@link VectorOperators#MUL
1695      *    MUL}{@code , s, m)}.
1696      *
1697      * @param e the input scalar
1698      * @param m the mask controlling lane selection
1699      * @return the result of muling each lane of this vector to the scalar
1700      * @see #mul(Vector,VectorMask)
1701      * @see #broadcast($type$)
1702      * @see #mul($type$)
1703      * @see VectorOperators#MUL
1704      * @see #lanewise(VectorOperators.Binary,Vector)
1705      * @see #lanewise(VectorOperators.Binary,$type$)
1706      */
1707     @ForceInline
1708     public final $abstractvectortype$ mul($type$ e,
1709                                           VectorMask<$Boxtype$> m) {
1710         return lanewise(MUL, e, m);
1711     }
1712 
1713     /**
1714      * {@inheritDoc} <!--workaround-->
1715 #if[FP]
1716      * @apiNote Because the underlying scalar operator is an IEEE
1717      * floating point number, division by zero in fact will
1718      * not throw an exception, but will yield a signed
1719      * infinity or NaN.
1720 #else[FP]
1721      * @apiNote If there is a zero divisor, {@code
1722      * ArithmeticException} will be thrown.
1723 #end[FP]
1724      */
1725     @Override
1726     @ForceInline
1727     public final $abstractvectortype$ div(Vector<$Boxtype$> v) {
1728         return lanewise(DIV, v);
1729     }
1730 
1731     /**
1732      * Divides this vector by the broadcast of an input scalar.
1733      *
1734      * This is a lane-wise binary operation which applies
1735      * the primitive division operation ({@code /}) to each lane.
1736      *
1737      * This method is also equivalent to the expression
1738      * {@link #lanewise(VectorOperators.Binary,$type$)
1739      *    lanewise}{@code (}{@link VectorOperators#DIV
1740      *    DIV}{@code , e)}.
1741      *
1742 #if[FP]
1743      * @apiNote Because the underlying scalar operator is an IEEE
1744      * floating point number, division by zero in fact will
1745      * not throw an exception, but will yield a signed
1746      * infinity or NaN.
1747 #else[FP]
1748      * @apiNote If there is a zero divisor, {@code
1749      * ArithmeticException} will be thrown.
1750 #end[FP]
1751      *
1752      * @param e the input scalar
1753      * @return the result of dividing each lane of this vector by the scalar
1754      * @see #div(Vector)
1755      * @see #broadcast($type$)
1756      * @see #div($type$,VectorMask)
1757      * @see VectorOperators#DIV
1758      * @see #lanewise(VectorOperators.Binary,Vector)
1759      * @see #lanewise(VectorOperators.Binary,$type$)
1760      */
1761     @ForceInline
1762     public final $abstractvectortype$ div($type$ e) {
1763         return lanewise(DIV, e);
1764     }
1765 
1766     /**
1767      * {@inheritDoc} <!--workaround-->
1768      * @see #div($type$,VectorMask)
1769 #if[FP]
1770      * @apiNote Because the underlying scalar operator is an IEEE
1771      * floating point number, division by zero in fact will
1772      * not throw an exception, but will yield a signed
1773      * infinity or NaN.
1774 #else[FP]
1775      * @apiNote If there is a zero divisor, {@code
1776      * ArithmeticException} will be thrown.
1777 #end[FP]
1778      */
1779     @Override
1780     @ForceInline
1781     public final $abstractvectortype$ div(Vector<$Boxtype$> v,
1782                                           VectorMask<$Boxtype$> m) {
1783         return lanewise(DIV, v, m);
1784     }
1785 
1786     /**
1787      * Divides this vector by the broadcast of an input scalar,
1788      * selecting lane elements controlled by a mask.
1789      *
1790      * This is a masked lane-wise binary operation which applies
1791      * the primitive division operation ({@code /}) to each lane.
1792      *
1793      * This method is also equivalent to the expression
1794      * {@link #lanewise(VectorOperators.Binary,$type$,VectorMask)
1795      *    lanewise}{@code (}{@link VectorOperators#DIV
1796      *    DIV}{@code , s, m)}.
1797      *
1798 #if[FP]
1799      * @apiNote Because the underlying scalar operator is an IEEE
1800      * floating point number, division by zero in fact will
1801      * not throw an exception, but will yield a signed
1802      * infinity or NaN.
1803 #else[FP]
1804      * @apiNote If there is a zero divisor, {@code
1805      * ArithmeticException} will be thrown.
1806 #end[FP]
1807      *
1808      * @param e the input scalar
1809      * @param m the mask controlling lane selection
1810      * @return the result of dividing each lane of this vector by the scalar
1811      * @see #div(Vector,VectorMask)
1812      * @see #broadcast($type$)
1813      * @see #div($type$)
1814      * @see VectorOperators#DIV
1815      * @see #lanewise(VectorOperators.Binary,Vector)
1816      * @see #lanewise(VectorOperators.Binary,$type$)
1817      */
1818     @ForceInline
1819     public final $abstractvectortype$ div($type$ e,
1820                                           VectorMask<$Boxtype$> m) {
1821         return lanewise(DIV, e, m);
1822     }
1823 
1824     /// END OF FULL-SERVICE BINARY METHODS
1825 
1826     /// SECOND-TIER BINARY METHODS
1827     //
1828     // There are no masked versions.
1829 
1830     /**
1831      * {@inheritDoc} <!--workaround-->
1832 #if[FP]
1833      * @apiNote
1834      * For this method, floating point negative
1835      * zero {@code -0.0} is treated as a value distinct from, and less
1836      * than the default value (positive zero).
1837 #end[FP]
1838      */
1839     @Override
1840     @ForceInline
1841     public final $abstractvectortype$ min(Vector<$Boxtype$> v) {
1842         return lanewise(MIN, v);
1843     }
1844 
1845     // FIXME:  "broadcast of an input scalar" is really wordy.  Reduce?
1846     /**
1847      * Computes the smaller of this vector and the broadcast of an input scalar.
1848      *
1849      * This is a lane-wise binary operation which applies the
1850      * operation {@code Math.min()} to each pair of
1851      * corresponding lane values.
1852      *
1853      * This method is also equivalent to the expression
1854      * {@link #lanewise(VectorOperators.Binary,$type$)
1855      *    lanewise}{@code (}{@link VectorOperators#MIN
1856      *    MIN}{@code , e)}.
1857      *
1858      * @param e the input scalar
1859      * @return the result of multiplying this vector by the given scalar
1860      * @see #min(Vector)
1861      * @see #broadcast($type$)
1862      * @see VectorOperators#MIN
1863      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1864 #if[FP]
1865      * @apiNote
1866      * For this method, floating point negative
1867      * zero {@code -0.0} is treated as a value distinct from, and less
1868      * than the default value (positive zero).
1869 #end[FP]
1870      */
1871     @ForceInline
1872     public final $abstractvectortype$ min($type$ e) {
1873         return lanewise(MIN, e);
1874     }
1875 
1876     /**
1877      * {@inheritDoc} <!--workaround-->
1878 #if[FP]
1879      * @apiNote
1880      * For this method, floating point negative
1881      * zero {@code -0.0} is treated as a value distinct from, and less
1882      * than the default value (positive zero).
1883 #end[FP]
1884      */
1885     @Override
1886     @ForceInline
1887     public final $abstractvectortype$ max(Vector<$Boxtype$> v) {
1888         return lanewise(MAX, v);
1889     }
1890 
1891     /**
1892      * Computes the larger of this vector and the broadcast of an input scalar.
1893      *
1894      * This is a lane-wise binary operation which applies the
1895      * operation {@code Math.max()} to each pair of
1896      * corresponding lane values.
1897      *
1898      * This method is also equivalent to the expression
1899      * {@link #lanewise(VectorOperators.Binary,$type$)
1900      *    lanewise}{@code (}{@link VectorOperators#MAX
1901      *    MAX}{@code , e)}.
1902      *
1903      * @param e the input scalar
1904      * @return the result of multiplying this vector by the given scalar
1905      * @see #max(Vector)
1906      * @see #broadcast($type$)
1907      * @see VectorOperators#MAX
1908      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
1909 #if[FP]
1910      * @apiNote
1911      * For this method, floating point negative
1912      * zero {@code -0.0} is treated as a value distinct from, and less
1913      * than the default value (positive zero).
1914 #end[FP]
1915      */
1916     @ForceInline
1917     public final $abstractvectortype$ max($type$ e) {
1918         return lanewise(MAX, e);
1919     }
1920 
1921 #if[BITWISE]
1922     // common bitwise operators: and, or, not (with scalar versions)
1923     /**
1924      * Computes the bitwise logical conjunction ({@code &})
1925      * of this vector and a second input vector.
1926      *
1927      * This is a lane-wise binary operation which applies the
1928      * the primitive bitwise "and" operation ({@code &})
1929      * to each pair of corresponding lane values.
1930      *
1931      * This method is also equivalent to the expression
1932      * {@link #lanewise(VectorOperators.Binary,Vector)
1933      *    lanewise}{@code (}{@link VectorOperators#AND
1934      *    AND}{@code , v)}.
1935      *
1936      * <p>
1937      * This is not a full-service named operation like
1938      * {@link #add(Vector) add}.  A masked version of
1939      * this operation is not directly available
1940      * but may be obtained via the masked version of
1941      * {@code lanewise}.
1942      *
1943      * @param v a second input vector
1944      * @return the bitwise {@code &} of this vector and the second input vector
1945      * @see #and($type$)
1946      * @see #or(Vector)
1947      * @see #not()
1948      * @see VectorOperators#AND
1949      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1950      */
1951     @ForceInline
1952     public final $abstractvectortype$ and(Vector<$Boxtype$> v) {
1953         return lanewise(AND, v);
1954     }
1955 
1956     /**
1957      * Computes the bitwise logical conjunction ({@code &})
1958      * of this vector and a scalar.
1959      *
1960      * This is a lane-wise binary operation which applies the
1961      * the primitive bitwise "and" operation ({@code &})
1962      * to each pair of corresponding lane values.
1963      *
1964      * This method is also equivalent to the expression
1965      * {@link #lanewise(VectorOperators.Binary,Vector)
1966      *    lanewise}{@code (}{@link VectorOperators#AND
1967      *    AND}{@code , e)}.
1968      *
1969      * @param e an input scalar
1970      * @return the bitwise {@code &} of this vector and scalar
1971      * @see #and(Vector)
1972      * @see VectorOperators#AND
1973      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
1974      */
1975     @ForceInline
1976     public final $abstractvectortype$ and($type$ e) {
1977         return lanewise(AND, e);
1978     }
1979 
1980     /**
1981      * Computes the bitwise logical disjunction ({@code |})
1982      * of this vector and a second input vector.
1983      *
1984      * This is a lane-wise binary operation which applies the
1985      * the primitive bitwise "or" operation ({@code |})
1986      * to each pair of corresponding lane values.
1987      *
1988      * This method is also equivalent to the expression
1989      * {@link #lanewise(VectorOperators.Binary,Vector)
1990      *    lanewise}{@code (}{@link VectorOperators#OR
1991      *    AND}{@code , v)}.
1992      *
1993      * <p>
1994      * This is not a full-service named operation like
1995      * {@link #add(Vector) add}.  A masked version of
1996      * this operation is not directly available
1997      * but may be obtained via the masked version of
1998      * {@code lanewise}.
1999      *
2000      * @param v a second input vector
2001      * @return the bitwise {@code |} of this vector and the second input vector
2002      * @see #or($type$)
2003      * @see #and(Vector)
2004      * @see #not()
2005      * @see VectorOperators#OR
2006      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
2007      */
2008     @ForceInline
2009     public final $abstractvectortype$ or(Vector<$Boxtype$> v) {
2010         return lanewise(OR, v);
2011     }
2012 
2013     /**
2014      * Computes the bitwise logical disjunction ({@code |})
2015      * of this vector and a scalar.
2016      *
2017      * This is a lane-wise binary operation which applies the
2018      * the primitive bitwise "or" operation ({@code |})
2019      * to each pair of corresponding lane values.
2020      *
2021      * This method is also equivalent to the expression
2022      * {@link #lanewise(VectorOperators.Binary,Vector)
2023      *    lanewise}{@code (}{@link VectorOperators#OR
2024      *    OR}{@code , e)}.
2025      *
2026      * @param e an input scalar
2027      * @return the bitwise {@code |} of this vector and scalar
2028      * @see #or(Vector)
2029      * @see VectorOperators#OR
2030      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
2031      */
2032     @ForceInline
2033     public final $abstractvectortype$ or($type$ e) {
2034         return lanewise(OR, e);
2035     }
2036 
2037 #end[BITWISE]
2038 
2039 #if[FP]
2040     // common FP operator: pow
2041     /**
2042      * Raises this vector to the power of a second input vector.
2043      *
2044      * This is a lane-wise binary operation which applies an operation
2045      * conforming to the specification of
2046      * {@link Math#pow Math.pow(a,b)}
2047      * to each pair of corresponding lane values.
2048 #if[intOrFloat]
2049      * The operation is adapted to cast the operands and the result,
2050      * specifically widening {@code float} operands to {@code double}
2051      * operands and narrowing the {@code double} result to a {@code float}
2052      * result.
2053 #end[intOrFloat]
2054      *
2055      * This method is also equivalent to the expression
2056      * {@link #lanewise(VectorOperators.Binary,Vector)
2057      *    lanewise}{@code (}{@link VectorOperators#POW
2058      *    POW}{@code , b)}.
2059      *
2060      * <p>
2061      * This is not a full-service named operation like
2062      * {@link #add(Vector) add}.  A masked version of
2063      * this operation is not directly available
2064      * but may be obtained via the masked version of
2065      * {@code lanewise}.
2066      *
2067      * @param b a vector exponent by which to raise this vector
2068      * @return the {@code b}-th power of this vector
2069      * @see #pow($type$)
2070      * @see VectorOperators#POW
2071      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
2072      */
2073     @ForceInline
2074     public final $abstractvectortype$ pow(Vector<$Boxtype$> b) {
2075         return lanewise(POW, b);
2076     }
2077 
2078     /**
2079      * Raises this vector to a scalar power.
2080      *
2081      * This is a lane-wise binary operation which applies an operation
2082      * conforming to the specification of
2083      * {@link Math#pow Math.pow(a,b)}
2084      * to each pair of corresponding lane values.
2085 #if[intOrFloat]
2086      * The operation is adapted to cast the operands and the result,
2087      * specifically widening {@code float} operands to {@code double}
2088      * operands and narrowing the {@code double} result to a {@code float}
2089      * result.
2090 #end[intOrFloat]
2091      *
2092      * This method is also equivalent to the expression
2093      * {@link #lanewise(VectorOperators.Binary,Vector)
2094      *    lanewise}{@code (}{@link VectorOperators#POW
2095      *    POW}{@code , b)}.
2096      *
2097      * @param b a scalar exponent by which to raise this vector
2098      * @return the {@code b}-th power of this vector
2099      * @see #pow(Vector)
2100      * @see VectorOperators#POW
2101      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
2102      */
2103     @ForceInline
2104     public final $abstractvectortype$ pow($type$ b) {
2105         return lanewise(POW, b);
2106     }
2107 #end[FP]
2108 
2109     /// UNARY METHODS
2110 
2111     /**
2112      * {@inheritDoc} <!--workaround-->
2113      */
2114     @Override
2115     @ForceInline
2116     public final
2117     $abstractvectortype$ neg() {
2118         return lanewise(NEG);
2119     }
2120 
2121     /**
2122      * {@inheritDoc} <!--workaround-->
2123      */
2124     @Override
2125     @ForceInline
2126     public final
2127     $abstractvectortype$ abs() {
2128         return lanewise(ABS);
2129     }
2130 
2131 #if[!FP]
2132 #if[!intOrLong]
2133     static int bitCount($type$ a) {
2134 #if[short]
2135         return Integer.bitCount((int)a & 0xFFFF);
2136 #else[short]
2137         return Integer.bitCount((int)a & 0xFF);
2138 #end[short]
2139     }
2140 #end[!intOrLong]
2141 #end[!FP]
2142 #if[!FP]
2143 #if[!intOrLong]
2144     static int numberOfTrailingZeros($type$ a) {
2145 #if[short]
2146         return a != 0 ? Integer.numberOfTrailingZeros(a) : 16;
2147 #else[short]
2148         return a != 0 ? Integer.numberOfTrailingZeros(a) : 8;
2149 #end[short]
2150     }
2151 #end[!intOrLong]
2152 #end[!FP]
2153 #if[!FP]
2154 #if[!intOrLong]
2155     static int numberOfLeadingZeros($type$ a) {
2156 #if[short]
2157         return a >= 0 ? Integer.numberOfLeadingZeros(a) - 16 : 0;
2158 #else[short]
2159         return a >= 0 ? Integer.numberOfLeadingZeros(a) - 24 : 0;
2160 #end[short]
2161     }
2162 
2163     static $type$ reverse($type$ a) {
2164         if (a == 0 || a == -1) return a;
2165 
2166 #if[short]
2167         $type$ b = rotateLeft(a, 8);
2168         b = ($type$) (((b & 0x5555) << 1) | ((b & 0xAAAA) >>> 1));
2169         b = ($type$) (((b & 0x3333) << 2) | ((b & 0xCCCC) >>> 2));
2170         b = ($type$) (((b & 0x0F0F) << 4) | ((b & 0xF0F0) >>> 4));
2171 #else[short]
2172         $type$ b = rotateLeft(a, 4);
2173         b = ($type$) (((b & 0x55) << 1) | ((b & 0xAA) >>> 1));
2174         b = ($type$) (((b & 0x33) << 2) | ((b & 0xCC) >>> 2));
2175 #end[short]
2176         return b;
2177     }
2178 #end[!intOrLong]
2179 #end[!FP]
2180 
2181 #if[BITWISE]
2182     // not (~)
2183     /**
2184      * Computes the bitwise logical complement ({@code ~})
2185      * of this vector.
2186      *
2187      * This is a lane-wise binary operation which applies the
2188      * the primitive bitwise "not" operation ({@code ~})
2189      * to each lane value.
2190      *
2191      * This method is also equivalent to the expression
2192      * {@link #lanewise(VectorOperators.Unary)
2193      *    lanewise}{@code (}{@link VectorOperators#NOT
2194      *    NOT}{@code )}.
2195      *
2196      * <p>
2197      * This is not a full-service named operation like
2198      * {@link #add(Vector) add}.  A masked version of
2199      * this operation is not directly available
2200      * but may be obtained via the masked version of
2201      * {@code lanewise}.
2202      *
2203      * @return the bitwise complement {@code ~} of this vector
2204      * @see #and(Vector)
2205      * @see VectorOperators#NOT
2206      * @see #lanewise(VectorOperators.Unary,VectorMask)
2207      */
2208     @ForceInline
2209     public final $abstractvectortype$ not() {
2210         return lanewise(NOT);
2211     }
2212 #end[BITWISE]
2213 
2214 #if[FP]
2215     // sqrt
2216     /**
2217      * Computes the square root of this vector.
2218      *
2219      * This is a lane-wise unary operation which applies an operation
2220      * conforming to the specification of
2221      * {@link Math#sqrt Math.sqrt(a)}
2222      * to each lane value.
2223 #if[intOrFloat]
2224      * The operation is adapted to cast the operand and the result,
2225      * specifically widening the {@code float} operand to a {@code double}
2226      * operand and narrowing the {@code double} result to a {@code float}
2227      * result.
2228 #end[intOrFloat]
2229      *
2230      * This method is also equivalent to the expression
2231      * {@link #lanewise(VectorOperators.Unary)
2232      *    lanewise}{@code (}{@link VectorOperators#SQRT
2233      *    SQRT}{@code )}.
2234      *
2235      * @return the square root of this vector
2236      * @see VectorOperators#SQRT
2237      * @see #lanewise(VectorOperators.Unary,VectorMask)
2238      */
2239     @ForceInline
2240     public final $abstractvectortype$ sqrt() {
2241         return lanewise(SQRT);
2242     }
2243 #end[FP]
2244 
2245     /// COMPARISONS
2246 
2247     /**
2248      * {@inheritDoc} <!--workaround-->
2249      */
2250     @Override
2251     @ForceInline
2252     public final
2253     VectorMask<$Boxtype$> eq(Vector<$Boxtype$> v) {
2254         return compare(EQ, v);
2255     }
2256 
2257     /**
2258      * Tests if this vector is equal to an input scalar.
2259      *
2260      * This is a lane-wise binary test operation which applies
2261      * the primitive equals operation ({@code ==}) to each lane.
2262      * The result is the same as {@code compare(VectorOperators.Comparison.EQ, e)}.
2263      *
2264      * @param e the input scalar
2265      * @return the result mask of testing if this vector
2266      *         is equal to {@code e}
2267      * @see #compare(VectorOperators.Comparison,$type$)
2268      */
2269     @ForceInline
2270     public final
2271     VectorMask<$Boxtype$> eq($type$ e) {
2272         return compare(EQ, e);
2273     }
2274 
2275     /**
2276      * {@inheritDoc} <!--workaround-->
2277      */
2278     @Override
2279     @ForceInline
2280     public final
2281     VectorMask<$Boxtype$> lt(Vector<$Boxtype$> v) {
2282         return compare(LT, v);
2283     }
2284 
2285     /**
2286      * Tests if this vector is less than an input scalar.
2287      *
2288      * This is a lane-wise binary test operation which applies
2289      * the primitive less than operation ({@code <}) to each lane.
2290      * The result is the same as {@code compare(VectorOperators.LT, e)}.
2291      *
2292      * @param e the input scalar
2293      * @return the mask result of testing if this vector
2294      *         is less than the input scalar
2295      * @see #compare(VectorOperators.Comparison,$type$)
2296      */
2297     @ForceInline
2298     public final
2299     VectorMask<$Boxtype$> lt($type$ e) {
2300         return compare(LT, e);
2301     }
2302 
2303     /**
2304      * {@inheritDoc} <!--workaround-->
2305      */
2306     @Override
2307     public abstract
2308     VectorMask<$Boxtype$> test(VectorOperators.Test op);
2309 
2310     /*package-private*/
2311     @ForceInline
2312     final
2313     <M extends VectorMask<$Boxtype$>>
2314     M testTemplate(Class<M> maskType, Test op) {
2315         $Type$Species vsp = vspecies();
2316         if (opKind(op, VO_SPECIAL)) {
2317 #if[FP]
2318             $Bitstype$Vector bits = this.viewAsIntegralLanes();
2319 #end[FP]
2320             VectorMask<$Boxbitstype$> m;
2321             if (op == IS_DEFAULT) {
2322                 m = {#if[FP]?bits.}compare(EQ, ($bitstype$) 0);
2323             } else if (op == IS_NEGATIVE) {
2324                 m = {#if[FP]?bits.}compare(LT, ($bitstype$) 0);
2325             }
2326 #if[FP]
2327             else if (op == IS_FINITE ||
2328                      op == IS_NAN ||
2329                      op == IS_INFINITE) {
2330                 // first kill the sign:
2331                 bits = bits.and($Boxbitstype$.MAX_VALUE);
2332                 // next find the bit pattern for infinity:
2333                 $bitstype$ infbits = ($bitstype$) toBits($Boxtype$.POSITIVE_INFINITY);
2334                 // now compare:
2335                 if (op == IS_FINITE) {
2336                     m = bits.compare(LT, infbits);
2337                 } else if (op == IS_NAN) {
2338                     m = bits.compare(GT, infbits);
2339                 } else {
2340                     m = bits.compare(EQ, infbits);
2341                 }
2342             }
2343 #end[FP]
2344             else {
2345                 throw new AssertionError(op);
2346             }
2347             return maskType.cast(m{#if[FP]?.cast(vsp)});
2348         }
2349         int opc = opCode(op);
2350         throw new AssertionError(op);
2351     }
2352 
2353     /**
2354      * {@inheritDoc} <!--workaround-->
2355      */
2356     @Override
2357     public abstract
2358     VectorMask<$Boxtype$> test(VectorOperators.Test op,
2359                                   VectorMask<$Boxtype$> m);
2360 
2361     /*package-private*/
2362     @ForceInline
2363     final
2364     <M extends VectorMask<$Boxtype$>>
2365     M testTemplate(Class<M> maskType, Test op, M mask) {
2366         $Type$Species vsp = vspecies();
2367         mask.check(maskType, this);
2368         if (opKind(op, VO_SPECIAL)) {
2369 #if[FP]
2370             $Bitstype$Vector bits = this.viewAsIntegralLanes();
2371             VectorMask<$Boxbitstype$> m = mask.cast($Bitstype$Vector.species(shape()));
2372 #else[FP]
2373             VectorMask<$Boxbitstype$> m = mask;
2374 #end[FP]
2375             if (op == IS_DEFAULT) {
2376                 m = {#if[FP]?bits.}compare(EQ, ($bitstype$) 0, m);
2377             } else if (op == IS_NEGATIVE) {
2378                 m = {#if[FP]?bits.}compare(LT, ($bitstype$) 0, m);
2379             }
2380 #if[FP]
2381             else if (op == IS_FINITE ||
2382                      op == IS_NAN ||
2383                      op == IS_INFINITE) {
2384                 // first kill the sign:
2385                 bits = bits.and($Boxbitstype$.MAX_VALUE);
2386                 // next find the bit pattern for infinity:
2387                 $bitstype$ infbits = ($bitstype$) toBits($Boxtype$.POSITIVE_INFINITY);
2388                 // now compare:
2389                 if (op == IS_FINITE) {
2390                     m = bits.compare(LT, infbits, m);
2391                 } else if (op == IS_NAN) {
2392                     m = bits.compare(GT, infbits, m);
2393                 } else {
2394                     m = bits.compare(EQ, infbits, m);
2395                 }
2396             }
2397 #end[FP]
2398             else {
2399                 throw new AssertionError(op);
2400             }
2401             return maskType.cast(m{#if[FP]?.cast(vsp)});
2402         }
2403         int opc = opCode(op);
2404         throw new AssertionError(op);
2405     }
2406 
2407     /**
2408      * {@inheritDoc} <!--workaround-->
2409      */
2410     @Override
2411     public abstract
2412     VectorMask<$Boxtype$> compare(VectorOperators.Comparison op, Vector<$Boxtype$> v);
2413 
2414     /*package-private*/
2415     @ForceInline
2416     final
2417     <M extends VectorMask<$Boxtype$>>
2418     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v) {
2419         $abstractvectortype$ that = ($abstractvectortype$) v;
2420         that.check(this);
2421         int opc = opCode(op);
2422         return VectorSupport.compare(
2423             opc, getClass(), maskType, $type$.class, length(),
2424             this, that, null,
2425             (cond, v0, v1, m1) -> {
2426                 AbstractMask<$Boxtype$> m
2427                     = v0.bTest(cond, v1, (cond_, i, a, b)
2428                                -> compareWithOp(cond, a, b));
2429                 @SuppressWarnings("unchecked")
2430                 M m2 = (M) m;
2431                 return m2;
2432             });
2433     }
2434 
2435     /*package-private*/
2436     @ForceInline
2437     final
2438     <M extends VectorMask<$Boxtype$>>
2439     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v, M m) {
2440         $abstractvectortype$ that = ($abstractvectortype$) v;
2441         that.check(this);
2442         m.check(maskType, this);
2443         int opc = opCode(op);
2444         return VectorSupport.compare(
2445             opc, getClass(), maskType, $type$.class, length(),
2446             this, that, m,
2447             (cond, v0, v1, m1) -> {
2448                 AbstractMask<$Boxtype$> cmpM
2449                     = v0.bTest(cond, v1, (cond_, i, a, b)
2450                                -> compareWithOp(cond, a, b));
2451                 @SuppressWarnings("unchecked")
2452                 M m2 = (M) cmpM.and(m1);
2453                 return m2;
2454             });
2455     }
2456 
2457     @ForceInline
2458     private static boolean compareWithOp(int cond, $type$ a, $type$ b) {
2459         return switch (cond) {
2460             case BT_eq -> a == b;
2461             case BT_ne -> a != b;
2462             case BT_lt -> a < b;
2463             case BT_le -> a <= b;
2464             case BT_gt -> a > b;
2465             case BT_ge -> a >= b;
2466 #if[!FP]
2467             case BT_ult -> $Boxtype$.compareUnsigned(a, b) < 0;
2468             case BT_ule -> $Boxtype$.compareUnsigned(a, b) <= 0;
2469             case BT_ugt -> $Boxtype$.compareUnsigned(a, b) > 0;
2470             case BT_uge -> $Boxtype$.compareUnsigned(a, b) >= 0;
2471 #end[!FP]
2472             default -> throw new AssertionError();
2473         };
2474     }
2475 
2476     /**
2477      * Tests this vector by comparing it with an input scalar,
2478      * according to the given comparison operation.
2479      *
2480      * This is a lane-wise binary test operation which applies
2481      * the comparison operation to each lane.
2482      * <p>
2483      * The result is the same as
2484      * {@code compare(op, broadcast(species(), e))}.
2485      * That is, the scalar may be regarded as broadcast to
2486      * a vector of the same species, and then compared
2487      * against the original vector, using the selected
2488      * comparison operation.
2489      *
2490      * @param op the operation used to compare lane values
2491      * @param e the input scalar
2492      * @return the mask result of testing lane-wise if this vector
2493      *         compares to the input, according to the selected
2494      *         comparison operator
2495      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector)
2496      * @see #eq($type$)
2497      * @see #lt($type$)
2498      */
2499     public abstract
2500     VectorMask<$Boxtype$> compare(Comparison op, $type$ e);
2501 
2502     /*package-private*/
2503     @ForceInline
2504     final
2505     <M extends VectorMask<$Boxtype$>>
2506     M compareTemplate(Class<M> maskType, Comparison op, $type$ e) {
2507         return compareTemplate(maskType, op, broadcast(e));
2508     }
2509 
2510     /**
2511      * Tests this vector by comparing it with an input scalar,
2512      * according to the given comparison operation,
2513      * in lanes selected by a mask.
2514      *
2515      * This is a masked lane-wise binary test operation which applies
2516      * to each pair of corresponding lane values.
2517      *
2518      * The returned result is equal to the expression
2519      * {@code compare(op,s).and(m)}.
2520      *
2521      * @param op the operation used to compare lane values
2522      * @param e the input scalar
2523      * @param m the mask controlling lane selection
2524      * @return the mask result of testing lane-wise if this vector
2525      *         compares to the input, according to the selected
2526      *         comparison operator,
2527      *         and only in the lanes selected by the mask
2528      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector,VectorMask)
2529      */
2530     @ForceInline
2531     public final VectorMask<$Boxtype$> compare(VectorOperators.Comparison op,
2532                                                $type$ e,
2533                                                VectorMask<$Boxtype$> m) {
2534         return compare(op, broadcast(e), m);
2535     }
2536 
2537 #if[!long]
2538     /**
2539      * {@inheritDoc} <!--workaround-->
2540      */
2541     @Override
2542     public abstract
2543     VectorMask<$Boxtype$> compare(Comparison op, long e);
2544 
2545     /*package-private*/
2546     @ForceInline
2547     final
2548     <M extends VectorMask<$Boxtype$>>
2549     M compareTemplate(Class<M> maskType, Comparison op, long e) {
2550         return compareTemplate(maskType, op, broadcast(e));
2551     }
2552 
2553     /**
2554      * {@inheritDoc} <!--workaround-->
2555      */
2556     @Override
2557     @ForceInline
2558     public final
2559     VectorMask<$Boxtype$> compare(Comparison op, long e, VectorMask<$Boxtype$> m) {
2560         return compare(op, broadcast(e), m);
2561     }
2562 
2563 
2564 #end[!long]
2565 
2566     /**
2567      * {@inheritDoc} <!--workaround-->
2568      */
2569     @Override public abstract
2570     $abstractvectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m);
2571 
2572     /*package-private*/
2573     @ForceInline
2574     final
2575     <M extends VectorMask<$Boxtype$>>
2576     $abstractvectortype$
2577     blendTemplate(Class<M> maskType, $abstractvectortype$ v, M m) {
2578         v.check(this);
2579         return VectorSupport.blend(
2580             getClass(), maskType, $type$.class, length(),
2581             this, v, m,
2582             (v0, v1, m_) -> v0.bOp(v1, m_, (i, a, b) -> b));
2583     }
2584 
2585     /**
2586      * {@inheritDoc} <!--workaround-->
2587      */
2588     @Override public abstract $abstractvectortype$ addIndex(int scale);
2589 
2590     /*package-private*/
2591     @ForceInline
2592     final $abstractvectortype$ addIndexTemplate(int scale) {
2593         $Type$Species vsp = vspecies();
2594         // make sure VLENGTH*scale doesn't overflow:
2595         vsp.checkScale(scale);
2596         return VectorSupport.indexVector(
2597             getClass(), $type$.class, length(),
2598             this, scale, vsp,
2599             (v, scale_, s)
2600             -> {
2601                 // If the platform doesn't support an INDEX
2602                 // instruction directly, load IOTA from memory
2603                 // and multiply.
2604                 $abstractvectortype$ iota = s.iota();
2605                 $type$ sc = ($type$) scale_;
2606                 return v.add(sc == 1 ? iota : iota.mul(sc));
2607             });
2608     }
2609 
2610     /**
2611      * Replaces selected lanes of this vector with
2612      * a scalar value
2613      * under the control of a mask.
2614      *
2615      * This is a masked lane-wise binary operation which
2616      * selects each lane value from one or the other input.
2617      *
2618      * The returned result is equal to the expression
2619      * {@code blend(broadcast(e),m)}.
2620      *
2621      * @param e the input scalar, containing the replacement lane value
2622      * @param m the mask controlling lane selection of the scalar
2623      * @return the result of blending the lane elements of this vector with
2624      *         the scalar value
2625      */
2626     @ForceInline
2627     public final $abstractvectortype$ blend($type$ e,
2628                                             VectorMask<$Boxtype$> m) {
2629         return blend(broadcast(e), m);
2630     }
2631 
2632 #if[!long]
2633     /**
2634      * Replaces selected lanes of this vector with
2635      * a scalar value
2636      * under the control of a mask.
2637      *
2638      * This is a masked lane-wise binary operation which
2639      * selects each lane value from one or the other input.
2640      *
2641      * The returned result is equal to the expression
2642      * {@code blend(broadcast(e),m)}.
2643      *
2644      * @param e the input scalar, containing the replacement lane value
2645      * @param m the mask controlling lane selection of the scalar
2646      * @return the result of blending the lane elements of this vector with
2647      *         the scalar value
2648      */
2649     @ForceInline
2650     public final $abstractvectortype$ blend(long e,
2651                                             VectorMask<$Boxtype$> m) {
2652         return blend(broadcast(e), m);
2653     }
2654 #end[!long]
2655 
2656     /**
2657      * {@inheritDoc} <!--workaround-->
2658      */
2659     @Override
2660     public abstract
2661     $abstractvectortype$ slice(int origin, Vector<$Boxtype$> v1);
2662 
2663     /*package-private*/
2664     final
2665     @ForceInline
2666     $abstractvectortype$ sliceTemplate(int origin, Vector<$Boxtype$> v1) {
2667         $abstractvectortype$ that = ($abstractvectortype$) v1;
2668         that.check(this);
2669         Objects.checkIndex(origin, length() + 1);
2670         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2671         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin))));
2672         iota = iotaShuffle(origin, 1, true);
2673         return that.rearrange(iota).blend(this.rearrange(iota), blendMask);
2674     }
2675 
2676     /**
2677      * {@inheritDoc} <!--workaround-->
2678      */
2679     @Override
2680     @ForceInline
2681     public final
2682     $abstractvectortype$ slice(int origin,
2683                                Vector<$Boxtype$> w,
2684                                VectorMask<$Boxtype$> m) {
2685         return broadcast(0).blend(slice(origin, w), m);
2686     }
2687 
2688     /**
2689      * {@inheritDoc} <!--workaround-->
2690      */
2691     @Override
2692     public abstract
2693     $abstractvectortype$ slice(int origin);
2694 
2695     /*package-private*/
2696     final
2697     @ForceInline
2698     $abstractvectortype$ sliceTemplate(int origin) {
2699         Objects.checkIndex(origin, length() + 1);
2700         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2701         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin))));
2702         iota = iotaShuffle(origin, 1, true);
2703         return vspecies().zero().blend(this.rearrange(iota), blendMask);
2704     }
2705 
2706     /**
2707      * {@inheritDoc} <!--workaround-->
2708      */
2709     @Override
2710     public abstract
2711     $abstractvectortype$ unslice(int origin, Vector<$Boxtype$> w, int part);
2712 
2713     /*package-private*/
2714     final
2715     @ForceInline
2716     $abstractvectortype$
2717     unsliceTemplate(int origin, Vector<$Boxtype$> w, int part) {
2718         $abstractvectortype$ that = ($abstractvectortype$) w;
2719         that.check(this);
2720         Objects.checkIndex(origin, length() + 1);
2721         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2722         VectorMask<$Boxtype$> blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT,
2723                                                                   (broadcast(($type$)(origin))));
2724         iota = iotaShuffle(-origin, 1, true);
2725         return that.blend(this.rearrange(iota), blendMask);
2726     }
2727 
2728     /*package-private*/
2729     final
2730     @ForceInline
2731     <M extends VectorMask<$Boxtype$>>
2732     $abstractvectortype$
2733     unsliceTemplate(Class<M> maskType, int origin, Vector<$Boxtype$> w, int part, M m) {
2734         $abstractvectortype$ that = ($abstractvectortype$) w;
2735         that.check(this);
2736         $abstractvectortype$ slice = that.sliceTemplate(origin, that);
2737         slice = slice.blendTemplate(maskType, this, m);
2738         return slice.unsliceTemplate(origin, w, part);
2739     }
2740 
2741     /**
2742      * {@inheritDoc} <!--workaround-->
2743      */
2744     @Override
2745     public abstract
2746     $abstractvectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m);
2747 
2748     /**
2749      * {@inheritDoc} <!--workaround-->
2750      */
2751     @Override
2752     public abstract
2753     $abstractvectortype$ unslice(int origin);
2754 
2755     /*package-private*/
2756     final
2757     @ForceInline
2758     $abstractvectortype$
2759     unsliceTemplate(int origin) {
2760         Objects.checkIndex(origin, length() + 1);
2761         VectorShuffle<$Boxtype$> iota = iotaShuffle();
2762         VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.GE,
2763                                                                   (broadcast(($type$)(origin))));
2764         iota = iotaShuffle(-origin, 1, true);
2765         return vspecies().zero().blend(this.rearrange(iota), blendMask);
2766     }
2767 
2768     private ArrayIndexOutOfBoundsException
2769     wrongPartForSlice(int part) {
2770         String msg = String.format("bad part number %d for slice operation",
2771                                    part);
2772         return new ArrayIndexOutOfBoundsException(msg);
2773     }
2774 
2775     /**
2776      * {@inheritDoc} <!--workaround-->
2777      */
2778     @Override
2779     public abstract
2780     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> m);
2781 
2782     /*package-private*/
2783     @ForceInline
2784     final
2785     <S extends VectorShuffle<$Boxtype$>>
2786     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype, S shuffle) {
2787         shuffle.checkIndexes();
2788         return VectorSupport.rearrangeOp(
2789             getClass(), shuffletype, null, $type$.class, length(),
2790             this, shuffle, null,
2791             (v1, s_, m_) -> v1.uOp((i, a) -> {
2792                 int ei = s_.laneSource(i);
2793                 return v1.lane(ei);
2794             }));
2795     }
2796 
2797     /**
2798      * {@inheritDoc} <!--workaround-->
2799      */
2800     @Override
2801     public abstract
2802     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2803                                    VectorMask<$Boxtype$> m);
2804 
2805     /*package-private*/
2806     @ForceInline
2807     final
2808     <S extends VectorShuffle<$Boxtype$>, M extends VectorMask<$Boxtype$>>
2809     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2810                                            Class<M> masktype,
2811                                            S shuffle,
2812                                            M m) {
2813 
2814         m.check(masktype, this);
2815         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2816         if (m.andNot(valid).anyTrue()) {
2817             shuffle.checkIndexes();
2818             throw new AssertionError();
2819         }
2820         return VectorSupport.rearrangeOp(
2821                    getClass(), shuffletype, masktype, $type$.class, length(),
2822                    this, shuffle, m,
2823                    (v1, s_, m_) -> v1.uOp((i, a) -> {
2824                         int ei = s_.laneSource(i);
2825                         return ei < 0  || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
2826                    }));
2827     }
2828 
2829     /**
2830      * {@inheritDoc} <!--workaround-->
2831      */
2832     @Override
2833     public abstract
2834     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2835                                    Vector<$Boxtype$> v);
2836 
2837     /*package-private*/
2838     @ForceInline
2839     final
2840     <S extends VectorShuffle<$Boxtype$>>
2841     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2842                                            S shuffle,
2843                                            $abstractvectortype$ v) {
2844         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2845         @SuppressWarnings("unchecked")
2846         S ws = (S) shuffle.wrapIndexes();
2847         $abstractvectortype$ r0 =
2848             VectorSupport.rearrangeOp(
2849                 getClass(), shuffletype, null, $type$.class, length(),
2850                 this, ws, null,
2851                 (v0, s_, m_) -> v0.uOp((i, a) -> {
2852                     int ei = s_.laneSource(i);
2853                     return v0.lane(ei);
2854                 }));
2855         $abstractvectortype$ r1 =
2856             VectorSupport.rearrangeOp(
2857                 getClass(), shuffletype, null, $type$.class, length(),
2858                 v, ws, null,
2859                 (v1, s_, m_) -> v1.uOp((i, a) -> {
2860                     int ei = s_.laneSource(i);
2861                     return v1.lane(ei);
2862                 }));
2863         return r1.blend(r0, valid);
2864     }
2865 
2866     @ForceInline
2867     private final
2868     VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) {
2869         $type$[] a = toArray();
2870         int[] sa = new int[a.length];
2871         for (int i = 0; i < a.length; i++) {
2872             sa[i] = (int) a[i];
2873         }
2874         return VectorShuffle.fromArray(dsp, sa, 0);
2875     }
2876 
2877     /*package-private*/
2878     @ForceInline
2879     final
2880     VectorShuffle<$Boxtype$> toShuffleTemplate(Class<?> shuffleType) {
2881         $Type$Species vsp = vspecies();
2882         return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
2883                                      getClass(), $type$.class, length(),
2884                                      shuffleType, byte.class, length(),
2885                                      this, vsp,
2886                                      $Type$Vector::toShuffle0);
2887     }
2888 
2889     /**
2890      * {@inheritDoc} <!--workaround-->
2891      * @since 19
2892      */
2893     @Override
2894     public abstract
2895     $Type$Vector compress(VectorMask<$Boxtype$> m);
2896 
2897     /*package-private*/
2898     @ForceInline
2899     final
2900     <M extends AbstractMask<$Boxtype$>>
2901     $Type$Vector compressTemplate(Class<M> masktype, M m) {
2902       m.check(masktype, this);
2903       return ($Type$Vector) VectorSupport.comExpOp(VectorSupport.VECTOR_OP_COMPRESS, getClass(), masktype,
2904                                                    $type$.class, length(), this, m,
2905                                                    (v1, m1) -> compressHelper(v1, m1));
2906     }
2907 
2908     /**
2909      * {@inheritDoc} <!--workaround-->
2910      * @since 19
2911      */
2912     @Override
2913     public abstract
2914     $Type$Vector expand(VectorMask<$Boxtype$> m);
2915 
2916     /*package-private*/
2917     @ForceInline
2918     final
2919     <M extends AbstractMask<$Boxtype$>>
2920     $Type$Vector expandTemplate(Class<M> masktype, M m) {
2921       m.check(masktype, this);
2922       return ($Type$Vector) VectorSupport.comExpOp(VectorSupport.VECTOR_OP_EXPAND, getClass(), masktype,
2923                                                    $type$.class, length(), this, m,
2924                                                    (v1, m1) -> expandHelper(v1, m1));
2925     }
2926 
2927 
2928     /**
2929      * {@inheritDoc} <!--workaround-->
2930      */
2931     @Override
2932     public abstract
2933     $abstractvectortype$ selectFrom(Vector<$Boxtype$> v);
2934 
2935     /*package-private*/
2936     @ForceInline
2937     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v) {
2938         return v.rearrange(this.toShuffle());
2939     }
2940 
2941     /**
2942      * {@inheritDoc} <!--workaround-->
2943      */
2944     @Override
2945     public abstract
2946     $abstractvectortype$ selectFrom(Vector<$Boxtype$> s, VectorMask<$Boxtype$> m);
2947 
2948     /*package-private*/
2949     @ForceInline
2950     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v,
2951                                                   AbstractMask<$Boxtype$> m) {
2952         return v.rearrange(this.toShuffle(), m);
2953     }
2954 
2955     /// Ternary operations
2956 
2957 #if[BITWISE]
2958     /**
2959      * Blends together the bits of two vectors under
2960      * the control of a third, which supplies mask bits.
2961      *
2962      * This is a lane-wise ternary operation which performs
2963      * a bitwise blending operation {@code (a&~c)|(b&c)}
2964      * to each lane.
2965      *
2966      * This method is also equivalent to the expression
2967      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2968      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2969      *    BITWISE_BLEND}{@code , bits, mask)}.
2970      *
2971      * @param bits input bits to blend into the current vector
2972      * @param mask a bitwise mask to enable blending of the input bits
2973      * @return the bitwise blend of the given bits into the current vector,
2974      *         under control of the bitwise mask
2975      * @see #bitwiseBlend($type$,$type$)
2976      * @see #bitwiseBlend($type$,Vector)
2977      * @see #bitwiseBlend(Vector,$type$)
2978      * @see VectorOperators#BITWISE_BLEND
2979      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
2980      */
2981     @ForceInline
2982     public final
2983     $abstractvectortype$ bitwiseBlend(Vector<$Boxtype$> bits, Vector<$Boxtype$> mask) {
2984         return lanewise(BITWISE_BLEND, bits, mask);
2985     }
2986 
2987     /**
2988      * Blends together the bits of a vector and a scalar under
2989      * the control of another scalar, which supplies mask bits.
2990      *
2991      * This is a lane-wise ternary operation which performs
2992      * a bitwise blending operation {@code (a&~c)|(b&c)}
2993      * to each lane.
2994      *
2995      * This method is also equivalent to the expression
2996      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
2997      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
2998      *    BITWISE_BLEND}{@code , bits, mask)}.
2999      *
3000      * @param bits input bits to blend into the current vector
3001      * @param mask a bitwise mask to enable blending of the input bits
3002      * @return the bitwise blend of the given bits into the current vector,
3003      *         under control of the bitwise mask
3004      * @see #bitwiseBlend(Vector,Vector)
3005      * @see VectorOperators#BITWISE_BLEND
3006      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
3007      */
3008     @ForceInline
3009     public final
3010     $abstractvectortype$ bitwiseBlend($type$ bits, $type$ mask) {
3011         return lanewise(BITWISE_BLEND, bits, mask);
3012     }
3013 
3014     /**
3015      * Blends together the bits of a vector and a scalar under
3016      * the control of another vector, which supplies mask bits.
3017      *
3018      * This is a lane-wise ternary operation which performs
3019      * a bitwise blending operation {@code (a&~c)|(b&c)}
3020      * to each lane.
3021      *
3022      * This method is also equivalent to the expression
3023      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
3024      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
3025      *    BITWISE_BLEND}{@code , bits, mask)}.
3026      *
3027      * @param bits input bits to blend into the current vector
3028      * @param mask a bitwise mask to enable blending of the input bits
3029      * @return the bitwise blend of the given bits into the current vector,
3030      *         under control of the bitwise mask
3031      * @see #bitwiseBlend(Vector,Vector)
3032      * @see VectorOperators#BITWISE_BLEND
3033      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
3034      */
3035     @ForceInline
3036     public final
3037     $abstractvectortype$ bitwiseBlend($type$ bits, Vector<$Boxtype$> mask) {
3038         return lanewise(BITWISE_BLEND, bits, mask);
3039     }
3040 
3041     /**
3042      * Blends together the bits of two vectors under
3043      * the control of a scalar, which supplies mask bits.
3044      *
3045      * This is a lane-wise ternary operation which performs
3046      * a bitwise blending operation {@code (a&~c)|(b&c)}
3047      * to each lane.
3048      *
3049      * This method is also equivalent to the expression
3050      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
3051      *    lanewise}{@code (}{@link VectorOperators#BITWISE_BLEND
3052      *    BITWISE_BLEND}{@code , bits, mask)}.
3053      *
3054      * @param bits input bits to blend into the current vector
3055      * @param mask a bitwise mask to enable blending of the input bits
3056      * @return the bitwise blend of the given bits into the current vector,
3057      *         under control of the bitwise mask
3058      * @see #bitwiseBlend(Vector,Vector)
3059      * @see VectorOperators#BITWISE_BLEND
3060      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
3061      */
3062     @ForceInline
3063     public final
3064     $abstractvectortype$ bitwiseBlend(Vector<$Boxtype$> bits, $type$ mask) {
3065         return lanewise(BITWISE_BLEND, bits, mask);
3066     }
3067 #end[BITWISE]
3068 
3069 #if[FP]
3070     /**
3071      * Multiplies this vector by a second input vector, and sums
3072      * the result with a third.
3073      *
3074      * Extended precision is used for the intermediate result,
3075      * avoiding possible loss of precision from rounding once
3076      * for each of the two operations.
3077      * The result is numerically close to {@code this.mul(b).add(c)},
3078      * and is typically closer to the true mathematical result.
3079      *
3080      * This is a lane-wise ternary operation which applies an operation
3081      * conforming to the specification of
3082      * {@link Math#fma($type$,$type$,$type$) Math.fma(a,b,c)}
3083      * to each lane.
3084 #if[intOrFloat]
3085      * The operation is adapted to cast the operands and the result,
3086      * specifically widening {@code float} operands to {@code double}
3087      * operands and narrowing the {@code double} result to a {@code float}
3088      * result.
3089 #end[intOrFloat]
3090      *
3091      * This method is also equivalent to the expression
3092      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
3093      *    lanewise}{@code (}{@link VectorOperators#FMA
3094      *    FMA}{@code , b, c)}.
3095      *
3096      * @param b the second input vector, supplying multiplier values
3097      * @param c the third input vector, supplying addend values
3098      * @return the product of this vector and the second input vector
3099      *         summed with the third input vector, using extended precision
3100      *         for the intermediate result
3101      * @see #fma($type$,$type$)
3102      * @see VectorOperators#FMA
3103      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
3104      */
3105     @ForceInline
3106     public final
3107     $abstractvectortype$ fma(Vector<$Boxtype$> b, Vector<$Boxtype$> c) {
3108         return lanewise(FMA, b, c);
3109     }
3110 
3111     /**
3112      * Multiplies this vector by a scalar multiplier, and sums
3113      * the result with a scalar addend.
3114      *
3115      * Extended precision is used for the intermediate result,
3116      * avoiding possible loss of precision from rounding once
3117      * for each of the two operations.
3118      * The result is numerically close to {@code this.mul(b).add(c)},
3119      * and is typically closer to the true mathematical result.
3120      *
3121      * This is a lane-wise ternary operation which applies an operation
3122      * conforming to the specification of
3123      * {@link Math#fma($type$,$type$,$type$) Math.fma(a,b,c)}
3124      * to each lane.
3125 #if[intOrFloat]
3126      * The operation is adapted to cast the operands and the result,
3127      * specifically widening {@code float} operands to {@code double}
3128      * operands and narrowing the {@code double} result to a {@code float}
3129      * result.
3130 #end[intOrFloat]
3131      *
3132      * This method is also equivalent to the expression
3133      * {@link #lanewise(VectorOperators.Ternary,Vector,Vector)
3134      *    lanewise}{@code (}{@link VectorOperators#FMA
3135      *    FMA}{@code , b, c)}.
3136      *
3137      * @param b the scalar multiplier
3138      * @param c the scalar addend
3139      * @return the product of this vector and the scalar multiplier
3140      *         summed with scalar addend, using extended precision
3141      *         for the intermediate result
3142      * @see #fma(Vector,Vector)
3143      * @see VectorOperators#FMA
3144      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
3145      */
3146     @ForceInline
3147     public final
3148     $abstractvectortype$ fma($type$ b, $type$ c) {
3149         return lanewise(FMA, b, c);
3150     }
3151 
3152     // Don't bother with (Vector,$type$) and ($type$,Vector) overloadings.
3153 #end[FP]
3154 
3155     // Type specific horizontal reductions
3156 
3157     /**
3158      * Returns a value accumulated from all the lanes of this vector.
3159      *
3160      * This is an associative cross-lane reduction operation which
3161      * applies the specified operation to all the lane elements.
3162      * <p>
3163      * A few reduction operations do not support arbitrary reordering
3164      * of their operands, yet are included here because of their
3165      * usefulness.
3166      * <ul>
3167      * <li>
3168      * In the case of {@code FIRST_NONZERO}, the reduction returns
3169      * the value from the lowest-numbered non-zero lane.
3170 #if[FP]
3171      * (As with {@code MAX} and {@code MIN}, floating point negative
3172      * zero {@code -0.0} is treated as a value distinct from
3173      * the default value, positive zero. So a first-nonzero lane reduction
3174      * might return {@code -0.0} even in the presence of non-zero
3175      * lane values.)
3176      * <li>
3177      * In the case of {@code ADD} and {@code MUL}, the
3178      * precise result will reflect the choice of an arbitrary order
3179      * of operations, which may even vary over time.
3180      * For further details see the section
3181      * <a href="VectorOperators.html#fp_assoc">Operations on floating point vectors</a>.
3182 #end[FP]
3183      * <li>
3184      * All other reduction operations are fully commutative and
3185      * associative.  The implementation can choose any order of
3186      * processing, yet it will always produce the same result.
3187      * </ul>
3188      *
3189      * @param op the operation used to combine lane values
3190      * @return the accumulated result
3191      * @throws UnsupportedOperationException if this vector does
3192      *         not support the requested operation
3193      * @see #reduceLanes(VectorOperators.Associative,VectorMask)
3194      * @see #add(Vector)
3195      * @see #mul(Vector)
3196      * @see #min(Vector)
3197      * @see #max(Vector)
3198 #if[BITWISE]
3199      * @see #and(Vector)
3200      * @see #or(Vector)
3201      * @see VectorOperators#XOR
3202 #end[BITWISE]
3203      * @see VectorOperators#FIRST_NONZERO
3204      */
3205     public abstract $type$ reduceLanes(VectorOperators.Associative op);
3206 
3207     /**
3208      * Returns a value accumulated from selected lanes of this vector,
3209      * controlled by a mask.
3210      *
3211      * This is an associative cross-lane reduction operation which
3212      * applies the specified operation to the selected lane elements.
3213      * <p>
3214      * If no elements are selected, an operation-specific identity
3215      * value is returned.
3216      * <ul>
3217      * <li>
3218      * If the operation is
3219 #if[BITWISE]
3220      *  {@code ADD}, {@code XOR}, {@code OR},
3221 #else[BITWISE]
3222      *  {@code ADD}
3223 #end[BITWISE]
3224      * or {@code FIRST_NONZERO},
3225      * then the identity value is {#if[FP]?positive }zero, the default {@code $type$} value.
3226      * <li>
3227      * If the operation is {@code MUL},
3228      * then the identity value is one.
3229 #if[BITWISE]
3230      * <li>
3231      * If the operation is {@code AND},
3232      * then the identity value is minus one (all bits set).
3233      * <li>
3234      * If the operation is {@code MAX},
3235      * then the identity value is {@code $Boxtype$.MIN_VALUE}.
3236      * <li>
3237      * If the operation is {@code MIN},
3238      * then the identity value is {@code $Boxtype$.MAX_VALUE}.
3239 #end[BITWISE]
3240 #if[FP]
3241      * <li>
3242      * If the operation is {@code MAX},
3243      * then the identity value is {@code $Boxtype$.NEGATIVE_INFINITY}.
3244      * <li>
3245      * If the operation is {@code MIN},
3246      * then the identity value is {@code $Boxtype$.POSITIVE_INFINITY}.
3247 #end[FP]
3248      * </ul>
3249      * <p>
3250      * A few reduction operations do not support arbitrary reordering
3251      * of their operands, yet are included here because of their
3252      * usefulness.
3253      * <ul>
3254      * <li>
3255      * In the case of {@code FIRST_NONZERO}, the reduction returns
3256      * the value from the lowest-numbered non-zero lane.
3257 #if[FP]
3258      * (As with {@code MAX} and {@code MIN}, floating point negative
3259      * zero {@code -0.0} is treated as a value distinct from
3260      * the default value, positive zero. So a first-nonzero lane reduction
3261      * might return {@code -0.0} even in the presence of non-zero
3262      * lane values.)
3263      * <li>
3264      * In the case of {@code ADD} and {@code MUL}, the
3265      * precise result will reflect the choice of an arbitrary order
3266      * of operations, which may even vary over time.
3267      * For further details see the section
3268      * <a href="VectorOperators.html#fp_assoc">Operations on floating point vectors</a>.
3269 #end[FP]
3270      * <li>
3271      * All other reduction operations are fully commutative and
3272      * associative.  The implementation can choose any order of
3273      * processing, yet it will always produce the same result.
3274      * </ul>
3275      *
3276      * @param op the operation used to combine lane values
3277      * @param m the mask controlling lane selection
3278      * @return the reduced result accumulated from the selected lane values
3279      * @throws UnsupportedOperationException if this vector does
3280      *         not support the requested operation
3281      * @see #reduceLanes(VectorOperators.Associative)
3282      */
3283     public abstract $type$ reduceLanes(VectorOperators.Associative op,
3284                                        VectorMask<$Boxtype$> m);
3285 
3286     /*package-private*/
3287     @ForceInline
3288     final
3289     $type$ reduceLanesTemplate(VectorOperators.Associative op,
3290                                Class<? extends VectorMask<$Boxtype$>> maskClass,
3291                                VectorMask<$Boxtype$> m) {
3292         m.check(maskClass, this);
3293         if (op == FIRST_NONZERO) {
3294             // FIXME:  The JIT should handle this.
3295             $abstractvectortype$ v = broadcast(($type$) 0).blend(this, m);
3296             return v.reduceLanesTemplate(op);
3297         }
3298         int opc = opCode(op);
3299         return fromBits(VectorSupport.reductionCoerced(
3300             opc, getClass(), maskClass, $type$.class, length(),
3301             this, m,
3302             REDUCE_IMPL.find(op, opc, $abstractvectortype$::reductionOperations)));
3303     }
3304 
3305     /*package-private*/
3306     @ForceInline
3307     final
3308     $type$ reduceLanesTemplate(VectorOperators.Associative op) {
3309         if (op == FIRST_NONZERO) {
3310             // FIXME:  The JIT should handle this.
3311             VectorMask<$Boxbitstype$> thisNZ
3312                 = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
3313             int ft = thisNZ.firstTrue();
3314             return ft < length() ? this.lane(ft) : ($type$) 0;
3315         }
3316         int opc = opCode(op);
3317         return fromBits(VectorSupport.reductionCoerced(
3318             opc, getClass(), null, $type$.class, length(),
3319             this, null,
3320             REDUCE_IMPL.find(op, opc, $abstractvectortype$::reductionOperations)));
3321     }
3322 
3323     private static final
3324     ImplCache<Associative, ReductionOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
3325         REDUCE_IMPL = new ImplCache<>(Associative.class, $Type$Vector.class);
3326 
3327     private static ReductionOperation<$abstractvectortype$, VectorMask<$Boxtype$>> reductionOperations(int opc_) {
3328         switch (opc_) {
3329             case VECTOR_OP_ADD: return (v, m) ->
3330                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a + b)));
3331             case VECTOR_OP_MUL: return (v, m) ->
3332                     toBits(v.rOp(($type$)1, m, (i, a, b) -> ($type$)(a * b)));
3333             case VECTOR_OP_MIN: return (v, m) ->
3334                     toBits(v.rOp(MAX_OR_INF, m, (i, a, b) -> ($type$) Math.min(a, b)));
3335             case VECTOR_OP_MAX: return (v, m) ->
3336                     toBits(v.rOp(MIN_OR_INF, m, (i, a, b) -> ($type$) Math.max(a, b)));
3337 #if[BITWISE]
3338             case VECTOR_OP_AND: return (v, m) ->
3339                     toBits(v.rOp(($type$)-1, m, (i, a, b) -> ($type$)(a & b)));
3340             case VECTOR_OP_OR: return (v, m) ->
3341                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a | b)));
3342             case VECTOR_OP_XOR: return (v, m) ->
3343                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a ^ b)));
3344 #end[BITWISE]
3345             default: return null;
3346         }
3347     }
3348 
3349 #if[FP]
3350     private static final $type$ MIN_OR_INF = $Boxtype$.NEGATIVE_INFINITY;
3351     private static final $type$ MAX_OR_INF = $Boxtype$.POSITIVE_INFINITY;
3352 #else[FP]
3353     private static final $type$ MIN_OR_INF = $Boxtype$.MIN_VALUE;
3354     private static final $type$ MAX_OR_INF = $Boxtype$.MAX_VALUE;
3355 #end[FP]
3356 
3357     public @Override abstract long reduceLanesToLong(VectorOperators.Associative op);
3358     public @Override abstract long reduceLanesToLong(VectorOperators.Associative op,
3359                                                      VectorMask<$Boxtype$> m);
3360 
3361     // Type specific accessors
3362 
3363     /**
3364      * Gets the lane element at lane index {@code i}
3365      *
3366      * @param i the lane index
3367      * @return the lane element at lane index {@code i}
3368      * @throws IllegalArgumentException if the index is is out of range
3369      * ({@code < 0 || >= length()})
3370      */
3371     public abstract $type$ lane(int i);
3372 
3373     /**
3374      * Replaces the lane element of this vector at lane index {@code i} with
3375      * value {@code e}.
3376      *
3377      * This is a cross-lane operation and behaves as if it returns the result
3378      * of blending this vector with an input vector that is the result of
3379      * broadcasting {@code e} and a mask that has only one lane set at lane
3380      * index {@code i}.
3381      *
3382      * @param i the lane index of the lane element to be replaced
3383      * @param e the value to be placed
3384      * @return the result of replacing the lane element of this vector at lane
3385      * index {@code i} with value {@code e}.
3386      * @throws IllegalArgumentException if the index is is out of range
3387      * ({@code < 0 || >= length()})
3388      */
3389     public abstract $abstractvectortype$ withLane(int i, $type$ e);
3390 
3391     // Memory load operations
3392 
3393     /**
3394      * Returns an array of type {@code $type$[]}
3395      * containing all the lane values.
3396      * The array length is the same as the vector length.
3397      * The array elements are stored in lane order.
3398      * <p>
3399      * This method behaves as if it stores
3400      * this vector into an allocated array
3401      * (using {@link #intoArray($type$[], int) intoArray})
3402      * and returns the array as follows:
3403      * <pre>{@code
3404      *   $type$[] a = new $type$[this.length()];
3405      *   this.intoArray(a, 0);
3406      *   return a;
3407      * }</pre>
3408      *
3409      * @return an array containing the lane values of this vector
3410      */
3411     @ForceInline
3412     @Override
3413     public final $type$[] toArray() {
3414         $type$[] a = new $type$[vspecies().laneCount()];
3415         intoArray(a, 0);
3416         return a;
3417     }
3418 
3419 #if[int]
3420     /**
3421      * {@inheritDoc} <!--workaround-->
3422      * This is an alias for {@link #toArray()}
3423      * When this method is used on used on vectors
3424      * of type {@code $abstractvectortype$},
3425      * there will be no loss of range or precision.
3426      */
3427     @ForceInline
3428     @Override
3429     public final int[] toIntArray() {
3430         return toArray();
3431     }
3432 #else[int]
3433     /** {@inheritDoc} <!--workaround-->
3434 #if[!FP]
3435 #if[!long]
3436      * @implNote
3437      * When this method is used on used on vectors
3438      * of type {@code $abstractvectortype$},
3439      * there will be no loss of precision or range,
3440      * and so no {@code UnsupportedOperationException} will
3441      * be thrown.
3442 #end[!long]
3443 #end[!FP]
3444      */
3445     @ForceInline
3446     @Override
3447     public final int[] toIntArray() {
3448         $type$[] a = toArray();
3449         int[] res = new int[a.length];
3450         for (int i = 0; i < a.length; i++) {
3451             $type$ e = a[i];
3452             res[i] = (int) $Type$Species.toIntegralChecked(e, true);
3453         }
3454         return res;
3455     }
3456 #end[int]
3457 
3458 #if[long]
3459     /**
3460      * {@inheritDoc} <!--workaround-->
3461      * This is an alias for {@link #toArray()}
3462      * When this method is used on used on vectors
3463      * of type {@code $abstractvectortype$},
3464      * there will be no loss of range or precision.
3465      */
3466     @ForceInline
3467     @Override
3468     public final long[] toLongArray() {
3469         return toArray();
3470     }
3471 #else[long]
3472     /** {@inheritDoc} <!--workaround-->
3473 #if[!FP]
3474      * @implNote
3475      * When this method is used on used on vectors
3476      * of type {@code $abstractvectortype$},
3477      * there will be no loss of precision or range,
3478      * and so no {@code UnsupportedOperationException} will
3479      * be thrown.
3480 #end[!FP]
3481      */
3482     @ForceInline
3483     @Override
3484     public final long[] toLongArray() {
3485         $type$[] a = toArray();
3486         long[] res = new long[a.length];
3487         for (int i = 0; i < a.length; i++) {
3488             $type$ e = a[i];
3489             res[i] = $Type$Species.toIntegralChecked(e, false);
3490         }
3491         return res;
3492     }
3493 #end[long]
3494 
3495 #if[double]
3496     /** {@inheritDoc} <!--workaround-->
3497      * @implNote
3498      * This is an alias for {@link #toArray()}
3499      * When this method is used on used on vectors
3500      * of type {@code $abstractvectortype$},
3501      * there will be no loss of precision.
3502      */
3503     @ForceInline
3504     @Override
3505     public final double[] toDoubleArray() {
3506         return toArray();
3507     }
3508 #else[double]
3509     /** {@inheritDoc} <!--workaround-->
3510 #if[long]
3511      * @implNote
3512      * When this method is used on used on vectors
3513      * of type {@code $abstractvectortype$},
3514      * up to nine bits of precision may be lost
3515      * for lane values of large magnitude.
3516 #else[long]
3517      * @implNote
3518      * When this method is used on used on vectors
3519      * of type {@code $abstractvectortype$},
3520      * there will be no loss of precision.
3521 #end[long]
3522      */
3523     @ForceInline
3524     @Override
3525     public final double[] toDoubleArray() {
3526         $type$[] a = toArray();
3527         double[] res = new double[a.length];
3528         for (int i = 0; i < a.length; i++) {
3529             res[i] = (double) a[i];
3530         }
3531         return res;
3532     }
3533 #end[double]
3534 
3535     /**
3536      * Loads a vector from an array of type {@code $type$[]}
3537      * starting at an offset.
3538      * For each vector lane, where {@code N} is the vector lane index, the
3539      * array element at index {@code offset + N} is placed into the
3540      * resulting vector at lane index {@code N}.
3541      *
3542      * @param species species of desired vector
3543      * @param a the array
3544      * @param offset the offset into the array
3545      * @return the vector loaded from an array
3546      * @throws IndexOutOfBoundsException
3547      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3548      *         for any lane {@code N} in the vector
3549      */
3550     @ForceInline
3551     public static
3552     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3553                                    $type$[] a, int offset) {
3554         offset = checkFromIndexSize(offset, species.length(), a.length);
3555         $Type$Species vsp = ($Type$Species) species;
3556         return vsp.dummyVector().fromArray0(a, offset);
3557     }
3558 
3559     /**
3560      * Loads a vector from an array of type {@code $type$[]}
3561      * starting at an offset and using a mask.
3562      * Lanes where the mask is unset are filled with the default
3563      * value of {@code $type$} ({#if[FP]?positive }zero).
3564      * For each vector lane, where {@code N} is the vector lane index,
3565      * if the mask lane at index {@code N} is set then the array element at
3566      * index {@code offset + N} is placed into the resulting vector at lane index
3567      * {@code N}, otherwise the default element value is placed into the
3568      * resulting vector at lane index {@code N}.
3569      *
3570      * @param species species of desired vector
3571      * @param a the array
3572      * @param offset the offset into the array
3573      * @param m the mask controlling lane selection
3574      * @return the vector loaded from an array
3575      * @throws IndexOutOfBoundsException
3576      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3577      *         for any lane {@code N} in the vector
3578      *         where the mask is set
3579      */
3580     @ForceInline
3581     public static
3582     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3583                                    $type$[] a, int offset,
3584                                    VectorMask<$Boxtype$> m) {
3585         $Type$Species vsp = ($Type$Species) species;
3586         if (offset >= 0 && offset <= (a.length - species.length())) {
3587             return vsp.dummyVector().fromArray0(a, offset, m);
3588         }
3589 
3590         // FIXME: optimize
3591         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3592         return vsp.vOp(m, i -> a[offset + i]);
3593     }
3594 
3595     /**
3596      * Gathers a new vector composed of elements from an array of type
3597      * {@code $type$[]},
3598      * using indexes obtained by adding a fixed {@code offset} to a
3599      * series of secondary offsets from an <em>index map</em>.
3600      * The index map is a contiguous sequence of {@code VLENGTH}
3601      * elements in a second array of {@code int}s, starting at a given
3602      * {@code mapOffset}.
3603      * <p>
3604      * For each vector lane, where {@code N} is the vector lane index,
3605      * the lane is loaded from the array
3606      * element {@code a[f(N)]}, where {@code f(N)} is the
3607      * index mapping expression
3608      * {@code offset + indexMap[mapOffset + N]]}.
3609      *
3610      * @param species species of desired vector
3611      * @param a the array
3612      * @param offset the offset into the array, may be negative if relative
3613      * indexes in the index map compensate to produce a value within the
3614      * array bounds
3615      * @param indexMap the index map
3616      * @param mapOffset the offset into the index map
3617      * @return the vector loaded from the indexed elements of the array
3618      * @throws IndexOutOfBoundsException
3619      *         if {@code mapOffset+N < 0}
3620      *         or if {@code mapOffset+N >= indexMap.length},
3621      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3622      *         is an invalid index into {@code a},
3623      *         for any lane {@code N} in the vector
3624      * @see $abstractvectortype$#toIntArray()
3625      */
3626 #if[byteOrShort]
3627     @ForceInline
3628     public static
3629     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3630                                    $type$[] a, int offset,
3631                                    int[] indexMap, int mapOffset) {
3632         $Type$Species vsp = ($Type$Species) species;
3633         return vsp.vOp(n -> a[offset + indexMap[mapOffset + n]]);
3634     }
3635 #else[byteOrShort]
3636     @ForceInline
3637     public static
3638     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3639                                    $type$[] a, int offset,
3640                                    int[] indexMap, int mapOffset) {
3641         $Type$Species vsp = ($Type$Species) species;
3642         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3643         Objects.requireNonNull(a);
3644         Objects.requireNonNull(indexMap);
3645         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();
3646 
3647 #if[longOrDouble]
3648         if (vsp.laneCount() == 1) {
3649           return $abstractvectortype$.fromArray(vsp, a, offset + indexMap[mapOffset]);
3650         }
3651 
3652         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3653         IntVector vix;
3654         if (isp.laneCount() != vsp.laneCount()) {
3655             // For $Type$MaxVector,  if vector length is non-power-of-two or
3656             // 2048 bits, indexShape of $Type$ species is S_MAX_BIT.
3657             // Assume that vector length is 2048, then the lane count of $Type$
3658             // vector is 32. When converting $Type$ species to int species,
3659             // indexShape is still S_MAX_BIT, but the lane count of int vector
3660             // is 64. So when loading index vector (IntVector), only lower half
3661             // of index data is needed.
3662             vix = IntVector
3663                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3664                 .add(offset);
3665         } else {
3666             vix = IntVector
3667                 .fromArray(isp, indexMap, mapOffset)
3668                 .add(offset);
3669         }
3670 #else[longOrDouble]
3671         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3672         IntVector vix = IntVector
3673             .fromArray(isp, indexMap, mapOffset)
3674             .add(offset);
3675 #end[longOrDouble]
3676 
3677         vix = VectorIntrinsics.checkIndex(vix, a.length);
3678 
3679         return VectorSupport.loadWithMap(
3680             vectorType, null, $type$.class, vsp.laneCount(),
3681             isp.vectorType(),
3682             a, ARRAY_BASE, vix, null,
3683             a, offset, indexMap, mapOffset, vsp,
3684             (c, idx, iMap, idy, s, vm) ->
3685             s.vOp(n -> c[idx + iMap[idy+n]]));
3686     }
3687 #end[byteOrShort]
3688 
3689     /**
3690      * Gathers a new vector composed of elements from an array of type
3691      * {@code $type$[]},
3692      * under the control of a mask, and
3693      * using indexes obtained by adding a fixed {@code offset} to a
3694      * series of secondary offsets from an <em>index map</em>.
3695      * The index map is a contiguous sequence of {@code VLENGTH}
3696      * elements in a second array of {@code int}s, starting at a given
3697      * {@code mapOffset}.
3698      * <p>
3699      * For each vector lane, where {@code N} is the vector lane index,
3700      * if the lane is set in the mask,
3701      * the lane is loaded from the array
3702      * element {@code a[f(N)]}, where {@code f(N)} is the
3703      * index mapping expression
3704      * {@code offset + indexMap[mapOffset + N]]}.
3705      * Unset lanes in the resulting vector are set to zero.
3706      *
3707      * @param species species of desired vector
3708      * @param a the array
3709      * @param offset the offset into the array, may be negative if relative
3710      * indexes in the index map compensate to produce a value within the
3711      * array bounds
3712      * @param indexMap the index map
3713      * @param mapOffset the offset into the index map
3714      * @param m the mask controlling lane selection
3715      * @return the vector loaded from the indexed elements of the array
3716      * @throws IndexOutOfBoundsException
3717      *         if {@code mapOffset+N < 0}
3718      *         or if {@code mapOffset+N >= indexMap.length},
3719      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3720      *         is an invalid index into {@code a},
3721      *         for any lane {@code N} in the vector
3722      *         where the mask is set
3723      * @see $abstractvectortype$#toIntArray()
3724      */
3725 #if[byteOrShort]
3726     @ForceInline
3727     public static
3728     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3729                                    $type$[] a, int offset,
3730                                    int[] indexMap, int mapOffset,
3731                                    VectorMask<$Boxtype$> m) {
3732         $Type$Species vsp = ($Type$Species) species;
3733         return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
3734     }
3735 #else[byteOrShort]
3736     @ForceInline
3737     public static
3738     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3739                                    $type$[] a, int offset,
3740                                    int[] indexMap, int mapOffset,
3741                                    VectorMask<$Boxtype$> m) {
3742         if (m.allTrue()) {
3743             return fromArray(species, a, offset, indexMap, mapOffset);
3744         }
3745         else {
3746             $Type$Species vsp = ($Type$Species) species;
3747             return vsp.dummyVector().fromArray0(a, offset, indexMap, mapOffset, m);
3748         }
3749     }
3750 #end[byteOrShort]
3751 
3752 #if[short]
3753     /**
3754      * Loads a vector from an array of type {@code char[]}
3755      * starting at an offset.
3756      * For each vector lane, where {@code N} is the vector lane index, the
3757      * array element at index {@code offset + N}
3758      * is first cast to a {@code short} value and then
3759      * placed into the resulting vector at lane index {@code N}.
3760      *
3761      * @param species species of desired vector
3762      * @param a the array
3763      * @param offset the offset into the array
3764      * @return the vector loaded from an array
3765      * @throws IndexOutOfBoundsException
3766      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3767      *         for any lane {@code N} in the vector
3768      */
3769     @ForceInline
3770     public static
3771     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3772                                        char[] a, int offset) {
3773         offset = checkFromIndexSize(offset, species.length(), a.length);
3774         $Type$Species vsp = ($Type$Species) species;
3775         return vsp.dummyVector().fromCharArray0(a, offset);
3776     }
3777 
3778     /**
3779      * Loads a vector from an array of type {@code char[]}
3780      * starting at an offset and using a mask.
3781      * Lanes where the mask is unset are filled with the default
3782      * value of {@code $type$} ({#if[FP]?positive }zero).
3783      * For each vector lane, where {@code N} is the vector lane index,
3784      * if the mask lane at index {@code N} is set then the array element at
3785      * index {@code offset + N}
3786      * is first cast to a {@code short} value and then
3787      * placed into the resulting vector at lane index
3788      * {@code N}, otherwise the default element value is placed into the
3789      * resulting vector at lane index {@code N}.
3790      *
3791      * @param species species of desired vector
3792      * @param a the array
3793      * @param offset the offset into the array
3794      * @param m the mask controlling lane selection
3795      * @return the vector loaded from an array
3796      * @throws IndexOutOfBoundsException
3797      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3798      *         for any lane {@code N} in the vector
3799      *         where the mask is set
3800      */
3801     @ForceInline
3802     public static
3803     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3804                                        char[] a, int offset,
3805                                        VectorMask<$Boxtype$> m) {
3806         $Type$Species vsp = ($Type$Species) species;
3807         if (offset >= 0 && offset <= (a.length - species.length())) {
3808             return vsp.dummyVector().fromCharArray0(a, offset, m);
3809         }
3810 
3811         // FIXME: optimize
3812         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3813         return vsp.vOp(m, i -> (short) a[offset + i]);
3814     }
3815 
3816     /**
3817      * Gathers a new vector composed of elements from an array of type
3818      * {@code char[]},
3819      * using indexes obtained by adding a fixed {@code offset} to a
3820      * series of secondary offsets from an <em>index map</em>.
3821      * The index map is a contiguous sequence of {@code VLENGTH}
3822      * elements in a second array of {@code int}s, starting at a given
3823      * {@code mapOffset}.
3824      * <p>
3825      * For each vector lane, where {@code N} is the vector lane index,
3826      * the lane is loaded from the expression
3827      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3828      * index mapping expression
3829      * {@code offset + indexMap[mapOffset + N]]}.
3830      *
3831      * @param species species of desired vector
3832      * @param a the array
3833      * @param offset the offset into the array, may be negative if relative
3834      * indexes in the index map compensate to produce a value within the
3835      * array bounds
3836      * @param indexMap the index map
3837      * @param mapOffset the offset into the index map
3838      * @return the vector loaded from the indexed elements of the array
3839      * @throws IndexOutOfBoundsException
3840      *         if {@code mapOffset+N < 0}
3841      *         or if {@code mapOffset+N >= indexMap.length},
3842      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3843      *         is an invalid index into {@code a},
3844      *         for any lane {@code N} in the vector
3845      * @see $abstractvectortype$#toIntArray()
3846      */
3847     @ForceInline
3848     public static
3849     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3850                                        char[] a, int offset,
3851                                        int[] indexMap, int mapOffset) {
3852         // FIXME: optimize
3853         $Type$Species vsp = ($Type$Species) species;
3854         return vsp.vOp(n -> (short) a[offset + indexMap[mapOffset + n]]);
3855     }
3856 
3857     /**
3858      * Gathers a new vector composed of elements from an array of type
3859      * {@code char[]},
3860      * under the control of a mask, and
3861      * using indexes obtained by adding a fixed {@code offset} to a
3862      * series of secondary offsets from an <em>index map</em>.
3863      * The index map is a contiguous sequence of {@code VLENGTH}
3864      * elements in a second array of {@code int}s, starting at a given
3865      * {@code mapOffset}.
3866      * <p>
3867      * For each vector lane, where {@code N} is the vector lane index,
3868      * if the lane is set in the mask,
3869      * the lane is loaded from the expression
3870      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3871      * index mapping expression
3872      * {@code offset + indexMap[mapOffset + N]]}.
3873      * Unset lanes in the resulting vector are set to zero.
3874      *
3875      * @param species species of desired vector
3876      * @param a the array
3877      * @param offset the offset into the array, may be negative if relative
3878      * indexes in the index map compensate to produce a value within the
3879      * array bounds
3880      * @param indexMap the index map
3881      * @param mapOffset the offset into the index map
3882      * @param m the mask controlling lane selection
3883      * @return the vector loaded from the indexed elements of the array
3884      * @throws IndexOutOfBoundsException
3885      *         if {@code mapOffset+N < 0}
3886      *         or if {@code mapOffset+N >= indexMap.length},
3887      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3888      *         is an invalid index into {@code a},
3889      *         for any lane {@code N} in the vector
3890      *         where the mask is set
3891      * @see $abstractvectortype$#toIntArray()
3892      */
3893     @ForceInline
3894     public static
3895     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3896                                        char[] a, int offset,
3897                                        int[] indexMap, int mapOffset,
3898                                        VectorMask<$Boxtype$> m) {
3899         // FIXME: optimize
3900         $Type$Species vsp = ($Type$Species) species;
3901         return vsp.vOp(m, n -> (short) a[offset + indexMap[mapOffset + n]]);
3902     }
3903 #end[short]
3904 
3905 #if[byte]
3906     /**
3907      * Loads a vector from an array of type {@code boolean[]}
3908      * starting at an offset.
3909      * For each vector lane, where {@code N} is the vector lane index, the
3910      * array element at index {@code offset + N}
3911      * is first converted to a {@code byte} value and then
3912      * placed into the resulting vector at lane index {@code N}.
3913      * <p>
3914      * A {@code boolean} value is converted to a {@code byte} value by applying the
3915      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3916      *
3917      * @param species species of desired vector
3918      * @param a the array
3919      * @param offset the offset into the array
3920      * @return the vector loaded from an array
3921      * @throws IndexOutOfBoundsException
3922      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3923      *         for any lane {@code N} in the vector
3924      */
3925     @ForceInline
3926     public static
3927     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3928                                           boolean[] a, int offset) {
3929         offset = checkFromIndexSize(offset, species.length(), a.length);
3930         $Type$Species vsp = ($Type$Species) species;
3931         return vsp.dummyVector().fromBooleanArray0(a, offset);
3932     }
3933 
3934     /**
3935      * Loads a vector from an array of type {@code boolean[]}
3936      * starting at an offset and using a mask.
3937      * Lanes where the mask is unset are filled with the default
3938      * value of {@code $type$} ({#if[FP]?positive }zero).
3939      * For each vector lane, where {@code N} is the vector lane index,
3940      * if the mask lane at index {@code N} is set then the array element at
3941      * index {@code offset + N}
3942      * is first converted to a {@code byte} value and then
3943      * placed into the resulting vector at lane index
3944      * {@code N}, otherwise the default element value is placed into the
3945      * resulting vector at lane index {@code N}.
3946      * <p>
3947      * A {@code boolean} value is converted to a {@code byte} value by applying the
3948      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3949      *
3950      * @param species species of desired vector
3951      * @param a the array
3952      * @param offset the offset into the array
3953      * @param m the mask controlling lane selection
3954      * @return the vector loaded from an array
3955      * @throws IndexOutOfBoundsException
3956      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3957      *         for any lane {@code N} in the vector
3958      *         where the mask is set
3959      */
3960     @ForceInline
3961     public static
3962     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3963                                           boolean[] a, int offset,
3964                                           VectorMask<$Boxtype$> m) {
3965         $Type$Species vsp = ($Type$Species) species;
3966         if (offset >= 0 && offset <= (a.length - species.length())) {
3967             $abstractvectortype$ zero = vsp.zero();
3968             return vsp.dummyVector().fromBooleanArray0(a, offset, m);
3969         }
3970 
3971         // FIXME: optimize
3972         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3973         return vsp.vOp(m, i -> (byte) (a[offset + i] ? 1 : 0));
3974     }
3975 
3976     /**
3977      * Gathers a new vector composed of elements from an array of type
3978      * {@code boolean[]},
3979      * using indexes obtained by adding a fixed {@code offset} to a
3980      * series of secondary offsets from an <em>index map</em>.
3981      * The index map is a contiguous sequence of {@code VLENGTH}
3982      * elements in a second array of {@code int}s, starting at a given
3983      * {@code mapOffset}.
3984      * <p>
3985      * For each vector lane, where {@code N} is the vector lane index,
3986      * the lane is loaded from the expression
3987      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
3988      * index mapping expression
3989      * {@code offset + indexMap[mapOffset + N]]}.
3990      *
3991      * @param species species of desired vector
3992      * @param a the array
3993      * @param offset the offset into the array, may be negative if relative
3994      * indexes in the index map compensate to produce a value within the
3995      * array bounds
3996      * @param indexMap the index map
3997      * @param mapOffset the offset into the index map
3998      * @return the vector loaded from the indexed elements of the array
3999      * @throws IndexOutOfBoundsException
4000      *         if {@code mapOffset+N < 0}
4001      *         or if {@code mapOffset+N >= indexMap.length},
4002      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4003      *         is an invalid index into {@code a},
4004      *         for any lane {@code N} in the vector
4005      * @see $abstractvectortype$#toIntArray()
4006      */
4007     @ForceInline
4008     public static
4009     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
4010                                           boolean[] a, int offset,
4011                                           int[] indexMap, int mapOffset) {
4012         // FIXME: optimize
4013         $Type$Species vsp = ($Type$Species) species;
4014         return vsp.vOp(n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
4015     }
4016 
4017     /**
4018      * Gathers a new vector composed of elements from an array of type
4019      * {@code boolean[]},
4020      * under the control of a mask, and
4021      * using indexes obtained by adding a fixed {@code offset} to a
4022      * series of secondary offsets from an <em>index map</em>.
4023      * The index map is a contiguous sequence of {@code VLENGTH}
4024      * elements in a second array of {@code int}s, starting at a given
4025      * {@code mapOffset}.
4026      * <p>
4027      * For each vector lane, where {@code N} is the vector lane index,
4028      * if the lane is set in the mask,
4029      * the lane is loaded from the expression
4030      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
4031      * index mapping expression
4032      * {@code offset + indexMap[mapOffset + N]]}.
4033      * Unset lanes in the resulting vector are set to zero.
4034      *
4035      * @param species species of desired vector
4036      * @param a the array
4037      * @param offset the offset into the array, may be negative if relative
4038      * indexes in the index map compensate to produce a value within the
4039      * array bounds
4040      * @param indexMap the index map
4041      * @param mapOffset the offset into the index map
4042      * @param m the mask controlling lane selection
4043      * @return the vector loaded from the indexed elements of the array
4044      * @throws IndexOutOfBoundsException
4045      *         if {@code mapOffset+N < 0}
4046      *         or if {@code mapOffset+N >= indexMap.length},
4047      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4048      *         is an invalid index into {@code a},
4049      *         for any lane {@code N} in the vector
4050      *         where the mask is set
4051      * @see $abstractvectortype$#toIntArray()
4052      */
4053     @ForceInline
4054     public static
4055     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
4056                                           boolean[] a, int offset,
4057                                           int[] indexMap, int mapOffset,
4058                                           VectorMask<$Boxtype$> m) {
4059         // FIXME: optimize
4060         $Type$Species vsp = ($Type$Species) species;
4061         return vsp.vOp(m, n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
4062     }
4063 #end[byte]
4064 
4065     /**
4066      * Loads a vector from a {@linkplain MemorySegment memory segment}
4067      * starting at an offset into the memory segment.
4068      * Bytes are composed into primitive lane elements according
4069      * to the specified byte order.
4070      * The vector is arranged into lanes according to
4071      * <a href="Vector.html#lane-order">memory ordering</a>.
4072      * <p>
4073      * This method behaves as if it returns the result of calling
4074      * {@link #fromMemorySegment(VectorSpecies,MemorySegment,long,ByteOrder,VectorMask)
4075      * fromMemorySegment()} as follows:
4076      * <pre>{@code
4077      * var m = species.maskAll(true);
4078      * return fromMemorySegment(species, ms, offset, bo, m);
4079      * }</pre>
4080      *
4081      * @param species species of desired vector
4082      * @param ms the memory segment
4083      * @param offset the offset into the memory segment
4084      * @param bo the intended byte order
4085      * @return a vector loaded from the memory segment
4086      * @throws IndexOutOfBoundsException
4087      *         if {@code offset+N*$sizeInBytes$ < 0}
4088      *         or {@code offset+N*$sizeInBytes$ >= ms.byteSize()}
4089      *         for any lane {@code N} in the vector
4090      * @throws IllegalArgumentException if the memory segment is a heap segment that is
4091      *         not backed by a {@code byte[]} array.
4092      * @throws IllegalStateException if the memory segment's session is not alive,
4093      *         or if access occurs from a thread other than the thread owning the session.
4094      * @since 19
4095      */
4096     @ForceInline
4097     public static
4098     $abstractvectortype$ fromMemorySegment(VectorSpecies<$Boxtype$> species,
4099                                            MemorySegment ms, long offset,
4100                                            ByteOrder bo) {
4101         offset = checkFromIndexSize(offset, species.vectorByteSize(), ms.byteSize());
4102         $Type$Species vsp = ($Type$Species) species;
4103         return vsp.dummyVector().fromMemorySegment0(ms, offset).maybeSwap(bo);
4104     }
4105 
4106     /**
4107      * Loads a vector from a {@linkplain MemorySegment memory segment}
4108      * starting at an offset into the memory segment
4109      * and using a mask.
4110      * Lanes where the mask is unset are filled with the default
4111      * value of {@code $type$} ({#if[FP]?positive }zero).
4112      * Bytes are composed into primitive lane elements according
4113      * to the specified byte order.
4114      * The vector is arranged into lanes according to
4115      * <a href="Vector.html#lane-order">memory ordering</a>.
4116      * <p>
4117      * The following pseudocode illustrates the behavior:
4118      * <pre>{@code
4119      * var slice = ms.asSlice(offset);
4120      * $type$[] ar = new $type$[species.length()];
4121      * for (int n = 0; n < ar.length; n++) {
4122      *     if (m.laneIsSet(n)) {
4123      *         ar[n] = slice.getAtIndex(ValuaLayout.JAVA_$TYPE$.withBitAlignment(8), n);
4124      *     }
4125      * }
4126      * $abstractvectortype$ r = $abstractvectortype$.fromArray(species, ar, 0);
4127      * }</pre>
4128      * @implNote
4129 #if[!byte]
4130      * This operation is likely to be more efficient if
4131      * the specified byte order is the same as
4132      * {@linkplain ByteOrder#nativeOrder()
4133      * the platform native order},
4134      * since this method will not need to reorder
4135      * the bytes of lane values.
4136 #else[!byte]
4137      * The byte order argument is ignored.
4138 #end[!byte]
4139      *
4140      * @param species species of desired vector
4141      * @param ms the memory segment
4142      * @param offset the offset into the memory segment
4143      * @param bo the intended byte order
4144      * @param m the mask controlling lane selection
4145      * @return a vector loaded from the memory segment
4146      * @throws IndexOutOfBoundsException
4147      *         if {@code offset+N*$sizeInBytes$ < 0}
4148      *         or {@code offset+N*$sizeInBytes$ >= ms.byteSize()}
4149      *         for any lane {@code N} in the vector
4150      *         where the mask is set
4151      * @throws IllegalArgumentException if the memory segment is a heap segment that is
4152      *         not backed by a {@code byte[]} array.
4153      * @throws IllegalStateException if the memory segment's session is not alive,
4154      *         or if access occurs from a thread other than the thread owning the session.
4155      * @since 19
4156      */
4157     @ForceInline
4158     public static
4159     $abstractvectortype$ fromMemorySegment(VectorSpecies<$Boxtype$> species,
4160                                            MemorySegment ms, long offset,
4161                                            ByteOrder bo,
4162                                            VectorMask<$Boxtype$> m) {
4163         $Type$Species vsp = ($Type$Species) species;
4164         if (offset >= 0 && offset <= (ms.byteSize() - species.vectorByteSize())) {
4165             return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
4166         }
4167 
4168         // FIXME: optimize
4169         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, ms.byteSize());
4170         return vsp.ldLongOp(ms, offset, m, $abstractvectortype$::memorySegmentGet);
4171     }
4172 
4173     // Memory store operations
4174 
4175     /**
4176      * Stores this vector into an array of type {@code $type$[]}
4177      * starting at an offset.
4178      * <p>
4179      * For each vector lane, where {@code N} is the vector lane index,
4180      * the lane element at index {@code N} is stored into the array
4181      * element {@code a[offset+N]}.
4182      *
4183      * @param a the array, of type {@code $type$[]}
4184      * @param offset the offset into the array
4185      * @throws IndexOutOfBoundsException
4186      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4187      *         for any lane {@code N} in the vector
4188      */
4189     @ForceInline
4190     public final
4191     void intoArray($type$[] a, int offset) {
4192         offset = checkFromIndexSize(offset, length(), a.length);
4193         $Type$Species vsp = vspecies();
4194         VectorSupport.store(
4195             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4196             a, arrayAddress(a, offset),
4197             this,
4198             a, offset,
4199             (arr, off, v)
4200             -> v.stOp(arr, (int) off,
4201                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4202     }
4203 
4204     /**
4205      * Stores this vector into an array of type {@code $type$[]}
4206      * starting at offset and using a mask.
4207      * <p>
4208      * For each vector lane, where {@code N} is the vector lane index,
4209      * the lane element at index {@code N} is stored into the array
4210      * element {@code a[offset+N]}.
4211      * If the mask lane at {@code N} is unset then the corresponding
4212      * array element {@code a[offset+N]} is left unchanged.
4213      * <p>
4214      * Array range checking is done for lanes where the mask is set.
4215      * Lanes where the mask is unset are not stored and do not need
4216      * to correspond to legitimate elements of {@code a}.
4217      * That is, unset lanes may correspond to array indexes less than
4218      * zero or beyond the end of the array.
4219      *
4220      * @param a the array, of type {@code $type$[]}
4221      * @param offset the offset into the array
4222      * @param m the mask controlling lane storage
4223      * @throws IndexOutOfBoundsException
4224      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4225      *         for any lane {@code N} in the vector
4226      *         where the mask is set
4227      */
4228     @ForceInline
4229     public final
4230     void intoArray($type$[] a, int offset,
4231                    VectorMask<$Boxtype$> m) {
4232         if (m.allTrue()) {
4233             intoArray(a, offset);
4234         } else {
4235             $Type$Species vsp = vspecies();
4236             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4237             intoArray0(a, offset, m);
4238         }
4239     }
4240 
4241     /**
4242      * Scatters this vector into an array of type {@code $type$[]}
4243      * using indexes obtained by adding a fixed {@code offset} to a
4244      * series of secondary offsets from an <em>index map</em>.
4245      * The index map is a contiguous sequence of {@code VLENGTH}
4246      * elements in a second array of {@code int}s, starting at a given
4247      * {@code mapOffset}.
4248      * <p>
4249      * For each vector lane, where {@code N} is the vector lane index,
4250      * the lane element at index {@code N} is stored into the array
4251      * element {@code a[f(N)]}, where {@code f(N)} is the
4252      * index mapping expression
4253      * {@code offset + indexMap[mapOffset + N]]}.
4254      *
4255      * @param a the array
4256      * @param offset an offset to combine with the index map offsets
4257      * @param indexMap the index map
4258      * @param mapOffset the offset into the index map
4259      * @throws IndexOutOfBoundsException
4260      *         if {@code mapOffset+N < 0}
4261      *         or if {@code mapOffset+N >= indexMap.length},
4262      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4263      *         is an invalid index into {@code a},
4264      *         for any lane {@code N} in the vector
4265      * @see $abstractvectortype$#toIntArray()
4266      */
4267 #if[byteOrShort]
4268     @ForceInline
4269     public final
4270     void intoArray($type$[] a, int offset,
4271                    int[] indexMap, int mapOffset) {
4272         stOp(a, offset,
4273              (arr, off, i, e) -> {
4274                  int j = indexMap[mapOffset + i];
4275                  arr[off + j] = e;
4276              });
4277     }
4278 #else[byteOrShort]
4279     @ForceInline
4280     public final
4281     void intoArray($type$[] a, int offset,
4282                    int[] indexMap, int mapOffset) {
4283         $Type$Species vsp = vspecies();
4284         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4285 #if[longOrDouble]
4286         if (vsp.laneCount() == 1) {
4287             intoArray(a, offset + indexMap[mapOffset]);
4288             return;
4289         }
4290 
4291         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4292         IntVector vix;
4293         if (isp.laneCount() != vsp.laneCount()) {
4294             // For $Type$MaxVector,  if vector length  is 2048 bits, indexShape
4295             // of $Type$ species is S_MAX_BIT. and the lane count of $Type$
4296             // vector is 32. When converting $Type$ species to int species,
4297             // indexShape is still S_MAX_BIT, but the lane count of int vector
4298             // is 64. So when loading index vector (IntVector), only lower half
4299             // of index data is needed.
4300             vix = IntVector
4301                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4302                 .add(offset);
4303         } else {
4304             vix = IntVector
4305                 .fromArray(isp, indexMap, mapOffset)
4306                 .add(offset);
4307         }
4308 
4309 #else[longOrDouble]
4310         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4311         IntVector vix = IntVector
4312             .fromArray(isp, indexMap, mapOffset)
4313             .add(offset);
4314 #end[longOrDouble]
4315 
4316         vix = VectorIntrinsics.checkIndex(vix, a.length);
4317 
4318         VectorSupport.storeWithMap(
4319             vsp.vectorType(), null, vsp.elementType(), vsp.laneCount(),
4320             isp.vectorType(),
4321             a, arrayAddress(a, 0), vix,
4322             this, null,
4323             a, offset, indexMap, mapOffset,
4324             (arr, off, v, map, mo, vm)
4325             -> v.stOp(arr, off,
4326                       (arr_, off_, i, e) -> {
4327                           int j = map[mo + i];
4328                           arr[off + j] = e;
4329                       }));
4330     }
4331 #end[byteOrShort]
4332 
4333     /**
4334      * Scatters this vector into an array of type {@code $type$[]},
4335      * under the control of a mask, and
4336      * using indexes obtained by adding a fixed {@code offset} to a
4337      * series of secondary offsets from an <em>index map</em>.
4338      * The index map is a contiguous sequence of {@code VLENGTH}
4339      * elements in a second array of {@code int}s, starting at a given
4340      * {@code mapOffset}.
4341      * <p>
4342      * For each vector lane, where {@code N} is the vector lane index,
4343      * if the mask lane at index {@code N} is set then
4344      * the lane element at index {@code N} is stored into the array
4345      * element {@code a[f(N)]}, where {@code f(N)} is the
4346      * index mapping expression
4347      * {@code offset + indexMap[mapOffset + N]]}.
4348      *
4349      * @param a the array
4350      * @param offset an offset to combine with the index map offsets
4351      * @param indexMap the index map
4352      * @param mapOffset the offset into the index map
4353      * @param m the mask
4354      * @throws IndexOutOfBoundsException
4355      *         if {@code mapOffset+N < 0}
4356      *         or if {@code mapOffset+N >= indexMap.length},
4357      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4358      *         is an invalid index into {@code a},
4359      *         for any lane {@code N} in the vector
4360      *         where the mask is set
4361      * @see $abstractvectortype$#toIntArray()
4362      */
4363 #if[byteOrShort]
4364     @ForceInline
4365     public final
4366     void intoArray($type$[] a, int offset,
4367                    int[] indexMap, int mapOffset,
4368                    VectorMask<$Boxtype$> m) {
4369         stOp(a, offset, m,
4370              (arr, off, i, e) -> {
4371                  int j = indexMap[mapOffset + i];
4372                  arr[off + j] = e;
4373              });
4374     }
4375 #else[byteOrShort]
4376     @ForceInline
4377     public final
4378     void intoArray($type$[] a, int offset,
4379                    int[] indexMap, int mapOffset,
4380                    VectorMask<$Boxtype$> m) {
4381         if (m.allTrue()) {
4382             intoArray(a, offset, indexMap, mapOffset);
4383         }
4384         else {
4385             intoArray0(a, offset, indexMap, mapOffset, m);
4386         }
4387     }
4388 #end[byteOrShort]
4389 
4390 #if[short]
4391     /**
4392      * Stores this vector into an array of type {@code char[]}
4393      * starting at an offset.
4394      * <p>
4395      * For each vector lane, where {@code N} is the vector lane index,
4396      * the lane element at index {@code N}
4397      * is first cast to a {@code char} value and then
4398      * stored into the array element {@code a[offset+N]}.
4399      *
4400      * @param a the array, of type {@code char[]}
4401      * @param offset the offset into the array
4402      * @throws IndexOutOfBoundsException
4403      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4404      *         for any lane {@code N} in the vector
4405      */
4406     @ForceInline
4407     public final
4408     void intoCharArray(char[] a, int offset) {
4409         offset = checkFromIndexSize(offset, length(), a.length);
4410         $Type$Species vsp = vspecies();
4411         VectorSupport.store(
4412             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4413             a, charArrayAddress(a, offset),
4414             this,
4415             a, offset,
4416             (arr, off, v)
4417             -> v.stOp(arr, (int) off,
4418                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
4419     }
4420 
4421     /**
4422      * Stores this vector into an array of type {@code char[]}
4423      * starting at offset and using a mask.
4424      * <p>
4425      * For each vector lane, where {@code N} is the vector lane index,
4426      * the lane element at index {@code N}
4427      * is first cast to a {@code char} value and then
4428      * stored into the array element {@code a[offset+N]}.
4429      * If the mask lane at {@code N} is unset then the corresponding
4430      * array element {@code a[offset+N]} is left unchanged.
4431      * <p>
4432      * Array range checking is done for lanes where the mask is set.
4433      * Lanes where the mask is unset are not stored and do not need
4434      * to correspond to legitimate elements of {@code a}.
4435      * That is, unset lanes may correspond to array indexes less than
4436      * zero or beyond the end of the array.
4437      *
4438      * @param a the array, of type {@code char[]}
4439      * @param offset the offset into the array
4440      * @param m the mask controlling lane storage
4441      * @throws IndexOutOfBoundsException
4442      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4443      *         for any lane {@code N} in the vector
4444      *         where the mask is set
4445      */
4446     @ForceInline
4447     public final
4448     void intoCharArray(char[] a, int offset,
4449                        VectorMask<$Boxtype$> m) {
4450         if (m.allTrue()) {
4451             intoCharArray(a, offset);
4452         } else {
4453             $Type$Species vsp = vspecies();
4454             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4455             intoCharArray0(a, offset, m);
4456         }
4457     }
4458 
4459     /**
4460      * Scatters this vector into an array of type {@code char[]}
4461      * using indexes obtained by adding a fixed {@code offset} to a
4462      * series of secondary offsets from an <em>index map</em>.
4463      * The index map is a contiguous sequence of {@code VLENGTH}
4464      * elements in a second array of {@code int}s, starting at a given
4465      * {@code mapOffset}.
4466      * <p>
4467      * For each vector lane, where {@code N} is the vector lane index,
4468      * the lane element at index {@code N}
4469      * is first cast to a {@code char} value and then
4470      * stored into the array
4471      * element {@code a[f(N)]}, where {@code f(N)} is the
4472      * index mapping expression
4473      * {@code offset + indexMap[mapOffset + N]]}.
4474      *
4475      * @param a the array
4476      * @param offset an offset to combine with the index map offsets
4477      * @param indexMap the index map
4478      * @param mapOffset the offset into the index map
4479      * @throws IndexOutOfBoundsException
4480      *         if {@code mapOffset+N < 0}
4481      *         or if {@code mapOffset+N >= indexMap.length},
4482      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4483      *         is an invalid index into {@code a},
4484      *         for any lane {@code N} in the vector
4485      * @see $abstractvectortype$#toIntArray()
4486      */
4487     @ForceInline
4488     public final
4489     void intoCharArray(char[] a, int offset,
4490                        int[] indexMap, int mapOffset) {
4491         // FIXME: optimize
4492         stOp(a, offset,
4493              (arr, off, i, e) -> {
4494                  int j = indexMap[mapOffset + i];
4495                  arr[off + j] = (char) e;
4496              });
4497     }
4498 
4499     /**
4500      * Scatters this vector into an array of type {@code char[]},
4501      * under the control of a mask, and
4502      * using indexes obtained by adding a fixed {@code offset} to a
4503      * series of secondary offsets from an <em>index map</em>.
4504      * The index map is a contiguous sequence of {@code VLENGTH}
4505      * elements in a second array of {@code int}s, starting at a given
4506      * {@code mapOffset}.
4507      * <p>
4508      * For each vector lane, where {@code N} is the vector lane index,
4509      * if the mask lane at index {@code N} is set then
4510      * the lane element at index {@code N}
4511      * is first cast to a {@code char} value and then
4512      * stored into the array
4513      * element {@code a[f(N)]}, where {@code f(N)} is the
4514      * index mapping expression
4515      * {@code offset + indexMap[mapOffset + N]]}.
4516      *
4517      * @param a the array
4518      * @param offset an offset to combine with the index map offsets
4519      * @param indexMap the index map
4520      * @param mapOffset the offset into the index map
4521      * @param m the mask
4522      * @throws IndexOutOfBoundsException
4523      *         if {@code mapOffset+N < 0}
4524      *         or if {@code mapOffset+N >= indexMap.length},
4525      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4526      *         is an invalid index into {@code a},
4527      *         for any lane {@code N} in the vector
4528      *         where the mask is set
4529      * @see $abstractvectortype$#toIntArray()
4530      */
4531     @ForceInline
4532     public final
4533     void intoCharArray(char[] a, int offset,
4534                        int[] indexMap, int mapOffset,
4535                        VectorMask<$Boxtype$> m) {
4536         // FIXME: optimize
4537         stOp(a, offset, m,
4538              (arr, off, i, e) -> {
4539                  int j = indexMap[mapOffset + i];
4540                  arr[off + j] = (char) e;
4541              });
4542     }
4543 #end[short]
4544 
4545 #if[byte]
4546     /**
4547      * Stores this vector into an array of type {@code boolean[]}
4548      * starting at an offset.
4549      * <p>
4550      * For each vector lane, where {@code N} is the vector lane index,
4551      * the lane element at index {@code N}
4552      * is first converted to a {@code boolean} value and then
4553      * stored into the array element {@code a[offset+N]}.
4554      * <p>
4555      * A {@code byte} value is converted to a {@code boolean} value by applying the
4556      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4557      *
4558      * @param a the array
4559      * @param offset the offset into the array
4560      * @throws IndexOutOfBoundsException
4561      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4562      *         for any lane {@code N} in the vector
4563      */
4564     @ForceInline
4565     public final
4566     void intoBooleanArray(boolean[] a, int offset) {
4567         offset = checkFromIndexSize(offset, length(), a.length);
4568         $Type$Species vsp = vspecies();
4569         ByteVector normalized = this.and((byte) 1);
4570         VectorSupport.store(
4571             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4572             a, booleanArrayAddress(a, offset),
4573             normalized,
4574             a, offset,
4575             (arr, off, v)
4576             -> v.stOp(arr, (int) off,
4577                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4578     }
4579 
4580     /**
4581      * Stores this vector into an array of type {@code boolean[]}
4582      * starting at offset and using a mask.
4583      * <p>
4584      * For each vector lane, where {@code N} is the vector lane index,
4585      * the lane element at index {@code N}
4586      * is first converted to a {@code boolean} value and then
4587      * stored into the array element {@code a[offset+N]}.
4588      * If the mask lane at {@code N} is unset then the corresponding
4589      * array element {@code a[offset+N]} is left unchanged.
4590      * <p>
4591      * A {@code byte} value is converted to a {@code boolean} value by applying the
4592      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4593      * <p>
4594      * Array range checking is done for lanes where the mask is set.
4595      * Lanes where the mask is unset are not stored and do not need
4596      * to correspond to legitimate elements of {@code a}.
4597      * That is, unset lanes may correspond to array indexes less than
4598      * zero or beyond the end of the array.
4599      *
4600      * @param a the array
4601      * @param offset the offset into the array
4602      * @param m the mask controlling lane storage
4603      * @throws IndexOutOfBoundsException
4604      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4605      *         for any lane {@code N} in the vector
4606      *         where the mask is set
4607      */
4608     @ForceInline
4609     public final
4610     void intoBooleanArray(boolean[] a, int offset,
4611                           VectorMask<$Boxtype$> m) {
4612         if (m.allTrue()) {
4613             intoBooleanArray(a, offset);
4614         } else {
4615             $Type$Species vsp = vspecies();
4616             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4617             intoBooleanArray0(a, offset, m);
4618         }
4619     }
4620 
4621     /**
4622      * Scatters this vector into an array of type {@code boolean[]}
4623      * using indexes obtained by adding a fixed {@code offset} to a
4624      * series of secondary offsets from an <em>index map</em>.
4625      * The index map is a contiguous sequence of {@code VLENGTH}
4626      * elements in a second array of {@code int}s, starting at a given
4627      * {@code mapOffset}.
4628      * <p>
4629      * For each vector lane, where {@code N} is the vector lane index,
4630      * the lane element at index {@code N}
4631      * is first converted to a {@code boolean} value and then
4632      * stored into the array
4633      * element {@code a[f(N)]}, where {@code f(N)} is the
4634      * index mapping expression
4635      * {@code offset + indexMap[mapOffset + N]]}.
4636      * <p>
4637      * A {@code byte} value is converted to a {@code boolean} value by applying the
4638      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4639      *
4640      * @param a the array
4641      * @param offset an offset to combine with the index map offsets
4642      * @param indexMap the index map
4643      * @param mapOffset the offset into the index map
4644      * @throws IndexOutOfBoundsException
4645      *         if {@code mapOffset+N < 0}
4646      *         or if {@code mapOffset+N >= indexMap.length},
4647      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4648      *         is an invalid index into {@code a},
4649      *         for any lane {@code N} in the vector
4650      * @see $abstractvectortype$#toIntArray()
4651      */
4652     @ForceInline
4653     public final
4654     void intoBooleanArray(boolean[] a, int offset,
4655                           int[] indexMap, int mapOffset) {
4656         // FIXME: optimize
4657         stOp(a, offset,
4658              (arr, off, i, e) -> {
4659                  int j = indexMap[mapOffset + i];
4660                  arr[off + j] = (e & 1) != 0;
4661              });
4662     }
4663 
4664     /**
4665      * Scatters this vector into an array of type {@code boolean[]},
4666      * under the control of a mask, and
4667      * using indexes obtained by adding a fixed {@code offset} to a
4668      * series of secondary offsets from an <em>index map</em>.
4669      * The index map is a contiguous sequence of {@code VLENGTH}
4670      * elements in a second array of {@code int}s, starting at a given
4671      * {@code mapOffset}.
4672      * <p>
4673      * For each vector lane, where {@code N} is the vector lane index,
4674      * if the mask lane at index {@code N} is set then
4675      * the lane element at index {@code N}
4676      * is first converted to a {@code boolean} value and then
4677      * stored into the array
4678      * element {@code a[f(N)]}, where {@code f(N)} is the
4679      * index mapping expression
4680      * {@code offset + indexMap[mapOffset + N]]}.
4681      * <p>
4682      * A {@code byte} value is converted to a {@code boolean} value by applying the
4683      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4684      *
4685      * @param a the array
4686      * @param offset an offset to combine with the index map offsets
4687      * @param indexMap the index map
4688      * @param mapOffset the offset into the index map
4689      * @param m the mask
4690      * @throws IndexOutOfBoundsException
4691      *         if {@code mapOffset+N < 0}
4692      *         or if {@code mapOffset+N >= indexMap.length},
4693      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4694      *         is an invalid index into {@code a},
4695      *         for any lane {@code N} in the vector
4696      *         where the mask is set
4697      * @see $abstractvectortype$#toIntArray()
4698      */
4699     @ForceInline
4700     public final
4701     void intoBooleanArray(boolean[] a, int offset,
4702                           int[] indexMap, int mapOffset,
4703                           VectorMask<$Boxtype$> m) {
4704         // FIXME: optimize
4705         stOp(a, offset, m,
4706              (arr, off, i, e) -> {
4707                  int j = indexMap[mapOffset + i];
4708                  arr[off + j] = (e & 1) != 0;
4709              });
4710     }
4711 #end[byte]
4712 
4713     /**
4714      * {@inheritDoc} <!--workaround-->
4715      * @since 19
4716      */
4717     @Override
4718     @ForceInline
4719     public final
4720     void intoMemorySegment(MemorySegment ms, long offset,
4721                            ByteOrder bo) {
4722         if (ms.isReadOnly()) {
4723             throw new UnsupportedOperationException("Attempt to write a read-only segment");
4724         }
4725 
4726         offset = checkFromIndexSize(offset, byteSize(), ms.byteSize());
4727         maybeSwap(bo).intoMemorySegment0(ms, offset);
4728     }
4729 
4730     /**
4731      * {@inheritDoc} <!--workaround-->
4732      * @since 19
4733      */
4734     @Override
4735     @ForceInline
4736     public final
4737     void intoMemorySegment(MemorySegment ms, long offset,
4738                            ByteOrder bo,
4739                            VectorMask<$Boxtype$> m) {
4740         if (m.allTrue()) {
4741             intoMemorySegment(ms, offset, bo);
4742         } else {
4743             if (ms.isReadOnly()) {
4744                 throw new UnsupportedOperationException("Attempt to write a read-only segment");
4745             }
4746             $Type$Species vsp = vspecies();
4747             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, ms.byteSize());
4748             maybeSwap(bo).intoMemorySegment0(ms, offset, m);
4749         }
4750     }
4751 
4752     // ================================================
4753 
4754     // Low-level memory operations.
4755     //
4756     // Note that all of these operations *must* inline into a context
4757     // where the exact species of the involved vector is a
4758     // compile-time constant.  Otherwise, the intrinsic generation
4759     // will fail and performance will suffer.
4760     //
4761     // In many cases this is achieved by re-deriving a version of the
4762     // method in each concrete subclass (per species).  The re-derived
4763     // method simply calls one of these generic methods, with exact
4764     // parameters for the controlling metadata, which is either a
4765     // typed vector or constant species instance.
4766 
4767     // Unchecked loading operations in native byte order.
4768     // Caller is responsible for applying index checks, masking, and
4769     // byte swapping.
4770 
4771     /*package-private*/
4772     abstract
4773     $abstractvectortype$ fromArray0($type$[] a, int offset);
4774     @ForceInline
4775     final
4776     $abstractvectortype$ fromArray0Template($type$[] a, int offset) {
4777         $Type$Species vsp = vspecies();
4778         return VectorSupport.load(
4779             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4780             a, arrayAddress(a, offset),
4781             a, offset, vsp,
4782             (arr, off, s) -> s.ldOp(arr, (int) off,
4783                                     (arr_, off_, i) -> arr_[off_ + i]));
4784     }
4785 
4786     /*package-private*/
4787     abstract
4788     $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4789     @ForceInline
4790     final
4791     <M extends VectorMask<$Boxtype$>>
4792     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4793         m.check(species());
4794         $Type$Species vsp = vspecies();
4795         return VectorSupport.loadMasked(
4796             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4797             a, arrayAddress(a, offset), m,
4798             a, offset, vsp,
4799             (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
4800                                         (arr_, off_, i) -> arr_[off_ + i]));
4801     }
4802 
4803 #if[!byteOrShort]
4804     /*package-private*/
4805     abstract
4806     $abstractvectortype$ fromArray0($type$[] a, int offset,
4807                                     int[] indexMap, int mapOffset,
4808                                     VectorMask<$Boxtype$> m);
4809     @ForceInline
4810     final
4811     <M extends VectorMask<$Boxtype$>>
4812     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset,
4813                                             int[] indexMap, int mapOffset, M m) {
4814         $Type$Species vsp = vspecies();
4815         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4816         Objects.requireNonNull(a);
4817         Objects.requireNonNull(indexMap);
4818         m.check(vsp);
4819         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();
4820 
4821 #if[longOrDouble]
4822         if (vsp.laneCount() == 1) {
4823           return $abstractvectortype$.fromArray(vsp, a, offset + indexMap[mapOffset], m);
4824         }
4825 
4826         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
4827         IntVector vix;
4828         if (isp.laneCount() != vsp.laneCount()) {
4829             // For $Type$MaxVector,  if vector length is non-power-of-two or
4830             // 2048 bits, indexShape of $Type$ species is S_MAX_BIT.
4831             // Assume that vector length is 2048, then the lane count of $Type$
4832             // vector is 32. When converting $Type$ species to int species,
4833             // indexShape is still S_MAX_BIT, but the lane count of int vector
4834             // is 64. So when loading index vector (IntVector), only lower half
4835             // of index data is needed.
4836             vix = IntVector
4837                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4838                 .add(offset);
4839         } else {
4840             vix = IntVector
4841                 .fromArray(isp, indexMap, mapOffset)
4842                 .add(offset);
4843         }
4844 #else[longOrDouble]
4845         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
4846         IntVector vix = IntVector
4847             .fromArray(isp, indexMap, mapOffset)
4848             .add(offset);
4849 #end[longOrDouble]
4850 
4851         // FIXME: Check index under mask controlling.
4852         vix = VectorIntrinsics.checkIndex(vix, a.length);
4853 
4854         return VectorSupport.loadWithMap(
4855             vectorType, maskClass, $type$.class, vsp.laneCount(),
4856             isp.vectorType(),
4857             a, ARRAY_BASE, vix, m,
4858             a, offset, indexMap, mapOffset, vsp,
4859             (c, idx, iMap, idy, s, vm) ->
4860             s.vOp(vm, n -> c[idx + iMap[idy+n]]));
4861     }
4862 #end[!byteOrShort]
4863 
4864 #if[short]
4865     /*package-private*/
4866     abstract
4867     $abstractvectortype$ fromCharArray0(char[] a, int offset);
4868     @ForceInline
4869     final
4870     $abstractvectortype$ fromCharArray0Template(char[] a, int offset) {
4871         $Type$Species vsp = vspecies();
4872         return VectorSupport.load(
4873             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4874             a, charArrayAddress(a, offset),
4875             a, offset, vsp,
4876             (arr, off, s) -> s.ldOp(arr, (int) off,
4877                                     (arr_, off_, i) -> (short) arr_[off_ + i]));
4878     }
4879 
4880     /*package-private*/
4881     abstract
4882     $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
4883     @ForceInline
4884     final
4885     <M extends VectorMask<$Boxtype$>>
4886     $abstractvectortype$ fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
4887         m.check(species());
4888         $Type$Species vsp = vspecies();
4889         return VectorSupport.loadMasked(
4890                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4891                 a, charArrayAddress(a, offset), m,
4892                 a, offset, vsp,
4893                 (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
4894                                             (arr_, off_, i) -> (short) arr_[off_ + i]));
4895     }
4896 #end[short]
4897 
4898 #if[byte]
4899     /*package-private*/
4900     abstract
4901     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset);
4902     @ForceInline
4903     final
4904     $abstractvectortype$ fromBooleanArray0Template(boolean[] a, int offset) {
4905         $Type$Species vsp = vspecies();
4906         return VectorSupport.load(
4907             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4908             a, booleanArrayAddress(a, offset),
4909             a, offset, vsp,
4910             (arr, off, s) -> s.ldOp(arr, (int) off,
4911                                     (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4912     }
4913 
4914     /*package-private*/
4915     abstract
4916     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4917     @ForceInline
4918     final
4919     <M extends VectorMask<$Boxtype$>>
4920     $abstractvectortype$ fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4921         m.check(species());
4922         $Type$Species vsp = vspecies();
4923         return VectorSupport.loadMasked(
4924             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4925             a, booleanArrayAddress(a, offset), m,
4926             a, offset, vsp,
4927             (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
4928                                         (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4929     }
4930 #end[byte]
4931 
4932     abstract
4933     $abstractvectortype$ fromMemorySegment0(MemorySegment bb, long offset);
4934     @ForceInline
4935     final
4936     $abstractvectortype$ fromMemorySegment0Template(MemorySegment ms, long offset) {
4937         $Type$Species vsp = vspecies();
4938         return ScopedMemoryAccess.loadFromMemorySegment(
4939                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4940                 (MemorySegmentProxy) ms, offset, vsp,
4941                 (msp, off, s) -> {
4942                     return s.ldLongOp((MemorySegment) msp, off, $abstractvectortype$::memorySegmentGet);
4943                 });
4944     }
4945 
4946     abstract
4947     $abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m);
4948     @ForceInline
4949     final
4950     <M extends VectorMask<$Boxtype$>>
4951     $abstractvectortype$ fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
4952         $Type$Species vsp = vspecies();
4953         m.check(vsp);
4954         return ScopedMemoryAccess.loadFromMemorySegmentMasked(
4955                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4956                 (MemorySegmentProxy) ms, offset, m, vsp,
4957                 (msp, off, s, vm) -> {
4958                     return s.ldLongOp((MemorySegment) msp, off, vm, $abstractvectortype$::memorySegmentGet);
4959                 });
4960     }
4961 
4962     // Unchecked storing operations in native byte order.
4963     // Caller is responsible for applying index checks, masking, and
4964     // byte swapping.
4965 
4966     abstract
4967     void intoArray0($type$[] a, int offset);
4968     @ForceInline
4969     final
4970     void intoArray0Template($type$[] a, int offset) {
4971         $Type$Species vsp = vspecies();
4972         VectorSupport.store(
4973             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4974             a, arrayAddress(a, offset),
4975             this, a, offset,
4976             (arr, off, v)
4977             -> v.stOp(arr, (int) off,
4978                       (arr_, off_, i, e) -> arr_[off_+i] = e));
4979     }
4980 
4981     abstract
4982     void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4983     @ForceInline
4984     final
4985     <M extends VectorMask<$Boxtype$>>
4986     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4987         m.check(species());
4988         $Type$Species vsp = vspecies();
4989         VectorSupport.storeMasked(
4990             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4991             a, arrayAddress(a, offset),
4992             this, m, a, offset,
4993             (arr, off, v, vm)
4994             -> v.stOp(arr, (int) off, vm,
4995                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4996     }
4997 
4998 #if[!byteOrShort]
4999     abstract
5000     void intoArray0($type$[] a, int offset,
5001                     int[] indexMap, int mapOffset,
5002                     VectorMask<$Boxtype$> m);
5003     @ForceInline
5004     final
5005     <M extends VectorMask<$Boxtype$>>
5006     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset,
5007                             int[] indexMap, int mapOffset, M m) {
5008         m.check(species());
5009         $Type$Species vsp = vspecies();
5010         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
5011 #if[longOrDouble]
5012         if (vsp.laneCount() == 1) {
5013             intoArray(a, offset + indexMap[mapOffset], m);
5014             return;
5015         }
5016 
5017         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
5018         IntVector vix;
5019         if (isp.laneCount() != vsp.laneCount()) {
5020             // For $Type$MaxVector,  if vector length  is 2048 bits, indexShape
5021             // of $Type$ species is S_MAX_BIT. and the lane count of $Type$
5022             // vector is 32. When converting $Type$ species to int species,
5023             // indexShape is still S_MAX_BIT, but the lane count of int vector
5024             // is 64. So when loading index vector (IntVector), only lower half
5025             // of index data is needed.
5026             vix = IntVector
5027                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
5028                 .add(offset);
5029         } else {
5030             vix = IntVector
5031                 .fromArray(isp, indexMap, mapOffset)
5032                 .add(offset);
5033         }
5034 
5035 #else[longOrDouble]
5036         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
5037         IntVector vix = IntVector
5038             .fromArray(isp, indexMap, mapOffset)
5039             .add(offset);
5040 #end[longOrDouble]
5041 
5042         // FIXME: Check index under mask controlling.
5043         vix = VectorIntrinsics.checkIndex(vix, a.length);
5044 
5045         VectorSupport.storeWithMap(
5046             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5047             isp.vectorType(),
5048             a, arrayAddress(a, 0), vix,
5049             this, m,
5050             a, offset, indexMap, mapOffset,
5051             (arr, off, v, map, mo, vm)
5052             -> v.stOp(arr, off, vm,
5053                       (arr_, off_, i, e) -> {
5054                           int j = map[mo + i];
5055                           arr[off + j] = e;
5056                       }));
5057     }
5058 #end[!byteOrShort]
5059 
5060 #if[byte]
5061     abstract
5062     void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
5063     @ForceInline
5064     final
5065     <M extends VectorMask<$Boxtype$>>
5066     void intoBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
5067         m.check(species());
5068         $Type$Species vsp = vspecies();
5069         ByteVector normalized = this.and((byte) 1);
5070         VectorSupport.storeMasked(
5071             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5072             a, booleanArrayAddress(a, offset),
5073             normalized, m, a, offset,
5074             (arr, off, v, vm)
5075             -> v.stOp(arr, (int) off, vm,
5076                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
5077     }
5078 #end[byte]
5079 
5080     @ForceInline
5081     final
5082     void intoMemorySegment0(MemorySegment ms, long offset) {
5083         $Type$Species vsp = vspecies();
5084         ScopedMemoryAccess.storeIntoMemorySegment(
5085                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
5086                 this,
5087                 (MemorySegmentProxy) ms, offset,
5088                 (msp, off, v) -> {
5089                     v.stLongOp((MemorySegment) msp, off, $abstractvectortype$::memorySegmentSet);
5090                 });
5091     }
5092 
5093     abstract
5094     void intoMemorySegment0(MemorySegment bb, long offset, VectorMask<$Boxtype$> m);
5095     @ForceInline
5096     final
5097     <M extends VectorMask<$Boxtype$>>
5098     void intoMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
5099         $Type$Species vsp = vspecies();
5100         m.check(vsp);
5101         ScopedMemoryAccess.storeIntoMemorySegmentMasked(
5102                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5103                 this, m,
5104                 (MemorySegmentProxy) ms, offset,
5105                 (msp, off, v, vm) -> {
5106                     v.stLongOp((MemorySegment) msp, off, vm, $abstractvectortype$::memorySegmentSet);
5107                 });
5108     }
5109 
5110 #if[short]
5111     /*package-private*/
5112     abstract
5113     void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
5114     @ForceInline
5115     final
5116     <M extends VectorMask<$Boxtype$>>
5117     void intoCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
5118         m.check(species());
5119         $Type$Species vsp = vspecies();
5120         VectorSupport.storeMasked(
5121             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5122             a, charArrayAddress(a, offset),
5123             this, m, a, offset,
5124             (arr, off, v, vm)
5125             -> v.stOp(arr, (int) off, vm,
5126                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
5127     }
5128 #end[short]
5129 
5130     // End of low-level memory operations.
5131 
5132     private static
5133     void checkMaskFromIndexSize(int offset,
5134                                 $Type$Species vsp,
5135                                 VectorMask<$Boxtype$> m,
5136                                 int scale,
5137                                 int limit) {
5138         ((AbstractMask<$Boxtype$>)m)
5139             .checkIndexByLane(offset, limit, vsp.iota(), scale);
5140     }
5141 
5142     private static
5143     void checkMaskFromIndexSize(long offset,
5144                                 $Type$Species vsp,
5145                                 VectorMask<$Boxtype$> m,
5146                                 int scale,
5147                                 long limit) {
5148         ((AbstractMask<$Boxtype$>)m)
5149             .checkIndexByLane(offset, limit, vsp.iota(), scale);
5150     }
5151 
5152     @ForceInline
5153     private void conditionalStoreNYI(int offset,
5154                                      $Type$Species vsp,
5155                                      VectorMask<$Boxtype$> m,
5156                                      int scale,
5157                                      int limit) {
5158         if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
5159             String msg =
5160                 String.format("unimplemented: store @%d in [0..%d), %s in %s",
5161                               offset, limit, m, vsp);
5162             throw new AssertionError(msg);
5163         }
5164     }
5165 
5166     /*package-private*/
5167     @Override
5168     @ForceInline
5169     final
5170     $abstractvectortype$ maybeSwap(ByteOrder bo) {
5171 #if[!byte]
5172         if (bo != NATIVE_ENDIAN) {
5173             return this.reinterpretAsBytes()
5174                 .rearrange(swapBytesShuffle())
5175                 .reinterpretAs$Type$s();
5176         }
5177 #end[!byte]
5178         return this;
5179     }
5180 
5181     static final int ARRAY_SHIFT =
5182         31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_$TYPE$_INDEX_SCALE);
5183     static final long ARRAY_BASE =
5184         Unsafe.ARRAY_$TYPE$_BASE_OFFSET;
5185 
5186     @ForceInline
5187     static long arrayAddress($type$[] a, int index) {
5188         return ARRAY_BASE + (((long)index) << ARRAY_SHIFT);
5189     }
5190 
5191 #if[short]
5192     static final int ARRAY_CHAR_SHIFT =
5193             31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_CHAR_INDEX_SCALE);
5194     static final long ARRAY_CHAR_BASE =
5195             Unsafe.ARRAY_CHAR_BASE_OFFSET;
5196 
5197     @ForceInline
5198     static long charArrayAddress(char[] a, int index) {
5199         return ARRAY_CHAR_BASE + (((long)index) << ARRAY_CHAR_SHIFT);
5200     }
5201 #end[short]
5202 
5203 #if[byte]
5204     static final int ARRAY_BOOLEAN_SHIFT =
5205             31 - Integer.numberOfLeadingZeros(Unsafe.ARRAY_BOOLEAN_INDEX_SCALE);
5206     static final long ARRAY_BOOLEAN_BASE =
5207             Unsafe.ARRAY_BOOLEAN_BASE_OFFSET;
5208 
5209     @ForceInline
5210     static long booleanArrayAddress(boolean[] a, int index) {
5211         return ARRAY_BOOLEAN_BASE + (((long)index) << ARRAY_BOOLEAN_SHIFT);
5212     }
5213 #end[byte]
5214 
5215     @ForceInline
5216     static long byteArrayAddress(byte[] a, int index) {
5217         return Unsafe.ARRAY_BYTE_BASE_OFFSET + index;
5218     }
5219 
5220     // ================================================
5221 
5222     /// Reinterpreting view methods:
5223     //   lanewise reinterpret: viewAsXVector()
5224     //   keep shape, redraw lanes: reinterpretAsEs()
5225 
5226     /**
5227      * {@inheritDoc} <!--workaround-->
5228      */
5229     @ForceInline
5230     @Override
5231     public final ByteVector reinterpretAsBytes() {
5232 #if[byte]
5233         return this;
5234 #else[byte]
5235          // Going to ByteVector, pay close attention to byte order.
5236          assert(REGISTER_ENDIAN == ByteOrder.LITTLE_ENDIAN);
5237          return asByteVectorRaw();
5238          //return asByteVectorRaw().rearrange(swapBytesShuffle());
5239 #end[byte]
5240     }
5241 
5242     /**
5243      * {@inheritDoc} <!--workaround-->
5244      */
5245     @ForceInline
5246     @Override
5247     public final $Bitstype$Vector viewAsIntegralLanes() {
5248 #if[BITWISE]
5249         return this;
5250 #else[BITWISE]
5251         LaneType ilt = LaneType.$TYPE$.asIntegral();
5252         return ($Bitstype$Vector) asVectorRaw(ilt);
5253 #end[BITWISE]
5254     }
5255 
5256     /**
5257      * {@inheritDoc} <!--workaround-->
5258 #if[byteOrShort]
5259      *
5260      * @implNote This method always throws
5261      * {@code UnsupportedOperationException}, because there is no floating
5262      * point type of the same size as {@code $type$}.  The return type
5263      * of this method is arbitrarily designated as
5264      * {@code Vector<?>}.  Future versions of this API may change the return
5265      * type if additional floating point types become available.
5266 #end[byteOrShort]
5267      */
5268     @ForceInline
5269     @Override
5270     public final
5271     {#if[byteOrShort]?Vector<?>:$Fptype$Vector}
5272     viewAsFloatingLanes() {
5273 #if[FP]
5274         return this;
5275 #else[FP]
5276         LaneType flt = LaneType.$TYPE$.asFloating();
5277 #if[!byteOrShort]
5278         return ($Fptype$Vector) asVectorRaw(flt);
5279 #else[!byteOrShort]
5280         // asFloating() will throw UnsupportedOperationException for the unsupported type $type$
5281         throw new AssertionError("Cannot reach here");
5282 #end[!byteOrShort]
5283 #end[FP]
5284     }
5285 
5286     // ================================================
5287 
5288     /// Object methods: toString, equals, hashCode
5289     //
5290     // Object methods are defined as if via Arrays.toString, etc.,
5291     // is applied to the array of elements.  Two equal vectors
5292     // are required to have equal species and equal lane values.
5293 
5294     /**
5295      * Returns a string representation of this vector, of the form
5296      * {@code "[0,1,2...]"}, reporting the lane values of this vector,
5297      * in lane order.
5298      *
5299      * The string is produced as if by a call to {@link
5300      * java.util.Arrays#toString($type$[]) Arrays.toString()},
5301      * as appropriate to the {@code $type$} array returned by
5302      * {@link #toArray this.toArray()}.
5303      *
5304      * @return a string of the form {@code "[0,1,2...]"}
5305      * reporting the lane values of this vector
5306      */
5307     @Override
5308     @ForceInline
5309     public final
5310     String toString() {
5311         // now that toArray is strongly typed, we can define this
5312         return Arrays.toString(toArray());
5313     }
5314 
5315     /**
5316      * {@inheritDoc} <!--workaround-->
5317      */
5318     @Override
5319     @ForceInline
5320     public final
5321     boolean equals(Object obj) {
5322         if (obj instanceof Vector) {
5323             Vector<?> that = (Vector<?>) obj;
5324             if (this.species().equals(that.species())) {
5325                 return this.eq(that.check(this.species())).allTrue();
5326             }
5327         }
5328         return false;
5329     }
5330 
5331     /**
5332      * {@inheritDoc} <!--workaround-->
5333      */
5334     @Override
5335     @ForceInline
5336     public final
5337     int hashCode() {
5338         // now that toArray is strongly typed, we can define this
5339         return Objects.hash(species(), Arrays.hashCode(toArray()));
5340     }
5341 
5342     // ================================================
5343 
5344     // Species
5345 
5346     /**
5347      * Class representing {@link $abstractvectortype$}'s of the same {@link VectorShape VectorShape}.
5348      */
5349     /*package-private*/
5350     static final class $Type$Species extends AbstractSpecies<$Boxtype$> {
5351         private $Type$Species(VectorShape shape,
5352                 Class<? extends $abstractvectortype$> vectorType,
5353                 Class<? extends AbstractMask<$Boxtype$>> maskType,
5354                 Function<Object, $abstractvectortype$> vectorFactory) {
5355             super(shape, LaneType.of($type$.class),
5356                   vectorType, maskType,
5357                   vectorFactory);
5358             assert(this.elementSize() == $Boxtype$.SIZE);
5359         }
5360 
5361         // Specializing overrides:
5362 
5363         @Override
5364         @ForceInline
5365         public final Class<$Boxtype$> elementType() {
5366             return $type$.class;
5367         }
5368 
5369         @Override
5370         @ForceInline
5371         final Class<$Boxtype$> genericElementType() {
5372             return $Boxtype$.class;
5373         }
5374 
5375         @SuppressWarnings("unchecked")
5376         @Override
5377         @ForceInline
5378         public final Class<? extends $Type$Vector> vectorType() {
5379             return (Class<? extends $Type$Vector>) vectorType;
5380         }
5381 
5382         @Override
5383         @ForceInline
5384         public final long checkValue(long e) {
5385             longToElementBits(e);  // only for exception
5386             return e;
5387         }
5388 
5389         /*package-private*/
5390         @Override
5391         @ForceInline
5392         final $abstractvectortype$ broadcastBits(long bits) {
5393             return ($abstractvectortype$)
5394                 VectorSupport.fromBitsCoerced(
5395                     vectorType, $type$.class, laneCount,
5396                     bits, MODE_BROADCAST, this,
5397                     (bits_, s_) -> s_.rvOp(i -> bits_));
5398         }
5399 
5400         /*package-private*/
5401         @ForceInline
5402         {#if[long]?public }final $abstractvectortype$ broadcast($type$ e) {
5403             return broadcastBits(toBits(e));
5404         }
5405 
5406 #if[!long]
5407         @Override
5408         @ForceInline
5409         public final $abstractvectortype$ broadcast(long e) {
5410             return broadcastBits(longToElementBits(e));
5411         }
5412 #end[!long]
5413 
5414         /*package-private*/
5415         final @Override
5416         @ForceInline
5417         long longToElementBits(long value) {
5418 #if[long]
5419             // In this case, the conversion can never fail.
5420             return value;
5421 #else[long]
5422             // Do the conversion, and then test it for failure.
5423             $type$ e = ($type$) value;
5424             if ((long) e != value) {
5425                 throw badElementBits(value, e);
5426             }
5427             return toBits(e);
5428 #end[long]
5429         }
5430 
5431         /*package-private*/
5432         @ForceInline
5433         static long toIntegralChecked($type$ e, boolean convertToInt) {
5434             long value = convertToInt ? (int) e : (long) e;
5435             if (($type$) value != e) {
5436                 throw badArrayBits(e, convertToInt, value);
5437             }
5438             return value;
5439         }
5440 
5441         /* this non-public one is for internal conversions */
5442         @Override
5443         @ForceInline
5444         final $abstractvectortype$ fromIntValues(int[] values) {
5445             VectorIntrinsics.requireLength(values.length, laneCount);
5446             $type$[] va = new $type$[laneCount()];
5447             for (int i = 0; i < va.length; i++) {
5448                 int lv = values[i];
5449                 $type$ v = ($type$) lv;
5450                 va[i] = v;
5451                 if ((int)v != lv) {
5452                     throw badElementBits(lv, v);
5453                 }
5454             }
5455             return dummyVector().fromArray0(va, 0);
5456         }
5457 
5458         // Virtual constructors
5459 
5460         @ForceInline
5461         @Override final
5462         public $abstractvectortype$ fromArray(Object a, int offset) {
5463             // User entry point:  Be careful with inputs.
5464             return $abstractvectortype$
5465                 .fromArray(this, ($type$[]) a, offset);
5466         }
5467 
5468         @ForceInline
5469         @Override final
5470         $abstractvectortype$ dummyVector() {
5471             return ($abstractvectortype$) super.dummyVector();
5472         }
5473 
5474         /*package-private*/
5475         final @Override
5476         @ForceInline
5477         $abstractvectortype$ rvOp(RVOp f) {
5478             $type$[] res = new $type$[laneCount()];
5479             for (int i = 0; i < res.length; i++) {
5480                 $bitstype$ bits = {#if[!long]?($bitstype$)} f.apply(i);
5481                 res[i] = fromBits(bits);
5482             }
5483             return dummyVector().vectorFactory(res);
5484         }
5485 
5486         $Type$Vector vOp(FVOp f) {
5487             $type$[] res = new $type$[laneCount()];
5488             for (int i = 0; i < res.length; i++) {
5489                 res[i] = f.apply(i);
5490             }
5491             return dummyVector().vectorFactory(res);
5492         }
5493 
5494         $Type$Vector vOp(VectorMask<$Boxtype$> m, FVOp f) {
5495             $type$[] res = new $type$[laneCount()];
5496             boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
5497             for (int i = 0; i < res.length; i++) {
5498                 if (mbits[i]) {
5499                     res[i] = f.apply(i);
5500                 }
5501             }
5502             return dummyVector().vectorFactory(res);
5503         }
5504 
5505         /*package-private*/
5506         @ForceInline
5507         <M> $abstractvectortype$ ldOp(M memory, int offset,
5508                                       FLdOp<M> f) {
5509             return dummyVector().ldOp(memory, offset, f);
5510         }
5511 
5512         /*package-private*/
5513         @ForceInline
5514         <M> $abstractvectortype$ ldOp(M memory, int offset,
5515                                       VectorMask<$Boxtype$> m,
5516                                       FLdOp<M> f) {
5517             return dummyVector().ldOp(memory, offset, m, f);
5518         }
5519 
5520         /*package-private*/
5521         @ForceInline
5522         $abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
5523                                       FLdLongOp f) {
5524             return dummyVector().ldLongOp(memory, offset, f);
5525         }
5526 
5527         /*package-private*/
5528         @ForceInline
5529         $abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
5530                                       VectorMask<$Boxtype$> m,
5531                                       FLdLongOp f) {
5532             return dummyVector().ldLongOp(memory, offset, m, f);
5533         }
5534 
5535         /*package-private*/
5536         @ForceInline
5537         <M> void stOp(M memory, int offset, FStOp<M> f) {
5538             dummyVector().stOp(memory, offset, f);
5539         }
5540 
5541         /*package-private*/
5542         @ForceInline
5543         <M> void stOp(M memory, int offset,
5544                       AbstractMask<$Boxtype$> m,
5545                       FStOp<M> f) {
5546             dummyVector().stOp(memory, offset, m, f);
5547         }
5548 
5549         /*package-private*/
5550         @ForceInline
5551         void stLongOp(MemorySegment memory, long offset, FStLongOp f) {
5552             dummyVector().stLongOp(memory, offset, f);
5553         }
5554 
5555         /*package-private*/
5556         @ForceInline
5557         void stLongOp(MemorySegment memory, long offset,
5558                       AbstractMask<$Boxtype$> m,
5559                       FStLongOp f) {
5560             dummyVector().stLongOp(memory, offset, m, f);
5561         }
5562 
5563         // N.B. Make sure these constant vectors and
5564         // masks load up correctly into registers.
5565         //
5566         // Also, see if we can avoid all that switching.
5567         // Could we cache both vectors and both masks in
5568         // this species object?
5569 
5570         // Zero and iota vector access
5571         @Override
5572         @ForceInline
5573         public final $abstractvectortype$ zero() {
5574             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5575                 return $Type$MaxVector.ZERO;
5576             switch (vectorBitSize()) {
5577                 case 64: return $Type$64Vector.ZERO;
5578                 case 128: return $Type$128Vector.ZERO;
5579                 case 256: return $Type$256Vector.ZERO;
5580                 case 512: return $Type$512Vector.ZERO;
5581             }
5582             throw new AssertionError();
5583         }
5584 
5585         @Override
5586         @ForceInline
5587         public final $abstractvectortype$ iota() {
5588             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5589                 return $Type$MaxVector.IOTA;
5590             switch (vectorBitSize()) {
5591                 case 64: return $Type$64Vector.IOTA;
5592                 case 128: return $Type$128Vector.IOTA;
5593                 case 256: return $Type$256Vector.IOTA;
5594                 case 512: return $Type$512Vector.IOTA;
5595             }
5596             throw new AssertionError();
5597         }
5598 
5599         // Mask access
5600         @Override
5601         @ForceInline
5602         public final VectorMask<$Boxtype$> maskAll(boolean bit) {
5603             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5604                 return $Type$MaxVector.$Type$MaxMask.maskAll(bit);
5605             switch (vectorBitSize()) {
5606                 case 64: return $Type$64Vector.$Type$64Mask.maskAll(bit);
5607                 case 128: return $Type$128Vector.$Type$128Mask.maskAll(bit);
5608                 case 256: return $Type$256Vector.$Type$256Mask.maskAll(bit);
5609                 case 512: return $Type$512Vector.$Type$512Mask.maskAll(bit);
5610             }
5611             throw new AssertionError();
5612         }
5613     }
5614 
5615     /**
5616      * Finds a species for an element type of {@code $type$} and shape.
5617      *
5618      * @param s the shape
5619      * @return a species for an element type of {@code $type$} and shape
5620      * @throws IllegalArgumentException if no such species exists for the shape
5621      */
5622     static $Type$Species species(VectorShape s) {
5623         Objects.requireNonNull(s);
5624         switch (s.switchKey) {
5625             case VectorShape.SK_64_BIT: return ($Type$Species) SPECIES_64;
5626             case VectorShape.SK_128_BIT: return ($Type$Species) SPECIES_128;
5627             case VectorShape.SK_256_BIT: return ($Type$Species) SPECIES_256;
5628             case VectorShape.SK_512_BIT: return ($Type$Species) SPECIES_512;
5629             case VectorShape.SK_Max_BIT: return ($Type$Species) SPECIES_MAX;
5630             default: throw new IllegalArgumentException("Bad shape: " + s);
5631         }
5632     }
5633 
5634     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_64_BIT VectorShape.S_64_BIT}. */
5635     public static final VectorSpecies<$Boxtype$> SPECIES_64
5636         = new $Type$Species(VectorShape.S_64_BIT,
5637                             $Type$64Vector.class,
5638                             $Type$64Vector.$Type$64Mask.class,
5639                             $Type$64Vector::new);
5640 
5641     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
5642     public static final VectorSpecies<$Boxtype$> SPECIES_128
5643         = new $Type$Species(VectorShape.S_128_BIT,
5644                             $Type$128Vector.class,
5645                             $Type$128Vector.$Type$128Mask.class,
5646                             $Type$128Vector::new);
5647 
5648     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
5649     public static final VectorSpecies<$Boxtype$> SPECIES_256
5650         = new $Type$Species(VectorShape.S_256_BIT,
5651                             $Type$256Vector.class,
5652                             $Type$256Vector.$Type$256Mask.class,
5653                             $Type$256Vector::new);
5654 
5655     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
5656     public static final VectorSpecies<$Boxtype$> SPECIES_512
5657         = new $Type$Species(VectorShape.S_512_BIT,
5658                             $Type$512Vector.class,
5659                             $Type$512Vector.$Type$512Mask.class,
5660                             $Type$512Vector::new);
5661 
5662     /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
5663     public static final VectorSpecies<$Boxtype$> SPECIES_MAX
5664         = new $Type$Species(VectorShape.S_Max_BIT,
5665                             $Type$MaxVector.class,
5666                             $Type$MaxVector.$Type$MaxMask.class,
5667                             $Type$MaxVector::new);
5668 
5669     /**
5670      * Preferred species for {@link $Type$Vector}s.
5671      * A preferred species is a species of maximal bit-size for the platform.
5672      */
5673     public static final VectorSpecies<$Boxtype$> SPECIES_PREFERRED
5674         = ($Type$Species) VectorSpecies.ofPreferred($type$.class);
5675 }