< prev index next >

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template

Print this page

  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.nio.ByteOrder;
  29 import java.nio.ReadOnlyBufferException;
  30 import java.util.Arrays;
  31 import java.util.Objects;
  32 import java.util.function.BinaryOperator;
  33 import java.util.function.Function;
  34 import java.util.function.UnaryOperator;
  35 
  36 import jdk.internal.misc.ScopedMemoryAccess;
  37 import jdk.internal.misc.Unsafe;
  38 import jdk.internal.vm.annotation.ForceInline;
  39 import jdk.internal.vm.vector.VectorSupport;
  40 
  41 import static jdk.internal.vm.vector.VectorSupport.*;
  42 import static jdk.incubator.vector.VectorIntrinsics.*;
  43 
  44 import static jdk.incubator.vector.VectorOperators.*;
  45 
  46 #warn This file is preprocessed before being compiled
  47 
  48 /**
  49  * A specialized {@link Vector} representing an ordered immutable sequence of
  50  * {@code $type$} values.
  51  */
  52 @SuppressWarnings("cast")  // warning: redundant cast

 160     $abstractvectortype$ uOp(FUnOp f);
 161     @ForceInline
 162     final
 163     $abstractvectortype$ uOpTemplate(FUnOp f) {
 164         $type$[] vec = vec();
 165         $type$[] res = new $type$[length()];
 166         for (int i = 0; i < res.length; i++) {
 167             res[i] = f.apply(i, vec[i]);
 168         }
 169         return vectorFactory(res);
 170     }
 171 
 172     /*package-private*/
 173     abstract
 174     $abstractvectortype$ uOp(VectorMask<$Boxtype$> m,
 175                              FUnOp f);
 176     @ForceInline
 177     final
 178     $abstractvectortype$ uOpTemplate(VectorMask<$Boxtype$> m,
 179                                      FUnOp f) {



 180         $type$[] vec = vec();
 181         $type$[] res = new $type$[length()];
 182         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 183         for (int i = 0; i < res.length; i++) {
 184             res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
 185         }
 186         return vectorFactory(res);
 187     }
 188 
 189     // Binary operator
 190 
 191     /*package-private*/
 192     interface FBinOp {
 193         $type$ apply(int i, $type$ a, $type$ b);
 194     }
 195 
 196     /*package-private*/
 197     abstract
 198     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 199                              FBinOp f);

 203                                      FBinOp f) {
 204         $type$[] res = new $type$[length()];
 205         $type$[] vec1 = this.vec();
 206         $type$[] vec2 = (($abstractvectortype$)o).vec();
 207         for (int i = 0; i < res.length; i++) {
 208             res[i] = f.apply(i, vec1[i], vec2[i]);
 209         }
 210         return vectorFactory(res);
 211     }
 212 
 213     /*package-private*/
 214     abstract
 215     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 216                              VectorMask<$Boxtype$> m,
 217                              FBinOp f);
 218     @ForceInline
 219     final
 220     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 221                                      VectorMask<$Boxtype$> m,
 222                                      FBinOp f) {



 223         $type$[] res = new $type$[length()];
 224         $type$[] vec1 = this.vec();
 225         $type$[] vec2 = (($abstractvectortype$)o).vec();
 226         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 227         for (int i = 0; i < res.length; i++) {
 228             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
 229         }
 230         return vectorFactory(res);
 231     }
 232 
 233     // Ternary operator
 234 
 235     /*package-private*/
 236     interface FTriOp {
 237         $type$ apply(int i, $type$ a, $type$ b, $type$ c);
 238     }
 239 
 240     /*package-private*/
 241     abstract
 242     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,

 252         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 253         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 254         for (int i = 0; i < res.length; i++) {
 255             res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
 256         }
 257         return vectorFactory(res);
 258     }
 259 
 260     /*package-private*/
 261     abstract
 262     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 263                              Vector<$Boxtype$> o2,
 264                              VectorMask<$Boxtype$> m,
 265                              FTriOp f);
 266     @ForceInline
 267     final
 268     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 269                                      Vector<$Boxtype$> o2,
 270                                      VectorMask<$Boxtype$> m,
 271                                      FTriOp f) {



 272         $type$[] res = new $type$[length()];
 273         $type$[] vec1 = this.vec();
 274         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 275         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 276         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 277         for (int i = 0; i < res.length; i++) {
 278             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
 279         }
 280         return vectorFactory(res);
 281     }
 282 
 283     // Reduction operator
 284 
 285     /*package-private*/
 286     abstract
 287     $type$ rOp($type$ v, FBinOp f);















 288     @ForceInline
 289     final
 290     $type$ rOpTemplate($type$ v, FBinOp f) {
 291         $type$[] vec = vec();
 292         for (int i = 0; i < vec.length; i++) {
 293             v = f.apply(i, v, vec[i]);
 294         }
 295         return v;
 296     }
 297 
 298     // Memory reference
 299 
 300     /*package-private*/
 301     interface FLdOp<M> {
 302         $type$ apply(M memory, int offset, int i);
 303     }
 304 
 305     /*package-private*/
 306     @ForceInline
 307     final

 555     }
 556 #end[!long]
 557 
 558     // Unary lanewise support
 559 
 560     /**
 561      * {@inheritDoc} <!--workaround-->
 562      */
 563     public abstract
 564     $abstractvectortype$ lanewise(VectorOperators.Unary op);
 565 
 566     @ForceInline
 567     final
 568     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op) {
 569         if (opKind(op, VO_SPECIAL)) {
 570             if (op == ZOMO) {
 571                 return blend(broadcast(-1), compare(NE, 0));
 572             }
 573 #if[BITWISE]
 574             if (op == NOT) {
 575                 return broadcast(-1).lanewiseTemplate(XOR, this);
 576             } else if (op == NEG) {
 577                 // FIXME: Support this in the JIT.
 578                 return broadcast(0).lanewiseTemplate(SUB, this);
 579             }
 580 #end[BITWISE]
 581         }
 582         int opc = opCode(op);
 583         return VectorSupport.unaryOp(
 584             opc, getClass(), $type$.class, length(),
 585             this,
 586             UN_IMPL.find(op, opc, (opc_) -> {
 587               switch (opc_) {
 588                 case VECTOR_OP_NEG: return v0 ->
 589                         v0.uOp((i, a) -> ($type$) -a);
 590                 case VECTOR_OP_ABS: return v0 ->
 591                         v0.uOp((i, a) -> ($type$) Math.abs(a));
 592 #if[FP]
 593                 case VECTOR_OP_SIN: return v0 ->
 594                         v0.uOp((i, a) -> ($type$) Math.sin(a));
 595                 case VECTOR_OP_COS: return v0 ->
 596                         v0.uOp((i, a) -> ($type$) Math.cos(a));
 597                 case VECTOR_OP_TAN: return v0 ->
 598                         v0.uOp((i, a) -> ($type$) Math.tan(a));
 599                 case VECTOR_OP_ASIN: return v0 ->
 600                         v0.uOp((i, a) -> ($type$) Math.asin(a));
 601                 case VECTOR_OP_ACOS: return v0 ->
 602                         v0.uOp((i, a) -> ($type$) Math.acos(a));
 603                 case VECTOR_OP_ATAN: return v0 ->
 604                         v0.uOp((i, a) -> ($type$) Math.atan(a));
 605                 case VECTOR_OP_EXP: return v0 ->
 606                         v0.uOp((i, a) -> ($type$) Math.exp(a));
 607                 case VECTOR_OP_LOG: return v0 ->
 608                         v0.uOp((i, a) -> ($type$) Math.log(a));
 609                 case VECTOR_OP_LOG10: return v0 ->
 610                         v0.uOp((i, a) -> ($type$) Math.log10(a));
 611                 case VECTOR_OP_SQRT: return v0 ->
 612                         v0.uOp((i, a) -> ($type$) Math.sqrt(a));
 613                 case VECTOR_OP_CBRT: return v0 ->
 614                         v0.uOp((i, a) -> ($type$) Math.cbrt(a));
 615                 case VECTOR_OP_SINH: return v0 ->
 616                         v0.uOp((i, a) -> ($type$) Math.sinh(a));
 617                 case VECTOR_OP_COSH: return v0 ->
 618                         v0.uOp((i, a) -> ($type$) Math.cosh(a));
 619                 case VECTOR_OP_TANH: return v0 ->
 620                         v0.uOp((i, a) -> ($type$) Math.tanh(a));
 621                 case VECTOR_OP_EXPM1: return v0 ->
 622                         v0.uOp((i, a) -> ($type$) Math.expm1(a));
 623                 case VECTOR_OP_LOG1P: return v0 ->
 624                         v0.uOp((i, a) -> ($type$) Math.log1p(a));
 625 #end[FP]
 626                 default: return null;
 627               }}));
 628     }
 629     private static final
 630     ImplCache<Unary,UnaryOperator<$abstractvectortype$>> UN_IMPL
 631         = new ImplCache<>(Unary.class, $Type$Vector.class);
 632 
 633     /**
 634      * {@inheritDoc} <!--workaround-->
 635      */
 636     @ForceInline
 637     public final
 638     $abstractvectortype$ lanewise(VectorOperators.Unary op,
 639                                   VectorMask<$Boxtype$> m) {
 640         return blend(lanewise(op), m);






































































 641     }
 642 
 643     // Binary lanewise support
 644 
 645     /**
 646      * {@inheritDoc} <!--workaround-->
 647      * @see #lanewise(VectorOperators.Binary,$type$)
 648      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 649      */
 650     @Override
 651     public abstract
 652     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 653                                   Vector<$Boxtype$> v);
 654     @ForceInline
 655     final
 656     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 657                                           Vector<$Boxtype$> v) {
 658         $abstractvectortype$ that = ($abstractvectortype$) v;
 659         that.check(this);

 660         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 661             if (op == FIRST_NONZERO) {
 662                 // FIXME: Support this in the JIT.
 663                 VectorMask<$Boxbitstype$> thisNZ
 664                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 665                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 666                 op = OR_UNCHECKED;
 667 #if[FP]
 668                 // FIXME: Support OR_UNCHECKED on float/double also!
 669                 return this.viewAsIntegralLanes()
 670                     .lanewise(op, that.viewAsIntegralLanes())
 671                     .viewAsFloatingLanes();
 672 #end[FP]
 673             }
 674 #if[BITWISE]
 675 #if[!FP]
 676             if (opKind(op, VO_SHIFT)) {
 677                 // As per shift specification for Java, mask the shift count.
 678                 // This allows the JIT to ignore some ISA details.
 679                 that = that.lanewise(AND, SHIFT_MASK);
 680             }
 681 #end[!FP]
 682             if (op == AND_NOT) {
 683                 // FIXME: Support this in the JIT.
 684                 that = that.lanewise(NOT);
 685                 op = AND;
 686             } else if (op == DIV) {
 687                 VectorMask<$Boxtype$> eqz = that.eq(($type$)0);
 688                 if (eqz.anyTrue()) {
 689                     throw that.divZeroException();
 690                 }
 691             }
 692 #end[BITWISE]
 693         }

 694         int opc = opCode(op);
 695         return VectorSupport.binaryOp(
 696             opc, getClass(), $type$.class, length(),
 697             this, that,
 698             BIN_IMPL.find(op, opc, (opc_) -> {
 699               switch (opc_) {
 700                 case VECTOR_OP_ADD: return (v0, v1) ->
 701                         v0.bOp(v1, (i, a, b) -> ($type$)(a + b));
 702                 case VECTOR_OP_SUB: return (v0, v1) ->
 703                         v0.bOp(v1, (i, a, b) -> ($type$)(a - b));
 704                 case VECTOR_OP_MUL: return (v0, v1) ->
 705                         v0.bOp(v1, (i, a, b) -> ($type$)(a * b));
 706                 case VECTOR_OP_DIV: return (v0, v1) ->
 707                         v0.bOp(v1, (i, a, b) -> ($type$)(a / b));
 708                 case VECTOR_OP_MAX: return (v0, v1) ->
 709                         v0.bOp(v1, (i, a, b) -> ($type$)Math.max(a, b));
 710                 case VECTOR_OP_MIN: return (v0, v1) ->
 711                         v0.bOp(v1, (i, a, b) -> ($type$)Math.min(a, b));
 712 #if[BITWISE]
 713                 case VECTOR_OP_AND: return (v0, v1) ->
 714                         v0.bOp(v1, (i, a, b) -> ($type$)(a & b));
 715                 case VECTOR_OP_OR: return (v0, v1) ->
 716                         v0.bOp(v1, (i, a, b) -> ($type$)(a | b));
 717                 case VECTOR_OP_XOR: return (v0, v1) ->
 718                         v0.bOp(v1, (i, a, b) -> ($type$)(a ^ b));
 719                 case VECTOR_OP_LSHIFT: return (v0, v1) ->
 720                         v0.bOp(v1, (i, a, n) -> ($type$)(a << n));
 721                 case VECTOR_OP_RSHIFT: return (v0, v1) ->
 722                         v0.bOp(v1, (i, a, n) -> ($type$)(a >> n));
 723                 case VECTOR_OP_URSHIFT: return (v0, v1) ->
 724                         v0.bOp(v1, (i, a, n) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 725                 case VECTOR_OP_LROTATE: return (v0, v1) ->
 726                         v0.bOp(v1, (i, a, n) -> rotateLeft(a, (int)n));
 727                 case VECTOR_OP_RROTATE: return (v0, v1) ->
 728                         v0.bOp(v1, (i, a, n) -> rotateRight(a, (int)n));
 729 #end[BITWISE]
 730 #if[FP]
 731                 case VECTOR_OP_ATAN2: return (v0, v1) ->
 732                         v0.bOp(v1, (i, a, b) -> ($type$) Math.atan2(a, b));
 733                 case VECTOR_OP_POW: return (v0, v1) ->
 734                         v0.bOp(v1, (i, a, b) -> ($type$) Math.pow(a, b));
 735                 case VECTOR_OP_HYPOT: return (v0, v1) ->
 736                         v0.bOp(v1, (i, a, b) -> ($type$) Math.hypot(a, b));
 737 #end[FP]
 738                 default: return null;
 739                 }}));
 740     }
 741     private static final
 742     ImplCache<Binary,BinaryOperator<$abstractvectortype$>> BIN_IMPL
 743         = new ImplCache<>(Binary.class, $Type$Vector.class);
 744 
 745     /**
 746      * {@inheritDoc} <!--workaround-->
 747      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 748      */
 749     @ForceInline
 750     public final
 751     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 752                                   Vector<$Boxtype$> v,
 753                                   VectorMask<$Boxtype$> m) {
 754 #if[BITWISE]




 755         $abstractvectortype$ that = ($abstractvectortype$) v;
 756         if (op == DIV) {
 757             VectorMask<$Boxtype$> eqz = that.eq(($type$)0);
 758             if (eqz.and(m).anyTrue()) {
 759                 throw that.divZeroException();






























 760             }
 761             // suppress div/0 exceptions in unset lanes
 762             that = that.lanewise(NOT, eqz);
 763             return blend(lanewise(DIV, that), m);
 764         }











































 765 #end[BITWISE]
 766         return blend(lanewise(op, v), m);











 767     }

 768     // FIXME: Maybe all of the public final methods in this file (the
 769     // simple ones that just call lanewise) should be pushed down to
 770     // the X-VectorBits template.  They can't optimize properly at
 771     // this level, and must rely on inlining.  Does it work?
 772     // (If it works, of course keep the code here.)
 773 
 774     /**
 775      * Combines the lane values of this vector
 776      * with the value of a broadcast scalar.
 777      *
 778      * This is a lane-wise binary operation which applies
 779      * the selected operation to each lane.
 780      * The return value will be equal to this expression:
 781      * {@code this.lanewise(op, this.broadcast(e))}.
 782      *
 783      * @param op the operation used to process lane values
 784      * @param e the input scalar
 785      * @return the result of applying the operation lane-wise
 786      *         to the two input vectors
 787      * @throws UnsupportedOperationException if this vector does

 812      * This is a masked lane-wise binary operation which applies
 813      * the selected operation to each lane.
 814      * The return value will be equal to this expression:
 815      * {@code this.lanewise(op, this.broadcast(e), m)}.
 816      *
 817      * @param op the operation used to process lane values
 818      * @param e the input scalar
 819      * @param m the mask controlling lane selection
 820      * @return the result of applying the operation lane-wise
 821      *         to the input vector and the scalar
 822      * @throws UnsupportedOperationException if this vector does
 823      *         not support the requested operation
 824      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
 825      * @see #lanewise(VectorOperators.Binary,$type$)
 826      */
 827     @ForceInline
 828     public final
 829     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 830                                   $type$ e,
 831                                   VectorMask<$Boxtype$> m) {
 832         return blend(lanewise(op, e), m);








 833     }
 834 
 835 #if[!long]
 836     /**
 837      * {@inheritDoc} <!--workaround-->
 838      * @apiNote
 839      * When working with vector subtypes like {@code $abstractvectortype$},
 840      * {@linkplain #lanewise(VectorOperators.Binary,$type$)
 841      * the more strongly typed method}
 842      * is typically selected.  It can be explicitly selected
 843      * using a cast: {@code v.lanewise(op,($type$)e)}.
 844      * The two expressions will produce numerically identical results.
 845      */
 846     @ForceInline
 847     public final
 848     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 849                                   long e) {
 850         $type$ e1 = ($type$) e;
 851         if ((long)e1 != e
 852 #if[BITWISE]

 853             // allow shift ops to clip down their int parameters
 854             && !(opKind(op, VO_SHIFT) && (int)e1 == e)


 855 #end[BITWISE]
 856             ) {
 857             vspecies().checkValue(e);  // for exception
 858         }
 859         return lanewise(op, e1);
 860     }
 861 
 862     /**
 863      * {@inheritDoc} <!--workaround-->
 864      * @apiNote
 865      * When working with vector subtypes like {@code $abstractvectortype$},
 866      * {@linkplain #lanewise(VectorOperators.Binary,$type$,VectorMask)
 867      * the more strongly typed method}
 868      * is typically selected.  It can be explicitly selected
 869      * using a cast: {@code v.lanewise(op,($type$)e,m)}.
 870      * The two expressions will produce numerically identical results.
 871      */
 872     @ForceInline
 873     public final
 874     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 875                                   long e, VectorMask<$Boxtype$> m) {
 876         return blend(lanewise(op, e), m);










 877     }
 878 #end[!long]
 879 
 880 #if[BITWISE]
 881     /*package-private*/
 882     abstract $abstractvectortype$
 883     lanewiseShift(VectorOperators.Binary op, int e);
 884 
 885     /*package-private*/
 886     @ForceInline
 887     final $abstractvectortype$
 888     lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
 889         // Special handling for these.  FIXME: Refactor?
 890         assert(opKind(op, VO_SHIFT));
 891         // As per shift specification for Java, mask the shift count.
 892         e &= SHIFT_MASK;
 893         int opc = opCode(op);
 894         return VectorSupport.broadcastInt(
 895             opc, getClass(), $type$.class, length(),
 896             this, e,
 897             BIN_INT_IMPL.find(op, opc, (opc_) -> {
 898               switch (opc_) {
 899                 case VECTOR_OP_LSHIFT: return (v, n) ->
 900                         v.uOp((i, a) -> ($type$)(a << n));
 901                 case VECTOR_OP_RSHIFT: return (v, n) ->
 902                         v.uOp((i, a) -> ($type$)(a >> n));
 903                 case VECTOR_OP_URSHIFT: return (v, n) ->
 904                         v.uOp((i, a) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 905                 case VECTOR_OP_LROTATE: return (v, n) ->
 906                         v.uOp((i, a) -> rotateLeft(a, (int)n));
 907                 case VECTOR_OP_RROTATE: return (v, n) ->
 908                         v.uOp((i, a) -> rotateRight(a, (int)n));
 909                 default: return null;
 910                 }}));








 911     }

 912     private static final
 913     ImplCache<Binary,VectorBroadcastIntOp<$abstractvectortype$>> BIN_INT_IMPL
 914         = new ImplCache<>(Binary.class, $Type$Vector.class);
 915 
















 916     // As per shift specification for Java, mask the shift count.
 917     // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
 918     // The latter two maskings go beyond the JLS, but seem reasonable
 919     // since our lane types are first-class types, not just dressed
 920     // up ints.
 921     private static final int SHIFT_MASK = ($Boxtype$.SIZE - 1);
 922 #if[byteOrShort]
 923     // Also simulate >>> on sub-word variables with a mask.
 924     private static final int LSHR_SETUP_MASK = ((1 << $Boxtype$.SIZE) - 1);
 925 #else[byteOrShort]
 926     private static final $type$ LSHR_SETUP_MASK = -1;
 927 #end[byteOrShort]
 928 #end[BITWISE]
 929 
 930     // Ternary lanewise support
 931 
 932     // Ternary operators come in eight variations:
 933     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
 934     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
 935 

 955     final
 956     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
 957                                           Vector<$Boxtype$> v1,
 958                                           Vector<$Boxtype$> v2) {
 959         $abstractvectortype$ that = ($abstractvectortype$) v1;
 960         $abstractvectortype$ tother = ($abstractvectortype$) v2;
 961         // It's a word: https://www.dictionary.com/browse/tother
 962         // See also Chapter 11 of Dickens, Our Mutual Friend:
 963         // "Totherest Governor," replied Mr Riderhood...
 964         that.check(this);
 965         tother.check(this);
 966 #if[BITWISE]
 967         if (op == BITWISE_BLEND) {
 968             // FIXME: Support this in the JIT.
 969             that = this.lanewise(XOR, that).lanewise(AND, tother);
 970             return this.lanewise(XOR, that);
 971         }
 972 #end[BITWISE]
 973         int opc = opCode(op);
 974         return VectorSupport.ternaryOp(
 975             opc, getClass(), $type$.class, length(),
 976             this, that, tother,
 977             TERN_IMPL.find(op, opc, (opc_) -> {
 978               switch (opc_) {
 979 #if[FP]
 980                 case VECTOR_OP_FMA: return (v0, v1_, v2_) ->
 981                         v0.tOp(v1_, v2_, (i, a, b, c) -> Math.fma(a, b, c));
 982 #end[FP]
 983                 default: return null;
 984                 }}));
 985     }
 986     private static final
 987     ImplCache<Ternary,TernaryOperation<$abstractvectortype$>> TERN_IMPL
 988         = new ImplCache<>(Ternary.class, $Type$Vector.class);
 989 
 990     /**
 991      * {@inheritDoc} <!--workaround-->
 992      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
 993      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
 994      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
 995      */
 996     @ForceInline
 997     public final
 998     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
 999                                   Vector<$Boxtype$> v1,
1000                                   Vector<$Boxtype$> v2,
1001                                   VectorMask<$Boxtype$> m) {
1002         return blend(lanewise(op, v1, v2), m);









































1003     }
1004 
1005     /**
1006      * Combines the lane values of this vector
1007      * with the values of two broadcast scalars.
1008      *
1009      * This is a lane-wise ternary operation which applies
1010      * the selected operation to each lane.
1011      * The return value will be equal to this expression:
1012      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
1013      *
1014      * @param op the operation used to combine lane values
1015      * @param e1 the first input scalar
1016      * @param e2 the second input scalar
1017      * @return the result of applying the operation lane-wise
1018      *         to the input vector and the scalars
1019      * @throws UnsupportedOperationException if this vector does
1020      *         not support the requested operation
1021      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1022      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)

1039      * The return value will be equal to this expression:
1040      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
1041      *
1042      * @param op the operation used to combine lane values
1043      * @param e1 the first input scalar
1044      * @param e2 the second input scalar
1045      * @param m the mask controlling lane selection
1046      * @return the result of applying the operation lane-wise
1047      *         to the input vector and the scalars
1048      * @throws UnsupportedOperationException if this vector does
1049      *         not support the requested operation
1050      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1051      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1052      */
1053     @ForceInline
1054     public final
1055     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
1056                                   $type$ e1,
1057                                   $type$ e2,
1058                                   VectorMask<$Boxtype$> m) {
1059         return blend(lanewise(op, e1, e2), m);
1060     }
1061 
1062     /**
1063      * Combines the lane values of this vector
1064      * with the values of another vector and a broadcast scalar.
1065      *
1066      * This is a lane-wise ternary operation which applies
1067      * the selected operation to each lane.
1068      * The return value will be equal to this expression:
1069      * {@code this.lanewise(op, v1, this.broadcast(e2))}.
1070      *
1071      * @param op the operation used to combine lane values
1072      * @param v1 the other input vector
1073      * @param e2 the input scalar
1074      * @return the result of applying the operation lane-wise
1075      *         to the input vectors and the scalar
1076      * @throws UnsupportedOperationException if this vector does
1077      *         not support the requested operation
1078      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1079      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)

1097      * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
1098      *
1099      * @param op the operation used to combine lane values
1100      * @param v1 the other input vector
1101      * @param e2 the input scalar
1102      * @param m the mask controlling lane selection
1103      * @return the result of applying the operation lane-wise
1104      *         to the input vectors and the scalar
1105      * @throws UnsupportedOperationException if this vector does
1106      *         not support the requested operation
1107      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1108      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1109      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
1110      */
1111     @ForceInline
1112     public final
1113     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
1114                                   Vector<$Boxtype$> v1,
1115                                   $type$ e2,
1116                                   VectorMask<$Boxtype$> m) {
1117         return blend(lanewise(op, v1, e2), m);
1118     }
1119 
1120     /**
1121      * Combines the lane values of this vector
1122      * with the values of another vector and a broadcast scalar.
1123      *
1124      * This is a lane-wise ternary operation which applies
1125      * the selected operation to each lane.
1126      * The return value will be equal to this expression:
1127      * {@code this.lanewise(op, this.broadcast(e1), v2)}.
1128      *
1129      * @param op the operation used to combine lane values
1130      * @param e1 the input scalar
1131      * @param v2 the other input vector
1132      * @return the result of applying the operation lane-wise
1133      *         to the input vectors and the scalar
1134      * @throws UnsupportedOperationException if this vector does
1135      *         not support the requested operation
1136      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1137      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)

1154      * The return value will be equal to this expression:
1155      * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
1156      *
1157      * @param op the operation used to combine lane values
1158      * @param e1 the input scalar
1159      * @param v2 the other input vector
1160      * @param m the mask controlling lane selection
1161      * @return the result of applying the operation lane-wise
1162      *         to the input vectors and the scalar
1163      * @throws UnsupportedOperationException if this vector does
1164      *         not support the requested operation
1165      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1166      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
1167      */
1168     @ForceInline
1169     public final
1170     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
1171                                   $type$ e1,
1172                                   Vector<$Boxtype$> v2,
1173                                   VectorMask<$Boxtype$> m) {
1174         return blend(lanewise(op, e1, v2), m);
1175     }
1176 
1177     // (Thus endeth the Great and Mighty Ternary Ogdoad.)
1178     // https://en.wikipedia.org/wiki/Ogdoad
1179 
1180     /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1181     //
1182     // These include masked and non-masked versions.
1183     // This subclass adds broadcast (masked or not).
1184 
1185     /**
1186      * {@inheritDoc} <!--workaround-->
1187      * @see #add($type$)
1188      */
1189     @Override
1190     @ForceInline
1191     public final $abstractvectortype$ add(Vector<$Boxtype$> v) {
1192         return lanewise(ADD, v);
1193     }
1194 

1999     @Override
2000     @ForceInline
2001     public final
2002     VectorMask<$Boxtype$> test(VectorOperators.Test op,
2003                                   VectorMask<$Boxtype$> m) {
2004         return test(op).and(m);
2005     }
2006 
2007     /**
2008      * {@inheritDoc} <!--workaround-->
2009      */
2010     @Override
2011     public abstract
2012     VectorMask<$Boxtype$> compare(VectorOperators.Comparison op, Vector<$Boxtype$> v);
2013 
2014     /*package-private*/
2015     @ForceInline
2016     final
2017     <M extends VectorMask<$Boxtype$>>
2018     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v) {
2019         Objects.requireNonNull(v);
2020         $Type$Species vsp = vspecies();
2021         $abstractvectortype$ that = ($abstractvectortype$) v;
2022         that.check(this);
2023         int opc = opCode(op);
2024         return VectorSupport.compare(
2025             opc, getClass(), maskType, $type$.class, length(),
2026             this, that,
2027             (cond, v0, v1) -> {
2028                 AbstractMask<$Boxtype$> m
2029                     = v0.bTest(cond, v1, (cond_, i, a, b)
2030                                -> compareWithOp(cond, a, b));
2031                 @SuppressWarnings("unchecked")
2032                 M m2 = (M) m;
2033                 return m2;
2034             });
2035     }
2036 






















2037     @ForceInline
2038     private static boolean compareWithOp(int cond, $type$ a, $type$ b) {
2039         return switch (cond) {
2040             case BT_eq -> a == b;
2041             case BT_ne -> a != b;
2042             case BT_lt -> a < b;
2043             case BT_le -> a <= b;
2044             case BT_gt -> a > b;
2045             case BT_ge -> a >= b;
2046 #if[!FP]
2047             case BT_ult -> $Boxtype$.compareUnsigned(a, b) < 0;
2048             case BT_ule -> $Boxtype$.compareUnsigned(a, b) <= 0;
2049             case BT_ugt -> $Boxtype$.compareUnsigned(a, b) > 0;
2050             case BT_uge -> $Boxtype$.compareUnsigned(a, b) >= 0;
2051 #end[!FP]
2052             default -> throw new AssertionError();
2053         };
2054     }
2055 
2056     /**
2057      * {@inheritDoc} <!--workaround-->
2058      */
2059     @Override
2060     @ForceInline
2061     public final
2062     VectorMask<$Boxtype$> compare(VectorOperators.Comparison op,
2063                                   Vector<$Boxtype$> v,
2064                                   VectorMask<$Boxtype$> m) {
2065         return compare(op, v).and(m);
2066     }
2067 
2068     /**
2069      * Tests this vector by comparing it with an input scalar,
2070      * according to the given comparison operation.
2071      *
2072      * This is a lane-wise binary test operation which applies
2073      * the comparison operation to each lane.
2074      * <p>
2075      * The result is the same as
2076      * {@code compare(op, broadcast(species(), e))}.
2077      * That is, the scalar may be regarded as broadcast to
2078      * a vector of the same species, and then compared
2079      * against the original vector, using the selected
2080      * comparison operation.
2081      *
2082      * @param op the operation used to compare lane values
2083      * @param e the input scalar
2084      * @return the mask result of testing lane-wise if this vector
2085      *         compares to the input, according to the selected
2086      *         comparison operator
2087      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector)

2106      *
2107      * This is a masked lane-wise binary test operation which applies
2108      * to each pair of corresponding lane values.
2109      *
2110      * The returned result is equal to the expression
2111      * {@code compare(op,s).and(m)}.
2112      *
2113      * @param op the operation used to compare lane values
2114      * @param e the input scalar
2115      * @param m the mask controlling lane selection
2116      * @return the mask result of testing lane-wise if this vector
2117      *         compares to the input, according to the selected
2118      *         comparison operator,
2119      *         and only in the lanes selected by the mask
2120      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector,VectorMask)
2121      */
2122     @ForceInline
2123     public final VectorMask<$Boxtype$> compare(VectorOperators.Comparison op,
2124                                                $type$ e,
2125                                                VectorMask<$Boxtype$> m) {
2126         return compare(op, e).and(m);
2127     }
2128 
2129 #if[!long]
2130     /**
2131      * {@inheritDoc} <!--workaround-->
2132      */
2133     @Override
2134     public abstract
2135     VectorMask<$Boxtype$> compare(Comparison op, long e);
2136 
2137     /*package-private*/
2138     @ForceInline
2139     final
2140     <M extends VectorMask<$Boxtype$>>
2141     M compareTemplate(Class<M> maskType, Comparison op, long e) {
2142         return compareTemplate(maskType, op, broadcast(e));
2143     }
2144 
2145     /**
2146      * {@inheritDoc} <!--workaround-->

2361     wrongPartForSlice(int part) {
2362         String msg = String.format("bad part number %d for slice operation",
2363                                    part);
2364         return new ArrayIndexOutOfBoundsException(msg);
2365     }
2366 
2367     /**
2368      * {@inheritDoc} <!--workaround-->
2369      */
2370     @Override
2371     public abstract
2372     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> m);
2373 
2374     /*package-private*/
2375     @ForceInline
2376     final
2377     <S extends VectorShuffle<$Boxtype$>>
2378     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype, S shuffle) {
2379         shuffle.checkIndexes();
2380         return VectorSupport.rearrangeOp(
2381             getClass(), shuffletype, $type$.class, length(),
2382             this, shuffle,
2383             (v1, s_) -> v1.uOp((i, a) -> {
2384                 int ei = s_.laneSource(i);
2385                 return v1.lane(ei);
2386             }));
2387     }
2388 
2389     /**
2390      * {@inheritDoc} <!--workaround-->
2391      */
2392     @Override
2393     public abstract
2394     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2395                                    VectorMask<$Boxtype$> m);
2396 
2397     /*package-private*/
2398     @ForceInline
2399     final
2400     <S extends VectorShuffle<$Boxtype$>>
2401     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,

2402                                            S shuffle,
2403                                            VectorMask<$Boxtype$> m) {
2404         $abstractvectortype$ unmasked =
2405             VectorSupport.rearrangeOp(
2406                 getClass(), shuffletype, $type$.class, length(),
2407                 this, shuffle,
2408                 (v1, s_) -> v1.uOp((i, a) -> {
2409                     int ei = s_.laneSource(i);
2410                     return ei < 0 ? 0 : v1.lane(ei);
2411                 }));
2412         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2413         if (m.andNot(valid).anyTrue()) {
2414             shuffle.checkIndexes();
2415             throw new AssertionError();
2416         }
2417         return broadcast(($type$)0).blend(unmasked, m);






2418     }
2419 
2420     /**
2421      * {@inheritDoc} <!--workaround-->
2422      */
2423     @Override
2424     public abstract
2425     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2426                                    Vector<$Boxtype$> v);
2427 
2428     /*package-private*/
2429     @ForceInline
2430     final
2431     <S extends VectorShuffle<$Boxtype$>>
2432     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2433                                            S shuffle,
2434                                            $abstractvectortype$ v) {
2435         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2436         @SuppressWarnings("unchecked")
2437         S ws = (S) shuffle.wrapIndexes();
2438         $abstractvectortype$ r0 =
2439             VectorSupport.rearrangeOp(
2440                 getClass(), shuffletype, $type$.class, length(),
2441                 this, ws,
2442                 (v0, s_) -> v0.uOp((i, a) -> {
2443                     int ei = s_.laneSource(i);
2444                     return v0.lane(ei);
2445                 }));
2446         $abstractvectortype$ r1 =
2447             VectorSupport.rearrangeOp(
2448                 getClass(), shuffletype, $type$.class, length(),
2449                 v, ws,
2450                 (v1, s_) -> v1.uOp((i, a) -> {
2451                     int ei = s_.laneSource(i);
2452                     return v1.lane(ei);
2453                 }));
2454         return r1.blend(r0, valid);
2455     }
2456 
2457     @ForceInline
2458     private final
2459     VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) {
2460         $type$[] a = toArray();
2461         int[] sa = new int[a.length];
2462         for (int i = 0; i < a.length; i++) {
2463             sa[i] = (int) a[i];
2464         }
2465         return VectorShuffle.fromArray(dsp, sa, 0);
2466     }
2467 
2468     /*package-private*/
2469     @ForceInline
2470     final

2822      * <li>
2823      * All other reduction operations are fully commutative and
2824      * associative.  The implementation can choose any order of
2825      * processing, yet it will always produce the same result.
2826      * </ul>
2827      *
2828      * @param op the operation used to combine lane values
2829      * @param m the mask controlling lane selection
2830      * @return the reduced result accumulated from the selected lane values
2831      * @throws UnsupportedOperationException if this vector does
2832      *         not support the requested operation
2833      * @see #reduceLanes(VectorOperators.Associative)
2834      */
2835     public abstract $type$ reduceLanes(VectorOperators.Associative op,
2836                                        VectorMask<$Boxtype$> m);
2837 
2838     /*package-private*/
2839     @ForceInline
2840     final
2841     $type$ reduceLanesTemplate(VectorOperators.Associative op,

2842                                VectorMask<$Boxtype$> m) {
2843         $abstractvectortype$ v = reduceIdentityVector(op).blend(this, m);
2844         return v.reduceLanesTemplate(op);








2845     }
2846 
2847     /*package-private*/
2848     @ForceInline
2849     final
2850     $type$ reduceLanesTemplate(VectorOperators.Associative op) {
2851         if (op == FIRST_NONZERO) {
2852             // FIXME:  The JIT should handle this, and other scan ops alos.
2853             VectorMask<$Boxbitstype$> thisNZ
2854                 = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
2855             return this.lane(thisNZ.firstTrue());
2856         }
2857         int opc = opCode(op);
2858         return fromBits(VectorSupport.reductionCoerced(
2859             opc, getClass(), $type$.class, length(),
2860             this,
2861             REDUCE_IMPL.find(op, opc, (opc_) -> {
2862               switch (opc_) {
2863               case VECTOR_OP_ADD: return v ->
2864                       toBits(v.rOp(($type$)0, (i, a, b) -> ($type$)(a + b)));
2865               case VECTOR_OP_MUL: return v ->
2866                       toBits(v.rOp(($type$)1, (i, a, b) -> ($type$)(a * b)));
2867               case VECTOR_OP_MIN: return v ->
2868                       toBits(v.rOp(MAX_OR_INF, (i, a, b) -> ($type$) Math.min(a, b)));
2869               case VECTOR_OP_MAX: return v ->
2870                       toBits(v.rOp(MIN_OR_INF, (i, a, b) -> ($type$) Math.max(a, b)));







2871 #if[BITWISE]
2872               case VECTOR_OP_AND: return v ->
2873                       toBits(v.rOp(($type$)-1, (i, a, b) -> ($type$)(a & b)));
2874               case VECTOR_OP_OR: return v ->
2875                       toBits(v.rOp(($type$)0, (i, a, b) -> ($type$)(a | b)));
2876               case VECTOR_OP_XOR: return v ->
2877                       toBits(v.rOp(($type$)0, (i, a, b) -> ($type$)(a ^ b)));
2878 #end[BITWISE]
2879               default: return null;
2880               }})));
2881     }
2882     private static final
2883     ImplCache<Associative,Function<$abstractvectortype$,Long>> REDUCE_IMPL
2884         = new ImplCache<>(Associative.class, $Type$Vector.class);
2885 
2886     private
2887     @ForceInline
2888     $abstractvectortype$ reduceIdentityVector(VectorOperators.Associative op) {
2889         int opc = opCode(op);
2890         UnaryOperator<$abstractvectortype$> fn
2891             = REDUCE_ID_IMPL.find(op, opc, (opc_) -> {
2892                 switch (opc_) {
2893                 case VECTOR_OP_ADD:
2894 #if[BITWISE]
2895                 case VECTOR_OP_OR:
2896                 case VECTOR_OP_XOR:
2897 #end[BITWISE]
2898                     return v -> v.broadcast(0);
2899                 case VECTOR_OP_MUL:
2900                     return v -> v.broadcast(1);
2901 #if[BITWISE]
2902                 case VECTOR_OP_AND:
2903                     return v -> v.broadcast(-1);
2904 #end[BITWISE]

3158      * @param species species of desired vector
3159      * @param a the byte array
3160      * @param offset the offset into the array
3161      * @param bo the intended byte order
3162      * @param m the mask controlling lane selection
3163      * @return a vector loaded from a byte array
3164      * @throws IndexOutOfBoundsException
3165      *         if {@code offset+N*ESIZE < 0}
3166      *         or {@code offset+(N+1)*ESIZE > a.length}
3167      *         for any lane {@code N} in the vector
3168      *         where the mask is set
3169      */
3170     @ForceInline
3171     public static
3172     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3173                                        byte[] a, int offset,
3174                                        ByteOrder bo,
3175                                        VectorMask<$Boxtype$> m) {
3176         $Type$Species vsp = ($Type$Species) species;
3177         if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
3178             $abstractvectortype$ zero = vsp.zero();
3179             $abstractvectortype$ v = zero.fromByteArray0(a, offset);
3180             return zero.blend(v.maybeSwap(bo), m);
3181         }
3182 
3183         // FIXME: optimize
3184         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
3185         ByteBuffer wb = wrapper(a, bo);
3186         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
3187                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
3188     }
3189 
3190     /**
3191      * Loads a vector from an array of type {@code $type$[]}
3192      * starting at an offset.
3193      * For each vector lane, where {@code N} is the vector lane index, the
3194      * array element at index {@code offset + N} is placed into the
3195      * resulting vector at lane index {@code N}.
3196      *
3197      * @param species species of desired vector
3198      * @param a the array
3199      * @param offset the offset into the array
3200      * @return the vector loaded from an array

3222      * {@code N}, otherwise the default element value is placed into the
3223      * resulting vector at lane index {@code N}.
3224      *
3225      * @param species species of desired vector
3226      * @param a the array
3227      * @param offset the offset into the array
3228      * @param m the mask controlling lane selection
3229      * @return the vector loaded from an array
3230      * @throws IndexOutOfBoundsException
3231      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3232      *         for any lane {@code N} in the vector
3233      *         where the mask is set
3234      */
3235     @ForceInline
3236     public static
3237     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3238                                    $type$[] a, int offset,
3239                                    VectorMask<$Boxtype$> m) {
3240         $Type$Species vsp = ($Type$Species) species;
3241         if (offset >= 0 && offset <= (a.length - species.length())) {
3242             $abstractvectortype$ zero = vsp.zero();
3243             return zero.blend(zero.fromArray0(a, offset), m);
3244         }
3245 
3246         // FIXME: optimize
3247         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3248         return vsp.vOp(m, i -> a[offset + i]);
3249     }
3250 
3251     /**
3252      * Gathers a new vector composed of elements from an array of type
3253      * {@code $type$[]},
3254      * using indexes obtained by adding a fixed {@code offset} to a
3255      * series of secondary offsets from an <em>index map</em>.
3256      * The index map is a contiguous sequence of {@code VLENGTH}
3257      * elements in a second array of {@code int}s, starting at a given
3258      * {@code mapOffset}.
3259      * <p>
3260      * For each vector lane, where {@code N} is the vector lane index,
3261      * the lane is loaded from the array
3262      * element {@code a[f(N)]}, where {@code f(N)} is the
3263      * index mapping expression

3316             // is 64. So when loading index vector (IntVector), only lower half
3317             // of index data is needed.
3318             vix = IntVector
3319                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3320                 .add(offset);
3321         } else {
3322             vix = IntVector
3323                 .fromArray(isp, indexMap, mapOffset)
3324                 .add(offset);
3325         }
3326 #else[longOrDouble]
3327         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3328         IntVector vix = IntVector
3329             .fromArray(isp, indexMap, mapOffset)
3330             .add(offset);
3331 #end[longOrDouble]
3332 
3333         vix = VectorIntrinsics.checkIndex(vix, a.length);
3334 
3335         return VectorSupport.loadWithMap(
3336             vectorType, $type$.class, vsp.laneCount(),
3337             IntVector.species(vsp.indexShape()).vectorType(),
3338             a, ARRAY_BASE, vix,
3339             a, offset, indexMap, mapOffset, vsp,
3340             ($type$[] c, int idx, int[] iMap, int idy, $Type$Species s) ->
3341             s.vOp(n -> c[idx + iMap[idy+n]]));
3342         }
3343 #end[byteOrShort]
3344 
3345     /**
3346      * Gathers a new vector composed of elements from an array of type
3347      * {@code $type$[]},
3348      * under the control of a mask, and
3349      * using indexes obtained by adding a fixed {@code offset} to a
3350      * series of secondary offsets from an <em>index map</em>.
3351      * The index map is a contiguous sequence of {@code VLENGTH}
3352      * elements in a second array of {@code int}s, starting at a given
3353      * {@code mapOffset}.
3354      * <p>
3355      * For each vector lane, where {@code N} is the vector lane index,
3356      * if the lane is set in the mask,
3357      * the lane is loaded from the array
3358      * element {@code a[f(N)]}, where {@code f(N)} is the
3359      * index mapping expression
3360      * {@code offset + indexMap[mapOffset + N]]}.
3361      * Unset lanes in the resulting vector are set to zero.
3362      *

3382     @ForceInline
3383     public static
3384     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3385                                    $type$[] a, int offset,
3386                                    int[] indexMap, int mapOffset,
3387                                    VectorMask<$Boxtype$> m) {
3388         $Type$Species vsp = ($Type$Species) species;
3389         return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
3390     }
3391 #else[byteOrShort]
3392     @ForceInline
3393     public static
3394     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3395                                    $type$[] a, int offset,
3396                                    int[] indexMap, int mapOffset,
3397                                    VectorMask<$Boxtype$> m) {
3398         if (m.allTrue()) {
3399             return fromArray(species, a, offset, indexMap, mapOffset);
3400         }
3401         else {
3402             // FIXME: Cannot vectorize yet, if there's a mask.
3403             $Type$Species vsp = ($Type$Species) species;
3404             return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
3405         }
3406     }
3407 #end[byteOrShort]
3408 
3409 #if[short]
3410     /**
3411      * Loads a vector from an array of type {@code char[]}
3412      * starting at an offset.
3413      * For each vector lane, where {@code N} is the vector lane index, the
3414      * array element at index {@code offset + N}
3415      * is first cast to a {@code short} value and then
3416      * placed into the resulting vector at lane index {@code N}.
3417      *
3418      * @param species species of desired vector
3419      * @param a the array
3420      * @param offset the offset into the array
3421      * @return the vector loaded from an array
3422      * @throws IndexOutOfBoundsException
3423      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3424      *         for any lane {@code N} in the vector

3445      * {@code N}, otherwise the default element value is placed into the
3446      * resulting vector at lane index {@code N}.
3447      *
3448      * @param species species of desired vector
3449      * @param a the array
3450      * @param offset the offset into the array
3451      * @param m the mask controlling lane selection
3452      * @return the vector loaded from an array
3453      * @throws IndexOutOfBoundsException
3454      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3455      *         for any lane {@code N} in the vector
3456      *         where the mask is set
3457      */
3458     @ForceInline
3459     public static
3460     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3461                                        char[] a, int offset,
3462                                        VectorMask<$Boxtype$> m) {
3463         $Type$Species vsp = ($Type$Species) species;
3464         if (offset >= 0 && offset <= (a.length - species.length())) {
3465             $abstractvectortype$ zero = vsp.zero();
3466             return zero.blend(zero.fromCharArray0(a, offset), m);
3467         }
3468 
3469         // FIXME: optimize
3470         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3471         return vsp.vOp(m, i -> (short) a[offset + i]);
3472     }
3473 
3474     /**
3475      * Gathers a new vector composed of elements from an array of type
3476      * {@code char[]},
3477      * using indexes obtained by adding a fixed {@code offset} to a
3478      * series of secondary offsets from an <em>index map</em>.
3479      * The index map is a contiguous sequence of {@code VLENGTH}
3480      * elements in a second array of {@code int}s, starting at a given
3481      * {@code mapOffset}.
3482      * <p>
3483      * For each vector lane, where {@code N} is the vector lane index,
3484      * the lane is loaded from the expression
3485      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3486      * index mapping expression

3606      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3607      *
3608      * @param species species of desired vector
3609      * @param a the array
3610      * @param offset the offset into the array
3611      * @param m the mask controlling lane selection
3612      * @return the vector loaded from an array
3613      * @throws IndexOutOfBoundsException
3614      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3615      *         for any lane {@code N} in the vector
3616      *         where the mask is set
3617      */
3618     @ForceInline
3619     public static
3620     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3621                                           boolean[] a, int offset,
3622                                           VectorMask<$Boxtype$> m) {
3623         $Type$Species vsp = ($Type$Species) species;
3624         if (offset >= 0 && offset <= (a.length - species.length())) {
3625             $abstractvectortype$ zero = vsp.zero();
3626             return zero.blend(zero.fromBooleanArray0(a, offset), m);
3627         }
3628 
3629         // FIXME: optimize
3630         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3631         return vsp.vOp(m, i -> (byte) (a[offset + i] ? 1 : 0));
3632     }
3633 
3634     /**
3635      * Gathers a new vector composed of elements from an array of type
3636      * {@code boolean[]},
3637      * using indexes obtained by adding a fixed {@code offset} to a
3638      * series of secondary offsets from an <em>index map</em>.
3639      * The index map is a contiguous sequence of {@code VLENGTH}
3640      * elements in a second array of {@code int}s, starting at a given
3641      * {@code mapOffset}.
3642      * <p>
3643      * For each vector lane, where {@code N} is the vector lane index,
3644      * the lane is loaded from the expression
3645      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
3646      * index mapping expression

3797      * @param species species of desired vector
3798      * @param bb the byte buffer
3799      * @param offset the offset into the byte buffer
3800      * @param bo the intended byte order
3801      * @param m the mask controlling lane selection
3802      * @return a vector loaded from a byte buffer
3803      * @throws IndexOutOfBoundsException
3804      *         if {@code offset+N*$sizeInBytes$ < 0}
3805      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
3806      *         for any lane {@code N} in the vector
3807      *         where the mask is set
3808      */
3809     @ForceInline
3810     public static
3811     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
3812                                         ByteBuffer bb, int offset,
3813                                         ByteOrder bo,
3814                                         VectorMask<$Boxtype$> m) {
3815         $Type$Species vsp = ($Type$Species) species;
3816         if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
3817             $abstractvectortype$ zero = vsp.zero();
3818             $abstractvectortype$ v = zero.fromByteBuffer0(bb, offset);
3819             return zero.blend(v.maybeSwap(bo), m);
3820         }
3821 
3822         // FIXME: optimize
3823         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
3824         ByteBuffer wb = wrapper(bb, bo);
3825         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
3826                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
3827     }
3828 
3829     // Memory store operations
3830 
3831     /**
3832      * Stores this vector into an array of type {@code $type$[]}
3833      * starting at an offset.
3834      * <p>
3835      * For each vector lane, where {@code N} is the vector lane index,
3836      * the lane element at index {@code N} is stored into the array
3837      * element {@code a[offset+N]}.
3838      *
3839      * @param a the array, of type {@code $type$[]}

3871      * Lanes where the mask is unset are not stored and do not need
3872      * to correspond to legitimate elements of {@code a}.
3873      * That is, unset lanes may correspond to array indexes less than
3874      * zero or beyond the end of the array.
3875      *
3876      * @param a the array, of type {@code $type$[]}
3877      * @param offset the offset into the array
3878      * @param m the mask controlling lane storage
3879      * @throws IndexOutOfBoundsException
3880      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3881      *         for any lane {@code N} in the vector
3882      *         where the mask is set
3883      */
3884     @ForceInline
3885     public final
3886     void intoArray($type$[] a, int offset,
3887                    VectorMask<$Boxtype$> m) {
3888         if (m.allTrue()) {
3889             intoArray(a, offset);
3890         } else {
3891             // FIXME: optimize
3892             $Type$Species vsp = vspecies();
3893             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3894             stOp(a, offset, m, (arr, off, i, v) -> arr[off+i] = v);
3895         }
3896     }
3897 
3898     /**
3899      * Scatters this vector into an array of type {@code $type$[]}
3900      * using indexes obtained by adding a fixed {@code offset} to a
3901      * series of secondary offsets from an <em>index map</em>.
3902      * The index map is a contiguous sequence of {@code VLENGTH}
3903      * elements in a second array of {@code int}s, starting at a given
3904      * {@code mapOffset}.
3905      * <p>
3906      * For each vector lane, where {@code N} is the vector lane index,
3907      * the lane element at index {@code N} is stored into the array
3908      * element {@code a[f(N)]}, where {@code f(N)} is the
3909      * index mapping expression
3910      * {@code offset + indexMap[mapOffset + N]]}.
3911      *
3912      * @param a the array
3913      * @param offset an offset to combine with the index map offsets
3914      * @param indexMap the index map

3956             // of index data is needed.
3957             vix = IntVector
3958                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3959                 .add(offset);
3960         } else {
3961             vix = IntVector
3962                 .fromArray(isp, indexMap, mapOffset)
3963                 .add(offset);
3964         }
3965 
3966 #else[longOrDouble]
3967         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
3968         IntVector vix = IntVector
3969             .fromArray(isp, indexMap, mapOffset)
3970             .add(offset);
3971 #end[longOrDouble]
3972 
3973         vix = VectorIntrinsics.checkIndex(vix, a.length);
3974 
3975         VectorSupport.storeWithMap(
3976             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3977             isp.vectorType(),
3978             a, arrayAddress(a, 0), vix,
3979             this,
3980             a, offset, indexMap, mapOffset,
3981             (arr, off, v, map, mo)
3982             -> v.stOp(arr, off,
3983                       (arr_, off_, i, e) -> {
3984                           int j = map[mo + i];
3985                           arr[off + j] = e;
3986                       }));
3987     }
3988 #end[byteOrShort]
3989 
3990     /**
3991      * Scatters this vector into an array of type {@code $type$[]},
3992      * under the control of a mask, and
3993      * using indexes obtained by adding a fixed {@code offset} to a
3994      * series of secondary offsets from an <em>index map</em>.
3995      * The index map is a contiguous sequence of {@code VLENGTH}
3996      * elements in a second array of {@code int}s, starting at a given
3997      * {@code mapOffset}.
3998      * <p>
3999      * For each vector lane, where {@code N} is the vector lane index,
4000      * if the mask lane at index {@code N} is set then
4001      * the lane element at index {@code N} is stored into the array

4022     public final
4023     void intoArray($type$[] a, int offset,
4024                    int[] indexMap, int mapOffset,
4025                    VectorMask<$Boxtype$> m) {
4026         stOp(a, offset, m,
4027              (arr, off, i, e) -> {
4028                  int j = indexMap[mapOffset + i];
4029                  arr[off + j] = e;
4030              });
4031     }
4032 #else[byteOrShort]
4033     @ForceInline
4034     public final
4035     void intoArray($type$[] a, int offset,
4036                    int[] indexMap, int mapOffset,
4037                    VectorMask<$Boxtype$> m) {
4038         if (m.allTrue()) {
4039             intoArray(a, offset, indexMap, mapOffset);
4040         }
4041         else {
4042             // FIXME: Cannot vectorize yet, if there's a mask.
4043             stOp(a, offset, m,
4044                  (arr, off, i, e) -> {
4045                      int j = indexMap[mapOffset + i];
4046                      arr[off + j] = e;
4047                  });
4048         }
4049     }
4050 #end[byteOrShort]
4051 
4052 #if[short]
4053     /**
4054      * Stores this vector into an array of type {@code char[]}
4055      * starting at an offset.
4056      * <p>
4057      * For each vector lane, where {@code N} is the vector lane index,
4058      * the lane element at index {@code N}
4059      * is first cast to a {@code char} value and then
4060      * stored into the array element {@code a[offset+N]}.
4061      *
4062      * @param a the array, of type {@code char[]}
4063      * @param offset the offset into the array
4064      * @throws IndexOutOfBoundsException
4065      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4066      *         for any lane {@code N} in the vector
4067      */

4095      * Lanes where the mask is unset are not stored and do not need
4096      * to correspond to legitimate elements of {@code a}.
4097      * That is, unset lanes may correspond to array indexes less than
4098      * zero or beyond the end of the array.
4099      *
4100      * @param a the array, of type {@code char[]}
4101      * @param offset the offset into the array
4102      * @param m the mask controlling lane storage
4103      * @throws IndexOutOfBoundsException
4104      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4105      *         for any lane {@code N} in the vector
4106      *         where the mask is set
4107      */
4108     @ForceInline
4109     public final
4110     void intoCharArray(char[] a, int offset,
4111                        VectorMask<$Boxtype$> m) {
4112         if (m.allTrue()) {
4113             intoCharArray(a, offset);
4114         } else {
4115             // FIXME: optimize
4116             $Type$Species vsp = vspecies();
4117             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4118             stOp(a, offset, m, (arr, off, i, v) -> arr[off+i] = (char) v);
4119         }
4120     }
4121 
4122     /**
4123      * Scatters this vector into an array of type {@code char[]}
4124      * using indexes obtained by adding a fixed {@code offset} to a
4125      * series of secondary offsets from an <em>index map</em>.
4126      * The index map is a contiguous sequence of {@code VLENGTH}
4127      * elements in a second array of {@code int}s, starting at a given
4128      * {@code mapOffset}.
4129      * <p>
4130      * For each vector lane, where {@code N} is the vector lane index,
4131      * the lane element at index {@code N}
4132      * is first cast to a {@code char} value and then
4133      * stored into the array
4134      * element {@code a[f(N)]}, where {@code f(N)} is the
4135      * index mapping expression
4136      * {@code offset + indexMap[mapOffset + N]]}.
4137      *
4138      * @param a the array

4258      * Lanes where the mask is unset are not stored and do not need
4259      * to correspond to legitimate elements of {@code a}.
4260      * That is, unset lanes may correspond to array indexes less than
4261      * zero or beyond the end of the array.
4262      *
4263      * @param a the array
4264      * @param offset the offset into the array
4265      * @param m the mask controlling lane storage
4266      * @throws IndexOutOfBoundsException
4267      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4268      *         for any lane {@code N} in the vector
4269      *         where the mask is set
4270      */
4271     @ForceInline
4272     public final
4273     void intoBooleanArray(boolean[] a, int offset,
4274                           VectorMask<$Boxtype$> m) {
4275         if (m.allTrue()) {
4276             intoBooleanArray(a, offset);
4277         } else {
4278             // FIXME: optimize
4279             $Type$Species vsp = vspecies();
4280             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4281             stOp(a, offset, m, (arr, off, i, e) -> arr[off+i] = (e & 1) != 0);
4282         }
4283     }
4284 
4285     /**
4286      * Scatters this vector into an array of type {@code boolean[]}
4287      * using indexes obtained by adding a fixed {@code offset} to a
4288      * series of secondary offsets from an <em>index map</em>.
4289      * The index map is a contiguous sequence of {@code VLENGTH}
4290      * elements in a second array of {@code int}s, starting at a given
4291      * {@code mapOffset}.
4292      * <p>
4293      * For each vector lane, where {@code N} is the vector lane index,
4294      * the lane element at index {@code N}
4295      * is first converted to a {@code boolean} value and then
4296      * stored into the array
4297      * element {@code a[f(N)]}, where {@code f(N)} is the
4298      * index mapping expression
4299      * {@code offset + indexMap[mapOffset + N]]}.
4300      * <p>
4301      * A {@code byte} value is converted to a {@code boolean} value by applying the

4381     @ForceInline
4382     public final
4383     void intoByteArray(byte[] a, int offset,
4384                        ByteOrder bo) {
4385         offset = checkFromIndexSize(offset, byteSize(), a.length);
4386         maybeSwap(bo).intoByteArray0(a, offset);
4387     }
4388 
4389     /**
4390      * {@inheritDoc} <!--workaround-->
4391      */
4392     @Override
4393     @ForceInline
4394     public final
4395     void intoByteArray(byte[] a, int offset,
4396                        ByteOrder bo,
4397                        VectorMask<$Boxtype$> m) {
4398         if (m.allTrue()) {
4399             intoByteArray(a, offset, bo);
4400         } else {
4401             // FIXME: optimize
4402             $Type$Species vsp = vspecies();
4403             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
4404             ByteBuffer wb = wrapper(a, bo);
4405             this.stOp(wb, offset, m,
4406                     (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
4407         }
4408     }
4409 
4410     /**
4411      * {@inheritDoc} <!--workaround-->
4412      */
4413     @Override
4414     @ForceInline
4415     public final
4416     void intoByteBuffer(ByteBuffer bb, int offset,
4417                         ByteOrder bo) {
4418         if (bb.isReadOnly()) {
4419             throw new ReadOnlyBufferException();
4420         }
4421         offset = checkFromIndexSize(offset, byteSize(), bb.limit());
4422         maybeSwap(bo).intoByteBuffer0(bb, offset);
4423     }
4424 
4425     /**
4426      * {@inheritDoc} <!--workaround-->
4427      */
4428     @Override
4429     @ForceInline
4430     public final
4431     void intoByteBuffer(ByteBuffer bb, int offset,
4432                         ByteOrder bo,
4433                         VectorMask<$Boxtype$> m) {
4434         if (m.allTrue()) {
4435             intoByteBuffer(bb, offset, bo);
4436         } else {
4437             // FIXME: optimize
4438             if (bb.isReadOnly()) {
4439                 throw new ReadOnlyBufferException();
4440             }
4441             $Type$Species vsp = vspecies();
4442             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4443             ByteBuffer wb = wrapper(bb, bo);
4444             this.stOp(wb, offset, m,
4445                     (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
4446         }
4447     }
4448 
4449     // ================================================
4450 
4451     // Low-level memory operations.
4452     //
4453     // Note that all of these operations *must* inline into a context
4454     // where the exact species of the involved vector is a
4455     // compile-time constant.  Otherwise, the intrinsic generation
4456     // will fail and performance will suffer.
4457     //
4458     // In many cases this is achieved by re-deriving a version of the
4459     // method in each concrete subclass (per species).  The re-derived
4460     // method simply calls one of these generic methods, with exact
4461     // parameters for the controlling metadata, which is either a
4462     // typed vector or constant species instance.
4463 
4464     // Unchecked loading operations in native byte order.
4465     // Caller is responsible for applying index checks, masking, and
4466     // byte swapping.
4467 
4468     /*package-private*/
4469     abstract
4470     $abstractvectortype$ fromArray0($type$[] a, int offset);
4471     @ForceInline
4472     final
4473     $abstractvectortype$ fromArray0Template($type$[] a, int offset) {
4474         $Type$Species vsp = vspecies();
4475         return VectorSupport.load(
4476             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4477             a, arrayAddress(a, offset),
4478             a, offset, vsp,
4479             (arr, off, s) -> s.ldOp(arr, off,
4480                                     (arr_, off_, i) -> arr_[off_ + i]));
4481     }
4482 














































































4483 #if[short]
4484     /*package-private*/
4485     abstract
4486     $abstractvectortype$ fromCharArray0(char[] a, int offset);
4487     @ForceInline
4488     final
4489     $abstractvectortype$ fromCharArray0Template(char[] a, int offset) {
4490         $Type$Species vsp = vspecies();
4491         return VectorSupport.load(
4492             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4493             a, charArrayAddress(a, offset),
4494             a, offset, vsp,
4495             (arr, off, s) -> s.ldOp(arr, off,
4496                                     (arr_, off_, i) -> (short) arr_[off_ + i]));
4497     }

















4498 #end[short]
4499 
4500 #if[byte]
4501     /*package-private*/
4502     abstract
4503     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset);
4504     @ForceInline
4505     final
4506     $abstractvectortype$ fromBooleanArray0Template(boolean[] a, int offset) {
4507         $Type$Species vsp = vspecies();
4508         return VectorSupport.load(
4509             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4510             a, booleanArrayAddress(a, offset),
4511             a, offset, vsp,
4512             (arr, off, s) -> s.ldOp(arr, off,
4513                                     (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4514     }

















4515 #end[byte]
4516 
4517     @Override
4518     abstract
4519     $abstractvectortype$ fromByteArray0(byte[] a, int offset);
4520     @ForceInline
4521     final
4522     $abstractvectortype$ fromByteArray0Template(byte[] a, int offset) {
4523         $Type$Species vsp = vspecies();
4524         return VectorSupport.load(
4525             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4526             a, byteArrayAddress(a, offset),
4527             a, offset, vsp,
4528             (arr, off, s) -> {
4529                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4530                 return s.ldOp(wb, off,
4531                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4532             });
4533     }
4534 



















4535     abstract
4536     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset);
4537     @ForceInline
4538     final
4539     $abstractvectortype$ fromByteBuffer0Template(ByteBuffer bb, int offset) {
4540         $Type$Species vsp = vspecies();
4541         return ScopedMemoryAccess.loadFromByteBuffer(
4542                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4543                 bb, offset, vsp,
4544                 (buf, off, s) -> {
4545                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4546                     return s.ldOp(wb, off,
4547                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4548                 });
4549     }
4550 


















4551     // Unchecked storing operations in native byte order.
4552     // Caller is responsible for applying index checks, masking, and
4553     // byte swapping.
4554 
4555     abstract
4556     void intoArray0($type$[] a, int offset);
4557     @ForceInline
4558     final
4559     void intoArray0Template($type$[] a, int offset) {
4560         $Type$Species vsp = vspecies();
4561         VectorSupport.store(
4562             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4563             a, arrayAddress(a, offset),
4564             this, a, offset,
4565             (arr, off, v)
4566             -> v.stOp(arr, off,
4567                       (arr_, off_, i, e) -> arr_[off_+i] = e));
4568     }
4569 



































































































4570     abstract
4571     void intoByteArray0(byte[] a, int offset);
4572     @ForceInline
4573     final
4574     void intoByteArray0Template(byte[] a, int offset) {
4575         $Type$Species vsp = vspecies();
4576         VectorSupport.store(
4577             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4578             a, byteArrayAddress(a, offset),
4579             this, a, offset,
4580             (arr, off, v) -> {
4581                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4582                 v.stOp(wb, off,
4583                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
4584             });
4585     }
4586 



















4587     @ForceInline
4588     final
4589     void intoByteBuffer0(ByteBuffer bb, int offset) {
4590         $Type$Species vsp = vspecies();
4591         ScopedMemoryAccess.storeIntoByteBuffer(
4592                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4593                 this, bb, offset,
4594                 (buf, off, v) -> {
4595                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4596                     v.stOp(wb, off,
4597                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
4598                 });
4599     }
4600 






































4601     // End of low-level memory operations.
4602 
4603     private static
4604     void checkMaskFromIndexSize(int offset,
4605                                 $Type$Species vsp,
4606                                 VectorMask<$Boxtype$> m,
4607                                 int scale,
4608                                 int limit) {
4609         ((AbstractMask<$Boxtype$>)m)
4610             .checkIndexByLane(offset, limit, vsp.iota(), scale);
4611     }
4612 
4613     @ForceInline
4614     private void conditionalStoreNYI(int offset,
4615                                      $Type$Species vsp,
4616                                      VectorMask<$Boxtype$> m,
4617                                      int scale,
4618                                      int limit) {
4619         if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
4620             String msg =

4956             $type$[] res = new $type$[laneCount()];
4957             boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
4958             for (int i = 0; i < res.length; i++) {
4959                 if (mbits[i]) {
4960                     res[i] = f.apply(i);
4961                 }
4962             }
4963             return dummyVector().vectorFactory(res);
4964         }
4965 
4966         /*package-private*/
4967         @ForceInline
4968         <M> $abstractvectortype$ ldOp(M memory, int offset,
4969                                       FLdOp<M> f) {
4970             return dummyVector().ldOp(memory, offset, f);
4971         }
4972 
4973         /*package-private*/
4974         @ForceInline
4975         <M> $abstractvectortype$ ldOp(M memory, int offset,
4976                                       AbstractMask<$Boxtype$> m,
4977                                       FLdOp<M> f) {
4978             return dummyVector().ldOp(memory, offset, m, f);
4979         }
4980 
4981         /*package-private*/
4982         @ForceInline
4983         <M> void stOp(M memory, int offset, FStOp<M> f) {
4984             dummyVector().stOp(memory, offset, f);
4985         }
4986 
4987         /*package-private*/
4988         @ForceInline
4989         <M> void stOp(M memory, int offset,
4990                       AbstractMask<$Boxtype$> m,
4991                       FStOp<M> f) {
4992             dummyVector().stOp(memory, offset, m, f);
4993         }
4994 
4995         // N.B. Make sure these constant vectors and
4996         // masks load up correctly into registers.

  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.nio.ByteOrder;
  29 import java.nio.ReadOnlyBufferException;
  30 import java.util.Arrays;
  31 import java.util.Objects;

  32 import java.util.function.Function;
  33 import java.util.function.UnaryOperator;
  34 
  35 import jdk.internal.misc.ScopedMemoryAccess;
  36 import jdk.internal.misc.Unsafe;
  37 import jdk.internal.vm.annotation.ForceInline;
  38 import jdk.internal.vm.vector.VectorSupport;
  39 
  40 import static jdk.internal.vm.vector.VectorSupport.*;
  41 import static jdk.incubator.vector.VectorIntrinsics.*;
  42 
  43 import static jdk.incubator.vector.VectorOperators.*;
  44 
  45 #warn This file is preprocessed before being compiled
  46 
  47 /**
  48  * A specialized {@link Vector} representing an ordered immutable sequence of
  49  * {@code $type$} values.
  50  */
  51 @SuppressWarnings("cast")  // warning: redundant cast

 159     $abstractvectortype$ uOp(FUnOp f);
 160     @ForceInline
 161     final
 162     $abstractvectortype$ uOpTemplate(FUnOp f) {
 163         $type$[] vec = vec();
 164         $type$[] res = new $type$[length()];
 165         for (int i = 0; i < res.length; i++) {
 166             res[i] = f.apply(i, vec[i]);
 167         }
 168         return vectorFactory(res);
 169     }
 170 
 171     /*package-private*/
 172     abstract
 173     $abstractvectortype$ uOp(VectorMask<$Boxtype$> m,
 174                              FUnOp f);
 175     @ForceInline
 176     final
 177     $abstractvectortype$ uOpTemplate(VectorMask<$Boxtype$> m,
 178                                      FUnOp f) {
 179         if (m == null) {
 180             return uOpTemplate(f);
 181         }
 182         $type$[] vec = vec();
 183         $type$[] res = new $type$[length()];
 184         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 185         for (int i = 0; i < res.length; i++) {
 186             res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
 187         }
 188         return vectorFactory(res);
 189     }
 190 
 191     // Binary operator
 192 
 193     /*package-private*/
 194     interface FBinOp {
 195         $type$ apply(int i, $type$ a, $type$ b);
 196     }
 197 
 198     /*package-private*/
 199     abstract
 200     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 201                              FBinOp f);

 205                                      FBinOp f) {
 206         $type$[] res = new $type$[length()];
 207         $type$[] vec1 = this.vec();
 208         $type$[] vec2 = (($abstractvectortype$)o).vec();
 209         for (int i = 0; i < res.length; i++) {
 210             res[i] = f.apply(i, vec1[i], vec2[i]);
 211         }
 212         return vectorFactory(res);
 213     }
 214 
 215     /*package-private*/
 216     abstract
 217     $abstractvectortype$ bOp(Vector<$Boxtype$> o,
 218                              VectorMask<$Boxtype$> m,
 219                              FBinOp f);
 220     @ForceInline
 221     final
 222     $abstractvectortype$ bOpTemplate(Vector<$Boxtype$> o,
 223                                      VectorMask<$Boxtype$> m,
 224                                      FBinOp f) {
 225         if (m == null) {
 226             return bOpTemplate(o, f);
 227         }
 228         $type$[] res = new $type$[length()];
 229         $type$[] vec1 = this.vec();
 230         $type$[] vec2 = (($abstractvectortype$)o).vec();
 231         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 232         for (int i = 0; i < res.length; i++) {
 233             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
 234         }
 235         return vectorFactory(res);
 236     }
 237 
 238     // Ternary operator
 239 
 240     /*package-private*/
 241     interface FTriOp {
 242         $type$ apply(int i, $type$ a, $type$ b, $type$ c);
 243     }
 244 
 245     /*package-private*/
 246     abstract
 247     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,

 257         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 258         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 259         for (int i = 0; i < res.length; i++) {
 260             res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
 261         }
 262         return vectorFactory(res);
 263     }
 264 
 265     /*package-private*/
 266     abstract
 267     $abstractvectortype$ tOp(Vector<$Boxtype$> o1,
 268                              Vector<$Boxtype$> o2,
 269                              VectorMask<$Boxtype$> m,
 270                              FTriOp f);
 271     @ForceInline
 272     final
 273     $abstractvectortype$ tOpTemplate(Vector<$Boxtype$> o1,
 274                                      Vector<$Boxtype$> o2,
 275                                      VectorMask<$Boxtype$> m,
 276                                      FTriOp f) {
 277         if (m == null) {
 278             return tOpTemplate(o1, o2, f);
 279         }
 280         $type$[] res = new $type$[length()];
 281         $type$[] vec1 = this.vec();
 282         $type$[] vec2 = (($abstractvectortype$)o1).vec();
 283         $type$[] vec3 = (($abstractvectortype$)o2).vec();
 284         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 285         for (int i = 0; i < res.length; i++) {
 286             res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
 287         }
 288         return vectorFactory(res);
 289     }
 290 
 291     // Reduction operator
 292 
 293     /*package-private*/
 294     abstract
 295     $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f);
 296 
 297     @ForceInline
 298     final
 299     $type$ rOpTemplate($type$ v, VectorMask<$Boxtype$> m, FBinOp f) {
 300         if (m == null) {
 301             return rOpTemplate(v, f);
 302         }
 303         $type$[] vec = vec();
 304         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 305         for (int i = 0; i < vec.length; i++) {
 306             v = mbits[i] ? f.apply(i, v, vec[i]) : v;
 307         }
 308         return v;
 309     }
 310 
 311     @ForceInline
 312     final
 313     $type$ rOpTemplate($type$ v, FBinOp f) {
 314         $type$[] vec = vec();
 315         for (int i = 0; i < vec.length; i++) {
 316             v = f.apply(i, v, vec[i]);
 317         }
 318         return v;
 319     }
 320 
 321     // Memory reference
 322 
 323     /*package-private*/
 324     interface FLdOp<M> {
 325         $type$ apply(M memory, int offset, int i);
 326     }
 327 
 328     /*package-private*/
 329     @ForceInline
 330     final

 578     }
 579 #end[!long]
 580 
 581     // Unary lanewise support
 582 
 583     /**
 584      * {@inheritDoc} <!--workaround-->
 585      */
 586     public abstract
 587     $abstractvectortype$ lanewise(VectorOperators.Unary op);
 588 
 589     @ForceInline
 590     final
 591     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op) {
 592         if (opKind(op, VO_SPECIAL)) {
 593             if (op == ZOMO) {
 594                 return blend(broadcast(-1), compare(NE, 0));
 595             }
 596 #if[BITWISE]
 597             if (op == NOT) {
 598                 return broadcast(-1).lanewise(XOR, this);
 599             } else if (op == NEG) {
 600                 // FIXME: Support this in the JIT.
 601                 return broadcast(0).lanewise(SUB, this);
 602             }
 603 #end[BITWISE]
 604         }
 605         int opc = opCode(op);
 606         return VectorSupport.unaryOp(
 607             opc, getClass(), null, $type$.class, length(),
 608             this, null,
 609             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));









































 610     }



 611 
 612     /**
 613      * {@inheritDoc} <!--workaround-->
 614      */
 615     @Override
 616     public abstract
 617     $abstractvectortype$ lanewise(VectorOperators.Unary op,
 618                                   VectorMask<$Boxtype$> m);
 619     @ForceInline
 620     final
 621     $abstractvectortype$ lanewiseTemplate(VectorOperators.Unary op,
 622                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
 623                                           VectorMask<$Boxtype$> m) {
 624         m.check(maskClass, this);
 625         if (opKind(op, VO_SPECIAL)) {
 626             if (op == ZOMO) {
 627                 return blend(broadcast(-1), compare(NE, 0, m));
 628             }
 629 #if[BITWISE]
 630             if (op == NOT) {
 631                 return lanewise(XOR, broadcast(-1), m);
 632             } else if (op == NEG) {
 633                 return lanewise(NOT, m).lanewise(ADD, broadcast(1), m);
 634             }
 635 #end[BITWISE]
 636         }
 637         int opc = opCode(op);
 638         return VectorSupport.unaryOp(
 639             opc, getClass(), maskClass, $type$.class, length(),
 640             this, m,
 641             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));
 642     }
 643 
 644     private static final
 645     ImplCache<Unary, UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 646         UN_IMPL = new ImplCache<>(Unary.class, $Type$Vector.class);
 647 
 648     private static UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> unaryOperations(int opc_) {
 649         switch (opc_) {
 650             case VECTOR_OP_NEG: return (v0, m) ->
 651                     v0.uOp(m, (i, a) -> ($type$) -a);
 652             case VECTOR_OP_ABS: return (v0, m) ->
 653                     v0.uOp(m, (i, a) -> ($type$) Math.abs(a));
 654 #if[FP]
 655             case VECTOR_OP_SIN: return (v0, m) ->
 656                     v0.uOp(m, (i, a) -> ($type$) Math.sin(a));
 657             case VECTOR_OP_COS: return (v0, m) ->
 658                     v0.uOp(m, (i, a) -> ($type$) Math.cos(a));
 659             case VECTOR_OP_TAN: return (v0, m) ->
 660                     v0.uOp(m, (i, a) -> ($type$) Math.tan(a));
 661             case VECTOR_OP_ASIN: return (v0, m) ->
 662                     v0.uOp(m, (i, a) -> ($type$) Math.asin(a));
 663             case VECTOR_OP_ACOS: return (v0, m) ->
 664                     v0.uOp(m, (i, a) -> ($type$) Math.acos(a));
 665             case VECTOR_OP_ATAN: return (v0, m) ->
 666                     v0.uOp(m, (i, a) -> ($type$) Math.atan(a));
 667             case VECTOR_OP_EXP: return (v0, m) ->
 668                     v0.uOp(m, (i, a) -> ($type$) Math.exp(a));
 669             case VECTOR_OP_LOG: return (v0, m) ->
 670                     v0.uOp(m, (i, a) -> ($type$) Math.log(a));
 671             case VECTOR_OP_LOG10: return (v0, m) ->
 672                     v0.uOp(m, (i, a) -> ($type$) Math.log10(a));
 673             case VECTOR_OP_SQRT: return (v0, m) ->
 674                     v0.uOp(m, (i, a) -> ($type$) Math.sqrt(a));
 675             case VECTOR_OP_CBRT: return (v0, m) ->
 676                     v0.uOp(m, (i, a) -> ($type$) Math.cbrt(a));
 677             case VECTOR_OP_SINH: return (v0, m) ->
 678                     v0.uOp(m, (i, a) -> ($type$) Math.sinh(a));
 679             case VECTOR_OP_COSH: return (v0, m) ->
 680                     v0.uOp(m, (i, a) -> ($type$) Math.cosh(a));
 681             case VECTOR_OP_TANH: return (v0, m) ->
 682                     v0.uOp(m, (i, a) -> ($type$) Math.tanh(a));
 683             case VECTOR_OP_EXPM1: return (v0, m) ->
 684                     v0.uOp(m, (i, a) -> ($type$) Math.expm1(a));
 685             case VECTOR_OP_LOG1P: return (v0, m) ->
 686                     v0.uOp(m, (i, a) -> ($type$) Math.log1p(a));
 687 #end[FP]
 688             default: return null;
 689         }
 690     }
 691 
 692     // Binary lanewise support
 693 
 694     /**
 695      * {@inheritDoc} <!--workaround-->
 696      * @see #lanewise(VectorOperators.Binary,$type$)
 697      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 698      */
 699     @Override
 700     public abstract
 701     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 702                                   Vector<$Boxtype$> v);
 703     @ForceInline
 704     final
 705     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 706                                           Vector<$Boxtype$> v) {
 707         $abstractvectortype$ that = ($abstractvectortype$) v;
 708         that.check(this);
 709 
 710         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 711             if (op == FIRST_NONZERO) {
 712                 // FIXME: Support this in the JIT.
 713                 VectorMask<$Boxbitstype$> thisNZ
 714                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 715                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 716                 op = OR_UNCHECKED;
 717 #if[FP]
 718                 // FIXME: Support OR_UNCHECKED on float/double also!
 719                 return this.viewAsIntegralLanes()
 720                     .lanewise(op, that.viewAsIntegralLanes())
 721                     .viewAsFloatingLanes();
 722 #end[FP]
 723             }
 724 #if[BITWISE]
 725 #if[!FP]
 726             if (opKind(op, VO_SHIFT)) {
 727                 // As per shift specification for Java, mask the shift count.
 728                 // This allows the JIT to ignore some ISA details.
 729                 that = that.lanewise(AND, SHIFT_MASK);
 730             }
 731 #end[!FP]
 732             if (op == AND_NOT) {
 733                 // FIXME: Support this in the JIT.
 734                 that = that.lanewise(NOT);
 735                 op = AND;
 736             } else if (op == DIV) {
 737                 VectorMask<$Boxtype$> eqz = that.eq(($type$) 0);
 738                 if (eqz.anyTrue()) {
 739                     throw that.divZeroException();
 740                 }
 741             }
 742 #end[BITWISE]
 743         }
 744 
 745         int opc = opCode(op);
 746         return VectorSupport.binaryOp(
 747             opc, getClass(), null, $type$.class, length(),
 748             this, that, null,
 749             BIN_IMPL.find(op, opc, $abstractvectortype$::binaryOperations));









































 750     }



 751 
 752     /**
 753      * {@inheritDoc} <!--workaround-->
 754      * @see #lanewise(VectorOperators.Binary,$type$,VectorMask)
 755      */
 756     @Override
 757     public abstract
 758     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 759                                   Vector<$Boxtype$> v,
 760                                   VectorMask<$Boxtype$> m);
 761     @ForceInline
 762     final
 763     $abstractvectortype$ lanewiseTemplate(VectorOperators.Binary op,
 764                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
 765                                           Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 766         $abstractvectortype$ that = ($abstractvectortype$) v;
 767         that.check(this);
 768         m.check(maskClass, this);
 769 
 770         if (opKind(op, VO_SPECIAL {#if[!FP]? | VO_SHIFT})) {
 771             if (op == FIRST_NONZERO) {
 772 #if[FP]
 773                 return blend(lanewise(op, v), m);
 774 #else[FP]
 775                 // FIXME: Support this in the JIT.
 776                 VectorMask<$Boxbitstype$> thisNZ
 777                     = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
 778                 that = that.blend(($type$) 0, thisNZ.cast(vspecies()));
 779                 op = OR_UNCHECKED;
 780 #end[FP]
 781             }
 782 #if[BITWISE]
 783 #if[!FP]
 784             if (opKind(op, VO_SHIFT)) {
 785                 // As per shift specification for Java, mask the shift count.
 786                 // This allows the JIT to ignore some ISA details.
 787                 that = that.lanewise(AND, SHIFT_MASK);
 788             }
 789 #end[!FP]
 790             if (op == AND_NOT) {
 791                 // FIXME: Support this in the JIT.
 792                 that = that.lanewise(NOT);
 793                 op = AND;
 794             } else if (op == DIV) {
 795                 VectorMask<$Boxtype$> eqz = that.eq(($type$)0);
 796                 if (eqz.and(m).anyTrue()) {
 797                     throw that.divZeroException();
 798                 }
 799                 // suppress div/0 exceptions in unset lanes
 800                 that = that.lanewise(NOT, eqz);
 801             }
 802 #end[BITWISE]


 803         }
 804 
 805         int opc = opCode(op);
 806         return VectorSupport.binaryOp(
 807             opc, getClass(), maskClass, $type$.class, length(),
 808             this, that, m,
 809             BIN_IMPL.find(op, opc, $abstractvectortype$::binaryOperations));
 810     }
 811 
 812     private static final
 813     ImplCache<Binary, BinaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 814         BIN_IMPL = new ImplCache<>(Binary.class, $Type$Vector.class);
 815 
 816     private static BinaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> binaryOperations(int opc_) {
 817         switch (opc_) {
 818             case VECTOR_OP_ADD: return (v0, v1, vm) ->
 819                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a + b));
 820             case VECTOR_OP_SUB: return (v0, v1, vm) ->
 821                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a - b));
 822             case VECTOR_OP_MUL: return (v0, v1, vm) ->
 823                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a * b));
 824             case VECTOR_OP_DIV: return (v0, v1, vm) ->
 825                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a / b));
 826             case VECTOR_OP_MAX: return (v0, v1, vm) ->
 827                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.max(a, b));
 828             case VECTOR_OP_MIN: return (v0, v1, vm) ->
 829                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.min(a, b));
 830 #if[BITWISE]
 831             case VECTOR_OP_AND: return (v0, v1, vm) ->
 832                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a & b));
 833             case VECTOR_OP_OR: return (v0, v1, vm) ->
 834                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a | b));
 835             case VECTOR_OP_XOR: return (v0, v1, vm) ->
 836                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a ^ b));
 837             case VECTOR_OP_LSHIFT: return (v0, v1, vm) ->
 838                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a << n));
 839             case VECTOR_OP_RSHIFT: return (v0, v1, vm) ->
 840                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a >> n));
 841             case VECTOR_OP_URSHIFT: return (v0, v1, vm) ->
 842                     v0.bOp(v1, vm, (i, a, n) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 843             case VECTOR_OP_LROTATE: return (v0, v1, vm) ->
 844                     v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
 845             case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
 846                     v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
 847 #end[BITWISE]
 848 #if[FP]
 849             case VECTOR_OP_OR: return (v0, v1, vm) ->
 850                     v0.bOp(v1, vm, (i, a, b) -> fromBits(toBits(a) | toBits(b)));
 851             case VECTOR_OP_ATAN2: return (v0, v1, vm) ->
 852                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.atan2(a, b));
 853             case VECTOR_OP_POW: return (v0, v1, vm) ->
 854                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.pow(a, b));
 855             case VECTOR_OP_HYPOT: return (v0, v1, vm) ->
 856                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.hypot(a, b));
 857 #end[FP]
 858             default: return null;
 859         }
 860     }
 861 
 862     // FIXME: Maybe all of the public final methods in this file (the
 863     // simple ones that just call lanewise) should be pushed down to
 864     // the X-VectorBits template.  They can't optimize properly at
 865     // this level, and must rely on inlining.  Does it work?
 866     // (If it works, of course keep the code here.)
 867 
 868     /**
 869      * Combines the lane values of this vector
 870      * with the value of a broadcast scalar.
 871      *
 872      * This is a lane-wise binary operation which applies
 873      * the selected operation to each lane.
 874      * The return value will be equal to this expression:
 875      * {@code this.lanewise(op, this.broadcast(e))}.
 876      *
 877      * @param op the operation used to process lane values
 878      * @param e the input scalar
 879      * @return the result of applying the operation lane-wise
 880      *         to the two input vectors
 881      * @throws UnsupportedOperationException if this vector does

 906      * This is a masked lane-wise binary operation which applies
 907      * the selected operation to each lane.
 908      * The return value will be equal to this expression:
 909      * {@code this.lanewise(op, this.broadcast(e), m)}.
 910      *
 911      * @param op the operation used to process lane values
 912      * @param e the input scalar
 913      * @param m the mask controlling lane selection
 914      * @return the result of applying the operation lane-wise
 915      *         to the input vector and the scalar
 916      * @throws UnsupportedOperationException if this vector does
 917      *         not support the requested operation
 918      * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
 919      * @see #lanewise(VectorOperators.Binary,$type$)
 920      */
 921     @ForceInline
 922     public final
 923     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 924                                   $type$ e,
 925                                   VectorMask<$Boxtype$> m) {
 926 #if[BITWISE]
 927         if (opKind(op, VO_SHIFT) && ($type$)(int)e == e) {
 928             return lanewiseShift(op, (int) e, m);
 929         }
 930         if (op == AND_NOT) {
 931             op = AND; e = ($type$) ~e;
 932         }
 933 #end[BITWISE]
 934         return lanewise(op, broadcast(e), m);
 935     }
 936 
 937 #if[!long]
 938     /**
 939      * {@inheritDoc} <!--workaround-->
 940      * @apiNote
 941      * When working with vector subtypes like {@code $abstractvectortype$},
 942      * {@linkplain #lanewise(VectorOperators.Binary,$type$)
 943      * the more strongly typed method}
 944      * is typically selected.  It can be explicitly selected
 945      * using a cast: {@code v.lanewise(op,($type$)e)}.
 946      * The two expressions will produce numerically identical results.
 947      */
 948     @ForceInline
 949     public final
 950     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 951                                   long e) {
 952         $type$ e1 = ($type$) e;

 953 #if[BITWISE]
 954         if ((long)e1 != e
 955             // allow shift ops to clip down their int parameters
 956             && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
 957 #else[BITWISE]
 958         if ((long)e1 != e) {
 959 #end[BITWISE]

 960             vspecies().checkValue(e);  // for exception
 961         }
 962         return lanewise(op, e1);
 963     }
 964 
 965     /**
 966      * {@inheritDoc} <!--workaround-->
 967      * @apiNote
 968      * When working with vector subtypes like {@code $abstractvectortype$},
 969      * {@linkplain #lanewise(VectorOperators.Binary,$type$,VectorMask)
 970      * the more strongly typed method}
 971      * is typically selected.  It can be explicitly selected
 972      * using a cast: {@code v.lanewise(op,($type$)e,m)}.
 973      * The two expressions will produce numerically identical results.
 974      */
 975     @ForceInline
 976     public final
 977     $abstractvectortype$ lanewise(VectorOperators.Binary op,
 978                                   long e, VectorMask<$Boxtype$> m) {
 979         $type$ e1 = ($type$) e;
 980 #if[BITWISE]
 981         if ((long)e1 != e
 982             // allow shift ops to clip down their int parameters
 983             && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
 984 #else[BITWISE]
 985         if ((long)e1 != e) {
 986 #end[BITWISE]
 987             vspecies().checkValue(e);  // for exception
 988         }
 989         return lanewise(op, e1, m);
 990     }
 991 #end[!long]
 992 
 993 #if[BITWISE]
 994     /*package-private*/
 995     abstract $abstractvectortype$
 996     lanewiseShift(VectorOperators.Binary op, int e);
 997 
 998     /*package-private*/
 999     @ForceInline
1000     final $abstractvectortype$
1001     lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
1002         // Special handling for these.  FIXME: Refactor?
1003         assert(opKind(op, VO_SHIFT));
1004         // As per shift specification for Java, mask the shift count.
1005         e &= SHIFT_MASK;
1006         int opc = opCode(op);
1007         return VectorSupport.broadcastInt(
1008             opc, getClass(), null, $type$.class, length(),
1009             this, e, null,
1010             BIN_INT_IMPL.find(op, opc, $abstractvectortype$::broadcastIntOperations));
1011     }
1012 
1013     /*package-private*/
1014     abstract $abstractvectortype$
1015     lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m);
1016 
1017     /*package-private*/
1018     @ForceInline
1019     final $abstractvectortype$
1020     lanewiseShiftTemplate(VectorOperators.Binary op,
1021                           Class<? extends VectorMask<$Boxtype$>> maskClass,
1022                           int e, VectorMask<$Boxtype$> m) {
1023         m.check(maskClass, this);
1024         assert(opKind(op, VO_SHIFT));
1025         // As per shift specification for Java, mask the shift count.
1026         e &= SHIFT_MASK;
1027         int opc = opCode(op);
1028         return VectorSupport.broadcastInt(
1029             opc, getClass(), maskClass, $type$.class, length(),
1030             this, e, m,
1031             BIN_INT_IMPL.find(op, opc, $abstractvectortype$::broadcastIntOperations));
1032     }
1033 
1034     private static final
1035     ImplCache<Binary,VectorBroadcastIntOp<$abstractvectortype$, VectorMask<$Boxtype$>>> BIN_INT_IMPL
1036         = new ImplCache<>(Binary.class, $Type$Vector.class);
1037 
1038     private static VectorBroadcastIntOp<$abstractvectortype$, VectorMask<$Boxtype$>> broadcastIntOperations(int opc_) {
1039         switch (opc_) {
1040             case VECTOR_OP_LSHIFT: return (v, n, m) ->
1041                     v.uOp(m, (i, a) -> ($type$)(a << n));
1042             case VECTOR_OP_RSHIFT: return (v, n, m) ->
1043                     v.uOp(m, (i, a) -> ($type$)(a >> n));
1044             case VECTOR_OP_URSHIFT: return (v, n, m) ->
1045                     v.uOp(m, (i, a) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
1046             case VECTOR_OP_LROTATE: return (v, n, m) ->
1047                     v.uOp(m, (i, a) -> rotateLeft(a, (int)n));
1048             case VECTOR_OP_RROTATE: return (v, n, m) ->
1049                     v.uOp(m, (i, a) -> rotateRight(a, (int)n));
1050             default: return null;
1051         }
1052     }
1053 
1054     // As per shift specification for Java, mask the shift count.
1055     // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
1056     // The latter two maskings go beyond the JLS, but seem reasonable
1057     // since our lane types are first-class types, not just dressed
1058     // up ints.
1059     private static final int SHIFT_MASK = ($Boxtype$.SIZE - 1);
1060 #if[byteOrShort]
1061     // Also simulate >>> on sub-word variables with a mask.
1062     private static final int LSHR_SETUP_MASK = ((1 << $Boxtype$.SIZE) - 1);
1063 #else[byteOrShort]
1064     private static final $type$ LSHR_SETUP_MASK = -1;
1065 #end[byteOrShort]
1066 #end[BITWISE]
1067 
1068     // Ternary lanewise support
1069 
1070     // Ternary operators come in eight variations:
1071     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
1072     //   lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
1073 

1093     final
1094     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
1095                                           Vector<$Boxtype$> v1,
1096                                           Vector<$Boxtype$> v2) {
1097         $abstractvectortype$ that = ($abstractvectortype$) v1;
1098         $abstractvectortype$ tother = ($abstractvectortype$) v2;
1099         // It's a word: https://www.dictionary.com/browse/tother
1100         // See also Chapter 11 of Dickens, Our Mutual Friend:
1101         // "Totherest Governor," replied Mr Riderhood...
1102         that.check(this);
1103         tother.check(this);
1104 #if[BITWISE]
1105         if (op == BITWISE_BLEND) {
1106             // FIXME: Support this in the JIT.
1107             that = this.lanewise(XOR, that).lanewise(AND, tother);
1108             return this.lanewise(XOR, that);
1109         }
1110 #end[BITWISE]
1111         int opc = opCode(op);
1112         return VectorSupport.ternaryOp(
1113             opc, getClass(), null, $type$.class, length(),
1114             this, that, tother, null,
1115             TERN_IMPL.find(op, opc, $abstractvectortype$::ternaryOperations));







1116     }



1117 
1118     /**
1119      * {@inheritDoc} <!--workaround-->
1120      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1121      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
1122      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)
1123      */
1124     @Override
1125     public abstract
1126     $abstractvectortype$ lanewise(VectorOperators.Ternary op,
1127                                   Vector<$Boxtype$> v1,
1128                                   Vector<$Boxtype$> v2,
1129                                   VectorMask<$Boxtype$> m);
1130     @ForceInline
1131     final
1132     $abstractvectortype$ lanewiseTemplate(VectorOperators.Ternary op,
1133                                           Class<? extends VectorMask<$Boxtype$>> maskClass,
1134                                           Vector<$Boxtype$> v1,
1135                                           Vector<$Boxtype$> v2,
1136                                           VectorMask<$Boxtype$> m) {
1137         $abstractvectortype$ that = ($abstractvectortype$) v1;
1138         $abstractvectortype$ tother = ($abstractvectortype$) v2;
1139         // It's a word: https://www.dictionary.com/browse/tother
1140         // See also Chapter 11 of Dickens, Our Mutual Friend:
1141         // "Totherest Governor," replied Mr Riderhood...
1142         that.check(this);
1143         tother.check(this);
1144         m.check(maskClass, this);
1145 
1146 #if[BITWISE]
1147         if (op == BITWISE_BLEND) {
1148             // FIXME: Support this in the JIT.
1149             that = this.lanewise(XOR, that).lanewise(AND, tother);
1150             return this.lanewise(XOR, that, m);
1151         }
1152 #end[BITWISE]
1153         int opc = opCode(op);
1154         return VectorSupport.ternaryOp(
1155             opc, getClass(), maskClass, $type$.class, length(),
1156             this, that, tother, m,
1157             TERN_IMPL.find(op, opc, $abstractvectortype$::ternaryOperations));
1158     }
1159 
1160     private static final
1161     ImplCache<Ternary, TernaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
1162         TERN_IMPL = new ImplCache<>(Ternary.class, $Type$Vector.class);
1163 
1164     private static TernaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> ternaryOperations(int opc_) {
1165         switch (opc_) {
1166 #if[FP]
1167             case VECTOR_OP_FMA: return (v0, v1_, v2_, m) ->
1168                     v0.tOp(v1_, v2_, m, (i, a, b, c) -> Math.fma(a, b, c));
1169 #end[FP]
1170             default: return null;
1171         }
1172     }
1173 
1174     /**
1175      * Combines the lane values of this vector
1176      * with the values of two broadcast scalars.
1177      *
1178      * This is a lane-wise ternary operation which applies
1179      * the selected operation to each lane.
1180      * The return value will be equal to this expression:
1181      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
1182      *
1183      * @param op the operation used to combine lane values
1184      * @param e1 the first input scalar
1185      * @param e2 the second input scalar
1186      * @return the result of applying the operation lane-wise
1187      *         to the input vector and the scalars
1188      * @throws UnsupportedOperationException if this vector does
1189      *         not support the requested operation
1190      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1191      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)

1208      * The return value will be equal to this expression:
1209      * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
1210      *
1211      * @param op the operation used to combine lane values
1212      * @param e1 the first input scalar
1213      * @param e2 the second input scalar
1214      * @param m the mask controlling lane selection
1215      * @return the result of applying the operation lane-wise
1216      *         to the input vector and the scalars
1217      * @throws UnsupportedOperationException if this vector does
1218      *         not support the requested operation
1219      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1220      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1221      */
1222     @ForceInline
1223     public final
1224     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
1225                                   $type$ e1,
1226                                   $type$ e2,
1227                                   VectorMask<$Boxtype$> m) {
1228         return lanewise(op, broadcast(e1), broadcast(e2), m);
1229     }
1230 
1231     /**
1232      * Combines the lane values of this vector
1233      * with the values of another vector and a broadcast scalar.
1234      *
1235      * This is a lane-wise ternary operation which applies
1236      * the selected operation to each lane.
1237      * The return value will be equal to this expression:
1238      * {@code this.lanewise(op, v1, this.broadcast(e2))}.
1239      *
1240      * @param op the operation used to combine lane values
1241      * @param v1 the other input vector
1242      * @param e2 the input scalar
1243      * @return the result of applying the operation lane-wise
1244      *         to the input vectors and the scalar
1245      * @throws UnsupportedOperationException if this vector does
1246      *         not support the requested operation
1247      * @see #lanewise(VectorOperators.Ternary,$type$,$type$)
1248      * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)

1266      * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
1267      *
1268      * @param op the operation used to combine lane values
1269      * @param v1 the other input vector
1270      * @param e2 the input scalar
1271      * @param m the mask controlling lane selection
1272      * @return the result of applying the operation lane-wise
1273      *         to the input vectors and the scalar
1274      * @throws UnsupportedOperationException if this vector does
1275      *         not support the requested operation
1276      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1277      * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
1278      * @see #lanewise(VectorOperators.Ternary,Vector,$type$)
1279      */
1280     @ForceInline
1281     public final
1282     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
1283                                   Vector<$Boxtype$> v1,
1284                                   $type$ e2,
1285                                   VectorMask<$Boxtype$> m) {
1286         return lanewise(op, v1, broadcast(e2), m);
1287     }
1288 
1289     /**
1290      * Combines the lane values of this vector
1291      * with the values of another vector and a broadcast scalar.
1292      *
1293      * This is a lane-wise ternary operation which applies
1294      * the selected operation to each lane.
1295      * The return value will be equal to this expression:
1296      * {@code this.lanewise(op, this.broadcast(e1), v2)}.
1297      *
1298      * @param op the operation used to combine lane values
1299      * @param e1 the input scalar
1300      * @param v2 the other input vector
1301      * @return the result of applying the operation lane-wise
1302      *         to the input vectors and the scalar
1303      * @throws UnsupportedOperationException if this vector does
1304      *         not support the requested operation
1305      * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1306      * @see #lanewise(VectorOperators.Ternary,$type$,Vector,VectorMask)

1323      * The return value will be equal to this expression:
1324      * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
1325      *
1326      * @param op the operation used to combine lane values
1327      * @param e1 the input scalar
1328      * @param v2 the other input vector
1329      * @param m the mask controlling lane selection
1330      * @return the result of applying the operation lane-wise
1331      *         to the input vectors and the scalar
1332      * @throws UnsupportedOperationException if this vector does
1333      *         not support the requested operation
1334      * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1335      * @see #lanewise(VectorOperators.Ternary,$type$,Vector)
1336      */
1337     @ForceInline
1338     public final
1339     $abstractvectortype$ lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
1340                                   $type$ e1,
1341                                   Vector<$Boxtype$> v2,
1342                                   VectorMask<$Boxtype$> m) {
1343         return lanewise(op, broadcast(e1), v2, m);
1344     }
1345 
1346     // (Thus endeth the Great and Mighty Ternary Ogdoad.)
1347     // https://en.wikipedia.org/wiki/Ogdoad
1348 
1349     /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1350     //
1351     // These include masked and non-masked versions.
1352     // This subclass adds broadcast (masked or not).
1353 
1354     /**
1355      * {@inheritDoc} <!--workaround-->
1356      * @see #add($type$)
1357      */
1358     @Override
1359     @ForceInline
1360     public final $abstractvectortype$ add(Vector<$Boxtype$> v) {
1361         return lanewise(ADD, v);
1362     }
1363 

2168     @Override
2169     @ForceInline
2170     public final
2171     VectorMask<$Boxtype$> test(VectorOperators.Test op,
2172                                   VectorMask<$Boxtype$> m) {
2173         return test(op).and(m);
2174     }
2175 
2176     /**
2177      * {@inheritDoc} <!--workaround-->
2178      */
2179     @Override
2180     public abstract
2181     VectorMask<$Boxtype$> compare(VectorOperators.Comparison op, Vector<$Boxtype$> v);
2182 
2183     /*package-private*/
2184     @ForceInline
2185     final
2186     <M extends VectorMask<$Boxtype$>>
2187     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v) {


2188         $abstractvectortype$ that = ($abstractvectortype$) v;
2189         that.check(this);
2190         int opc = opCode(op);
2191         return VectorSupport.compare(
2192             opc, getClass(), maskType, $type$.class, length(),
2193             this, that, null,
2194             (cond, v0, v1, m1) -> {
2195                 AbstractMask<$Boxtype$> m
2196                     = v0.bTest(cond, v1, (cond_, i, a, b)
2197                                -> compareWithOp(cond, a, b));
2198                 @SuppressWarnings("unchecked")
2199                 M m2 = (M) m;
2200                 return m2;
2201             });
2202     }
2203 
2204     /*package-private*/
2205     @ForceInline
2206     final
2207     <M extends VectorMask<$Boxtype$>>
2208     M compareTemplate(Class<M> maskType, Comparison op, Vector<$Boxtype$> v, M m) {
2209         $abstractvectortype$ that = ($abstractvectortype$) v;
2210         that.check(this);
2211         m.check(maskType, this);
2212         int opc = opCode(op);
2213         return VectorSupport.compare(
2214             opc, getClass(), maskType, $type$.class, length(),
2215             this, that, m,
2216             (cond, v0, v1, m1) -> {
2217                 AbstractMask<$Boxtype$> cmpM
2218                     = v0.bTest(cond, v1, (cond_, i, a, b)
2219                                -> compareWithOp(cond, a, b));
2220                 @SuppressWarnings("unchecked")
2221                 M m2 = (M) cmpM.and(m1);
2222                 return m2;
2223             });
2224     }
2225 
2226     @ForceInline
2227     private static boolean compareWithOp(int cond, $type$ a, $type$ b) {
2228         return switch (cond) {
2229             case BT_eq -> a == b;
2230             case BT_ne -> a != b;
2231             case BT_lt -> a < b;
2232             case BT_le -> a <= b;
2233             case BT_gt -> a > b;
2234             case BT_ge -> a >= b;
2235 #if[!FP]
2236             case BT_ult -> $Boxtype$.compareUnsigned(a, b) < 0;
2237             case BT_ule -> $Boxtype$.compareUnsigned(a, b) <= 0;
2238             case BT_ugt -> $Boxtype$.compareUnsigned(a, b) > 0;
2239             case BT_uge -> $Boxtype$.compareUnsigned(a, b) >= 0;
2240 #end[!FP]
2241             default -> throw new AssertionError();
2242         };
2243     }
2244 












2245     /**
2246      * Tests this vector by comparing it with an input scalar,
2247      * according to the given comparison operation.
2248      *
2249      * This is a lane-wise binary test operation which applies
2250      * the comparison operation to each lane.
2251      * <p>
2252      * The result is the same as
2253      * {@code compare(op, broadcast(species(), e))}.
2254      * That is, the scalar may be regarded as broadcast to
2255      * a vector of the same species, and then compared
2256      * against the original vector, using the selected
2257      * comparison operation.
2258      *
2259      * @param op the operation used to compare lane values
2260      * @param e the input scalar
2261      * @return the mask result of testing lane-wise if this vector
2262      *         compares to the input, according to the selected
2263      *         comparison operator
2264      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector)

2283      *
2284      * This is a masked lane-wise binary test operation which applies
2285      * to each pair of corresponding lane values.
2286      *
2287      * The returned result is equal to the expression
2288      * {@code compare(op,s).and(m)}.
2289      *
2290      * @param op the operation used to compare lane values
2291      * @param e the input scalar
2292      * @param m the mask controlling lane selection
2293      * @return the mask result of testing lane-wise if this vector
2294      *         compares to the input, according to the selected
2295      *         comparison operator,
2296      *         and only in the lanes selected by the mask
2297      * @see $abstractvectortype$#compare(VectorOperators.Comparison,Vector,VectorMask)
2298      */
2299     @ForceInline
2300     public final VectorMask<$Boxtype$> compare(VectorOperators.Comparison op,
2301                                                $type$ e,
2302                                                VectorMask<$Boxtype$> m) {
2303         return compare(op, broadcast(e), m);
2304     }
2305 
2306 #if[!long]
2307     /**
2308      * {@inheritDoc} <!--workaround-->
2309      */
2310     @Override
2311     public abstract
2312     VectorMask<$Boxtype$> compare(Comparison op, long e);
2313 
2314     /*package-private*/
2315     @ForceInline
2316     final
2317     <M extends VectorMask<$Boxtype$>>
2318     M compareTemplate(Class<M> maskType, Comparison op, long e) {
2319         return compareTemplate(maskType, op, broadcast(e));
2320     }
2321 
2322     /**
2323      * {@inheritDoc} <!--workaround-->

2538     wrongPartForSlice(int part) {
2539         String msg = String.format("bad part number %d for slice operation",
2540                                    part);
2541         return new ArrayIndexOutOfBoundsException(msg);
2542     }
2543 
2544     /**
2545      * {@inheritDoc} <!--workaround-->
2546      */
2547     @Override
2548     public abstract
2549     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> m);
2550 
2551     /*package-private*/
2552     @ForceInline
2553     final
2554     <S extends VectorShuffle<$Boxtype$>>
2555     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype, S shuffle) {
2556         shuffle.checkIndexes();
2557         return VectorSupport.rearrangeOp(
2558             getClass(), shuffletype, null, $type$.class, length(),
2559             this, shuffle, null,
2560             (v1, s_, m_) -> v1.uOp((i, a) -> {
2561                 int ei = s_.laneSource(i);
2562                 return v1.lane(ei);
2563             }));
2564     }
2565 
2566     /**
2567      * {@inheritDoc} <!--workaround-->
2568      */
2569     @Override
2570     public abstract
2571     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2572                                    VectorMask<$Boxtype$> m);
2573 
2574     /*package-private*/
2575     @ForceInline
2576     final
2577     <S extends VectorShuffle<$Boxtype$>, M extends VectorMask<$Boxtype$>>
2578     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2579                                            Class<M> masktype,
2580                                            S shuffle,
2581                                            M m) {
2582 
2583         m.check(masktype, this);






2584         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2585         if (m.andNot(valid).anyTrue()) {
2586             shuffle.checkIndexes();
2587             throw new AssertionError();
2588         }
2589         return VectorSupport.rearrangeOp(
2590                    getClass(), shuffletype, masktype, $type$.class, length(),
2591                    this, shuffle, m,
2592                    (v1, s_, m_) -> v1.uOp((i, a) -> {
2593                         int ei = s_.laneSource(i);
2594                         return ei < 0  || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
2595                    }));
2596     }
2597 
2598     /**
2599      * {@inheritDoc} <!--workaround-->
2600      */
2601     @Override
2602     public abstract
2603     $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> s,
2604                                    Vector<$Boxtype$> v);
2605 
2606     /*package-private*/
2607     @ForceInline
2608     final
2609     <S extends VectorShuffle<$Boxtype$>>
2610     $abstractvectortype$ rearrangeTemplate(Class<S> shuffletype,
2611                                            S shuffle,
2612                                            $abstractvectortype$ v) {
2613         VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
2614         @SuppressWarnings("unchecked")
2615         S ws = (S) shuffle.wrapIndexes();
2616         $abstractvectortype$ r0 =
2617             VectorSupport.rearrangeOp(
2618                 getClass(), shuffletype, null, $type$.class, length(),
2619                 this, ws, null,
2620                 (v0, s_, m_) -> v0.uOp((i, a) -> {
2621                     int ei = s_.laneSource(i);
2622                     return v0.lane(ei);
2623                 }));
2624         $abstractvectortype$ r1 =
2625             VectorSupport.rearrangeOp(
2626                 getClass(), shuffletype, null, $type$.class, length(),
2627                 v, ws, null,
2628                 (v1, s_, m_) -> v1.uOp((i, a) -> {
2629                     int ei = s_.laneSource(i);
2630                     return v1.lane(ei);
2631                 }));
2632         return r1.blend(r0, valid);
2633     }
2634 
2635     @ForceInline
2636     private final
2637     VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) {
2638         $type$[] a = toArray();
2639         int[] sa = new int[a.length];
2640         for (int i = 0; i < a.length; i++) {
2641             sa[i] = (int) a[i];
2642         }
2643         return VectorShuffle.fromArray(dsp, sa, 0);
2644     }
2645 
2646     /*package-private*/
2647     @ForceInline
2648     final

3000      * <li>
3001      * All other reduction operations are fully commutative and
3002      * associative.  The implementation can choose any order of
3003      * processing, yet it will always produce the same result.
3004      * </ul>
3005      *
3006      * @param op the operation used to combine lane values
3007      * @param m the mask controlling lane selection
3008      * @return the reduced result accumulated from the selected lane values
3009      * @throws UnsupportedOperationException if this vector does
3010      *         not support the requested operation
3011      * @see #reduceLanes(VectorOperators.Associative)
3012      */
3013     public abstract $type$ reduceLanes(VectorOperators.Associative op,
3014                                        VectorMask<$Boxtype$> m);
3015 
3016     /*package-private*/
3017     @ForceInline
3018     final
3019     $type$ reduceLanesTemplate(VectorOperators.Associative op,
3020                                Class<? extends VectorMask<$Boxtype$>> maskClass,
3021                                VectorMask<$Boxtype$> m) {
3022         m.check(maskClass, this);
3023         if (op == FIRST_NONZERO) {
3024             $abstractvectortype$ v = reduceIdentityVector(op).blend(this, m);
3025             return v.reduceLanesTemplate(op);
3026         }
3027         int opc = opCode(op);
3028         return fromBits(VectorSupport.reductionCoerced(
3029             opc, getClass(), maskClass, $type$.class, length(),
3030             this, m,
3031             REDUCE_IMPL.find(op, opc, $abstractvectortype$::reductionOperations)));
3032     }
3033 
3034     /*package-private*/
3035     @ForceInline
3036     final
3037     $type$ reduceLanesTemplate(VectorOperators.Associative op) {
3038         if (op == FIRST_NONZERO) {
3039             // FIXME:  The JIT should handle this, and other scan ops alos.
3040             VectorMask<$Boxbitstype$> thisNZ
3041                 = this.viewAsIntegralLanes().compare(NE, ($bitstype$) 0);
3042             return this.lane(thisNZ.firstTrue());
3043         }
3044         int opc = opCode(op);
3045         return fromBits(VectorSupport.reductionCoerced(
3046             opc, getClass(), null, $type$.class, length(),
3047             this, null,
3048             REDUCE_IMPL.find(op, opc, $abstractvectortype$::reductionOperations)));
3049     }
3050 
3051     private static final
3052     ImplCache<Associative, ReductionOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
3053         REDUCE_IMPL = new ImplCache<>(Associative.class, $Type$Vector.class);
3054 
3055     private static ReductionOperation<$abstractvectortype$, VectorMask<$Boxtype$>> reductionOperations(int opc_) {
3056         switch (opc_) {
3057             case VECTOR_OP_ADD: return (v, m) ->
3058                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a + b)));
3059             case VECTOR_OP_MUL: return (v, m) ->
3060                     toBits(v.rOp(($type$)1, m, (i, a, b) -> ($type$)(a * b)));
3061             case VECTOR_OP_MIN: return (v, m) ->
3062                     toBits(v.rOp(MAX_OR_INF, m, (i, a, b) -> ($type$) Math.min(a, b)));
3063             case VECTOR_OP_MAX: return (v, m) ->
3064                     toBits(v.rOp(MIN_OR_INF, m, (i, a, b) -> ($type$) Math.max(a, b)));
3065 #if[BITWISE]
3066             case VECTOR_OP_AND: return (v, m) ->
3067                     toBits(v.rOp(($type$)-1, m, (i, a, b) -> ($type$)(a & b)));
3068             case VECTOR_OP_OR: return (v, m) ->
3069                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a | b)));
3070             case VECTOR_OP_XOR: return (v, m) ->
3071                     toBits(v.rOp(($type$)0, m, (i, a, b) -> ($type$)(a ^ b)));
3072 #end[BITWISE]
3073             default: return null;
3074         }
3075     }



3076 
3077     private
3078     @ForceInline
3079     $abstractvectortype$ reduceIdentityVector(VectorOperators.Associative op) {
3080         int opc = opCode(op);
3081         UnaryOperator<$abstractvectortype$> fn
3082             = REDUCE_ID_IMPL.find(op, opc, (opc_) -> {
3083                 switch (opc_) {
3084                 case VECTOR_OP_ADD:
3085 #if[BITWISE]
3086                 case VECTOR_OP_OR:
3087                 case VECTOR_OP_XOR:
3088 #end[BITWISE]
3089                     return v -> v.broadcast(0);
3090                 case VECTOR_OP_MUL:
3091                     return v -> v.broadcast(1);
3092 #if[BITWISE]
3093                 case VECTOR_OP_AND:
3094                     return v -> v.broadcast(-1);
3095 #end[BITWISE]

3349      * @param species species of desired vector
3350      * @param a the byte array
3351      * @param offset the offset into the array
3352      * @param bo the intended byte order
3353      * @param m the mask controlling lane selection
3354      * @return a vector loaded from a byte array
3355      * @throws IndexOutOfBoundsException
3356      *         if {@code offset+N*ESIZE < 0}
3357      *         or {@code offset+(N+1)*ESIZE > a.length}
3358      *         for any lane {@code N} in the vector
3359      *         where the mask is set
3360      */
3361     @ForceInline
3362     public static
3363     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3364                                        byte[] a, int offset,
3365                                        ByteOrder bo,
3366                                        VectorMask<$Boxtype$> m) {
3367         $Type$Species vsp = ($Type$Species) species;
3368         if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
3369             return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);


3370         }
3371 
3372         // FIXME: optimize
3373         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
3374         ByteBuffer wb = wrapper(a, bo);
3375         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
3376                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
3377     }
3378 
3379     /**
3380      * Loads a vector from an array of type {@code $type$[]}
3381      * starting at an offset.
3382      * For each vector lane, where {@code N} is the vector lane index, the
3383      * array element at index {@code offset + N} is placed into the
3384      * resulting vector at lane index {@code N}.
3385      *
3386      * @param species species of desired vector
3387      * @param a the array
3388      * @param offset the offset into the array
3389      * @return the vector loaded from an array

3411      * {@code N}, otherwise the default element value is placed into the
3412      * resulting vector at lane index {@code N}.
3413      *
3414      * @param species species of desired vector
3415      * @param a the array
3416      * @param offset the offset into the array
3417      * @param m the mask controlling lane selection
3418      * @return the vector loaded from an array
3419      * @throws IndexOutOfBoundsException
3420      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3421      *         for any lane {@code N} in the vector
3422      *         where the mask is set
3423      */
3424     @ForceInline
3425     public static
3426     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3427                                    $type$[] a, int offset,
3428                                    VectorMask<$Boxtype$> m) {
3429         $Type$Species vsp = ($Type$Species) species;
3430         if (offset >= 0 && offset <= (a.length - species.length())) {
3431             return vsp.dummyVector().fromArray0(a, offset, m);

3432         }
3433 
3434         // FIXME: optimize
3435         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3436         return vsp.vOp(m, i -> a[offset + i]);
3437     }
3438 
3439     /**
3440      * Gathers a new vector composed of elements from an array of type
3441      * {@code $type$[]},
3442      * using indexes obtained by adding a fixed {@code offset} to a
3443      * series of secondary offsets from an <em>index map</em>.
3444      * The index map is a contiguous sequence of {@code VLENGTH}
3445      * elements in a second array of {@code int}s, starting at a given
3446      * {@code mapOffset}.
3447      * <p>
3448      * For each vector lane, where {@code N} is the vector lane index,
3449      * the lane is loaded from the array
3450      * element {@code a[f(N)]}, where {@code f(N)} is the
3451      * index mapping expression

3504             // is 64. So when loading index vector (IntVector), only lower half
3505             // of index data is needed.
3506             vix = IntVector
3507                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3508                 .add(offset);
3509         } else {
3510             vix = IntVector
3511                 .fromArray(isp, indexMap, mapOffset)
3512                 .add(offset);
3513         }
3514 #else[longOrDouble]
3515         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3516         IntVector vix = IntVector
3517             .fromArray(isp, indexMap, mapOffset)
3518             .add(offset);
3519 #end[longOrDouble]
3520 
3521         vix = VectorIntrinsics.checkIndex(vix, a.length);
3522 
3523         return VectorSupport.loadWithMap(
3524             vectorType, null, $type$.class, vsp.laneCount(),
3525             isp.vectorType(),
3526             a, ARRAY_BASE, vix, null,
3527             a, offset, indexMap, mapOffset, vsp,
3528             (c, idx, iMap, idy, s, vm) ->
3529             s.vOp(n -> c[idx + iMap[idy+n]]));
3530     }
3531 #end[byteOrShort]
3532 
3533     /**
3534      * Gathers a new vector composed of elements from an array of type
3535      * {@code $type$[]},
3536      * under the control of a mask, and
3537      * using indexes obtained by adding a fixed {@code offset} to a
3538      * series of secondary offsets from an <em>index map</em>.
3539      * The index map is a contiguous sequence of {@code VLENGTH}
3540      * elements in a second array of {@code int}s, starting at a given
3541      * {@code mapOffset}.
3542      * <p>
3543      * For each vector lane, where {@code N} is the vector lane index,
3544      * if the lane is set in the mask,
3545      * the lane is loaded from the array
3546      * element {@code a[f(N)]}, where {@code f(N)} is the
3547      * index mapping expression
3548      * {@code offset + indexMap[mapOffset + N]]}.
3549      * Unset lanes in the resulting vector are set to zero.
3550      *

3570     @ForceInline
3571     public static
3572     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3573                                    $type$[] a, int offset,
3574                                    int[] indexMap, int mapOffset,
3575                                    VectorMask<$Boxtype$> m) {
3576         $Type$Species vsp = ($Type$Species) species;
3577         return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
3578     }
3579 #else[byteOrShort]
3580     @ForceInline
3581     public static
3582     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3583                                    $type$[] a, int offset,
3584                                    int[] indexMap, int mapOffset,
3585                                    VectorMask<$Boxtype$> m) {
3586         if (m.allTrue()) {
3587             return fromArray(species, a, offset, indexMap, mapOffset);
3588         }
3589         else {

3590             $Type$Species vsp = ($Type$Species) species;
3591             return vsp.dummyVector().fromArray0(a, offset, indexMap, mapOffset, m);
3592         }
3593     }
3594 #end[byteOrShort]
3595 
3596 #if[short]
3597     /**
3598      * Loads a vector from an array of type {@code char[]}
3599      * starting at an offset.
3600      * For each vector lane, where {@code N} is the vector lane index, the
3601      * array element at index {@code offset + N}
3602      * is first cast to a {@code short} value and then
3603      * placed into the resulting vector at lane index {@code N}.
3604      *
3605      * @param species species of desired vector
3606      * @param a the array
3607      * @param offset the offset into the array
3608      * @return the vector loaded from an array
3609      * @throws IndexOutOfBoundsException
3610      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3611      *         for any lane {@code N} in the vector

3632      * {@code N}, otherwise the default element value is placed into the
3633      * resulting vector at lane index {@code N}.
3634      *
3635      * @param species species of desired vector
3636      * @param a the array
3637      * @param offset the offset into the array
3638      * @param m the mask controlling lane selection
3639      * @return the vector loaded from an array
3640      * @throws IndexOutOfBoundsException
3641      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3642      *         for any lane {@code N} in the vector
3643      *         where the mask is set
3644      */
3645     @ForceInline
3646     public static
3647     $abstractvectortype$ fromCharArray(VectorSpecies<$Boxtype$> species,
3648                                        char[] a, int offset,
3649                                        VectorMask<$Boxtype$> m) {
3650         $Type$Species vsp = ($Type$Species) species;
3651         if (offset >= 0 && offset <= (a.length - species.length())) {
3652             return vsp.dummyVector().fromCharArray0(a, offset, m);

3653         }
3654 
3655         // FIXME: optimize
3656         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3657         return vsp.vOp(m, i -> (short) a[offset + i]);
3658     }
3659 
3660     /**
3661      * Gathers a new vector composed of elements from an array of type
3662      * {@code char[]},
3663      * using indexes obtained by adding a fixed {@code offset} to a
3664      * series of secondary offsets from an <em>index map</em>.
3665      * The index map is a contiguous sequence of {@code VLENGTH}
3666      * elements in a second array of {@code int}s, starting at a given
3667      * {@code mapOffset}.
3668      * <p>
3669      * For each vector lane, where {@code N} is the vector lane index,
3670      * the lane is loaded from the expression
3671      * {@code (short) a[f(N)]}, where {@code f(N)} is the
3672      * index mapping expression

3792      * expression {@code (byte) (b ? 1 : 0)}, where {@code b} is the {@code boolean} value.
3793      *
3794      * @param species species of desired vector
3795      * @param a the array
3796      * @param offset the offset into the array
3797      * @param m the mask controlling lane selection
3798      * @return the vector loaded from an array
3799      * @throws IndexOutOfBoundsException
3800      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3801      *         for any lane {@code N} in the vector
3802      *         where the mask is set
3803      */
3804     @ForceInline
3805     public static
3806     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3807                                           boolean[] a, int offset,
3808                                           VectorMask<$Boxtype$> m) {
3809         $Type$Species vsp = ($Type$Species) species;
3810         if (offset >= 0 && offset <= (a.length - species.length())) {
3811             $abstractvectortype$ zero = vsp.zero();
3812             return vsp.dummyVector().fromBooleanArray0(a, offset, m);
3813         }
3814 
3815         // FIXME: optimize
3816         checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3817         return vsp.vOp(m, i -> (byte) (a[offset + i] ? 1 : 0));
3818     }
3819 
3820     /**
3821      * Gathers a new vector composed of elements from an array of type
3822      * {@code boolean[]},
3823      * using indexes obtained by adding a fixed {@code offset} to a
3824      * series of secondary offsets from an <em>index map</em>.
3825      * The index map is a contiguous sequence of {@code VLENGTH}
3826      * elements in a second array of {@code int}s, starting at a given
3827      * {@code mapOffset}.
3828      * <p>
3829      * For each vector lane, where {@code N} is the vector lane index,
3830      * the lane is loaded from the expression
3831      * {@code (byte) (a[f(N)] ? 1 : 0)}, where {@code f(N)} is the
3832      * index mapping expression

3983      * @param species species of desired vector
3984      * @param bb the byte buffer
3985      * @param offset the offset into the byte buffer
3986      * @param bo the intended byte order
3987      * @param m the mask controlling lane selection
3988      * @return a vector loaded from a byte buffer
3989      * @throws IndexOutOfBoundsException
3990      *         if {@code offset+N*$sizeInBytes$ < 0}
3991      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
3992      *         for any lane {@code N} in the vector
3993      *         where the mask is set
3994      */
3995     @ForceInline
3996     public static
3997     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
3998                                         ByteBuffer bb, int offset,
3999                                         ByteOrder bo,
4000                                         VectorMask<$Boxtype$> m) {
4001         $Type$Species vsp = ($Type$Species) species;
4002         if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
4003             return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);


4004         }
4005 
4006         // FIXME: optimize
4007         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4008         ByteBuffer wb = wrapper(bb, bo);
4009         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
4010                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4011     }
4012 
4013     // Memory store operations
4014 
4015     /**
4016      * Stores this vector into an array of type {@code $type$[]}
4017      * starting at an offset.
4018      * <p>
4019      * For each vector lane, where {@code N} is the vector lane index,
4020      * the lane element at index {@code N} is stored into the array
4021      * element {@code a[offset+N]}.
4022      *
4023      * @param a the array, of type {@code $type$[]}

4055      * Lanes where the mask is unset are not stored and do not need
4056      * to correspond to legitimate elements of {@code a}.
4057      * That is, unset lanes may correspond to array indexes less than
4058      * zero or beyond the end of the array.
4059      *
4060      * @param a the array, of type {@code $type$[]}
4061      * @param offset the offset into the array
4062      * @param m the mask controlling lane storage
4063      * @throws IndexOutOfBoundsException
4064      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4065      *         for any lane {@code N} in the vector
4066      *         where the mask is set
4067      */
4068     @ForceInline
4069     public final
4070     void intoArray($type$[] a, int offset,
4071                    VectorMask<$Boxtype$> m) {
4072         if (m.allTrue()) {
4073             intoArray(a, offset);
4074         } else {

4075             $Type$Species vsp = vspecies();
4076             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4077             intoArray0(a, offset, m);
4078         }
4079     }
4080 
4081     /**
4082      * Scatters this vector into an array of type {@code $type$[]}
4083      * using indexes obtained by adding a fixed {@code offset} to a
4084      * series of secondary offsets from an <em>index map</em>.
4085      * The index map is a contiguous sequence of {@code VLENGTH}
4086      * elements in a second array of {@code int}s, starting at a given
4087      * {@code mapOffset}.
4088      * <p>
4089      * For each vector lane, where {@code N} is the vector lane index,
4090      * the lane element at index {@code N} is stored into the array
4091      * element {@code a[f(N)]}, where {@code f(N)} is the
4092      * index mapping expression
4093      * {@code offset + indexMap[mapOffset + N]]}.
4094      *
4095      * @param a the array
4096      * @param offset an offset to combine with the index map offsets
4097      * @param indexMap the index map

4139             // of index data is needed.
4140             vix = IntVector
4141                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4142                 .add(offset);
4143         } else {
4144             vix = IntVector
4145                 .fromArray(isp, indexMap, mapOffset)
4146                 .add(offset);
4147         }
4148 
4149 #else[longOrDouble]
4150         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4151         IntVector vix = IntVector
4152             .fromArray(isp, indexMap, mapOffset)
4153             .add(offset);
4154 #end[longOrDouble]
4155 
4156         vix = VectorIntrinsics.checkIndex(vix, a.length);
4157 
4158         VectorSupport.storeWithMap(
4159             vsp.vectorType(), null, vsp.elementType(), vsp.laneCount(),
4160             isp.vectorType(),
4161             a, arrayAddress(a, 0), vix,
4162             this, null,
4163             a, offset, indexMap, mapOffset,
4164             (arr, off, v, map, mo, vm)
4165             -> v.stOp(arr, off,
4166                       (arr_, off_, i, e) -> {
4167                           int j = map[mo + i];
4168                           arr[off + j] = e;
4169                       }));
4170     }
4171 #end[byteOrShort]
4172 
4173     /**
4174      * Scatters this vector into an array of type {@code $type$[]},
4175      * under the control of a mask, and
4176      * using indexes obtained by adding a fixed {@code offset} to a
4177      * series of secondary offsets from an <em>index map</em>.
4178      * The index map is a contiguous sequence of {@code VLENGTH}
4179      * elements in a second array of {@code int}s, starting at a given
4180      * {@code mapOffset}.
4181      * <p>
4182      * For each vector lane, where {@code N} is the vector lane index,
4183      * if the mask lane at index {@code N} is set then
4184      * the lane element at index {@code N} is stored into the array

4205     public final
4206     void intoArray($type$[] a, int offset,
4207                    int[] indexMap, int mapOffset,
4208                    VectorMask<$Boxtype$> m) {
4209         stOp(a, offset, m,
4210              (arr, off, i, e) -> {
4211                  int j = indexMap[mapOffset + i];
4212                  arr[off + j] = e;
4213              });
4214     }
4215 #else[byteOrShort]
4216     @ForceInline
4217     public final
4218     void intoArray($type$[] a, int offset,
4219                    int[] indexMap, int mapOffset,
4220                    VectorMask<$Boxtype$> m) {
4221         if (m.allTrue()) {
4222             intoArray(a, offset, indexMap, mapOffset);
4223         }
4224         else {
4225             intoArray0(a, offset, indexMap, mapOffset, m);





4226         }
4227     }
4228 #end[byteOrShort]
4229 
4230 #if[short]
4231     /**
4232      * Stores this vector into an array of type {@code char[]}
4233      * starting at an offset.
4234      * <p>
4235      * For each vector lane, where {@code N} is the vector lane index,
4236      * the lane element at index {@code N}
4237      * is first cast to a {@code char} value and then
4238      * stored into the array element {@code a[offset+N]}.
4239      *
4240      * @param a the array, of type {@code char[]}
4241      * @param offset the offset into the array
4242      * @throws IndexOutOfBoundsException
4243      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4244      *         for any lane {@code N} in the vector
4245      */

4273      * Lanes where the mask is unset are not stored and do not need
4274      * to correspond to legitimate elements of {@code a}.
4275      * That is, unset lanes may correspond to array indexes less than
4276      * zero or beyond the end of the array.
4277      *
4278      * @param a the array, of type {@code char[]}
4279      * @param offset the offset into the array
4280      * @param m the mask controlling lane storage
4281      * @throws IndexOutOfBoundsException
4282      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4283      *         for any lane {@code N} in the vector
4284      *         where the mask is set
4285      */
4286     @ForceInline
4287     public final
4288     void intoCharArray(char[] a, int offset,
4289                        VectorMask<$Boxtype$> m) {
4290         if (m.allTrue()) {
4291             intoCharArray(a, offset);
4292         } else {

4293             $Type$Species vsp = vspecies();
4294             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4295             intoCharArray0(a, offset, m);
4296         }
4297     }
4298 
4299     /**
4300      * Scatters this vector into an array of type {@code char[]}
4301      * using indexes obtained by adding a fixed {@code offset} to a
4302      * series of secondary offsets from an <em>index map</em>.
4303      * The index map is a contiguous sequence of {@code VLENGTH}
4304      * elements in a second array of {@code int}s, starting at a given
4305      * {@code mapOffset}.
4306      * <p>
4307      * For each vector lane, where {@code N} is the vector lane index,
4308      * the lane element at index {@code N}
4309      * is first cast to a {@code char} value and then
4310      * stored into the array
4311      * element {@code a[f(N)]}, where {@code f(N)} is the
4312      * index mapping expression
4313      * {@code offset + indexMap[mapOffset + N]]}.
4314      *
4315      * @param a the array

4435      * Lanes where the mask is unset are not stored and do not need
4436      * to correspond to legitimate elements of {@code a}.
4437      * That is, unset lanes may correspond to array indexes less than
4438      * zero or beyond the end of the array.
4439      *
4440      * @param a the array
4441      * @param offset the offset into the array
4442      * @param m the mask controlling lane storage
4443      * @throws IndexOutOfBoundsException
4444      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4445      *         for any lane {@code N} in the vector
4446      *         where the mask is set
4447      */
4448     @ForceInline
4449     public final
4450     void intoBooleanArray(boolean[] a, int offset,
4451                           VectorMask<$Boxtype$> m) {
4452         if (m.allTrue()) {
4453             intoBooleanArray(a, offset);
4454         } else {

4455             $Type$Species vsp = vspecies();
4456             checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
4457             intoBooleanArray0(a, offset, m);
4458         }
4459     }
4460 
4461     /**
4462      * Scatters this vector into an array of type {@code boolean[]}
4463      * using indexes obtained by adding a fixed {@code offset} to a
4464      * series of secondary offsets from an <em>index map</em>.
4465      * The index map is a contiguous sequence of {@code VLENGTH}
4466      * elements in a second array of {@code int}s, starting at a given
4467      * {@code mapOffset}.
4468      * <p>
4469      * For each vector lane, where {@code N} is the vector lane index,
4470      * the lane element at index {@code N}
4471      * is first converted to a {@code boolean} value and then
4472      * stored into the array
4473      * element {@code a[f(N)]}, where {@code f(N)} is the
4474      * index mapping expression
4475      * {@code offset + indexMap[mapOffset + N]]}.
4476      * <p>
4477      * A {@code byte} value is converted to a {@code boolean} value by applying the

4557     @ForceInline
4558     public final
4559     void intoByteArray(byte[] a, int offset,
4560                        ByteOrder bo) {
4561         offset = checkFromIndexSize(offset, byteSize(), a.length);
4562         maybeSwap(bo).intoByteArray0(a, offset);
4563     }
4564 
4565     /**
4566      * {@inheritDoc} <!--workaround-->
4567      */
4568     @Override
4569     @ForceInline
4570     public final
4571     void intoByteArray(byte[] a, int offset,
4572                        ByteOrder bo,
4573                        VectorMask<$Boxtype$> m) {
4574         if (m.allTrue()) {
4575             intoByteArray(a, offset, bo);
4576         } else {

4577             $Type$Species vsp = vspecies();
4578             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
4579             maybeSwap(bo).intoByteArray0(a, offset, m);


4580         }
4581     }
4582 
4583     /**
4584      * {@inheritDoc} <!--workaround-->
4585      */
4586     @Override
4587     @ForceInline
4588     public final
4589     void intoByteBuffer(ByteBuffer bb, int offset,
4590                         ByteOrder bo) {
4591         if (ScopedMemoryAccess.isReadOnly(bb)) {
4592             throw new ReadOnlyBufferException();
4593         }
4594         offset = checkFromIndexSize(offset, byteSize(), bb.limit());
4595         maybeSwap(bo).intoByteBuffer0(bb, offset);
4596     }
4597 
4598     /**
4599      * {@inheritDoc} <!--workaround-->
4600      */
4601     @Override
4602     @ForceInline
4603     public final
4604     void intoByteBuffer(ByteBuffer bb, int offset,
4605                         ByteOrder bo,
4606                         VectorMask<$Boxtype$> m) {
4607         if (m.allTrue()) {
4608             intoByteBuffer(bb, offset, bo);
4609         } else {

4610             if (bb.isReadOnly()) {
4611                 throw new ReadOnlyBufferException();
4612             }
4613             $Type$Species vsp = vspecies();
4614             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4615             maybeSwap(bo).intoByteBuffer0(bb, offset, m);


4616         }
4617     }
4618 
4619     // ================================================
4620 
4621     // Low-level memory operations.
4622     //
4623     // Note that all of these operations *must* inline into a context
4624     // where the exact species of the involved vector is a
4625     // compile-time constant.  Otherwise, the intrinsic generation
4626     // will fail and performance will suffer.
4627     //
4628     // In many cases this is achieved by re-deriving a version of the
4629     // method in each concrete subclass (per species).  The re-derived
4630     // method simply calls one of these generic methods, with exact
4631     // parameters for the controlling metadata, which is either a
4632     // typed vector or constant species instance.
4633 
4634     // Unchecked loading operations in native byte order.
4635     // Caller is responsible for applying index checks, masking, and
4636     // byte swapping.
4637 
4638     /*package-private*/
4639     abstract
4640     $abstractvectortype$ fromArray0($type$[] a, int offset);
4641     @ForceInline
4642     final
4643     $abstractvectortype$ fromArray0Template($type$[] a, int offset) {
4644         $Type$Species vsp = vspecies();
4645         return VectorSupport.load(
4646             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4647             a, arrayAddress(a, offset),
4648             a, offset, vsp,
4649             (arr, off, s) -> s.ldOp(arr, off,
4650                                     (arr_, off_, i) -> arr_[off_ + i]));
4651     }
4652 
4653     /*package-private*/
4654     abstract
4655     $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4656     @ForceInline
4657     final
4658     <M extends VectorMask<$Boxtype$>>
4659     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4660         m.check(species());
4661         $Type$Species vsp = vspecies();
4662         return VectorSupport.loadMasked(
4663             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4664             a, arrayAddress(a, offset), m,
4665             a, offset, vsp,
4666             (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4667                                         (arr_, off_, i) -> arr_[off_ + i]));
4668     }
4669 
4670 #if[!byteOrShort]
4671     /*package-private*/
4672     abstract
4673     $abstractvectortype$ fromArray0($type$[] a, int offset,
4674                                     int[] indexMap, int mapOffset,
4675                                     VectorMask<$Boxtype$> m);
4676     @ForceInline
4677     final
4678     <M extends VectorMask<$Boxtype$>>
4679     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset,
4680                                             int[] indexMap, int mapOffset, M m) {
4681         $Type$Species vsp = vspecies();
4682         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4683         Objects.requireNonNull(a);
4684         Objects.requireNonNull(indexMap);
4685         m.check(vsp);
4686         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();
4687 
4688 #if[longOrDouble]
4689         if (vsp.laneCount() == 1) {
4690           return $abstractvectortype$.fromArray(vsp, a, offset + indexMap[mapOffset], m);
4691         }
4692 
4693         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
4694         IntVector vix;
4695         if (isp.laneCount() != vsp.laneCount()) {
4696             // For $Type$MaxVector,  if vector length is non-power-of-two or
4697             // 2048 bits, indexShape of $Type$ species is S_MAX_BIT.
4698             // Assume that vector length is 2048, then the lane count of $Type$
4699             // vector is 32. When converting $Type$ species to int species,
4700             // indexShape is still S_MAX_BIT, but the lane count of int vector
4701             // is 64. So when loading index vector (IntVector), only lower half
4702             // of index data is needed.
4703             vix = IntVector
4704                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4705                 .add(offset);
4706         } else {
4707             vix = IntVector
4708                 .fromArray(isp, indexMap, mapOffset)
4709                 .add(offset);
4710         }
4711 #else[longOrDouble]
4712         // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
4713         IntVector vix = IntVector
4714             .fromArray(isp, indexMap, mapOffset)
4715             .add(offset);
4716 #end[longOrDouble]
4717 
4718         // FIXME: Check index under mask controlling.
4719         vix = VectorIntrinsics.checkIndex(vix, a.length);
4720 
4721         return VectorSupport.loadWithMap(
4722             vectorType, maskClass, $type$.class, vsp.laneCount(),
4723             isp.vectorType(),
4724             a, ARRAY_BASE, vix, m,
4725             a, offset, indexMap, mapOffset, vsp,
4726             (c, idx, iMap, idy, s, vm) ->
4727             s.vOp(vm, n -> c[idx + iMap[idy+n]]));
4728     }
4729 #end[!byteOrShort]
4730 
4731 #if[short]
4732     /*package-private*/
4733     abstract
4734     $abstractvectortype$ fromCharArray0(char[] a, int offset);
4735     @ForceInline
4736     final
4737     $abstractvectortype$ fromCharArray0Template(char[] a, int offset) {
4738         $Type$Species vsp = vspecies();
4739         return VectorSupport.load(
4740             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4741             a, charArrayAddress(a, offset),
4742             a, offset, vsp,
4743             (arr, off, s) -> s.ldOp(arr, off,
4744                                     (arr_, off_, i) -> (short) arr_[off_ + i]));
4745     }
4746 
4747     /*package-private*/
4748     abstract
4749     $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
4750     @ForceInline
4751     final
4752     <M extends VectorMask<$Boxtype$>>
4753     $abstractvectortype$ fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
4754         m.check(species());
4755         $Type$Species vsp = vspecies();
4756         return VectorSupport.loadMasked(
4757                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4758                 a, charArrayAddress(a, offset), m,
4759                 a, offset, vsp,
4760                 (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4761                                             (arr_, off_, i) -> (short) arr_[off_ + i]));
4762     }
4763 #end[short]
4764 
4765 #if[byte]
4766     /*package-private*/
4767     abstract
4768     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset);
4769     @ForceInline
4770     final
4771     $abstractvectortype$ fromBooleanArray0Template(boolean[] a, int offset) {
4772         $Type$Species vsp = vspecies();
4773         return VectorSupport.load(
4774             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4775             a, booleanArrayAddress(a, offset),
4776             a, offset, vsp,
4777             (arr, off, s) -> s.ldOp(arr, off,
4778                                     (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4779     }
4780 
4781     /*package-private*/
4782     abstract
4783     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4784     @ForceInline
4785     final
4786     <M extends VectorMask<$Boxtype$>>
4787     $abstractvectortype$ fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4788         m.check(species());
4789         $Type$Species vsp = vspecies();
4790         return VectorSupport.loadMasked(
4791             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4792             a, booleanArrayAddress(a, offset), m,
4793             a, offset, vsp,
4794             (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4795                                         (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4796     }
4797 #end[byte]
4798 
4799     @Override
4800     abstract
4801     $abstractvectortype$ fromByteArray0(byte[] a, int offset);
4802     @ForceInline
4803     final
4804     $abstractvectortype$ fromByteArray0Template(byte[] a, int offset) {
4805         $Type$Species vsp = vspecies();
4806         return VectorSupport.load(
4807             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4808             a, byteArrayAddress(a, offset),
4809             a, offset, vsp,
4810             (arr, off, s) -> {
4811                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4812                 return s.ldOp(wb, off,
4813                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4814             });
4815     }
4816 
4817     abstract
4818     $abstractvectortype$ fromByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
4819     @ForceInline
4820     final
4821     <M extends VectorMask<$Boxtype$>>
4822     $abstractvectortype$ fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
4823         $Type$Species vsp = vspecies();
4824         m.check(vsp);
4825         return VectorSupport.loadMasked(
4826             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4827             a, byteArrayAddress(a, offset), m,
4828             a, offset, vsp,
4829             (arr, off, s, vm) -> {
4830                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4831                 return s.ldOp(wb, off, vm,
4832                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4833             });
4834     }
4835 
4836     abstract
4837     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset);
4838     @ForceInline
4839     final
4840     $abstractvectortype$ fromByteBuffer0Template(ByteBuffer bb, int offset) {
4841         $Type$Species vsp = vspecies();
4842         return ScopedMemoryAccess.loadFromByteBuffer(
4843                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4844                 bb, offset, vsp,
4845                 (buf, off, s) -> {
4846                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4847                     return s.ldOp(wb, off,
4848                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4849                 });
4850     }
4851 
4852     abstract
4853     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
4854     @ForceInline
4855     final
4856     <M extends VectorMask<$Boxtype$>>
4857     $abstractvectortype$ fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
4858         $Type$Species vsp = vspecies();
4859         m.check(vsp);
4860         return ScopedMemoryAccess.loadFromByteBufferMasked(
4861                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4862                 bb, offset, m, vsp,
4863                 (buf, off, s, vm) -> {
4864                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4865                     return s.ldOp(wb, off, vm,
4866                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4867                 });
4868     }
4869 
4870     // Unchecked storing operations in native byte order.
4871     // Caller is responsible for applying index checks, masking, and
4872     // byte swapping.
4873 
4874     abstract
4875     void intoArray0($type$[] a, int offset);
4876     @ForceInline
4877     final
4878     void intoArray0Template($type$[] a, int offset) {
4879         $Type$Species vsp = vspecies();
4880         VectorSupport.store(
4881             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4882             a, arrayAddress(a, offset),
4883             this, a, offset,
4884             (arr, off, v)
4885             -> v.stOp(arr, off,
4886                       (arr_, off_, i, e) -> arr_[off_+i] = e));
4887     }
4888 
4889     abstract
4890     void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4891     @ForceInline
4892     final
4893     <M extends VectorMask<$Boxtype$>>
4894     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4895         m.check(species());
4896         $Type$Species vsp = vspecies();
4897         VectorSupport.storeMasked(
4898             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4899             a, arrayAddress(a, offset),
4900             this, m, a, offset,
4901             (arr, off, v, vm)
4902             -> v.stOp(arr, off, vm,
4903                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4904     }
4905 
4906 #if[!byteOrShort]
4907     abstract
4908     void intoArray0($type$[] a, int offset,
4909                     int[] indexMap, int mapOffset,
4910                     VectorMask<$Boxtype$> m);
4911     @ForceInline
4912     final
4913     <M extends VectorMask<$Boxtype$>>
4914     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset,
4915                             int[] indexMap, int mapOffset, M m) {
4916         m.check(species());
4917         $Type$Species vsp = vspecies();
4918         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4919 #if[longOrDouble]
4920         if (vsp.laneCount() == 1) {
4921             intoArray(a, offset + indexMap[mapOffset], m);
4922             return;
4923         }
4924 
4925         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4926         IntVector vix;
4927         if (isp.laneCount() != vsp.laneCount()) {
4928             // For $Type$MaxVector,  if vector length  is 2048 bits, indexShape
4929             // of $Type$ species is S_MAX_BIT. and the lane count of $Type$
4930             // vector is 32. When converting $Type$ species to int species,
4931             // indexShape is still S_MAX_BIT, but the lane count of int vector
4932             // is 64. So when loading index vector (IntVector), only lower half
4933             // of index data is needed.
4934             vix = IntVector
4935                 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
4936                 .add(offset);
4937         } else {
4938             vix = IntVector
4939                 .fromArray(isp, indexMap, mapOffset)
4940                 .add(offset);
4941         }
4942 
4943 #else[longOrDouble]
4944         // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
4945         IntVector vix = IntVector
4946             .fromArray(isp, indexMap, mapOffset)
4947             .add(offset);
4948 #end[longOrDouble]
4949 
4950         // FIXME: Check index under mask controlling.
4951         vix = VectorIntrinsics.checkIndex(vix, a.length);
4952 
4953         VectorSupport.storeWithMap(
4954             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4955             isp.vectorType(),
4956             a, arrayAddress(a, 0), vix,
4957             this, m,
4958             a, offset, indexMap, mapOffset,
4959             (arr, off, v, map, mo, vm)
4960             -> v.stOp(arr, off, vm,
4961                       (arr_, off_, i, e) -> {
4962                           int j = map[mo + i];
4963                           arr[off + j] = e;
4964                       }));
4965     }
4966 #end[!byteOrShort]
4967 
4968 #if[byte]
4969     abstract
4970     void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4971     @ForceInline
4972     final
4973     <M extends VectorMask<$Boxtype$>>
4974     void intoBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4975         m.check(species());
4976         $Type$Species vsp = vspecies();
4977         ByteVector normalized = this.and((byte) 1);
4978         VectorSupport.storeMasked(
4979             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4980             a, booleanArrayAddress(a, offset),
4981             normalized, m, a, offset,
4982             (arr, off, v, vm)
4983             -> v.stOp(arr, off, vm,
4984                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4985     }
4986 #end[byte]
4987 
4988     abstract
4989     void intoByteArray0(byte[] a, int offset);
4990     @ForceInline
4991     final
4992     void intoByteArray0Template(byte[] a, int offset) {
4993         $Type$Species vsp = vspecies();
4994         VectorSupport.store(
4995             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4996             a, byteArrayAddress(a, offset),
4997             this, a, offset,
4998             (arr, off, v) -> {
4999                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
5000                 v.stOp(wb, off,
5001                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5002             });
5003     }
5004 
5005     abstract
5006     void intoByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
5007     @ForceInline
5008     final
5009     <M extends VectorMask<$Boxtype$>>
5010     void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
5011         $Type$Species vsp = vspecies();
5012         m.check(vsp);
5013         VectorSupport.storeMasked(
5014             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5015             a, byteArrayAddress(a, offset),
5016             this, m, a, offset,
5017             (arr, off, v, vm) -> {
5018                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
5019                 v.stOp(wb, off, vm,
5020                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5021             });
5022     }
5023 
5024     @ForceInline
5025     final
5026     void intoByteBuffer0(ByteBuffer bb, int offset) {
5027         $Type$Species vsp = vspecies();
5028         ScopedMemoryAccess.storeIntoByteBuffer(
5029                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
5030                 this, bb, offset,
5031                 (buf, off, v) -> {
5032                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
5033                     v.stOp(wb, off,
5034                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5035                 });
5036     }
5037 
5038     abstract
5039     void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
5040     @ForceInline
5041     final
5042     <M extends VectorMask<$Boxtype$>>
5043     void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
5044         $Type$Species vsp = vspecies();
5045         m.check(vsp);
5046         ScopedMemoryAccess.storeIntoByteBufferMasked(
5047                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5048                 this, m, bb, offset,
5049                 (buf, off, v, vm) -> {
5050                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
5051                     v.stOp(wb, off, vm,
5052                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5053                 });
5054     }
5055 
5056 #if[short]
5057     /*package-private*/
5058     abstract
5059     void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
5060     @ForceInline
5061     final
5062     <M extends VectorMask<$Boxtype$>>
5063     void intoCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
5064         m.check(species());
5065         $Type$Species vsp = vspecies();
5066         VectorSupport.storeMasked(
5067             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5068             a, charArrayAddress(a, offset),
5069             this, m, a, offset,
5070             (arr, off, v, vm)
5071             -> v.stOp(arr, off, vm,
5072                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
5073     }
5074 #end[short]
5075 
5076     // End of low-level memory operations.
5077 
5078     private static
5079     void checkMaskFromIndexSize(int offset,
5080                                 $Type$Species vsp,
5081                                 VectorMask<$Boxtype$> m,
5082                                 int scale,
5083                                 int limit) {
5084         ((AbstractMask<$Boxtype$>)m)
5085             .checkIndexByLane(offset, limit, vsp.iota(), scale);
5086     }
5087 
5088     @ForceInline
5089     private void conditionalStoreNYI(int offset,
5090                                      $Type$Species vsp,
5091                                      VectorMask<$Boxtype$> m,
5092                                      int scale,
5093                                      int limit) {
5094         if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
5095             String msg =

5431             $type$[] res = new $type$[laneCount()];
5432             boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
5433             for (int i = 0; i < res.length; i++) {
5434                 if (mbits[i]) {
5435                     res[i] = f.apply(i);
5436                 }
5437             }
5438             return dummyVector().vectorFactory(res);
5439         }
5440 
5441         /*package-private*/
5442         @ForceInline
5443         <M> $abstractvectortype$ ldOp(M memory, int offset,
5444                                       FLdOp<M> f) {
5445             return dummyVector().ldOp(memory, offset, f);
5446         }
5447 
5448         /*package-private*/
5449         @ForceInline
5450         <M> $abstractvectortype$ ldOp(M memory, int offset,
5451                                       VectorMask<$Boxtype$> m,
5452                                       FLdOp<M> f) {
5453             return dummyVector().ldOp(memory, offset, m, f);
5454         }
5455 
5456         /*package-private*/
5457         @ForceInline
5458         <M> void stOp(M memory, int offset, FStOp<M> f) {
5459             dummyVector().stOp(memory, offset, f);
5460         }
5461 
5462         /*package-private*/
5463         @ForceInline
5464         <M> void stOp(M memory, int offset,
5465                       AbstractMask<$Boxtype$> m,
5466                       FStOp<M> f) {
5467             dummyVector().stOp(memory, offset, m, f);
5468         }
5469 
5470         // N.B. Make sure these constant vectors and
5471         // masks load up correctly into registers.
< prev index next >