< prev index next >

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template

Print this page

   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.nio.ByteOrder;
  29 import java.nio.ReadOnlyBufferException;
  30 import java.util.Arrays;
  31 import java.util.Objects;
  32 import java.util.function.Function;
  33 import java.util.function.UnaryOperator;
  34 



  35 import jdk.internal.misc.ScopedMemoryAccess;
  36 import jdk.internal.misc.Unsafe;
  37 import jdk.internal.vm.annotation.ForceInline;
  38 import jdk.internal.vm.vector.VectorSupport;
  39 
  40 import static jdk.internal.vm.vector.VectorSupport.*;
  41 import static jdk.incubator.vector.VectorIntrinsics.*;
  42 
  43 import static jdk.incubator.vector.VectorOperators.*;
  44 
  45 #warn This file is preprocessed before being compiled
  46 
  47 /**
  48  * A specialized {@link Vector} representing an ordered immutable sequence of
  49  * {@code $type$} values.
  50  */
  51 @SuppressWarnings("cast")  // warning: redundant cast
  52 public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
  53 
  54     $abstractvectortype$($type$[] vec) {
  55         super(vec);
  56     }
  57 
  58 #if[FP]
  59     static final int FORBID_OPCODE_KIND = VO_NOFP;
  60 #else[FP]
  61     static final int FORBID_OPCODE_KIND = VO_ONLYFP;
  62 #end[FP]
  63 


  64     @ForceInline
  65     static int opCode(Operator op) {
  66         return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
  67     }
  68     @ForceInline
  69     static int opCode(Operator op, int requireKind) {
  70         requireKind |= VO_OPCODE_VALID;
  71         return VectorOperators.opCode(op, requireKind, FORBID_OPCODE_KIND);
  72     }
  73     @ForceInline
  74     static boolean opKind(Operator op, int bit) {
  75         return VectorOperators.opKind(op, bit);
  76     }
  77 
  78     // Virtualized factories and operators,
  79     // coded with portable definitions.
  80     // These are all @ForceInline in case
  81     // they need to be used performantly.
  82     // The various shape-specific subclasses
  83     // also specialize them by wrapping

 338         return vectorFactory(res);
 339     }
 340 
 341     /*package-private*/
 342     @ForceInline
 343     final
 344     <M> $abstractvectortype$ ldOp(M memory, int offset,
 345                                   VectorMask<$Boxtype$> m,
 346                                   FLdOp<M> f) {
 347         //$type$[] vec = vec();
 348         $type$[] res = new $type$[length()];
 349         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 350         for (int i = 0; i < res.length; i++) {
 351             if (mbits[i]) {
 352                 res[i] = f.apply(memory, offset, i);
 353             }
 354         }
 355         return vectorFactory(res);
 356     }
 357 







































 358     interface FStOp<M> {
 359         void apply(M memory, int offset, int i, $type$ a);
 360     }
 361 
 362     /*package-private*/
 363     @ForceInline
 364     final
 365     <M> void stOp(M memory, int offset,
 366                   FStOp<M> f) {
 367         $type$[] vec = vec();
 368         for (int i = 0; i < vec.length; i++) {
 369             f.apply(memory, offset, i, vec[i]);
 370         }
 371     }
 372 
 373     /*package-private*/
 374     @ForceInline
 375     final
 376     <M> void stOp(M memory, int offset,
 377                   VectorMask<$Boxtype$> m,
 378                   FStOp<M> f) {
 379         $type$[] vec = vec();
 380         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 381         for (int i = 0; i < vec.length; i++) {
 382             if (mbits[i]) {
 383                 f.apply(memory, offset, i, vec[i]);
 384             }
 385         }
 386     }
 387 


































 388     // Binary test
 389 
 390     /*package-private*/
 391     interface FBinTest {
 392         boolean apply(int cond, int i, $type$ a, $type$ b);
 393     }
 394 
 395     /*package-private*/
 396     @ForceInline
 397     final
 398     AbstractMask<$Boxtype$> bTest(int cond,
 399                                   Vector<$Boxtype$> o,
 400                                   FBinTest f) {
 401         $type$[] vec1 = vec();
 402         $type$[] vec2 = (($abstractvectortype$)o).vec();
 403         boolean[] bits = new boolean[length()];
 404         for (int i = 0; i < length(); i++){
 405             bits[i] = f.apply(cond, i, vec1[i], vec2[i]);
 406         }
 407         return maskFactory(bits);

 428 #end[intOrLong]
 429     }
 430 #end[BITWISE]
 431 
 432     /*package-private*/
 433     @Override
 434     abstract $Type$Species vspecies();
 435 
 436     /*package-private*/
 437     @ForceInline
 438     static long toBits($type$ e) {
 439         return {#if[FP]? $Type$.$type$ToRaw$Bitstype$Bits(e): e};
 440     }
 441 
 442     /*package-private*/
 443     @ForceInline
 444     static $type$ fromBits(long bits) {
 445         return {#if[FP]?$Type$.$bitstype$BitsTo$Type$}(($bitstype$)bits);
 446     }
 447 






























 448     // Static factories (other than memory operations)
 449 
 450     // Note: A surprising behavior in javadoc
 451     // sometimes makes a lone /** {@inheritDoc} */
 452     // comment drop the method altogether,
 453     // apparently if the method mentions an
 454     // parameter or return type of Vector<$Boxtype$>
 455     // instead of Vector<E> as originally specified.
 456     // Adding an empty HTML fragment appears to
 457     // nudge javadoc into providing the desired
 458     // inherited documentation.  We use the HTML
 459     // comment <!--workaround--> for this.
 460 
 461     /**
 462      * Returns a vector of the given species
 463      * where all lane elements are set to
 464      * zero, the default primitive value.
 465      *
 466      * @param species species of the desired zero vector
 467      * @return a zero vector

 629             }
 630 #end[BITWISE]
 631         }
 632         int opc = opCode(op);
 633         return VectorSupport.unaryOp(
 634             opc, getClass(), maskClass, $type$.class, length(),
 635             this, m,
 636             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));
 637     }
 638 
 639     private static final
 640     ImplCache<Unary, UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 641         UN_IMPL = new ImplCache<>(Unary.class, $Type$Vector.class);
 642 
 643     private static UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> unaryOperations(int opc_) {
 644         switch (opc_) {
 645             case VECTOR_OP_NEG: return (v0, m) ->
 646                     v0.uOp(m, (i, a) -> ($type$) -a);
 647             case VECTOR_OP_ABS: return (v0, m) ->
 648                     v0.uOp(m, (i, a) -> ($type$) Math.abs(a));






























 649 #if[FP]
 650             case VECTOR_OP_SIN: return (v0, m) ->
 651                     v0.uOp(m, (i, a) -> ($type$) Math.sin(a));
 652             case VECTOR_OP_COS: return (v0, m) ->
 653                     v0.uOp(m, (i, a) -> ($type$) Math.cos(a));
 654             case VECTOR_OP_TAN: return (v0, m) ->
 655                     v0.uOp(m, (i, a) -> ($type$) Math.tan(a));
 656             case VECTOR_OP_ASIN: return (v0, m) ->
 657                     v0.uOp(m, (i, a) -> ($type$) Math.asin(a));
 658             case VECTOR_OP_ACOS: return (v0, m) ->
 659                     v0.uOp(m, (i, a) -> ($type$) Math.acos(a));
 660             case VECTOR_OP_ATAN: return (v0, m) ->
 661                     v0.uOp(m, (i, a) -> ($type$) Math.atan(a));
 662             case VECTOR_OP_EXP: return (v0, m) ->
 663                     v0.uOp(m, (i, a) -> ($type$) Math.exp(a));
 664             case VECTOR_OP_LOG: return (v0, m) ->
 665                     v0.uOp(m, (i, a) -> ($type$) Math.log(a));
 666             case VECTOR_OP_LOG10: return (v0, m) ->
 667                     v0.uOp(m, (i, a) -> ($type$) Math.log10(a));
 668             case VECTOR_OP_SQRT: return (v0, m) ->

 822                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.max(a, b));
 823             case VECTOR_OP_MIN: return (v0, v1, vm) ->
 824                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.min(a, b));
 825 #if[BITWISE]
 826             case VECTOR_OP_AND: return (v0, v1, vm) ->
 827                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a & b));
 828             case VECTOR_OP_OR: return (v0, v1, vm) ->
 829                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a | b));
 830             case VECTOR_OP_XOR: return (v0, v1, vm) ->
 831                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a ^ b));
 832             case VECTOR_OP_LSHIFT: return (v0, v1, vm) ->
 833                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a << n));
 834             case VECTOR_OP_RSHIFT: return (v0, v1, vm) ->
 835                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a >> n));
 836             case VECTOR_OP_URSHIFT: return (v0, v1, vm) ->
 837                     v0.bOp(v1, vm, (i, a, n) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 838             case VECTOR_OP_LROTATE: return (v0, v1, vm) ->
 839                     v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
 840             case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
 841                     v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));






 842 #end[BITWISE]
 843 #if[FP]
 844             case VECTOR_OP_OR: return (v0, v1, vm) ->
 845                     v0.bOp(v1, vm, (i, a, b) -> fromBits(toBits(a) | toBits(b)));
 846             case VECTOR_OP_ATAN2: return (v0, v1, vm) ->
 847                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.atan2(a, b));
 848             case VECTOR_OP_POW: return (v0, v1, vm) ->
 849                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.pow(a, b));
 850             case VECTOR_OP_HYPOT: return (v0, v1, vm) ->
 851                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.hypot(a, b));
 852 #end[FP]
 853             default: return null;
 854         }
 855     }
 856 
 857     // FIXME: Maybe all of the public final methods in this file (the
 858     // simple ones that just call lanewise) should be pushed down to
 859     // the X-VectorBits template.  They can't optimize properly at
 860     // this level, and must rely on inlining.  Does it work?
 861     // (If it works, of course keep the code here.)

1970     /**
1971      * {@inheritDoc} <!--workaround-->
1972      */
1973     @Override
1974     @ForceInline
1975     public final
1976     $abstractvectortype$ neg() {
1977         return lanewise(NEG);
1978     }
1979 
1980     /**
1981      * {@inheritDoc} <!--workaround-->
1982      */
1983     @Override
1984     @ForceInline
1985     public final
1986     $abstractvectortype$ abs() {
1987         return lanewise(ABS);
1988     }
1989 


















































1990 #if[BITWISE]
1991     // not (~)
1992     /**
1993      * Computes the bitwise logical complement ({@code ~})
1994      * of this vector.
1995      *
1996      * This is a lane-wise binary operation which applies the
1997      * the primitive bitwise "not" operation ({@code ~})
1998      * to each lane value.
1999      *
2000      * This method is also equivalent to the expression
2001      * {@link #lanewise(VectorOperators.Unary)
2002      *    lanewise}{@code (}{@link VectorOperators#NOT
2003      *    NOT}{@code )}.
2004      *
2005      * <p>
2006      * This is not a full-service named operation like
2007      * {@link #add(Vector) add}.  A masked version of
2008      * this operation is not directly available
2009      * but may be obtained via the masked version of

2678         $type$[] a = toArray();
2679         int[] sa = new int[a.length];
2680         for (int i = 0; i < a.length; i++) {
2681             sa[i] = (int) a[i];
2682         }
2683         return VectorShuffle.fromArray(dsp, sa, 0);
2684     }
2685 
2686     /*package-private*/
2687     @ForceInline
2688     final
2689     VectorShuffle<$Boxtype$> toShuffleTemplate(Class<?> shuffleType) {
2690         $Type$Species vsp = vspecies();
2691         return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
2692                                      getClass(), $type$.class, length(),
2693                                      shuffleType, byte.class, length(),
2694                                      this, vsp,
2695                                      $Type$Vector::toShuffle0);
2696     }
2697 







































2698     /**
2699      * {@inheritDoc} <!--workaround-->
2700      */
2701     @Override
2702     public abstract
2703     $abstractvectortype$ selectFrom(Vector<$Boxtype$> v);
2704 
2705     /*package-private*/
2706     @ForceInline
2707     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v) {
2708         return v.rearrange(this.toShuffle());
2709     }
2710 
2711     /**
2712      * {@inheritDoc} <!--workaround-->
2713      */
2714     @Override
2715     public abstract
2716     $abstractvectortype$ selectFrom(Vector<$Boxtype$> s, VectorMask<$Boxtype$> m);
2717 

3285      * for lane values of large magnitude.
3286 #else[long]
3287      * @implNote
3288      * When this method is used on used on vectors
3289      * of type {@code $abstractvectortype$},
3290      * there will be no loss of precision.
3291 #end[long]
3292      */
3293     @ForceInline
3294     @Override
3295     public final double[] toDoubleArray() {
3296         $type$[] a = toArray();
3297         double[] res = new double[a.length];
3298         for (int i = 0; i < a.length; i++) {
3299             res[i] = (double) a[i];
3300         }
3301         return res;
3302     }
3303 #end[double]
3304 
3305     /**
3306      * Loads a vector from a byte array starting at an offset.
3307      * Bytes are composed into primitive lane elements according
3308      * to the specified byte order.
3309      * The vector is arranged into lanes according to
3310      * <a href="Vector.html#lane-order">memory ordering</a>.
3311      * <p>
3312      * This method behaves as if it returns the result of calling
3313      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3314      * fromByteBuffer()} as follows:
3315      * <pre>{@code
3316      * var bb = ByteBuffer.wrap(a);
3317      * var m = species.maskAll(true);
3318      * return fromByteBuffer(species, bb, offset, bo, m);
3319      * }</pre>
3320      *
3321      * @param species species of desired vector
3322      * @param a the byte array
3323      * @param offset the offset into the array
3324      * @param bo the intended byte order
3325      * @return a vector loaded from a byte array
3326      * @throws IndexOutOfBoundsException
3327      *         if {@code offset+N*ESIZE < 0}
3328      *         or {@code offset+(N+1)*ESIZE > a.length}
3329      *         for any lane {@code N} in the vector
3330      */
3331     @ForceInline
3332     public static
3333     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3334                                        byte[] a, int offset,
3335                                        ByteOrder bo) {
3336         offset = checkFromIndexSize(offset, species.vectorByteSize(), a.length);
3337         $Type$Species vsp = ($Type$Species) species;
3338         return vsp.dummyVector().fromByteArray0(a, offset).maybeSwap(bo);
3339     }
3340 
3341     /**
3342      * Loads a vector from a byte array starting at an offset
3343      * and using a mask.
3344      * Lanes where the mask is unset are filled with the default
3345      * value of {@code $type$} ({#if[FP]?positive }zero).
3346      * Bytes are composed into primitive lane elements according
3347      * to the specified byte order.
3348      * The vector is arranged into lanes according to
3349      * <a href="Vector.html#lane-order">memory ordering</a>.
3350      * <p>
3351      * This method behaves as if it returns the result of calling
3352      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3353      * fromByteBuffer()} as follows:
3354      * <pre>{@code
3355      * var bb = ByteBuffer.wrap(a);
3356      * return fromByteBuffer(species, bb, offset, bo, m);
3357      * }</pre>
3358      *
3359      * @param species species of desired vector
3360      * @param a the byte array
3361      * @param offset the offset into the array
3362      * @param bo the intended byte order
3363      * @param m the mask controlling lane selection
3364      * @return a vector loaded from a byte array
3365      * @throws IndexOutOfBoundsException
3366      *         if {@code offset+N*ESIZE < 0}
3367      *         or {@code offset+(N+1)*ESIZE > a.length}
3368      *         for any lane {@code N} in the vector
3369      *         where the mask is set
3370      */
3371     @ForceInline
3372     public static
3373     $abstractvectortype$ fromByteArray(VectorSpecies<$Boxtype$> species,
3374                                        byte[] a, int offset,
3375                                        ByteOrder bo,
3376                                        VectorMask<$Boxtype$> m) {
3377         $Type$Species vsp = ($Type$Species) species;
3378         if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
3379             return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
3380         }
3381 
3382         // FIXME: optimize
3383         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
3384         ByteBuffer wb = wrapper(a, bo);
3385         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
3386                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
3387     }
3388 
3389     /**
3390      * Loads a vector from an array of type {@code $type$[]}
3391      * starting at an offset.
3392      * For each vector lane, where {@code N} is the vector lane index, the
3393      * array element at index {@code offset + N} is placed into the
3394      * resulting vector at lane index {@code N}.
3395      *
3396      * @param species species of desired vector
3397      * @param a the array
3398      * @param offset the offset into the array
3399      * @return the vector loaded from an array
3400      * @throws IndexOutOfBoundsException
3401      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3402      *         for any lane {@code N} in the vector
3403      */
3404     @ForceInline
3405     public static
3406     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3407                                    $type$[] a, int offset) {
3408         offset = checkFromIndexSize(offset, species.length(), a.length);

3900      *         or if {@code mapOffset+N >= indexMap.length},
3901      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
3902      *         is an invalid index into {@code a},
3903      *         for any lane {@code N} in the vector
3904      *         where the mask is set
3905      * @see $abstractvectortype$#toIntArray()
3906      */
3907     @ForceInline
3908     public static
3909     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
3910                                           boolean[] a, int offset,
3911                                           int[] indexMap, int mapOffset,
3912                                           VectorMask<$Boxtype$> m) {
3913         // FIXME: optimize
3914         $Type$Species vsp = ($Type$Species) species;
3915         return vsp.vOp(m, n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
3916     }
3917 #end[byte]
3918 
3919     /**
3920      * Loads a vector from a {@linkplain ByteBuffer byte buffer}
3921      * starting at an offset into the byte buffer.
3922      * Bytes are composed into primitive lane elements according
3923      * to the specified byte order.
3924      * The vector is arranged into lanes according to
3925      * <a href="Vector.html#lane-order">memory ordering</a>.
3926      * <p>
3927      * This method behaves as if it returns the result of calling
3928      * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3929      * fromByteBuffer()} as follows:
3930      * <pre>{@code
3931      * var m = species.maskAll(true);
3932      * return fromByteBuffer(species, bb, offset, bo, m);
3933      * }</pre>
3934      *
3935      * @param species species of desired vector
3936      * @param bb the byte buffer
3937      * @param offset the offset into the byte buffer
3938      * @param bo the intended byte order
3939      * @return a vector loaded from a byte buffer
3940      * @throws IndexOutOfBoundsException
3941      *         if {@code offset+N*$sizeInBytes$ < 0}
3942      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
3943      *         for any lane {@code N} in the vector





3944      */
3945     @ForceInline
3946     public static
3947     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
3948                                         ByteBuffer bb, int offset,
3949                                         ByteOrder bo) {
3950         offset = checkFromIndexSize(offset, species.vectorByteSize(), bb.limit());
3951         $Type$Species vsp = ($Type$Species) species;
3952         return vsp.dummyVector().fromByteBuffer0(bb, offset).maybeSwap(bo);
3953     }
3954 
3955     /**
3956      * Loads a vector from a {@linkplain ByteBuffer byte buffer}
3957      * starting at an offset into the byte buffer
3958      * and using a mask.
3959      * Lanes where the mask is unset are filled with the default
3960      * value of {@code $type$} ({#if[FP]?positive }zero).
3961      * Bytes are composed into primitive lane elements according
3962      * to the specified byte order.
3963      * The vector is arranged into lanes according to
3964      * <a href="Vector.html#lane-order">memory ordering</a>.
3965      * <p>
3966      * The following pseudocode illustrates the behavior:
3967      * <pre>{@code
3968      * $Type$Buffer eb = bb.duplicate()
3969      *     .position(offset){#if[byte]?;}
3970 #if[!byte]
3971      *     .order(bo).as$Type$Buffer();
3972 #end[!byte]
3973      * $type$[] ar = new $type$[species.length()];
3974      * for (int n = 0; n < ar.length; n++) {
3975      *     if (m.laneIsSet(n)) {
3976      *         ar[n] = eb.get(n);
3977      *     }
3978      * }
3979      * $abstractvectortype$ r = $abstractvectortype$.fromArray(species, ar, 0);
3980      * }</pre>
3981      * @implNote
3982 #if[!byte]
3983      * This operation is likely to be more efficient if
3984      * the specified byte order is the same as
3985      * {@linkplain ByteOrder#nativeOrder()
3986      * the platform native order},
3987      * since this method will not need to reorder
3988      * the bytes of lane values.
3989 #else[!byte]
3990      * The byte order argument is ignored.
3991 #end[!byte]
3992      *
3993      * @param species species of desired vector
3994      * @param bb the byte buffer
3995      * @param offset the offset into the byte buffer
3996      * @param bo the intended byte order
3997      * @param m the mask controlling lane selection
3998      * @return a vector loaded from a byte buffer
3999      * @throws IndexOutOfBoundsException
4000      *         if {@code offset+N*$sizeInBytes$ < 0}
4001      *         or {@code offset+N*$sizeInBytes$ >= bb.limit()}
4002      *         for any lane {@code N} in the vector
4003      *         where the mask is set





4004      */
4005     @ForceInline
4006     public static
4007     $abstractvectortype$ fromByteBuffer(VectorSpecies<$Boxtype$> species,
4008                                         ByteBuffer bb, int offset,
4009                                         ByteOrder bo,
4010                                         VectorMask<$Boxtype$> m) {
4011         $Type$Species vsp = ($Type$Species) species;
4012         if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
4013             return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
4014         }
4015 
4016         // FIXME: optimize
4017         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4018         ByteBuffer wb = wrapper(bb, bo);
4019         return vsp.ldOp(wb, offset, (AbstractMask<$Boxtype$>)m,
4020                    (wb_, o, i)  -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4021     }
4022 
4023     // Memory store operations
4024 
4025     /**
4026      * Stores this vector into an array of type {@code $type$[]}
4027      * starting at an offset.
4028      * <p>
4029      * For each vector lane, where {@code N} is the vector lane index,
4030      * the lane element at index {@code N} is stored into the array
4031      * element {@code a[offset+N]}.
4032      *
4033      * @param a the array, of type {@code $type$[]}
4034      * @param offset the offset into the array
4035      * @throws IndexOutOfBoundsException
4036      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4037      *         for any lane {@code N} in the vector
4038      */
4039     @ForceInline
4040     public final
4041     void intoArray($type$[] a, int offset) {
4042         offset = checkFromIndexSize(offset, length(), a.length);
4043         $Type$Species vsp = vspecies();
4044         VectorSupport.store(
4045             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4046             a, arrayAddress(a, offset),
4047             this,
4048             a, offset,
4049             (arr, off, v)
4050             -> v.stOp(arr, off,
4051                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4052     }
4053 
4054     /**
4055      * Stores this vector into an array of type {@code $type$[]}
4056      * starting at offset and using a mask.
4057      * <p>
4058      * For each vector lane, where {@code N} is the vector lane index,
4059      * the lane element at index {@code N} is stored into the array
4060      * element {@code a[offset+N]}.
4061      * If the mask lane at {@code N} is unset then the corresponding
4062      * array element {@code a[offset+N]} is left unchanged.
4063      * <p>
4064      * Array range checking is done for lanes where the mask is set.
4065      * Lanes where the mask is unset are not stored and do not need
4066      * to correspond to legitimate elements of {@code a}.
4067      * That is, unset lanes may correspond to array indexes less than
4068      * zero or beyond the end of the array.
4069      *
4070      * @param a the array, of type {@code $type$[]}

4247      * is first cast to a {@code char} value and then
4248      * stored into the array element {@code a[offset+N]}.
4249      *
4250      * @param a the array, of type {@code char[]}
4251      * @param offset the offset into the array
4252      * @throws IndexOutOfBoundsException
4253      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4254      *         for any lane {@code N} in the vector
4255      */
4256     @ForceInline
4257     public final
4258     void intoCharArray(char[] a, int offset) {
4259         offset = checkFromIndexSize(offset, length(), a.length);
4260         $Type$Species vsp = vspecies();
4261         VectorSupport.store(
4262             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4263             a, charArrayAddress(a, offset),
4264             this,
4265             a, offset,
4266             (arr, off, v)
4267             -> v.stOp(arr, off,
4268                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
4269     }
4270 
4271     /**
4272      * Stores this vector into an array of type {@code char[]}
4273      * starting at offset and using a mask.
4274      * <p>
4275      * For each vector lane, where {@code N} is the vector lane index,
4276      * the lane element at index {@code N}
4277      * is first cast to a {@code char} value and then
4278      * stored into the array element {@code a[offset+N]}.
4279      * If the mask lane at {@code N} is unset then the corresponding
4280      * array element {@code a[offset+N]} is left unchanged.
4281      * <p>
4282      * Array range checking is done for lanes where the mask is set.
4283      * Lanes where the mask is unset are not stored and do not need
4284      * to correspond to legitimate elements of {@code a}.
4285      * That is, unset lanes may correspond to array indexes less than
4286      * zero or beyond the end of the array.
4287      *

4406      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4407      *
4408      * @param a the array
4409      * @param offset the offset into the array
4410      * @throws IndexOutOfBoundsException
4411      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4412      *         for any lane {@code N} in the vector
4413      */
4414     @ForceInline
4415     public final
4416     void intoBooleanArray(boolean[] a, int offset) {
4417         offset = checkFromIndexSize(offset, length(), a.length);
4418         $Type$Species vsp = vspecies();
4419         ByteVector normalized = this.and((byte) 1);
4420         VectorSupport.store(
4421             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4422             a, booleanArrayAddress(a, offset),
4423             normalized,
4424             a, offset,
4425             (arr, off, v)
4426             -> v.stOp(arr, off,
4427                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4428     }
4429 
4430     /**
4431      * Stores this vector into an array of type {@code boolean[]}
4432      * starting at offset and using a mask.
4433      * <p>
4434      * For each vector lane, where {@code N} is the vector lane index,
4435      * the lane element at index {@code N}
4436      * is first converted to a {@code boolean} value and then
4437      * stored into the array element {@code a[offset+N]}.
4438      * If the mask lane at {@code N} is unset then the corresponding
4439      * array element {@code a[offset+N]} is left unchanged.
4440      * <p>
4441      * A {@code byte} value is converted to a {@code boolean} value by applying the
4442      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4443      * <p>
4444      * Array range checking is done for lanes where the mask is set.
4445      * Lanes where the mask is unset are not stored and do not need
4446      * to correspond to legitimate elements of {@code a}.

4545      *         for any lane {@code N} in the vector
4546      *         where the mask is set
4547      * @see $abstractvectortype$#toIntArray()
4548      */
4549     @ForceInline
4550     public final
4551     void intoBooleanArray(boolean[] a, int offset,
4552                           int[] indexMap, int mapOffset,
4553                           VectorMask<$Boxtype$> m) {
4554         // FIXME: optimize
4555         stOp(a, offset, m,
4556              (arr, off, i, e) -> {
4557                  int j = indexMap[mapOffset + i];
4558                  arr[off + j] = (e & 1) != 0;
4559              });
4560     }
4561 #end[byte]
4562 
4563     /**
4564      * {@inheritDoc} <!--workaround-->

4565      */
4566     @Override
4567     @ForceInline
4568     public final
4569     void intoByteArray(byte[] a, int offset,
4570                        ByteOrder bo) {
4571         offset = checkFromIndexSize(offset, byteSize(), a.length);
4572         maybeSwap(bo).intoByteArray0(a, offset);
4573     }
4574 
4575     /**
4576      * {@inheritDoc} <!--workaround-->
4577      */
4578     @Override
4579     @ForceInline
4580     public final
4581     void intoByteArray(byte[] a, int offset,
4582                        ByteOrder bo,
4583                        VectorMask<$Boxtype$> m) {
4584         if (m.allTrue()) {
4585             intoByteArray(a, offset, bo);
4586         } else {
4587             $Type$Species vsp = vspecies();
4588             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, a.length);
4589             maybeSwap(bo).intoByteArray0(a, offset, m);
4590         }
4591     }
4592 
4593     /**
4594      * {@inheritDoc} <!--workaround-->
4595      */
4596     @Override
4597     @ForceInline
4598     public final
4599     void intoByteBuffer(ByteBuffer bb, int offset,
4600                         ByteOrder bo) {
4601         if (ScopedMemoryAccess.isReadOnly(bb)) {
4602             throw new ReadOnlyBufferException();
4603         }
4604         offset = checkFromIndexSize(offset, byteSize(), bb.limit());
4605         maybeSwap(bo).intoByteBuffer0(bb, offset);
4606     }
4607 
4608     /**
4609      * {@inheritDoc} <!--workaround-->

4610      */
4611     @Override
4612     @ForceInline
4613     public final
4614     void intoByteBuffer(ByteBuffer bb, int offset,
4615                         ByteOrder bo,
4616                         VectorMask<$Boxtype$> m) {
4617         if (m.allTrue()) {
4618             intoByteBuffer(bb, offset, bo);
4619         } else {
4620             if (bb.isReadOnly()) {
4621                 throw new ReadOnlyBufferException();
4622             }
4623             $Type$Species vsp = vspecies();
4624             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, bb.limit());
4625             maybeSwap(bo).intoByteBuffer0(bb, offset, m);
4626         }
4627     }
4628 
4629     // ================================================
4630 
4631     // Low-level memory operations.
4632     //
4633     // Note that all of these operations *must* inline into a context
4634     // where the exact species of the involved vector is a
4635     // compile-time constant.  Otherwise, the intrinsic generation
4636     // will fail and performance will suffer.
4637     //
4638     // In many cases this is achieved by re-deriving a version of the
4639     // method in each concrete subclass (per species).  The re-derived
4640     // method simply calls one of these generic methods, with exact
4641     // parameters for the controlling metadata, which is either a
4642     // typed vector or constant species instance.
4643 
4644     // Unchecked loading operations in native byte order.
4645     // Caller is responsible for applying index checks, masking, and
4646     // byte swapping.
4647 
4648     /*package-private*/
4649     abstract
4650     $abstractvectortype$ fromArray0($type$[] a, int offset);
4651     @ForceInline
4652     final
4653     $abstractvectortype$ fromArray0Template($type$[] a, int offset) {
4654         $Type$Species vsp = vspecies();
4655         return VectorSupport.load(
4656             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4657             a, arrayAddress(a, offset),
4658             a, offset, vsp,
4659             (arr, off, s) -> s.ldOp(arr, off,
4660                                     (arr_, off_, i) -> arr_[off_ + i]));
4661     }
4662 
4663     /*package-private*/
4664     abstract
4665     $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4666     @ForceInline
4667     final
4668     <M extends VectorMask<$Boxtype$>>
4669     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4670         m.check(species());
4671         $Type$Species vsp = vspecies();
4672         return VectorSupport.loadMasked(
4673             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4674             a, arrayAddress(a, offset), m,
4675             a, offset, vsp,
4676             (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4677                                         (arr_, off_, i) -> arr_[off_ + i]));
4678     }
4679 
4680 #if[!byteOrShort]
4681     /*package-private*/
4682     abstract
4683     $abstractvectortype$ fromArray0($type$[] a, int offset,
4684                                     int[] indexMap, int mapOffset,
4685                                     VectorMask<$Boxtype$> m);
4686     @ForceInline
4687     final
4688     <M extends VectorMask<$Boxtype$>>
4689     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset,
4690                                             int[] indexMap, int mapOffset, M m) {
4691         $Type$Species vsp = vspecies();
4692         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4693         Objects.requireNonNull(a);
4694         Objects.requireNonNull(indexMap);
4695         m.check(vsp);
4696         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();

4733             isp.vectorType(),
4734             a, ARRAY_BASE, vix, m,
4735             a, offset, indexMap, mapOffset, vsp,
4736             (c, idx, iMap, idy, s, vm) ->
4737             s.vOp(vm, n -> c[idx + iMap[idy+n]]));
4738     }
4739 #end[!byteOrShort]
4740 
4741 #if[short]
4742     /*package-private*/
4743     abstract
4744     $abstractvectortype$ fromCharArray0(char[] a, int offset);
4745     @ForceInline
4746     final
4747     $abstractvectortype$ fromCharArray0Template(char[] a, int offset) {
4748         $Type$Species vsp = vspecies();
4749         return VectorSupport.load(
4750             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4751             a, charArrayAddress(a, offset),
4752             a, offset, vsp,
4753             (arr, off, s) -> s.ldOp(arr, off,
4754                                     (arr_, off_, i) -> (short) arr_[off_ + i]));
4755     }
4756 
4757     /*package-private*/
4758     abstract
4759     $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
4760     @ForceInline
4761     final
4762     <M extends VectorMask<$Boxtype$>>
4763     $abstractvectortype$ fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
4764         m.check(species());
4765         $Type$Species vsp = vspecies();
4766         return VectorSupport.loadMasked(
4767                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4768                 a, charArrayAddress(a, offset), m,
4769                 a, offset, vsp,
4770                 (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4771                                             (arr_, off_, i) -> (short) arr_[off_ + i]));
4772     }
4773 #end[short]
4774 
4775 #if[byte]
4776     /*package-private*/
4777     abstract
4778     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset);
4779     @ForceInline
4780     final
4781     $abstractvectortype$ fromBooleanArray0Template(boolean[] a, int offset) {
4782         $Type$Species vsp = vspecies();
4783         return VectorSupport.load(
4784             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4785             a, booleanArrayAddress(a, offset),
4786             a, offset, vsp,
4787             (arr, off, s) -> s.ldOp(arr, off,
4788                                     (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4789     }
4790 
4791     /*package-private*/
4792     abstract
4793     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4794     @ForceInline
4795     final
4796     <M extends VectorMask<$Boxtype$>>
4797     $abstractvectortype$ fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4798         m.check(species());
4799         $Type$Species vsp = vspecies();
4800         return VectorSupport.loadMasked(
4801             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4802             a, booleanArrayAddress(a, offset), m,
4803             a, offset, vsp,
4804             (arr, off, s, vm) -> s.ldOp(arr, off, vm,
4805                                         (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4806     }
4807 #end[byte]
4808 
4809     @Override
4810     abstract
4811     $abstractvectortype$ fromByteArray0(byte[] a, int offset);
4812     @ForceInline
4813     final
4814     $abstractvectortype$ fromByteArray0Template(byte[] a, int offset) {
4815         $Type$Species vsp = vspecies();
4816         return VectorSupport.load(
4817             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4818             a, byteArrayAddress(a, offset),
4819             a, offset, vsp,
4820             (arr, off, s) -> {
4821                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4822                 return s.ldOp(wb, off,
4823                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4824             });
4825     }
4826 
4827     abstract
4828     $abstractvectortype$ fromByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
4829     @ForceInline
4830     final
4831     <M extends VectorMask<$Boxtype$>>
4832     $abstractvectortype$ fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
4833         $Type$Species vsp = vspecies();
4834         m.check(vsp);
4835         return VectorSupport.loadMasked(
4836             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4837             a, byteArrayAddress(a, offset), m,
4838             a, offset, vsp,
4839             (arr, off, s, vm) -> {
4840                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
4841                 return s.ldOp(wb, off, vm,
4842                         (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4843             });
4844     }
4845 
4846     abstract
4847     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset);
4848     @ForceInline
4849     final
4850     $abstractvectortype$ fromByteBuffer0Template(ByteBuffer bb, int offset) {
4851         $Type$Species vsp = vspecies();
4852         return ScopedMemoryAccess.loadFromByteBuffer(
4853                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4854                 bb, offset, vsp,
4855                 (buf, off, s) -> {
4856                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4857                     return s.ldOp(wb, off,
4858                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4859                 });
4860     }
4861 
4862     abstract
4863     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
4864     @ForceInline
4865     final
4866     <M extends VectorMask<$Boxtype$>>
4867     $abstractvectortype$ fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
4868         $Type$Species vsp = vspecies();
4869         m.check(vsp);
4870         return ScopedMemoryAccess.loadFromByteBufferMasked(
4871                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4872                 bb, offset, m, vsp,
4873                 (buf, off, s, vm) -> {
4874                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
4875                     return s.ldOp(wb, off, vm,
4876                             (wb_, o, i) -> wb_.get{#if[byte]?(:$Type$(}o + i * $sizeInBytes$));
4877                 });
4878     }
4879 
4880     // Unchecked storing operations in native byte order.
4881     // Caller is responsible for applying index checks, masking, and
4882     // byte swapping.
4883 
4884     abstract
4885     void intoArray0($type$[] a, int offset);
4886     @ForceInline
4887     final
4888     void intoArray0Template($type$[] a, int offset) {
4889         $Type$Species vsp = vspecies();
4890         VectorSupport.store(
4891             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4892             a, arrayAddress(a, offset),
4893             this, a, offset,
4894             (arr, off, v)
4895             -> v.stOp(arr, off,
4896                       (arr_, off_, i, e) -> arr_[off_+i] = e));
4897     }
4898 
4899     abstract
4900     void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4901     @ForceInline
4902     final
4903     <M extends VectorMask<$Boxtype$>>
4904     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4905         m.check(species());
4906         $Type$Species vsp = vspecies();
4907         VectorSupport.storeMasked(
4908             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4909             a, arrayAddress(a, offset),
4910             this, m, a, offset,
4911             (arr, off, v, vm)
4912             -> v.stOp(arr, off, vm,
4913                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4914     }
4915 
4916 #if[!byteOrShort]
4917     abstract
4918     void intoArray0($type$[] a, int offset,
4919                     int[] indexMap, int mapOffset,
4920                     VectorMask<$Boxtype$> m);
4921     @ForceInline
4922     final
4923     <M extends VectorMask<$Boxtype$>>
4924     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset,
4925                             int[] indexMap, int mapOffset, M m) {
4926         m.check(species());
4927         $Type$Species vsp = vspecies();
4928         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4929 #if[longOrDouble]
4930         if (vsp.laneCount() == 1) {
4931             intoArray(a, offset + indexMap[mapOffset], m);
4932             return;

4973                           arr[off + j] = e;
4974                       }));
4975     }
4976 #end[!byteOrShort]
4977 
4978 #if[byte]
4979     abstract
4980     void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4981     @ForceInline
4982     final
4983     <M extends VectorMask<$Boxtype$>>
4984     void intoBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4985         m.check(species());
4986         $Type$Species vsp = vspecies();
4987         ByteVector normalized = this.and((byte) 1);
4988         VectorSupport.storeMasked(
4989             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4990             a, booleanArrayAddress(a, offset),
4991             normalized, m, a, offset,
4992             (arr, off, v, vm)
4993             -> v.stOp(arr, off, vm,
4994                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4995     }
4996 #end[byte]
4997 
4998     abstract
4999     void intoByteArray0(byte[] a, int offset);
5000     @ForceInline
5001     final
5002     void intoByteArray0Template(byte[] a, int offset) {
5003         $Type$Species vsp = vspecies();
5004         VectorSupport.store(
5005             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
5006             a, byteArrayAddress(a, offset),
5007             this, a, offset,
5008             (arr, off, v) -> {
5009                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
5010                 v.stOp(wb, off,
5011                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5012             });
5013     }
5014 
5015     abstract
5016     void intoByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m);
5017     @ForceInline
5018     final
5019     <M extends VectorMask<$Boxtype$>>
5020     void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
5021         $Type$Species vsp = vspecies();
5022         m.check(vsp);
5023         VectorSupport.storeMasked(
5024             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5025             a, byteArrayAddress(a, offset),
5026             this, m, a, offset,
5027             (arr, off, v, vm) -> {
5028                 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
5029                 v.stOp(wb, off, vm,
5030                         (tb_, o, i, e) -> tb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5031             });
5032     }
5033 
5034     @ForceInline
5035     final
5036     void intoByteBuffer0(ByteBuffer bb, int offset) {
5037         $Type$Species vsp = vspecies();
5038         ScopedMemoryAccess.storeIntoByteBuffer(
5039                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
5040                 this, bb, offset,
5041                 (buf, off, v) -> {
5042                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
5043                     v.stOp(wb, off,
5044                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5045                 });
5046     }
5047 
5048     abstract
5049     void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m);
5050     @ForceInline
5051     final
5052     <M extends VectorMask<$Boxtype$>>
5053     void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
5054         $Type$Species vsp = vspecies();
5055         m.check(vsp);
5056         ScopedMemoryAccess.storeIntoByteBufferMasked(
5057                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5058                 this, m, bb, offset,
5059                 (buf, off, v, vm) -> {
5060                     ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
5061                     v.stOp(wb, off, vm,
5062                             (wb_, o, i, e) -> wb_.put{#if[byte]?(:$Type$(}o + i * $sizeInBytes$, e));
5063                 });
5064     }
5065 
5066 #if[short]
5067     /*package-private*/
5068     abstract
5069     void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
5070     @ForceInline
5071     final
5072     <M extends VectorMask<$Boxtype$>>
5073     void intoCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
5074         m.check(species());
5075         $Type$Species vsp = vspecies();
5076         VectorSupport.storeMasked(
5077             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5078             a, charArrayAddress(a, offset),
5079             this, m, a, offset,
5080             (arr, off, v, vm)
5081             -> v.stOp(arr, off, vm,
5082                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
5083     }
5084 #end[short]
5085 
5086     // End of low-level memory operations.
5087 
5088     private static
5089     void checkMaskFromIndexSize(int offset,
5090                                 $Type$Species vsp,
5091                                 VectorMask<$Boxtype$> m,
5092                                 int scale,
5093                                 int limit) {
5094         ((AbstractMask<$Boxtype$>)m)
5095             .checkIndexByLane(offset, limit, vsp.iota(), scale);
5096     }
5097 










5098     @ForceInline
5099     private void conditionalStoreNYI(int offset,
5100                                      $Type$Species vsp,
5101                                      VectorMask<$Boxtype$> m,
5102                                      int scale,
5103                                      int limit) {
5104         if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
5105             String msg =
5106                 String.format("unimplemented: store @%d in [0..%d), %s in %s",
5107                               offset, limit, m, vsp);
5108             throw new AssertionError(msg);
5109         }
5110     }
5111 
5112     /*package-private*/
5113     @Override
5114     @ForceInline
5115     final
5116     $abstractvectortype$ maybeSwap(ByteOrder bo) {
5117 #if[!byte]

5446                 }
5447             }
5448             return dummyVector().vectorFactory(res);
5449         }
5450 
5451         /*package-private*/
5452         @ForceInline
5453         <M> $abstractvectortype$ ldOp(M memory, int offset,
5454                                       FLdOp<M> f) {
5455             return dummyVector().ldOp(memory, offset, f);
5456         }
5457 
5458         /*package-private*/
5459         @ForceInline
5460         <M> $abstractvectortype$ ldOp(M memory, int offset,
5461                                       VectorMask<$Boxtype$> m,
5462                                       FLdOp<M> f) {
5463             return dummyVector().ldOp(memory, offset, m, f);
5464         }
5465 















5466         /*package-private*/
5467         @ForceInline
5468         <M> void stOp(M memory, int offset, FStOp<M> f) {
5469             dummyVector().stOp(memory, offset, f);
5470         }
5471 
5472         /*package-private*/
5473         @ForceInline
5474         <M> void stOp(M memory, int offset,
5475                       AbstractMask<$Boxtype$> m,
5476                       FStOp<M> f) {
5477             dummyVector().stOp(memory, offset, m, f);
5478         }














5479 
5480         // N.B. Make sure these constant vectors and
5481         // masks load up correctly into registers.
5482         //
5483         // Also, see if we can avoid all that switching.
5484         // Could we cache both vectors and both masks in
5485         // this species object?
5486 
5487         // Zero and iota vector access
5488         @Override
5489         @ForceInline
5490         public final $abstractvectortype$ zero() {
5491             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5492                 return $Type$MaxVector.ZERO;
5493             switch (vectorBitSize()) {
5494                 case 64: return $Type$64Vector.ZERO;
5495                 case 128: return $Type$128Vector.ZERO;
5496                 case 256: return $Type$256Vector.ZERO;
5497                 case 512: return $Type$512Vector.ZERO;
5498             }

   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 

  27 import java.nio.ByteOrder;

  28 import java.util.Arrays;
  29 import java.util.Objects;
  30 import java.util.function.Function;

  31 
  32 import jdk.incubator.foreign.MemorySegment;
  33 import jdk.incubator.foreign.ValueLayout;
  34 import jdk.internal.access.foreign.MemorySegmentProxy;
  35 import jdk.internal.misc.ScopedMemoryAccess;
  36 import jdk.internal.misc.Unsafe;
  37 import jdk.internal.vm.annotation.ForceInline;
  38 import jdk.internal.vm.vector.VectorSupport;
  39 
  40 import static jdk.internal.vm.vector.VectorSupport.*;
  41 import static jdk.incubator.vector.VectorIntrinsics.*;
  42 
  43 import static jdk.incubator.vector.VectorOperators.*;
  44 
  45 #warn This file is preprocessed before being compiled
  46 
  47 /**
  48  * A specialized {@link Vector} representing an ordered immutable sequence of
  49  * {@code $type$} values.
  50  */
  51 @SuppressWarnings("cast")  // warning: redundant cast
  52 public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
  53 
  54     $abstractvectortype$($type$[] vec) {
  55         super(vec);
  56     }
  57 
  58 #if[FP]
  59     static final int FORBID_OPCODE_KIND = VO_NOFP;
  60 #else[FP]
  61     static final int FORBID_OPCODE_KIND = VO_ONLYFP;
  62 #end[FP]
  63 
  64     static final ValueLayout.Of$Type$ ELEMENT_LAYOUT = ValueLayout.JAVA_$TYPE$.withBitAlignment(8);
  65 
  66     @ForceInline
  67     static int opCode(Operator op) {
  68         return VectorOperators.opCode(op, VO_OPCODE_VALID, FORBID_OPCODE_KIND);
  69     }
  70     @ForceInline
  71     static int opCode(Operator op, int requireKind) {
  72         requireKind |= VO_OPCODE_VALID;
  73         return VectorOperators.opCode(op, requireKind, FORBID_OPCODE_KIND);
  74     }
  75     @ForceInline
  76     static boolean opKind(Operator op, int bit) {
  77         return VectorOperators.opKind(op, bit);
  78     }
  79 
  80     // Virtualized factories and operators,
  81     // coded with portable definitions.
  82     // These are all @ForceInline in case
  83     // they need to be used performantly.
  84     // The various shape-specific subclasses
  85     // also specialize them by wrapping

 340         return vectorFactory(res);
 341     }
 342 
 343     /*package-private*/
 344     @ForceInline
 345     final
 346     <M> $abstractvectortype$ ldOp(M memory, int offset,
 347                                   VectorMask<$Boxtype$> m,
 348                                   FLdOp<M> f) {
 349         //$type$[] vec = vec();
 350         $type$[] res = new $type$[length()];
 351         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 352         for (int i = 0; i < res.length; i++) {
 353             if (mbits[i]) {
 354                 res[i] = f.apply(memory, offset, i);
 355             }
 356         }
 357         return vectorFactory(res);
 358     }
 359 
 360     /*package-private*/
 361     interface FLdLongOp {
 362         $type$ apply(MemorySegment memory, long offset, int i);
 363     }
 364 
 365     /*package-private*/
 366     @ForceInline
 367     final
 368     $abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
 369                                   FLdLongOp f) {
 370         //dummy; no vec = vec();
 371         $type$[] res = new $type$[length()];
 372         for (int i = 0; i < res.length; i++) {
 373             res[i] = f.apply(memory, offset, i);
 374         }
 375         return vectorFactory(res);
 376     }
 377 
 378     /*package-private*/
 379     @ForceInline
 380     final
 381     $abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
 382                                   VectorMask<$Boxtype$> m,
 383                                   FLdLongOp f) {
 384         //$type$[] vec = vec();
 385         $type$[] res = new $type$[length()];
 386         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 387         for (int i = 0; i < res.length; i++) {
 388             if (mbits[i]) {
 389                 res[i] = f.apply(memory, offset, i);
 390             }
 391         }
 392         return vectorFactory(res);
 393     }
 394 
 395     static $type$ memorySegmentGet(MemorySegment ms, long o, int i) {
 396         return ms.get(ELEMENT_LAYOUT, o + i * $sizeInBytes$L);
 397     }
 398 
 399     interface FStOp<M> {
 400         void apply(M memory, int offset, int i, $type$ a);
 401     }
 402 
 403     /*package-private*/
 404     @ForceInline
 405     final
 406     <M> void stOp(M memory, int offset,
 407                   FStOp<M> f) {
 408         $type$[] vec = vec();
 409         for (int i = 0; i < vec.length; i++) {
 410             f.apply(memory, offset, i, vec[i]);
 411         }
 412     }
 413 
 414     /*package-private*/
 415     @ForceInline
 416     final
 417     <M> void stOp(M memory, int offset,
 418                   VectorMask<$Boxtype$> m,
 419                   FStOp<M> f) {
 420         $type$[] vec = vec();
 421         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 422         for (int i = 0; i < vec.length; i++) {
 423             if (mbits[i]) {
 424                 f.apply(memory, offset, i, vec[i]);
 425             }
 426         }
 427     }
 428 
 429     interface FStLongOp {
 430         void apply(MemorySegment memory, long offset, int i, $type$ a);
 431     }
 432 
 433     /*package-private*/
 434     @ForceInline
 435     final
 436     void stLongOp(MemorySegment memory, long offset,
 437                   FStLongOp f) {
 438         $type$[] vec = vec();
 439         for (int i = 0; i < vec.length; i++) {
 440             f.apply(memory, offset, i, vec[i]);
 441         }
 442     }
 443 
 444     /*package-private*/
 445     @ForceInline
 446     final
 447     void stLongOp(MemorySegment memory, long offset,
 448                   VectorMask<$Boxtype$> m,
 449                   FStLongOp f) {
 450         $type$[] vec = vec();
 451         boolean[] mbits = ((AbstractMask<$Boxtype$>)m).getBits();
 452         for (int i = 0; i < vec.length; i++) {
 453             if (mbits[i]) {
 454                 f.apply(memory, offset, i, vec[i]);
 455             }
 456         }
 457     }
 458 
 459     static void memorySegmentSet(MemorySegment ms, long o, int i, $type$ e) {
 460         ms.set(ELEMENT_LAYOUT, o + i * $sizeInBytes$L, e);
 461     }
 462 
 463     // Binary test
 464 
 465     /*package-private*/
 466     interface FBinTest {
 467         boolean apply(int cond, int i, $type$ a, $type$ b);
 468     }
 469 
 470     /*package-private*/
 471     @ForceInline
 472     final
 473     AbstractMask<$Boxtype$> bTest(int cond,
 474                                   Vector<$Boxtype$> o,
 475                                   FBinTest f) {
 476         $type$[] vec1 = vec();
 477         $type$[] vec2 = (($abstractvectortype$)o).vec();
 478         boolean[] bits = new boolean[length()];
 479         for (int i = 0; i < length(); i++){
 480             bits[i] = f.apply(cond, i, vec1[i], vec2[i]);
 481         }
 482         return maskFactory(bits);

 503 #end[intOrLong]
 504     }
 505 #end[BITWISE]
 506 
 507     /*package-private*/
 508     @Override
 509     abstract $Type$Species vspecies();
 510 
 511     /*package-private*/
 512     @ForceInline
 513     static long toBits($type$ e) {
 514         return {#if[FP]? $Type$.$type$ToRaw$Bitstype$Bits(e): e};
 515     }
 516 
 517     /*package-private*/
 518     @ForceInline
 519     static $type$ fromBits(long bits) {
 520         return {#if[FP]?$Type$.$bitstype$BitsTo$Type$}(($bitstype$)bits);
 521     }
 522 
 523     static $abstractvectortype$ expandHelper(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 524         VectorSpecies<$Boxtype$> vsp = m.vectorSpecies();
 525         $abstractvectortype$ r  = ($abstractvectortype$) vsp.zero();
 526         $abstractvectortype$ vi = ($abstractvectortype$) v;
 527         if (m.allTrue()) {
 528             return vi;
 529         }
 530         for (int i = 0, j = 0; i < vsp.length(); i++) {
 531             if (m.laneIsSet(i)) {
 532                 r = r.withLane(i, vi.lane(j++));
 533             }
 534         }
 535         return r;
 536     }
 537 
 538     static $abstractvectortype$ compressHelper(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 539         VectorSpecies<$Boxtype$> vsp = m.vectorSpecies();
 540         $abstractvectortype$ r  = ($abstractvectortype$) vsp.zero();
 541         $abstractvectortype$ vi = ($abstractvectortype$) v;
 542         if (m.allTrue()) {
 543             return vi;
 544         }
 545         for (int i = 0, j = 0; i < vsp.length(); i++) {
 546             if (m.laneIsSet(i)) {
 547                 r = r.withLane(j++, vi.lane(i));
 548             }
 549         }
 550         return r;
 551     }
 552 
 553     // Static factories (other than memory operations)
 554 
 555     // Note: A surprising behavior in javadoc
 556     // sometimes makes a lone /** {@inheritDoc} */
 557     // comment drop the method altogether,
 558     // apparently if the method mentions an
 559     // parameter or return type of Vector<$Boxtype$>
 560     // instead of Vector<E> as originally specified.
 561     // Adding an empty HTML fragment appears to
 562     // nudge javadoc into providing the desired
 563     // inherited documentation.  We use the HTML
 564     // comment <!--workaround--> for this.
 565 
 566     /**
 567      * Returns a vector of the given species
 568      * where all lane elements are set to
 569      * zero, the default primitive value.
 570      *
 571      * @param species species of the desired zero vector
 572      * @return a zero vector

 734             }
 735 #end[BITWISE]
 736         }
 737         int opc = opCode(op);
 738         return VectorSupport.unaryOp(
 739             opc, getClass(), maskClass, $type$.class, length(),
 740             this, m,
 741             UN_IMPL.find(op, opc, $abstractvectortype$::unaryOperations));
 742     }
 743 
 744     private static final
 745     ImplCache<Unary, UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>>>
 746         UN_IMPL = new ImplCache<>(Unary.class, $Type$Vector.class);
 747 
 748     private static UnaryOperation<$abstractvectortype$, VectorMask<$Boxtype$>> unaryOperations(int opc_) {
 749         switch (opc_) {
 750             case VECTOR_OP_NEG: return (v0, m) ->
 751                     v0.uOp(m, (i, a) -> ($type$) -a);
 752             case VECTOR_OP_ABS: return (v0, m) ->
 753                     v0.uOp(m, (i, a) -> ($type$) Math.abs(a));
 754 #if[!FP]
 755 #if[intOrLong]
 756             case VECTOR_OP_BIT_COUNT: return (v0, m) ->
 757                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.bitCount(a));
 758             case VECTOR_OP_TZ_COUNT: return (v0, m) ->
 759                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.numberOfTrailingZeros(a));
 760             case VECTOR_OP_LZ_COUNT: return (v0, m) ->
 761                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.numberOfLeadingZeros(a));
 762             case VECTOR_OP_REVERSE: return (v0, m) ->
 763                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.reverse(a));
 764 #else[intOrLong]
 765             case VECTOR_OP_BIT_COUNT: return (v0, m) ->
 766                     v0.uOp(m, (i, a) -> ($type$) bitCount(a));
 767             case VECTOR_OP_TZ_COUNT: return (v0, m) ->
 768                     v0.uOp(m, (i, a) -> ($type$) numberOfTrailingZeros(a));
 769             case VECTOR_OP_LZ_COUNT: return (v0, m) ->
 770                     v0.uOp(m, (i, a) -> ($type$) numberOfLeadingZeros(a));
 771             case VECTOR_OP_REVERSE: return (v0, m) ->
 772                     v0.uOp(m, (i, a) -> reverse(a));
 773 #end[intOrLong]
 774 #if[BITWISE]
 775 #if[byte]
 776             case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
 777                     v0.uOp(m, (i, a) -> a);
 778 #else[byte]
 779             case VECTOR_OP_REVERSE_BYTES: return (v0, m) ->
 780                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.reverseBytes(a));
 781 #end[byte]
 782 #end[BITWISE]
 783 #end[!FP]
 784 #if[FP]
 785             case VECTOR_OP_SIN: return (v0, m) ->
 786                     v0.uOp(m, (i, a) -> ($type$) Math.sin(a));
 787             case VECTOR_OP_COS: return (v0, m) ->
 788                     v0.uOp(m, (i, a) -> ($type$) Math.cos(a));
 789             case VECTOR_OP_TAN: return (v0, m) ->
 790                     v0.uOp(m, (i, a) -> ($type$) Math.tan(a));
 791             case VECTOR_OP_ASIN: return (v0, m) ->
 792                     v0.uOp(m, (i, a) -> ($type$) Math.asin(a));
 793             case VECTOR_OP_ACOS: return (v0, m) ->
 794                     v0.uOp(m, (i, a) -> ($type$) Math.acos(a));
 795             case VECTOR_OP_ATAN: return (v0, m) ->
 796                     v0.uOp(m, (i, a) -> ($type$) Math.atan(a));
 797             case VECTOR_OP_EXP: return (v0, m) ->
 798                     v0.uOp(m, (i, a) -> ($type$) Math.exp(a));
 799             case VECTOR_OP_LOG: return (v0, m) ->
 800                     v0.uOp(m, (i, a) -> ($type$) Math.log(a));
 801             case VECTOR_OP_LOG10: return (v0, m) ->
 802                     v0.uOp(m, (i, a) -> ($type$) Math.log10(a));
 803             case VECTOR_OP_SQRT: return (v0, m) ->

 957                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.max(a, b));
 958             case VECTOR_OP_MIN: return (v0, v1, vm) ->
 959                     v0.bOp(v1, vm, (i, a, b) -> ($type$)Math.min(a, b));
 960 #if[BITWISE]
 961             case VECTOR_OP_AND: return (v0, v1, vm) ->
 962                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a & b));
 963             case VECTOR_OP_OR: return (v0, v1, vm) ->
 964                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a | b));
 965             case VECTOR_OP_XOR: return (v0, v1, vm) ->
 966                     v0.bOp(v1, vm, (i, a, b) -> ($type$)(a ^ b));
 967             case VECTOR_OP_LSHIFT: return (v0, v1, vm) ->
 968                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a << n));
 969             case VECTOR_OP_RSHIFT: return (v0, v1, vm) ->
 970                     v0.bOp(v1, vm, (i, a, n) -> ($type$)(a >> n));
 971             case VECTOR_OP_URSHIFT: return (v0, v1, vm) ->
 972                     v0.bOp(v1, vm, (i, a, n) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 973             case VECTOR_OP_LROTATE: return (v0, v1, vm) ->
 974                     v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
 975             case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
 976                     v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
 977 #if[intOrLong]
 978             case VECTOR_OP_COMPRESS_BITS: return (v0, v1, vm) ->
 979                     v0.bOp(v1, vm, (i, a, n) -> $Boxtype$.compress(a, n));
 980             case VECTOR_OP_EXPAND_BITS: return (v0, v1, vm) ->
 981                     v0.bOp(v1, vm, (i, a, n) -> $Boxtype$.expand(a, n));
 982 #end[intOrLong]
 983 #end[BITWISE]
 984 #if[FP]
 985             case VECTOR_OP_OR: return (v0, v1, vm) ->
 986                     v0.bOp(v1, vm, (i, a, b) -> fromBits(toBits(a) | toBits(b)));
 987             case VECTOR_OP_ATAN2: return (v0, v1, vm) ->
 988                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.atan2(a, b));
 989             case VECTOR_OP_POW: return (v0, v1, vm) ->
 990                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.pow(a, b));
 991             case VECTOR_OP_HYPOT: return (v0, v1, vm) ->
 992                     v0.bOp(v1, vm, (i, a, b) -> ($type$) Math.hypot(a, b));
 993 #end[FP]
 994             default: return null;
 995         }
 996     }
 997 
 998     // FIXME: Maybe all of the public final methods in this file (the
 999     // simple ones that just call lanewise) should be pushed down to
1000     // the X-VectorBits template.  They can't optimize properly at
1001     // this level, and must rely on inlining.  Does it work?
1002     // (If it works, of course keep the code here.)

2111     /**
2112      * {@inheritDoc} <!--workaround-->
2113      */
2114     @Override
2115     @ForceInline
2116     public final
2117     $abstractvectortype$ neg() {
2118         return lanewise(NEG);
2119     }
2120 
2121     /**
2122      * {@inheritDoc} <!--workaround-->
2123      */
2124     @Override
2125     @ForceInline
2126     public final
2127     $abstractvectortype$ abs() {
2128         return lanewise(ABS);
2129     }
2130 
2131 #if[!FP]
2132 #if[!intOrLong]
2133     static int bitCount($type$ a) {
2134 #if[short]
2135         return Integer.bitCount((int)a & 0xFFFF);
2136 #else[short]
2137         return Integer.bitCount((int)a & 0xFF);
2138 #end[short]
2139     }
2140 #end[!intOrLong]
2141 #end[!FP]
2142 #if[!FP]
2143 #if[!intOrLong]
2144     static int numberOfTrailingZeros($type$ a) {
2145 #if[short]
2146         return a != 0 ? Integer.numberOfTrailingZeros(a) : 16;
2147 #else[short]
2148         return a != 0 ? Integer.numberOfTrailingZeros(a) : 8;
2149 #end[short]
2150     }
2151 #end[!intOrLong]
2152 #end[!FP]
2153 #if[!FP]
2154 #if[!intOrLong]
2155     static int numberOfLeadingZeros($type$ a) {
2156 #if[short]
2157         return a >= 0 ? Integer.numberOfLeadingZeros(a) - 16 : 0;
2158 #else[short]
2159         return a >= 0 ? Integer.numberOfLeadingZeros(a) - 24 : 0;
2160 #end[short]
2161     }
2162 
2163     static $type$ reverse($type$ a) {
2164         if (a == 0 || a == -1) return a;
2165 
2166 #if[short]
2167         $type$ b = rotateLeft(a, 8);
2168         b = ($type$) (((b & 0x5555) << 1) | ((b & 0xAAAA) >>> 1));
2169         b = ($type$) (((b & 0x3333) << 2) | ((b & 0xCCCC) >>> 2));
2170         b = ($type$) (((b & 0x0F0F) << 4) | ((b & 0xF0F0) >>> 4));
2171 #else[short]
2172         $type$ b = rotateLeft(a, 4);
2173         b = ($type$) (((b & 0x55) << 1) | ((b & 0xAA) >>> 1));
2174         b = ($type$) (((b & 0x33) << 2) | ((b & 0xCC) >>> 2));
2175 #end[short]
2176         return b;
2177     }
2178 #end[!intOrLong]
2179 #end[!FP]
2180 
2181 #if[BITWISE]
2182     // not (~)
2183     /**
2184      * Computes the bitwise logical complement ({@code ~})
2185      * of this vector.
2186      *
2187      * This is a lane-wise binary operation which applies the
2188      * the primitive bitwise "not" operation ({@code ~})
2189      * to each lane value.
2190      *
2191      * This method is also equivalent to the expression
2192      * {@link #lanewise(VectorOperators.Unary)
2193      *    lanewise}{@code (}{@link VectorOperators#NOT
2194      *    NOT}{@code )}.
2195      *
2196      * <p>
2197      * This is not a full-service named operation like
2198      * {@link #add(Vector) add}.  A masked version of
2199      * this operation is not directly available
2200      * but may be obtained via the masked version of

2869         $type$[] a = toArray();
2870         int[] sa = new int[a.length];
2871         for (int i = 0; i < a.length; i++) {
2872             sa[i] = (int) a[i];
2873         }
2874         return VectorShuffle.fromArray(dsp, sa, 0);
2875     }
2876 
2877     /*package-private*/
2878     @ForceInline
2879     final
2880     VectorShuffle<$Boxtype$> toShuffleTemplate(Class<?> shuffleType) {
2881         $Type$Species vsp = vspecies();
2882         return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
2883                                      getClass(), $type$.class, length(),
2884                                      shuffleType, byte.class, length(),
2885                                      this, vsp,
2886                                      $Type$Vector::toShuffle0);
2887     }
2888 
2889     /**
2890      * {@inheritDoc} <!--workaround-->
2891      * @since 19
2892      */
2893     @Override
2894     public abstract
2895     $Type$Vector compress(VectorMask<$Boxtype$> m);
2896 
2897     /*package-private*/
2898     @ForceInline
2899     final
2900     <M extends AbstractMask<$Boxtype$>>
2901     $Type$Vector compressTemplate(Class<M> masktype, M m) {
2902       m.check(masktype, this);
2903       return ($Type$Vector) VectorSupport.comExpOp(VectorSupport.VECTOR_OP_COMPRESS, getClass(), masktype,
2904                                                    $type$.class, length(), this, m,
2905                                                    (v1, m1) -> compressHelper(v1, m1));
2906     }
2907 
2908     /**
2909      * {@inheritDoc} <!--workaround-->
2910      * @since 19
2911      */
2912     @Override
2913     public abstract
2914     $Type$Vector expand(VectorMask<$Boxtype$> m);
2915 
2916     /*package-private*/
2917     @ForceInline
2918     final
2919     <M extends AbstractMask<$Boxtype$>>
2920     $Type$Vector expandTemplate(Class<M> masktype, M m) {
2921       m.check(masktype, this);
2922       return ($Type$Vector) VectorSupport.comExpOp(VectorSupport.VECTOR_OP_EXPAND, getClass(), masktype,
2923                                                    $type$.class, length(), this, m,
2924                                                    (v1, m1) -> expandHelper(v1, m1));
2925     }
2926 
2927 
2928     /**
2929      * {@inheritDoc} <!--workaround-->
2930      */
2931     @Override
2932     public abstract
2933     $abstractvectortype$ selectFrom(Vector<$Boxtype$> v);
2934 
2935     /*package-private*/
2936     @ForceInline
2937     final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v) {
2938         return v.rearrange(this.toShuffle());
2939     }
2940 
2941     /**
2942      * {@inheritDoc} <!--workaround-->
2943      */
2944     @Override
2945     public abstract
2946     $abstractvectortype$ selectFrom(Vector<$Boxtype$> s, VectorMask<$Boxtype$> m);
2947 

3515      * for lane values of large magnitude.
3516 #else[long]
3517      * @implNote
3518      * When this method is used on used on vectors
3519      * of type {@code $abstractvectortype$},
3520      * there will be no loss of precision.
3521 #end[long]
3522      */
3523     @ForceInline
3524     @Override
3525     public final double[] toDoubleArray() {
3526         $type$[] a = toArray();
3527         double[] res = new double[a.length];
3528         for (int i = 0; i < a.length; i++) {
3529             res[i] = (double) a[i];
3530         }
3531         return res;
3532     }
3533 #end[double]
3534 




















































































3535     /**
3536      * Loads a vector from an array of type {@code $type$[]}
3537      * starting at an offset.
3538      * For each vector lane, where {@code N} is the vector lane index, the
3539      * array element at index {@code offset + N} is placed into the
3540      * resulting vector at lane index {@code N}.
3541      *
3542      * @param species species of desired vector
3543      * @param a the array
3544      * @param offset the offset into the array
3545      * @return the vector loaded from an array
3546      * @throws IndexOutOfBoundsException
3547      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
3548      *         for any lane {@code N} in the vector
3549      */
3550     @ForceInline
3551     public static
3552     $abstractvectortype$ fromArray(VectorSpecies<$Boxtype$> species,
3553                                    $type$[] a, int offset) {
3554         offset = checkFromIndexSize(offset, species.length(), a.length);

4046      *         or if {@code mapOffset+N >= indexMap.length},
4047      *         or if {@code f(N)=offset+indexMap[mapOffset+N]}
4048      *         is an invalid index into {@code a},
4049      *         for any lane {@code N} in the vector
4050      *         where the mask is set
4051      * @see $abstractvectortype$#toIntArray()
4052      */
4053     @ForceInline
4054     public static
4055     $abstractvectortype$ fromBooleanArray(VectorSpecies<$Boxtype$> species,
4056                                           boolean[] a, int offset,
4057                                           int[] indexMap, int mapOffset,
4058                                           VectorMask<$Boxtype$> m) {
4059         // FIXME: optimize
4060         $Type$Species vsp = ($Type$Species) species;
4061         return vsp.vOp(m, n -> (byte) (a[offset + indexMap[mapOffset + n]] ? 1 : 0));
4062     }
4063 #end[byte]
4064 
4065     /**
4066      * Loads a vector from a {@linkplain MemorySegment memory segment}
4067      * starting at an offset into the memory segment.
4068      * Bytes are composed into primitive lane elements according
4069      * to the specified byte order.
4070      * The vector is arranged into lanes according to
4071      * <a href="Vector.html#lane-order">memory ordering</a>.
4072      * <p>
4073      * This method behaves as if it returns the result of calling
4074      * {@link #fromMemorySegment(VectorSpecies,MemorySegment,long,ByteOrder,VectorMask)
4075      * fromMemorySegment()} as follows:
4076      * <pre>{@code
4077      * var m = species.maskAll(true);
4078      * return fromMemorySegment(species, ms, offset, bo, m);
4079      * }</pre>
4080      *
4081      * @param species species of desired vector
4082      * @param ms the memory segment
4083      * @param offset the offset into the memory segment
4084      * @param bo the intended byte order
4085      * @return a vector loaded from the memory segment
4086      * @throws IndexOutOfBoundsException
4087      *         if {@code offset+N*$sizeInBytes$ < 0}
4088      *         or {@code offset+N*$sizeInBytes$ >= ms.byteSize()}
4089      *         for any lane {@code N} in the vector
4090      * @throws IllegalArgumentException if the memory segment is a heap segment that is
4091      *         not backed by a {@code byte[]} array.
4092      * @throws IllegalStateException if the memory segment's session is not alive,
4093      *         or if access occurs from a thread other than the thread owning the session.
4094      * @since 19
4095      */
4096     @ForceInline
4097     public static
4098     $abstractvectortype$ fromMemorySegment(VectorSpecies<$Boxtype$> species,
4099                                            MemorySegment ms, long offset,
4100                                            ByteOrder bo) {
4101         offset = checkFromIndexSize(offset, species.vectorByteSize(), ms.byteSize());
4102         $Type$Species vsp = ($Type$Species) species;
4103         return vsp.dummyVector().fromMemorySegment0(ms, offset).maybeSwap(bo);
4104     }
4105 
4106     /**
4107      * Loads a vector from a {@linkplain MemorySegment memory segment}
4108      * starting at an offset into the memory segment
4109      * and using a mask.
4110      * Lanes where the mask is unset are filled with the default
4111      * value of {@code $type$} ({#if[FP]?positive }zero).
4112      * Bytes are composed into primitive lane elements according
4113      * to the specified byte order.
4114      * The vector is arranged into lanes according to
4115      * <a href="Vector.html#lane-order">memory ordering</a>.
4116      * <p>
4117      * The following pseudocode illustrates the behavior:
4118      * <pre>{@code
4119      * var slice = ms.asSlice(offset);




4120      * $type$[] ar = new $type$[species.length()];
4121      * for (int n = 0; n < ar.length; n++) {
4122      *     if (m.laneIsSet(n)) {
4123      *         ar[n] = slice.getAtIndex(ValuaLayout.JAVA_$TYPE$.withBitAlignment(8), n);
4124      *     }
4125      * }
4126      * $abstractvectortype$ r = $abstractvectortype$.fromArray(species, ar, 0);
4127      * }</pre>
4128      * @implNote
4129 #if[!byte]
4130      * This operation is likely to be more efficient if
4131      * the specified byte order is the same as
4132      * {@linkplain ByteOrder#nativeOrder()
4133      * the platform native order},
4134      * since this method will not need to reorder
4135      * the bytes of lane values.
4136 #else[!byte]
4137      * The byte order argument is ignored.
4138 #end[!byte]
4139      *
4140      * @param species species of desired vector
4141      * @param ms the memory segment
4142      * @param offset the offset into the memory segment
4143      * @param bo the intended byte order
4144      * @param m the mask controlling lane selection
4145      * @return a vector loaded from the memory segment
4146      * @throws IndexOutOfBoundsException
4147      *         if {@code offset+N*$sizeInBytes$ < 0}
4148      *         or {@code offset+N*$sizeInBytes$ >= ms.byteSize()}
4149      *         for any lane {@code N} in the vector
4150      *         where the mask is set
4151      * @throws IllegalArgumentException if the memory segment is a heap segment that is
4152      *         not backed by a {@code byte[]} array.
4153      * @throws IllegalStateException if the memory segment's session is not alive,
4154      *         or if access occurs from a thread other than the thread owning the session.
4155      * @since 19
4156      */
4157     @ForceInline
4158     public static
4159     $abstractvectortype$ fromMemorySegment(VectorSpecies<$Boxtype$> species,
4160                                            MemorySegment ms, long offset,
4161                                            ByteOrder bo,
4162                                            VectorMask<$Boxtype$> m) {
4163         $Type$Species vsp = ($Type$Species) species;
4164         if (offset >= 0 && offset <= (ms.byteSize() - species.vectorByteSize())) {
4165             return vsp.dummyVector().fromMemorySegment0(ms, offset, m).maybeSwap(bo);
4166         }
4167 
4168         // FIXME: optimize
4169         checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, ms.byteSize());
4170         return vsp.ldLongOp(ms, offset, m, $abstractvectortype$::memorySegmentGet);


4171     }
4172 
4173     // Memory store operations
4174 
4175     /**
4176      * Stores this vector into an array of type {@code $type$[]}
4177      * starting at an offset.
4178      * <p>
4179      * For each vector lane, where {@code N} is the vector lane index,
4180      * the lane element at index {@code N} is stored into the array
4181      * element {@code a[offset+N]}.
4182      *
4183      * @param a the array, of type {@code $type$[]}
4184      * @param offset the offset into the array
4185      * @throws IndexOutOfBoundsException
4186      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4187      *         for any lane {@code N} in the vector
4188      */
4189     @ForceInline
4190     public final
4191     void intoArray($type$[] a, int offset) {
4192         offset = checkFromIndexSize(offset, length(), a.length);
4193         $Type$Species vsp = vspecies();
4194         VectorSupport.store(
4195             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4196             a, arrayAddress(a, offset),
4197             this,
4198             a, offset,
4199             (arr, off, v)
4200             -> v.stOp(arr, (int) off,
4201                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4202     }
4203 
4204     /**
4205      * Stores this vector into an array of type {@code $type$[]}
4206      * starting at offset and using a mask.
4207      * <p>
4208      * For each vector lane, where {@code N} is the vector lane index,
4209      * the lane element at index {@code N} is stored into the array
4210      * element {@code a[offset+N]}.
4211      * If the mask lane at {@code N} is unset then the corresponding
4212      * array element {@code a[offset+N]} is left unchanged.
4213      * <p>
4214      * Array range checking is done for lanes where the mask is set.
4215      * Lanes where the mask is unset are not stored and do not need
4216      * to correspond to legitimate elements of {@code a}.
4217      * That is, unset lanes may correspond to array indexes less than
4218      * zero or beyond the end of the array.
4219      *
4220      * @param a the array, of type {@code $type$[]}

4397      * is first cast to a {@code char} value and then
4398      * stored into the array element {@code a[offset+N]}.
4399      *
4400      * @param a the array, of type {@code char[]}
4401      * @param offset the offset into the array
4402      * @throws IndexOutOfBoundsException
4403      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4404      *         for any lane {@code N} in the vector
4405      */
4406     @ForceInline
4407     public final
4408     void intoCharArray(char[] a, int offset) {
4409         offset = checkFromIndexSize(offset, length(), a.length);
4410         $Type$Species vsp = vspecies();
4411         VectorSupport.store(
4412             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4413             a, charArrayAddress(a, offset),
4414             this,
4415             a, offset,
4416             (arr, off, v)
4417             -> v.stOp(arr, (int) off,
4418                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
4419     }
4420 
4421     /**
4422      * Stores this vector into an array of type {@code char[]}
4423      * starting at offset and using a mask.
4424      * <p>
4425      * For each vector lane, where {@code N} is the vector lane index,
4426      * the lane element at index {@code N}
4427      * is first cast to a {@code char} value and then
4428      * stored into the array element {@code a[offset+N]}.
4429      * If the mask lane at {@code N} is unset then the corresponding
4430      * array element {@code a[offset+N]} is left unchanged.
4431      * <p>
4432      * Array range checking is done for lanes where the mask is set.
4433      * Lanes where the mask is unset are not stored and do not need
4434      * to correspond to legitimate elements of {@code a}.
4435      * That is, unset lanes may correspond to array indexes less than
4436      * zero or beyond the end of the array.
4437      *

4556      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4557      *
4558      * @param a the array
4559      * @param offset the offset into the array
4560      * @throws IndexOutOfBoundsException
4561      *         if {@code offset+N < 0} or {@code offset+N >= a.length}
4562      *         for any lane {@code N} in the vector
4563      */
4564     @ForceInline
4565     public final
4566     void intoBooleanArray(boolean[] a, int offset) {
4567         offset = checkFromIndexSize(offset, length(), a.length);
4568         $Type$Species vsp = vspecies();
4569         ByteVector normalized = this.and((byte) 1);
4570         VectorSupport.store(
4571             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4572             a, booleanArrayAddress(a, offset),
4573             normalized,
4574             a, offset,
4575             (arr, off, v)
4576             -> v.stOp(arr, (int) off,
4577                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
4578     }
4579 
4580     /**
4581      * Stores this vector into an array of type {@code boolean[]}
4582      * starting at offset and using a mask.
4583      * <p>
4584      * For each vector lane, where {@code N} is the vector lane index,
4585      * the lane element at index {@code N}
4586      * is first converted to a {@code boolean} value and then
4587      * stored into the array element {@code a[offset+N]}.
4588      * If the mask lane at {@code N} is unset then the corresponding
4589      * array element {@code a[offset+N]} is left unchanged.
4590      * <p>
4591      * A {@code byte} value is converted to a {@code boolean} value by applying the
4592      * expression {@code (b & 1) != 0} where {@code b} is the byte value.
4593      * <p>
4594      * Array range checking is done for lanes where the mask is set.
4595      * Lanes where the mask is unset are not stored and do not need
4596      * to correspond to legitimate elements of {@code a}.

4695      *         for any lane {@code N} in the vector
4696      *         where the mask is set
4697      * @see $abstractvectortype$#toIntArray()
4698      */
4699     @ForceInline
4700     public final
4701     void intoBooleanArray(boolean[] a, int offset,
4702                           int[] indexMap, int mapOffset,
4703                           VectorMask<$Boxtype$> m) {
4704         // FIXME: optimize
4705         stOp(a, offset, m,
4706              (arr, off, i, e) -> {
4707                  int j = indexMap[mapOffset + i];
4708                  arr[off + j] = (e & 1) != 0;
4709              });
4710     }
4711 #end[byte]
4712 
4713     /**
4714      * {@inheritDoc} <!--workaround-->
4715      * @since 19
4716      */
4717     @Override
4718     @ForceInline
4719     public final
4720     void intoMemorySegment(MemorySegment ms, long offset,
4721                            ByteOrder bo) {
4722         if (ms.isReadOnly()) {
4723             throw new UnsupportedOperationException("Attempt to write a read-only segment");

















4724         }

4725 
4726         offset = checkFromIndexSize(offset, byteSize(), ms.byteSize());
4727         maybeSwap(bo).intoMemorySegment0(ms, offset);











4728     }
4729 
4730     /**
4731      * {@inheritDoc} <!--workaround-->
4732      * @since 19
4733      */
4734     @Override
4735     @ForceInline
4736     public final
4737     void intoMemorySegment(MemorySegment ms, long offset,
4738                            ByteOrder bo,
4739                            VectorMask<$Boxtype$> m) {
4740         if (m.allTrue()) {
4741             intoMemorySegment(ms, offset, bo);
4742         } else {
4743             if (ms.isReadOnly()) {
4744                 throw new UnsupportedOperationException("Attempt to write a read-only segment");
4745             }
4746             $Type$Species vsp = vspecies();
4747             checkMaskFromIndexSize(offset, vsp, m, $sizeInBytes$, ms.byteSize());
4748             maybeSwap(bo).intoMemorySegment0(ms, offset, m);
4749         }
4750     }
4751 
4752     // ================================================
4753 
4754     // Low-level memory operations.
4755     //
4756     // Note that all of these operations *must* inline into a context
4757     // where the exact species of the involved vector is a
4758     // compile-time constant.  Otherwise, the intrinsic generation
4759     // will fail and performance will suffer.
4760     //
4761     // In many cases this is achieved by re-deriving a version of the
4762     // method in each concrete subclass (per species).  The re-derived
4763     // method simply calls one of these generic methods, with exact
4764     // parameters for the controlling metadata, which is either a
4765     // typed vector or constant species instance.
4766 
4767     // Unchecked loading operations in native byte order.
4768     // Caller is responsible for applying index checks, masking, and
4769     // byte swapping.
4770 
4771     /*package-private*/
4772     abstract
4773     $abstractvectortype$ fromArray0($type$[] a, int offset);
4774     @ForceInline
4775     final
4776     $abstractvectortype$ fromArray0Template($type$[] a, int offset) {
4777         $Type$Species vsp = vspecies();
4778         return VectorSupport.load(
4779             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4780             a, arrayAddress(a, offset),
4781             a, offset, vsp,
4782             (arr, off, s) -> s.ldOp(arr, (int) off,
4783                                     (arr_, off_, i) -> arr_[off_ + i]));
4784     }
4785 
4786     /*package-private*/
4787     abstract
4788     $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4789     @ForceInline
4790     final
4791     <M extends VectorMask<$Boxtype$>>
4792     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4793         m.check(species());
4794         $Type$Species vsp = vspecies();
4795         return VectorSupport.loadMasked(
4796             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4797             a, arrayAddress(a, offset), m,
4798             a, offset, vsp,
4799             (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
4800                                         (arr_, off_, i) -> arr_[off_ + i]));
4801     }
4802 
4803 #if[!byteOrShort]
4804     /*package-private*/
4805     abstract
4806     $abstractvectortype$ fromArray0($type$[] a, int offset,
4807                                     int[] indexMap, int mapOffset,
4808                                     VectorMask<$Boxtype$> m);
4809     @ForceInline
4810     final
4811     <M extends VectorMask<$Boxtype$>>
4812     $abstractvectortype$ fromArray0Template(Class<M> maskClass, $type$[] a, int offset,
4813                                             int[] indexMap, int mapOffset, M m) {
4814         $Type$Species vsp = vspecies();
4815         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
4816         Objects.requireNonNull(a);
4817         Objects.requireNonNull(indexMap);
4818         m.check(vsp);
4819         Class<? extends $abstractvectortype$> vectorType = vsp.vectorType();

4856             isp.vectorType(),
4857             a, ARRAY_BASE, vix, m,
4858             a, offset, indexMap, mapOffset, vsp,
4859             (c, idx, iMap, idy, s, vm) ->
4860             s.vOp(vm, n -> c[idx + iMap[idy+n]]));
4861     }
4862 #end[!byteOrShort]
4863 
4864 #if[short]
4865     /*package-private*/
4866     abstract
4867     $abstractvectortype$ fromCharArray0(char[] a, int offset);
4868     @ForceInline
4869     final
4870     $abstractvectortype$ fromCharArray0Template(char[] a, int offset) {
4871         $Type$Species vsp = vspecies();
4872         return VectorSupport.load(
4873             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4874             a, charArrayAddress(a, offset),
4875             a, offset, vsp,
4876             (arr, off, s) -> s.ldOp(arr, (int) off,
4877                                     (arr_, off_, i) -> (short) arr_[off_ + i]));
4878     }
4879 
4880     /*package-private*/
4881     abstract
4882     $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
4883     @ForceInline
4884     final
4885     <M extends VectorMask<$Boxtype$>>
4886     $abstractvectortype$ fromCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
4887         m.check(species());
4888         $Type$Species vsp = vspecies();
4889         return VectorSupport.loadMasked(
4890                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4891                 a, charArrayAddress(a, offset), m,
4892                 a, offset, vsp,
4893                 (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
4894                                             (arr_, off_, i) -> (short) arr_[off_ + i]));
4895     }
4896 #end[short]
4897 
4898 #if[byte]
4899     /*package-private*/
4900     abstract
4901     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset);
4902     @ForceInline
4903     final
4904     $abstractvectortype$ fromBooleanArray0Template(boolean[] a, int offset) {
4905         $Type$Species vsp = vspecies();
4906         return VectorSupport.load(
4907             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4908             a, booleanArrayAddress(a, offset),
4909             a, offset, vsp,
4910             (arr, off, s) -> s.ldOp(arr, (int) off,
4911                                     (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4912     }
4913 
4914     /*package-private*/
4915     abstract
4916     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
4917     @ForceInline
4918     final
4919     <M extends VectorMask<$Boxtype$>>
4920     $abstractvectortype$ fromBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
4921         m.check(species());
4922         $Type$Species vsp = vspecies();
4923         return VectorSupport.loadMasked(
4924             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4925             a, booleanArrayAddress(a, offset), m,
4926             a, offset, vsp,
4927             (arr, off, s, vm) -> s.ldOp(arr, (int) off, vm,
4928                                         (arr_, off_, i) -> (byte) (arr_[off_ + i] ? 1 : 0)));
4929     }
4930 #end[byte]
4931 

4932     abstract
4933     $abstractvectortype$ fromMemorySegment0(MemorySegment bb, long offset);
4934     @ForceInline
4935     final
4936     $abstractvectortype$ fromMemorySegment0Template(MemorySegment ms, long offset) {
4937         $Type$Species vsp = vspecies();
4938         return ScopedMemoryAccess.loadFromMemorySegment(




































4939                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4940                 (MemorySegmentProxy) ms, offset, vsp,
4941                 (msp, off, s) -> {
4942                     return s.ldLongOp((MemorySegment) msp, off, $abstractvectortype$::memorySegmentGet);


4943                 });
4944     }
4945 
4946     abstract
4947     $abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m);
4948     @ForceInline
4949     final
4950     <M extends VectorMask<$Boxtype$>>
4951     $abstractvectortype$ fromMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
4952         $Type$Species vsp = vspecies();
4953         m.check(vsp);
4954         return ScopedMemoryAccess.loadFromMemorySegmentMasked(
4955                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4956                 (MemorySegmentProxy) ms, offset, m, vsp,
4957                 (msp, off, s, vm) -> {
4958                     return s.ldLongOp((MemorySegment) msp, off, vm, $abstractvectortype$::memorySegmentGet);


4959                 });
4960     }
4961 
4962     // Unchecked storing operations in native byte order.
4963     // Caller is responsible for applying index checks, masking, and
4964     // byte swapping.
4965 
4966     abstract
4967     void intoArray0($type$[] a, int offset);
4968     @ForceInline
4969     final
4970     void intoArray0Template($type$[] a, int offset) {
4971         $Type$Species vsp = vspecies();
4972         VectorSupport.store(
4973             vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
4974             a, arrayAddress(a, offset),
4975             this, a, offset,
4976             (arr, off, v)
4977             -> v.stOp(arr, (int) off,
4978                       (arr_, off_, i, e) -> arr_[off_+i] = e));
4979     }
4980 
4981     abstract
4982     void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m);
4983     @ForceInline
4984     final
4985     <M extends VectorMask<$Boxtype$>>
4986     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset, M m) {
4987         m.check(species());
4988         $Type$Species vsp = vspecies();
4989         VectorSupport.storeMasked(
4990             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
4991             a, arrayAddress(a, offset),
4992             this, m, a, offset,
4993             (arr, off, v, vm)
4994             -> v.stOp(arr, (int) off, vm,
4995                       (arr_, off_, i, e) -> arr_[off_ + i] = e));
4996     }
4997 
4998 #if[!byteOrShort]
4999     abstract
5000     void intoArray0($type$[] a, int offset,
5001                     int[] indexMap, int mapOffset,
5002                     VectorMask<$Boxtype$> m);
5003     @ForceInline
5004     final
5005     <M extends VectorMask<$Boxtype$>>
5006     void intoArray0Template(Class<M> maskClass, $type$[] a, int offset,
5007                             int[] indexMap, int mapOffset, M m) {
5008         m.check(species());
5009         $Type$Species vsp = vspecies();
5010         IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
5011 #if[longOrDouble]
5012         if (vsp.laneCount() == 1) {
5013             intoArray(a, offset + indexMap[mapOffset], m);
5014             return;

5055                           arr[off + j] = e;
5056                       }));
5057     }
5058 #end[!byteOrShort]
5059 
5060 #if[byte]
5061     abstract
5062     void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m);
5063     @ForceInline
5064     final
5065     <M extends VectorMask<$Boxtype$>>
5066     void intoBooleanArray0Template(Class<M> maskClass, boolean[] a, int offset, M m) {
5067         m.check(species());
5068         $Type$Species vsp = vspecies();
5069         ByteVector normalized = this.and((byte) 1);
5070         VectorSupport.storeMasked(
5071             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5072             a, booleanArrayAddress(a, offset),
5073             normalized, m, a, offset,
5074             (arr, off, v, vm)
5075             -> v.stOp(arr, (int) off, vm,
5076                       (arr_, off_, i, e) -> arr_[off_ + i] = (e & 1) != 0));
5077     }
5078 #end[byte]
5079 


5080     @ForceInline
5081     final
5082     void intoMemorySegment0(MemorySegment ms, long offset) {
5083         $Type$Species vsp = vspecies();
5084         ScopedMemoryAccess.storeIntoMemorySegment(


































5085                 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
5086                 this,
5087                 (MemorySegmentProxy) ms, offset,
5088                 (msp, off, v) -> {
5089                     v.stLongOp((MemorySegment) msp, off, $abstractvectortype$::memorySegmentSet);

5090                 });
5091     }
5092 
5093     abstract
5094     void intoMemorySegment0(MemorySegment bb, long offset, VectorMask<$Boxtype$> m);
5095     @ForceInline
5096     final
5097     <M extends VectorMask<$Boxtype$>>
5098     void intoMemorySegment0Template(Class<M> maskClass, MemorySegment ms, long offset, M m) {
5099         $Type$Species vsp = vspecies();
5100         m.check(vsp);
5101         ScopedMemoryAccess.storeIntoMemorySegmentMasked(
5102                 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5103                 this, m,
5104                 (MemorySegmentProxy) ms, offset,
5105                 (msp, off, v, vm) -> {
5106                     v.stLongOp((MemorySegment) msp, off, vm, $abstractvectortype$::memorySegmentSet);

5107                 });
5108     }
5109 
5110 #if[short]
5111     /*package-private*/
5112     abstract
5113     void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m);
5114     @ForceInline
5115     final
5116     <M extends VectorMask<$Boxtype$>>
5117     void intoCharArray0Template(Class<M> maskClass, char[] a, int offset, M m) {
5118         m.check(species());
5119         $Type$Species vsp = vspecies();
5120         VectorSupport.storeMasked(
5121             vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
5122             a, charArrayAddress(a, offset),
5123             this, m, a, offset,
5124             (arr, off, v, vm)
5125             -> v.stOp(arr, (int) off, vm,
5126                       (arr_, off_, i, e) -> arr_[off_ + i] = (char) e));
5127     }
5128 #end[short]
5129 
5130     // End of low-level memory operations.
5131 
5132     private static
5133     void checkMaskFromIndexSize(int offset,
5134                                 $Type$Species vsp,
5135                                 VectorMask<$Boxtype$> m,
5136                                 int scale,
5137                                 int limit) {
5138         ((AbstractMask<$Boxtype$>)m)
5139             .checkIndexByLane(offset, limit, vsp.iota(), scale);
5140     }
5141 
5142     private static
5143     void checkMaskFromIndexSize(long offset,
5144                                 $Type$Species vsp,
5145                                 VectorMask<$Boxtype$> m,
5146                                 int scale,
5147                                 long limit) {
5148         ((AbstractMask<$Boxtype$>)m)
5149             .checkIndexByLane(offset, limit, vsp.iota(), scale);
5150     }
5151 
5152     @ForceInline
5153     private void conditionalStoreNYI(int offset,
5154                                      $Type$Species vsp,
5155                                      VectorMask<$Boxtype$> m,
5156                                      int scale,
5157                                      int limit) {
5158         if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
5159             String msg =
5160                 String.format("unimplemented: store @%d in [0..%d), %s in %s",
5161                               offset, limit, m, vsp);
5162             throw new AssertionError(msg);
5163         }
5164     }
5165 
5166     /*package-private*/
5167     @Override
5168     @ForceInline
5169     final
5170     $abstractvectortype$ maybeSwap(ByteOrder bo) {
5171 #if[!byte]

5500                 }
5501             }
5502             return dummyVector().vectorFactory(res);
5503         }
5504 
5505         /*package-private*/
5506         @ForceInline
5507         <M> $abstractvectortype$ ldOp(M memory, int offset,
5508                                       FLdOp<M> f) {
5509             return dummyVector().ldOp(memory, offset, f);
5510         }
5511 
5512         /*package-private*/
5513         @ForceInline
5514         <M> $abstractvectortype$ ldOp(M memory, int offset,
5515                                       VectorMask<$Boxtype$> m,
5516                                       FLdOp<M> f) {
5517             return dummyVector().ldOp(memory, offset, m, f);
5518         }
5519 
5520         /*package-private*/
5521         @ForceInline
5522         $abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
5523                                       FLdLongOp f) {
5524             return dummyVector().ldLongOp(memory, offset, f);
5525         }
5526 
5527         /*package-private*/
5528         @ForceInline
5529         $abstractvectortype$ ldLongOp(MemorySegment memory, long offset,
5530                                       VectorMask<$Boxtype$> m,
5531                                       FLdLongOp f) {
5532             return dummyVector().ldLongOp(memory, offset, m, f);
5533         }
5534 
5535         /*package-private*/
5536         @ForceInline
5537         <M> void stOp(M memory, int offset, FStOp<M> f) {
5538             dummyVector().stOp(memory, offset, f);
5539         }
5540 
5541         /*package-private*/
5542         @ForceInline
5543         <M> void stOp(M memory, int offset,
5544                       AbstractMask<$Boxtype$> m,
5545                       FStOp<M> f) {
5546             dummyVector().stOp(memory, offset, m, f);
5547         }
5548 
5549         /*package-private*/
5550         @ForceInline
5551         void stLongOp(MemorySegment memory, long offset, FStLongOp f) {
5552             dummyVector().stLongOp(memory, offset, f);
5553         }
5554 
5555         /*package-private*/
5556         @ForceInline
5557         void stLongOp(MemorySegment memory, long offset,
5558                       AbstractMask<$Boxtype$> m,
5559                       FStLongOp f) {
5560             dummyVector().stLongOp(memory, offset, m, f);
5561         }
5562 
5563         // N.B. Make sure these constant vectors and
5564         // masks load up correctly into registers.
5565         //
5566         // Also, see if we can avoid all that switching.
5567         // Could we cache both vectors and both masks in
5568         // this species object?
5569 
5570         // Zero and iota vector access
5571         @Override
5572         @ForceInline
5573         public final $abstractvectortype$ zero() {
5574             if ((Class<?>) vectorType() == $Type$MaxVector.class)
5575                 return $Type$MaxVector.ZERO;
5576             switch (vectorBitSize()) {
5577                 case 64: return $Type$64Vector.ZERO;
5578                 case 128: return $Type$128Vector.ZERO;
5579                 case 256: return $Type$256Vector.ZERO;
5580                 case 512: return $Type$512Vector.ZERO;
5581             }
< prev index next >