1 /*
   2  * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 /*
  25  * @test
  26  * @modules jdk.incubator.foreign jdk.incubator.vector java.base/jdk.internal.vm.annotation
  27 #if[MaxBit]
  28  * @run testng/othervm --add-opens jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED
  29  *      -XX:-TieredCompilation $vectorteststype$
  30 #else[MaxBit]
  31  * @run testng/othervm -XX:-TieredCompilation $vectorteststype$
  32 #end[MaxBit]
  33  *
  34  */
  35 
  36 #warn This file is preprocessed before being compiled
  37 
  38 import jdk.incubator.foreign.MemorySegment;
  39 import jdk.incubator.foreign.ResourceScope;
  40 import jdk.incubator.foreign.ValueLayout;
  41 import jdk.incubator.vector.$Type$Vector;
  42 import jdk.incubator.vector.VectorMask;
  43 #if[MaxBit]
  44 import jdk.incubator.vector.VectorShape;
  45 #end[MaxBit]
  46 import jdk.incubator.vector.VectorSpecies;
  47 import jdk.incubator.vector.VectorShuffle;
  48 import jdk.internal.vm.annotation.DontInline;
  49 import org.testng.Assert;
  50 import org.testng.annotations.DataProvider;
  51 import org.testng.annotations.Test;
  52 
  53 import java.nio.ByteOrder;
  54 import java.util.List;
  55 import java.util.function.*;
  56 
  57 @Test
  58 public class $vectorteststype$ extends AbstractVectorLoadStoreTest {
  59 #if[MaxBit]
  60     static final VectorSpecies<$Wideboxtype$> SPECIES =
  61                 $Type$Vector.SPECIES_MAX;
  62 #else[MaxBit]
  63     static final VectorSpecies<$Wideboxtype$> SPECIES =
  64                 $Type$Vector.SPECIES_$bits$;
  65 #end[MaxBit]
  66 
  67     static final int INVOC_COUNT = Integer.getInteger("jdk.incubator.vector.test.loop-iterations", 100);
  68 
  69 #if[MaxBit]
  70     static VectorShape getMaxBit() {
  71         return VectorShape.S_Max_BIT;
  72     }
  73 
  74     private static final int Max = 256;  // juts so we can do N/$bits$
  75 #end[MaxBit]
  76 
  77     static final int BUFFER_REPS = Integer.getInteger("jdk.incubator.vector.test.buffer-vectors", 25000 / $bits$);
  78 
  79     static void assertArraysEquals($type$[] r, $type$[] a, boolean[] mask) {
  80         int i = 0;
  81         try {
  82             for (; i < a.length; i++) {
  83                 Assert.assertEquals(r[i], mask[i % SPECIES.length()] ? a[i] : ($type$) 0);
  84             }
  85         } catch (AssertionError e) {
  86             Assert.assertEquals(r[i], mask[i % SPECIES.length()] ? a[i] : ($type$) 0, "at index #" + i);
  87         }
  88     }
  89 
  90     static final List<IntFunction<$type$[]>> $TYPE$_GENERATORS = List.of(
  91             withToString("$type$[i * 5]", (int s) -> {
  92                 return fill(s * BUFFER_REPS,
  93                             i -> ($type$)(i * 5));
  94             }),
  95             withToString("$type$[i + 1]", (int s) -> {
  96                 return fill(s * BUFFER_REPS,
  97                             i -> ((($type$)(i + 1) == 0) ? 1 : ($type$)(i + 1)));
  98             })
  99     );
 100 
 101     // Relative to array.length
 102     static final List<IntFunction<Integer>> INDEX_GENERATORS = List.of(
 103             withToString("-1", (int l) -> {
 104                 return -1;
 105             }),
 106             withToString("l", (int l) -> {
 107                 return l;
 108             }),
 109             withToString("l - 1", (int l) -> {
 110                 return l - 1;
 111             }),
 112             withToString("l + 1", (int l) -> {
 113                 return l + 1;
 114             }),
 115             withToString("l - speciesl + 1", (int l) -> {
 116                 return l - SPECIES.length() + 1;
 117             }),
 118             withToString("l + speciesl - 1", (int l) -> {
 119                 return l + SPECIES.length() - 1;
 120             }),
 121             withToString("l + speciesl", (int l) -> {
 122                 return l + SPECIES.length();
 123             }),
 124             withToString("l + speciesl + 1", (int l) -> {
 125                 return l + SPECIES.length() + 1;
 126             })
 127     );
 128 
 129     // Relative to byte[] array.length or MemorySegment.byteSize()
 130     static final List<IntFunction<Integer>> BYTE_INDEX_GENERATORS = List.of(
 131             withToString("-1", (int l) -> {
 132                 return -1;
 133             }),
 134             withToString("l", (int l) -> {
 135                 return l;
 136             }),
 137             withToString("l - 1", (int l) -> {
 138                 return l - 1;
 139             }),
 140             withToString("l + 1", (int l) -> {
 141                 return l + 1;
 142             }),
 143             withToString("l - speciesl*ebsize + 1", (int l) -> {
 144                 return l - SPECIES.vectorByteSize() + 1;
 145             }),
 146             withToString("l + speciesl*ebsize - 1", (int l) -> {
 147                 return l + SPECIES.vectorByteSize() - 1;
 148             }),
 149             withToString("l + speciesl*ebsize", (int l) -> {
 150                 return l + SPECIES.vectorByteSize();
 151             }),
 152             withToString("l + speciesl*ebsize + 1", (int l) -> {
 153                 return l + SPECIES.vectorByteSize() + 1;
 154             })
 155     );
 156 
 157     @DataProvider
 158     public Object[][] $type$Provider() {
 159         return $TYPE$_GENERATORS.stream().
 160                 map(f -> new Object[]{f}).
 161                 toArray(Object[][]::new);
 162     }
 163 
 164     @DataProvider
 165     public Object[][] maskProvider() {
 166         return BOOLEAN_MASK_GENERATORS.stream().
 167                 map(f -> new Object[]{f}).
 168                 toArray(Object[][]::new);
 169     }
 170 
 171     @DataProvider
 172     public Object[][] $type$ProviderForIOOBE() {
 173         var f = $TYPE$_GENERATORS.get(0);
 174         return INDEX_GENERATORS.stream().map(fi -> {
 175                     return new Object[] {f, fi};
 176                 }).
 177                 toArray(Object[][]::new);
 178     }
 179 
 180     @DataProvider
 181     public Object[][] $type$MaskProvider() {
 182         return BOOLEAN_MASK_GENERATORS.stream().
 183                 flatMap(fm -> $TYPE$_GENERATORS.stream().map(fa -> {
 184                     return new Object[] {fa, fm};
 185                 })).
 186                 toArray(Object[][]::new);
 187     }
 188 
 189     @DataProvider
 190     public Object[][] $type$MaskProviderForIOOBE() {
 191         var f = $TYPE$_GENERATORS.get(0);
 192         return BOOLEAN_MASK_GENERATORS.stream().
 193                 flatMap(fm -> INDEX_GENERATORS.stream().map(fi -> {
 194                     return new Object[] {f, fi, fm};
 195                 })).
 196                 toArray(Object[][]::new);
 197     }
 198 
 199     @DataProvider
 200     public Object[][] $type$MemorySegmentProvider() {
 201         return $TYPE$_GENERATORS.stream().
 202                 flatMap(fa -> MEMORY_SEGMENT_GENERATORS.stream().
 203                         flatMap(fb -> BYTE_ORDER_VALUES.stream().map(bo -> {
 204                             return new Object[]{fa, fb, bo};
 205                         }))).
 206                 toArray(Object[][]::new);
 207     }
 208 
 209     @DataProvider
 210     public Object[][] $type$MemorySegmentMaskProvider() {
 211         return BOOLEAN_MASK_GENERATORS.stream().
 212                 flatMap(fm -> $TYPE$_GENERATORS.stream().
 213                         flatMap(fa -> MEMORY_SEGMENT_GENERATORS.stream().
 214                                 flatMap(fb -> BYTE_ORDER_VALUES.stream().map(bo -> {
 215                             return new Object[]{fa, fb, fm, bo};
 216                         })))).
 217                 toArray(Object[][]::new);
 218     }
 219 
 220     @DataProvider
 221     public Object[][] $type$ByteProviderForIOOBE() {
 222         var f = $TYPE$_GENERATORS.get(0);
 223         return BYTE_INDEX_GENERATORS.stream().map(fi -> {
 224                     return new Object[] {f, fi};
 225                 }).
 226                 toArray(Object[][]::new);
 227     }
 228 
 229     @DataProvider
 230     public Object[][] $type$ByteMaskProviderForIOOBE() {
 231         var f = $TYPE$_GENERATORS.get(0);
 232         return BOOLEAN_MASK_GENERATORS.stream().
 233                 flatMap(fm -> BYTE_INDEX_GENERATORS.stream().map(fi -> {
 234                     return new Object[] {f, fi, fm};
 235                 })).
 236                 toArray(Object[][]::new);
 237     }
 238 
 239     static MemorySegment toSegment($type$[] a, IntFunction<MemorySegment> fb) {
 240         MemorySegment ms = fb.apply(a.length * SPECIES.elementSize() / 8);
 241         for (int i = 0; i < a.length; i++) {
 242             ms.set(ValueLayout.JAVA_$TYPE$, i * SPECIES.elementSize() / 8 , a[i]);
 243         }
 244         return ms;
 245     }
 246 
 247     static $type$[] segmentToArray(MemorySegment ms) {
 248         return ms.toArray(ValueLayout.JAVA_$TYPE$);
 249     }
 250 
 251 
 252     interface To$Type$F {
 253         $type$ apply(int i);
 254     }
 255 
 256     static $type$[] fill(int s , To$Type$F f) {
 257         return fill(new $type$[s], f);
 258     }
 259 
 260     static $type$[] fill($type$[] a, To$Type$F f) {
 261         for (int i = 0; i < a.length; i++) {
 262             a[i] = f.apply(i);
 263         }
 264         return a;
 265     }
 266 
 267     @DontInline
 268     static $abstractvectortype$ fromArray($type$[] a, int i) {
 269         return $abstractvectortype$.fromArray(SPECIES, a, i);
 270     }
 271 
 272     @DontInline
 273     static $abstractvectortype$ fromArray($type$[] a, int i, VectorMask<$Boxtype$> m) {
 274         return $abstractvectortype$.fromArray(SPECIES, a, i, m);
 275     }
 276 
 277     @DontInline
 278     static void intoArray($abstractvectortype$ v, $type$[] a, int i) {
 279         v.intoArray(a, i);
 280     }
 281 
 282     @DontInline
 283     static void intoArray($abstractvectortype$ v, $type$[] a, int i, VectorMask<$Boxtype$> m) {
 284         v.intoArray(a, i, m);
 285     }
 286 
 287     @DontInline
 288     static $abstractvectortype$ fromMemorySegment(MemorySegment a, int i, ByteOrder bo) {
 289         return $abstractvectortype$.fromMemorySegment(SPECIES, a, i, bo);
 290     }
 291 
 292     @DontInline
 293     static $abstractvectortype$ fromMemorySegment(MemorySegment a, int i, ByteOrder bo, VectorMask<$Boxtype$> m) {
 294         return $abstractvectortype$.fromMemorySegment(SPECIES, a, i, bo, m);
 295     }
 296 
 297     @DontInline
 298     static void intoMemorySegment($abstractvectortype$ v, MemorySegment a, int i, ByteOrder bo) {
 299         v.intoMemorySegment(a, i, bo);
 300     }
 301 
 302     @DontInline
 303     static void intoMemorySegment($abstractvectortype$ v, MemorySegment a, int i, ByteOrder bo, VectorMask<$Boxtype$> m) {
 304         v.intoMemorySegment(a, i, bo, m);
 305     }
 306 
 307     @Test(dataProvider = "$type$Provider")
 308     static void loadStoreArray(IntFunction<$type$[]> fa) {
 309         $type$[] a = fa.apply(SPECIES.length());
 310         $type$[] r = new $type$[a.length];
 311 
 312         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 313             for (int i = 0; i < a.length; i += SPECIES.length()) {
 314                 $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, i);
 315                 av.intoArray(r, i);
 316             }
 317         }
 318         Assert.assertEquals(r, a);
 319     }
 320 
 321     @Test(dataProvider = "$type$ProviderForIOOBE")
 322     static void loadArrayIOOBE(IntFunction<$type$[]> fa, IntFunction<Integer> fi) {
 323         $type$[] a = fa.apply(SPECIES.length());
 324         $type$[] r = new $type$[a.length];
 325 
 326         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 327             for (int i = 0; i < a.length; i += SPECIES.length()) {
 328                 $abstractvectortype$ av = fromArray(a, i);
 329                 av.intoArray(r, i);
 330             }
 331         }
 332 
 333         int index = fi.apply(a.length);
 334         boolean shouldFail = isIndexOutOfBounds(SPECIES.length(), index, a.length);
 335         try {
 336             fromArray(a, index);
 337             if (shouldFail) {
 338                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 339             }
 340         } catch (IndexOutOfBoundsException e) {
 341             if (!shouldFail) {
 342                 Assert.fail("Unexpected IndexOutOfBoundsException");
 343             }
 344         }
 345     }
 346 
 347     @Test(dataProvider = "$type$ProviderForIOOBE")
 348     static void storeArrayIOOBE(IntFunction<$type$[]> fa, IntFunction<Integer> fi) {
 349         $type$[] a = fa.apply(SPECIES.length());
 350         $type$[] r = new $type$[a.length];
 351 
 352         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 353             for (int i = 0; i < a.length; i += SPECIES.length()) {
 354                 $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, i);
 355                 intoArray(av, r, i);
 356             }
 357         }
 358 
 359         int index = fi.apply(a.length);
 360         boolean shouldFail = isIndexOutOfBounds(SPECIES.length(), index, a.length);
 361         try {
 362             $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, 0);
 363             intoArray(av, r, index);
 364             if (shouldFail) {
 365                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 366             }
 367         } catch (IndexOutOfBoundsException e) {
 368             if (!shouldFail) {
 369                 Assert.fail("Unexpected IndexOutOfBoundsException");
 370             }
 371         }
 372     }
 373 
 374 
 375     @Test(dataProvider = "$type$MaskProvider")
 376     static void loadStoreMaskArray(IntFunction<$type$[]> fa,
 377                                    IntFunction<boolean[]> fm) {
 378         $type$[] a = fa.apply(SPECIES.length());
 379         $type$[] r = new $type$[a.length];
 380         boolean[] mask = fm.apply(SPECIES.length());
 381         VectorMask<$Boxtype$> vmask = VectorMask.fromValues(SPECIES, mask);
 382 
 383         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 384             for (int i = 0; i < a.length; i += SPECIES.length()) {
 385                 $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, i, vmask);
 386                 av.intoArray(r, i);
 387             }
 388         }
 389         assertArraysEquals(r, a, mask);
 390 
 391 
 392         r = new $type$[a.length];
 393 
 394         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 395             for (int i = 0; i < a.length; i += SPECIES.length()) {
 396                 $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, i);
 397                 av.intoArray(r, i, vmask);
 398             }
 399         }
 400         assertArraysEquals(r, a, mask);
 401     }
 402 
 403     @Test(dataProvider = "$type$MaskProviderForIOOBE")
 404     static void loadArrayMaskIOOBE(IntFunction<$type$[]> fa, IntFunction<Integer> fi, IntFunction<boolean[]> fm) {
 405         $type$[] a = fa.apply(SPECIES.length());
 406         $type$[] r = new $type$[a.length];
 407         boolean[] mask = fm.apply(SPECIES.length());
 408         VectorMask<$Boxtype$> vmask = VectorMask.fromValues(SPECIES, mask);
 409 
 410         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 411             for (int i = 0; i < a.length; i += SPECIES.length()) {
 412                 $abstractvectortype$ av = fromArray(a, i, vmask);
 413                 av.intoArray(r, i);
 414             }
 415         }
 416 
 417         int index = fi.apply(a.length);
 418         boolean shouldFail = isIndexOutOfBoundsForMask(mask, index, a.length);
 419         try {
 420             fromArray(a, index, vmask);
 421             if (shouldFail) {
 422                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 423             }
 424         } catch (IndexOutOfBoundsException e) {
 425             if (!shouldFail) {
 426                 Assert.fail("Unexpected IndexOutOfBoundsException");
 427             }
 428         }
 429     }
 430 
 431     @Test(dataProvider = "$type$MaskProviderForIOOBE")
 432     static void storeArrayMaskIOOBE(IntFunction<$type$[]> fa, IntFunction<Integer> fi, IntFunction<boolean[]> fm) {
 433         $type$[] a = fa.apply(SPECIES.length());
 434         $type$[] r = new $type$[a.length];
 435         boolean[] mask = fm.apply(SPECIES.length());
 436         VectorMask<$Boxtype$> vmask = VectorMask.fromValues(SPECIES, mask);
 437 
 438         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 439             for (int i = 0; i < a.length; i += SPECIES.length()) {
 440                 $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, i);
 441                 intoArray(av, r, i, vmask);
 442             }
 443         }
 444 
 445         int index = fi.apply(a.length);
 446         boolean shouldFail = isIndexOutOfBoundsForMask(mask, index, a.length);
 447         try {
 448             $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, 0);
 449             intoArray(av, a, index, vmask);
 450             if (shouldFail) {
 451                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 452             }
 453         } catch (IndexOutOfBoundsException e) {
 454             if (!shouldFail) {
 455                 Assert.fail("Unexpected IndexOutOfBoundsException");
 456             }
 457         }
 458     }
 459 
 460 
 461     @Test(dataProvider = "$type$MaskProvider")
 462     static void loadStoreMask(IntFunction<$type$[]> fa,
 463                               IntFunction<boolean[]> fm) {
 464         boolean[] mask = fm.apply(SPECIES.length());
 465         boolean[] r = new boolean[mask.length];
 466 
 467         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 468             for (int i = 0; i < mask.length; i += SPECIES.length()) {
 469                 VectorMask<$Boxtype$> vmask = VectorMask.fromArray(SPECIES, mask, i);
 470                 vmask.intoArray(r, i);
 471             }
 472         }
 473         Assert.assertEquals(r, mask);
 474     }
 475 
 476 
 477     @Test(dataProvider = "$type$MemorySegmentProvider")
 478     static void loadStoreMemorySegment(IntFunction<$type$[]> fa,
 479                                        IntFunction<MemorySegment> fb,
 480                                        ByteOrder bo) {
 481         MemorySegment a = toSegment(fa.apply(SPECIES.length()), fb);
 482         MemorySegment r = fb.apply((int) a.byteSize());
 483 
 484         int l = (int) a.byteSize();
 485         int s = SPECIES.vectorByteSize();
 486 
 487         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 488             for (int i = 0; i < l; i += s) {
 489                 $abstractvectortype$ av = $abstractvectortype$.fromMemorySegment(SPECIES, a, i, bo);
 490                 av.intoMemorySegment(r, i, bo);
 491             }
 492         }
 493         long m = r.mismatch(a);
 494         Assert.assertEquals(m, -1, "Segments not equal");
 495     }
 496 
 497     @Test(dataProvider = "$type$ByteProviderForIOOBE")
 498     static void loadMemorySegmentIOOBE(IntFunction<$type$[]> fa, IntFunction<Integer> fi) {
 499         MemorySegment a = toSegment(fa.apply(SPECIES.length()), i -> MemorySegment.allocateNative(i, ResourceScope.newImplicitScope()));
 500         MemorySegment r = MemorySegment.allocateNative(a.byteSize(), ResourceScope.newImplicitScope());
 501 
 502         int l = (int) a.byteSize();
 503         int s = SPECIES.vectorByteSize();
 504 
 505         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 506             for (int i = 0; i < l; i += s) {
 507                 $abstractvectortype$ av = fromMemorySegment(a, i, ByteOrder.nativeOrder());
 508                 av.intoMemorySegment(r, i, ByteOrder.nativeOrder());
 509             }
 510         }
 511 
 512         int index = fi.apply((int) a.byteSize());
 513         boolean shouldFail = isIndexOutOfBounds(SPECIES.vectorByteSize(), index, (int) a.byteSize());
 514         try {
 515             fromMemorySegment(a, index, ByteOrder.nativeOrder());
 516             if (shouldFail) {
 517                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 518             }
 519         } catch (IndexOutOfBoundsException e) {
 520             if (!shouldFail) {
 521                 Assert.fail("Unexpected IndexOutOfBoundsException");
 522             }
 523         }
 524     }
 525 
 526     @Test(dataProvider = "$type$ByteProviderForIOOBE")
 527     static void storeMemorySegmentIOOBE(IntFunction<$type$[]> fa, IntFunction<Integer> fi) {
 528         MemorySegment a = toSegment(fa.apply(SPECIES.length()), i -> MemorySegment.allocateNative(i, ResourceScope.newImplicitScope()));
 529         MemorySegment r = MemorySegment.allocateNative(a.byteSize(), ResourceScope.newImplicitScope());
 530 
 531         int l = (int) a.byteSize();
 532         int s = SPECIES.vectorByteSize();
 533 
 534         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 535             for (int i = 0; i < l; i += s) {
 536                 $abstractvectortype$ av = $abstractvectortype$.fromMemorySegment(SPECIES, a, i, ByteOrder.nativeOrder());
 537                 intoMemorySegment(av, r, i, ByteOrder.nativeOrder());
 538             }
 539         }
 540 
 541         int index = fi.apply((int) a.byteSize());
 542         boolean shouldFail = isIndexOutOfBounds(SPECIES.vectorByteSize(), index, (int) a.byteSize());
 543         try {
 544             $abstractvectortype$ av = $abstractvectortype$.fromMemorySegment(SPECIES, a, 0, ByteOrder.nativeOrder());
 545             intoMemorySegment(av, r, index, ByteOrder.nativeOrder());
 546             if (shouldFail) {
 547                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 548             }
 549         } catch (IndexOutOfBoundsException e) {
 550             if (!shouldFail) {
 551                 Assert.fail("Unexpected IndexOutOfBoundsException");
 552             }
 553         }
 554     }
 555 
 556     @Test(dataProvider = "$type$MemorySegmentMaskProvider")
 557     static void loadStoreMemorySegmentMask(IntFunction<$type$[]> fa,
 558                                            IntFunction<MemorySegment> fb,
 559                                            IntFunction<boolean[]> fm,
 560                                            ByteOrder bo) {
 561         $type$[] _a = fa.apply(SPECIES.length());
 562         MemorySegment a = toSegment(_a, fb);
 563         MemorySegment r = fb.apply((int) a.byteSize());
 564         boolean[] mask = fm.apply(SPECIES.length());
 565         VectorMask<$Boxtype$> vmask = VectorMask.fromValues(SPECIES, mask);
 566 
 567         int l = (int) a.byteSize();
 568         int s = SPECIES.vectorByteSize();
 569 
 570         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 571             for (int i = 0; i < l; i += s) {
 572                 $abstractvectortype$ av = $abstractvectortype$.fromMemorySegment(SPECIES, a, i, bo, vmask);
 573                 av.intoMemorySegment(r, i, bo);
 574             }
 575         }
 576         assertArraysEquals(segmentToArray(r), _a, mask);
 577 
 578 
 579         r = fb.apply((int) a.byteSize());
 580 
 581         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 582             for (int i = 0; i < l; i += s) {
 583                 $abstractvectortype$ av = $abstractvectortype$.fromMemorySegment(SPECIES, a, i, bo);
 584                 av.intoMemorySegment(r, i, bo, vmask);
 585             }
 586         }
 587         assertArraysEquals(segmentToArray(r), _a, mask);
 588     }
 589 
 590     @Test(dataProvider = "$type$ByteMaskProviderForIOOBE")
 591     static void loadMemorySegmentMaskIOOBE(IntFunction<$type$[]> fa, IntFunction<Integer> fi, IntFunction<boolean[]> fm) {
 592         MemorySegment a = toSegment(fa.apply(SPECIES.length()), i -> MemorySegment.allocateNative(i, ResourceScope.newImplicitScope()));
 593         MemorySegment r = MemorySegment.allocateNative(a.byteSize(), ResourceScope.newImplicitScope());
 594         boolean[] mask = fm.apply(SPECIES.length());
 595         VectorMask<$Boxtype$> vmask = VectorMask.fromValues(SPECIES, mask);
 596 
 597         int l = (int) a.byteSize();
 598         int s = SPECIES.vectorByteSize();
 599 
 600         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 601             for (int i = 0; i < l; i += s) {
 602                 $abstractvectortype$ av = fromMemorySegment(a, i, ByteOrder.nativeOrder(), vmask);
 603                 av.intoMemorySegment(r, i, ByteOrder.nativeOrder());
 604             }
 605         }
 606 
 607         int index = fi.apply((int) a.byteSize());
 608         boolean shouldFail = isIndexOutOfBoundsForMask(mask, index, (int) a.byteSize(), SPECIES.elementSize() / 8);
 609         try {
 610             fromMemorySegment(a, index, ByteOrder.nativeOrder(), vmask);
 611             if (shouldFail) {
 612                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 613             }
 614         } catch (IndexOutOfBoundsException e) {
 615             if (!shouldFail) {
 616                 Assert.fail("Unexpected IndexOutOfBoundsException");
 617             }
 618         }
 619     }
 620 
 621     @Test(dataProvider = "$type$ByteMaskProviderForIOOBE")
 622     static void storeMemorySegmentMaskIOOBE(IntFunction<$type$[]> fa, IntFunction<Integer> fi, IntFunction<boolean[]> fm) {
 623         MemorySegment a = toSegment(fa.apply(SPECIES.length()), i -> MemorySegment.allocateNative(i, ResourceScope.newImplicitScope()));
 624         MemorySegment r = MemorySegment.allocateNative(a.byteSize(), ResourceScope.newImplicitScope());
 625         boolean[] mask = fm.apply(SPECIES.length());
 626         VectorMask<$Boxtype$> vmask = VectorMask.fromValues(SPECIES, mask);
 627 
 628         int l = (int) a.byteSize();
 629         int s = SPECIES.vectorByteSize();
 630 
 631         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 632             for (int i = 0; i < l; i += s) {
 633                 $abstractvectortype$ av = $abstractvectortype$.fromMemorySegment(SPECIES, a, i, ByteOrder.nativeOrder());
 634                 intoMemorySegment(av, r, i, ByteOrder.nativeOrder(), vmask);
 635             }
 636         }
 637 
 638         int index = fi.apply((int) a.byteSize());
 639         boolean shouldFail = isIndexOutOfBoundsForMask(mask, index, (int) a.byteSize(), SPECIES.elementSize() / 8);
 640         try {
 641             $abstractvectortype$ av = $abstractvectortype$.fromMemorySegment(SPECIES, a, 0, ByteOrder.nativeOrder());
 642             intoMemorySegment(av, a, index, ByteOrder.nativeOrder(), vmask);
 643             if (shouldFail) {
 644                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 645             }
 646         } catch (IndexOutOfBoundsException e) {
 647             if (!shouldFail) {
 648                 Assert.fail("Unexpected IndexOutOfBoundsException");
 649             }
 650         }
 651     }
 652 
 653     @Test(dataProvider = "$type$MemorySegmentProvider")
 654     static void loadStoreReadonlyMemorySegment(IntFunction<$type$[]> fa,
 655                                                IntFunction<MemorySegment> fb,
 656                                                ByteOrder bo) {
 657         MemorySegment a = toSegment(fa.apply(SPECIES.length()), fb).asReadOnly();
 658 
 659         Assert.assertThrows(
 660                 UnsupportedOperationException.class,
 661                 () -> SPECIES.zero().intoMemorySegment(a, 0, bo)
 662         );
 663 
 664         Assert.assertThrows(
 665                 UnsupportedOperationException.class,
 666                 () -> SPECIES.zero().intoMemorySegment(a, 0, bo, SPECIES.maskAll(true))
 667         );
 668 
 669         Assert.assertThrows(
 670                 UnsupportedOperationException.class,
 671                 () -> SPECIES.zero().intoMemorySegment(a, 0, bo, SPECIES.maskAll(false))
 672         );
 673 
 674         VectorMask<$Boxtype$> m = SPECIES.shuffleFromOp(i -> i % 2 == 0 ? 1 : -1)
 675                 .laneIsValid();
 676         Assert.assertThrows(
 677                 UnsupportedOperationException.class,
 678                 () -> SPECIES.zero().intoMemorySegment(a, 0, bo, m)
 679         );
 680     }
 681 
 682 
 683     @Test(dataProvider = "maskProvider")
 684     static void loadStoreMask(IntFunction<boolean[]> fm) {
 685         boolean[] a = fm.apply(SPECIES.length());
 686         boolean[] r = new boolean[a.length];
 687 
 688         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 689             for (int i = 0; i < a.length; i += SPECIES.length()) {
 690                 VectorMask<$Boxtype$> vmask = SPECIES.loadMask(a, i);
 691                 vmask.intoArray(r, i);
 692             }
 693         }
 694         Assert.assertEquals(r, a);
 695     }
 696 
 697 
 698     @Test
 699     static void loadStoreShuffle() {
 700         IntUnaryOperator fn = a -> a + 5;
 701         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 702             var shuffle = VectorShuffle.fromOp(SPECIES, fn);
 703             int [] r = shuffle.toArray();
 704 
 705             int [] a = expectedShuffle(SPECIES.length(), fn);
 706             Assert.assertEquals(r, a);
 707        }
 708     }
 709 
 710 
 711 #if[short]
 712     static void assertArraysEquals(char[] a, char[] r, boolean[] mask) {
 713         int i = 0;
 714         try {
 715             for (; i < a.length; i++) {
 716                 Assert.assertEquals(mask[i % SPECIES.length()] ? a[i] : (char) 0, r[i]);
 717             }
 718         } catch (AssertionError e) {
 719             Assert.assertEquals(mask[i % SPECIES.length()] ? a[i] : (char) 0, r[i], "at index #" + i);
 720         }
 721     }
 722 
 723     static final List<IntFunction<char[]>> CHAR_GENERATORS = List.of(
 724             withToString("char[i * 5]", (int s) -> {
 725                 return fillChar(s * BUFFER_REPS,
 726                             i -> (char)(i * 5));
 727             }),
 728             withToString("char[i + 1]", (int s) -> {
 729                 return fillChar(s * BUFFER_REPS,
 730                             i -> (((char)(i + 1) == 0) ? 1 : (char)(i + 1)));
 731             })
 732     );
 733 
 734     @DataProvider
 735     public Object[][] charProvider() {
 736         return CHAR_GENERATORS.stream().
 737                 map(f -> new Object[]{f}).
 738                 toArray(Object[][]::new);
 739     }
 740 
 741     @DataProvider
 742     public Object[][] charProviderForIOOBE() {
 743         var f = CHAR_GENERATORS.get(0);
 744         return INDEX_GENERATORS.stream().map(fi -> {
 745                     return new Object[] {f, fi};
 746                 }).
 747                 toArray(Object[][]::new);
 748     }
 749 
 750     @DataProvider
 751     public Object[][] charMaskProvider() {
 752         return BOOLEAN_MASK_GENERATORS.stream().
 753                 flatMap(fm -> CHAR_GENERATORS.stream().map(fa -> {
 754                     return new Object[] {fa, fm};
 755                 })).
 756                 toArray(Object[][]::new);
 757     }
 758 
 759     @DataProvider
 760     public Object[][] charMaskProviderForIOOBE() {
 761         var f = CHAR_GENERATORS.get(0);
 762         return BOOLEAN_MASK_GENERATORS.stream().
 763                 flatMap(fm -> INDEX_GENERATORS.stream().map(fi -> {
 764                     return new Object[] {f, fi, fm};
 765                 })).
 766                 toArray(Object[][]::new);
 767     }
 768 
 769     interface ToCharF {
 770         char apply(int i);
 771     }
 772 
 773     static char[] fillChar(int s , ToCharF f) {
 774         return fillChar(new char[s], f);
 775     }
 776 
 777     static char[] fillChar(char[] a, ToCharF f) {
 778         for (int i = 0; i < a.length; i++) {
 779             a[i] = f.apply(i);
 780         }
 781         return a;
 782     }
 783 
 784     @DontInline
 785     static $abstractvectortype$ fromCharArray(char[] a, int i) {
 786         return $abstractvectortype$.fromCharArray(SPECIES, a, i);
 787     }
 788 
 789     @DontInline
 790     static $abstractvectortype$ fromCharArray(char[] a, int i, VectorMask<$Boxtype$> m) {
 791         return $abstractvectortype$.fromCharArray(SPECIES, a, i, m);
 792     }
 793 
 794     @DontInline
 795     static void intoCharArray($abstractvectortype$ v, char[] a, int i) {
 796         v.intoCharArray(a, i);
 797     }
 798 
 799     @DontInline
 800     static void intoCharArray($abstractvectortype$ v, char[] a, int i, VectorMask<$Boxtype$> m) {
 801         v.intoCharArray(a, i, m);
 802     }
 803 
 804     @Test(dataProvider = "charProvider")
 805     static void loadStoreCharArray(IntFunction<char[]> fa) {
 806         char[] a = fa.apply(SPECIES.length());
 807         char[] r = new char[a.length];
 808 
 809         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 810             for (int i = 0; i < a.length; i += SPECIES.length()) {
 811                 $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, i);
 812                 av.intoCharArray(r, i);
 813             }
 814         }
 815         Assert.assertEquals(a, r);
 816     }
 817 
 818     @Test(dataProvider = "charProviderForIOOBE")
 819     static void loadCharArrayIOOBE(IntFunction<char[]> fa, IntFunction<Integer> fi) {
 820         char[] a = fa.apply(SPECIES.length());
 821         char[] r = new char[a.length];
 822 
 823         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 824             for (int i = 0; i < a.length; i += SPECIES.length()) {
 825                 $abstractvectortype$ av = fromCharArray(a, i);
 826                 av.intoCharArray(r, i);
 827             }
 828         }
 829 
 830         int index = fi.apply(a.length);
 831         boolean shouldFail = isIndexOutOfBounds(SPECIES.length(), index, a.length);
 832         try {
 833             fromCharArray(a, index);
 834             if (shouldFail) {
 835                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 836             }
 837         } catch (IndexOutOfBoundsException e) {
 838             if (!shouldFail) {
 839                 Assert.fail("Unexpected IndexOutOfBoundsException");
 840             }
 841         }
 842     }
 843 
 844     @Test(dataProvider = "charProviderForIOOBE")
 845     static void storeCharArrayIOOBE(IntFunction<char[]> fa, IntFunction<Integer> fi) {
 846         char[] a = fa.apply(SPECIES.length());
 847         char[] r = new char[a.length];
 848 
 849         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 850             for (int i = 0; i < a.length; i += SPECIES.length()) {
 851                 $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, i);
 852                 intoCharArray(av, r, i);
 853             }
 854         }
 855 
 856         int index = fi.apply(a.length);
 857         boolean shouldFail = isIndexOutOfBounds(SPECIES.length(), index, a.length);
 858         try {
 859             $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, 0);
 860             intoCharArray(av, r, index);
 861             if (shouldFail) {
 862                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 863             }
 864         } catch (IndexOutOfBoundsException e) {
 865             if (!shouldFail) {
 866                 Assert.fail("Unexpected IndexOutOfBoundsException");
 867             }
 868         }
 869     }
 870 
 871     @Test(dataProvider = "charMaskProvider")
 872     static void loadStoreMaskCharArray(IntFunction<char[]> fa,
 873                                        IntFunction<boolean[]> fm) {
 874         char[] a = fa.apply(SPECIES.length());
 875         char[] r = new char[a.length];
 876         boolean[] mask = fm.apply(SPECIES.length());
 877         VectorMask<$Boxtype$> vmask = VectorMask.fromValues(SPECIES, mask);
 878 
 879         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 880             for (int i = 0; i < a.length; i += SPECIES.length()) {
 881                 $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, i, vmask);
 882                 av.intoCharArray(r, i);
 883             }
 884         }
 885         assertArraysEquals(a, r, mask);
 886 
 887 
 888         r = new char[a.length];
 889 
 890         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 891             for (int i = 0; i < a.length; i += SPECIES.length()) {
 892                 $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, i);
 893                 av.intoCharArray(r, i, vmask);
 894             }
 895         }
 896         assertArraysEquals(a, r, mask);
 897     }
 898 
 899     @Test(dataProvider = "charMaskProviderForIOOBE")
 900     static void loadCharArrayMaskIOOBE(IntFunction<char[]> fa, IntFunction<Integer> fi, IntFunction<boolean[]> fm) {
 901         char[] a = fa.apply(SPECIES.length());
 902         char[] r = new char[a.length];
 903         boolean[] mask = fm.apply(SPECIES.length());
 904         VectorMask<$Boxtype$> vmask = VectorMask.fromValues(SPECIES, mask);
 905 
 906         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 907             for (int i = 0; i < a.length; i += SPECIES.length()) {
 908                 $abstractvectortype$ av = fromCharArray(a, i, vmask);
 909                 av.intoCharArray(r, i);
 910             }
 911         }
 912 
 913         int index = fi.apply(a.length);
 914         boolean shouldFail = isIndexOutOfBoundsForMask(mask, index, a.length);
 915         try {
 916             fromCharArray(a, index, vmask);
 917             if (shouldFail) {
 918                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 919             }
 920         } catch (IndexOutOfBoundsException e) {
 921             if (!shouldFail) {
 922                 Assert.fail("Unexpected IndexOutOfBoundsException");
 923             }
 924         }
 925     }
 926 
 927     @Test(dataProvider = "charMaskProviderForIOOBE")
 928     static void storeCharArrayMaskIOOBE(IntFunction<char[]> fa, IntFunction<Integer> fi, IntFunction<boolean[]> fm) {
 929         char[] a = fa.apply(SPECIES.length());
 930         char[] r = new char[a.length];
 931         boolean[] mask = fm.apply(SPECIES.length());
 932         VectorMask<$Boxtype$> vmask = VectorMask.fromValues(SPECIES, mask);
 933 
 934         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 935             for (int i = 0; i < a.length; i += SPECIES.length()) {
 936                 $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, i);
 937                 intoCharArray(av, r, i, vmask);
 938             }
 939         }
 940 
 941         int index = fi.apply(a.length);
 942         boolean shouldFail = isIndexOutOfBoundsForMask(mask, index, a.length);
 943         try {
 944             $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, 0);
 945             intoCharArray(av, a, index, vmask);
 946             if (shouldFail) {
 947                 Assert.fail("Failed to throw IndexOutOfBoundsException");
 948             }
 949         } catch (IndexOutOfBoundsException e) {
 950             if (!shouldFail) {
 951                 Assert.fail("Unexpected IndexOutOfBoundsException");
 952             }
 953         }
 954     }
 955 
 956 #end[short]
 957 
 958 #if[byte]
 959     static void assertArraysEquals(boolean[] r, byte[] a) {
 960         int i = 0;
 961         try {
 962             for (; i < a.length; i++) {
 963                 Assert.assertEquals(r[i], (a[i] & 1) == 1);
 964             }
 965         } catch (AssertionError e) {
 966             Assert.assertEquals(r[i], (a[i] & 1) == 1, "at index #" + i);
 967         }
 968     }
 969 
 970     static void assertArraysEquals(boolean[] r, boolean[] a, boolean[] mask) {
 971         int i = 0;
 972         try {
 973             for (; i < a.length; i++) {
 974                 Assert.assertEquals(r[i], mask[i % SPECIES.length()] && a[i]);
 975             }
 976         } catch (AssertionError e) {
 977             Assert.assertEquals(r[i], mask[i % SPECIES.length()] && a[i], "at index #" + i);
 978         }
 979     }
 980 
 981     static boolean[] convertToBooleanArray(byte[] a) {
 982         boolean[] r = new boolean[a.length];
 983 
 984         for (int i = 0; i < a.length; i++) {
 985             r[i] = (a[i] & 1) == 1;
 986         }
 987 
 988         return r;
 989     }
 990 
 991     @Test(dataProvider = "byteProvider")
 992     static void loadByteStoreBooleanArray(IntFunction<byte[]> fa) {
 993         byte[] a = fa.apply(SPECIES.length());
 994         boolean[] r = new boolean[a.length];
 995 
 996         for (int ic = 0; ic < INVOC_COUNT; ic++) {
 997             for (int i = 0; i < a.length; i += SPECIES.length()) {
 998                 ByteVector av = ByteVector.fromArray(SPECIES, a, i);
 999                 av.intoBooleanArray(r, i);
1000             }
1001         }
1002         assertArraysEquals(r, a);
1003     }
1004 
1005     @Test(dataProvider = "byteProvider")
1006     static void loadStoreBooleanArray(IntFunction<byte[]> fa) {
1007         boolean[] a = convertToBooleanArray(fa.apply(SPECIES.length()));
1008         boolean[] r = new boolean[a.length];
1009 
1010         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1011             for (int i = 0; i < a.length; i += SPECIES.length()) {
1012                 ByteVector av = ByteVector.fromBooleanArray(SPECIES, a, i);
1013                 av.intoBooleanArray(r, i);
1014             }
1015         }
1016         Assert.assertEquals(r, a);
1017     }
1018 
1019     @Test(dataProvider = "byteMaskProvider")
1020     static void loadStoreMaskBooleanArray(IntFunction<byte[]> fa,
1021                                           IntFunction<boolean[]> fm) {
1022         boolean[] a = convertToBooleanArray(fa.apply(SPECIES.length()));
1023         boolean[] r = new boolean[a.length];
1024         boolean[] mask = fm.apply(SPECIES.length());
1025         VectorMask<Byte> vmask = VectorMask.fromValues(SPECIES, mask);
1026 
1027         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1028             for (int i = 0; i < a.length; i += SPECIES.length()) {
1029                 ByteVector av = ByteVector.fromBooleanArray(SPECIES, a, i, vmask);
1030                 av.intoBooleanArray(r, i);
1031             }
1032         }
1033         assertArraysEquals(r, a, mask);
1034 
1035 
1036         r = new boolean[a.length];
1037 
1038         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1039             for (int i = 0; i < a.length; i += SPECIES.length()) {
1040                 ByteVector av = ByteVector.fromBooleanArray(SPECIES, a, i);
1041                 av.intoBooleanArray(r, i, vmask);
1042             }
1043         }
1044         assertArraysEquals(r, a, mask);
1045     }
1046 #end[byte]
1047 
1048 
1049     // Gather/Scatter load/store tests
1050 
1051     static void assertGatherArraysEquals($type$[] r, $type$[] a, int[] indexMap) {
1052         int i = 0;
1053         int j = 0;
1054         try {
1055             for (; i < a.length; i += SPECIES.length()) {
1056                 j = i;
1057                 for (; j < i + SPECIES.length(); j++) {
1058                     Assert.assertEquals(r[j], a[i + indexMap[j]]);
1059                 }
1060             }
1061         } catch (AssertionError e) {
1062             Assert.assertEquals(r[j], a[i + indexMap[j]], "at index #" + j);
1063         }
1064     }
1065 
1066     static void assertGatherArraysEquals($type$[] r, $type$[] a, int[] indexMap, boolean[] mask) {
1067         int i = 0;
1068         int j = 0;
1069         try {
1070             for (; i < a.length; i += SPECIES.length()) {
1071                 j = i;
1072                 for (; j < i + SPECIES.length(); j++) {
1073                     Assert.assertEquals(r[j], mask[j % SPECIES.length()] ? a[i + indexMap[j]]: ($type$) 0);
1074                 }
1075             }
1076         } catch (AssertionError e) {
1077             Assert.assertEquals(r[i], mask[j % SPECIES.length()] ? a[i + indexMap[j]]: ($type$) 0, "at index #" + j);
1078         }
1079     }
1080 
1081     static void assertScatterArraysEquals($type$[] r, $type$[] a, int[] indexMap, boolean[] mask) {
1082         $type$[] expected = new $type$[r.length];
1083 
1084         // Store before checking, since the same location may be stored to more than once
1085         for (int i = 0; i < a.length; i += SPECIES.length()) {
1086             for (int j = i; j < i + SPECIES.length(); j++) {
1087                 if (mask[j % SPECIES.length()]) {
1088                     expected[i + indexMap[j]] = a[j];
1089                 }
1090             }
1091         }
1092 
1093         Assert.assertEquals(r, expected);
1094     }
1095 
1096     static void assertScatterArraysEquals($type$[] r, $type$[] a, int[] indexMap) {
1097         $type$[] expected = new $type$[r.length];
1098 
1099         // Store before checking, since the same location may be stored to more than once
1100         for (int i = 0; i < a.length; i += SPECIES.length()) {
1101             for (int j = i; j < i + SPECIES.length(); j++) {
1102                 expected[i + indexMap[j]] = a[j];
1103             }
1104         }
1105 
1106         Assert.assertEquals(r, expected);
1107     }
1108 
1109     @DataProvider
1110     public Object[][] gatherScatterProvider() {
1111         return INT_INDEX_GENERATORS.stream().
1112                 flatMap(fs -> $TYPE$_GENERATORS.stream().map(fa -> {
1113                     return new Object[] {fa, fs};
1114                 })).
1115                 toArray(Object[][]::new);
1116     }
1117 
1118     @DataProvider
1119     public Object[][] gatherScatterMaskProvider() {
1120         return BOOLEAN_MASK_GENERATORS.stream().
1121           flatMap(fs -> INT_INDEX_GENERATORS.stream().flatMap(fm ->
1122             $TYPE$_GENERATORS.stream().map(fa -> {
1123                     return new Object[] {fa, fm, fs};
1124             }))).
1125             toArray(Object[][]::new);
1126     }
1127 
1128 
1129     @Test(dataProvider = "gatherScatterProvider")
1130     static void gather(IntFunction<$type$[]> fa, BiFunction<Integer,Integer,int[]> fs) {
1131         $type$[] a = fa.apply(SPECIES.length());
1132         int[] b = fs.apply(a.length, SPECIES.length());
1133         $type$[] r = new $type$[a.length];
1134 
1135         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1136             for (int i = 0; i < a.length; i += SPECIES.length()) {
1137                 $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, i, b, i);
1138                 av.intoArray(r, i);
1139             }
1140         }
1141 
1142         assertGatherArraysEquals(r, a, b);
1143     }
1144 
1145     @Test(dataProvider = "gatherScatterMaskProvider")
1146     static void gatherMask(IntFunction<$type$[]> fa, BiFunction<Integer,Integer,int[]> fs, IntFunction<boolean[]> fm) {
1147         $type$[] a = fa.apply(SPECIES.length());
1148         int[] b = fs.apply(a.length, SPECIES.length());
1149         $type$[] r = new $type$[a.length];
1150         boolean[] mask = fm.apply(SPECIES.length());
1151         VectorMask<$Wideboxtype$> vmask = VectorMask.fromArray(SPECIES, mask, 0);
1152 
1153         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1154             for (int i = 0; i < a.length; i += SPECIES.length()) {
1155                 $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, i, b, i, vmask);
1156                 av.intoArray(r, i);
1157             }
1158         }
1159 
1160         assertGatherArraysEquals(r, a, b, mask);
1161     }
1162 
1163     @Test(dataProvider = "gatherScatterProvider")
1164     static void scatter(IntFunction<$type$[]> fa, BiFunction<Integer,Integer,int[]> fs) {
1165         $type$[] a = fa.apply(SPECIES.length());
1166         int[] b = fs.apply(a.length, SPECIES.length());
1167         $type$[] r = new $type$[a.length];
1168 
1169         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1170             for (int i = 0; i < a.length; i += SPECIES.length()) {
1171                 $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, i);
1172                 av.intoArray(r, i, b, i);
1173             }
1174         }
1175 
1176         assertScatterArraysEquals(r, a, b);
1177     }
1178 
1179     @Test(dataProvider = "gatherScatterMaskProvider")
1180     static void scatterMask(IntFunction<$type$[]> fa, BiFunction<Integer,Integer,int[]> fs, IntFunction<boolean[]> fm) {
1181         $type$[] a = fa.apply(SPECIES.length());
1182         int[] b = fs.apply(a.length, SPECIES.length());
1183         $type$[] r = new $type$[a.length];
1184         boolean[] mask = fm.apply(SPECIES.length());
1185         VectorMask<$Wideboxtype$> vmask = VectorMask.fromArray(SPECIES, mask, 0);
1186 
1187         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1188             for (int i = 0; i < a.length; i += SPECIES.length()) {
1189                 $abstractvectortype$ av = $abstractvectortype$.fromArray(SPECIES, a, i);
1190                 av.intoArray(r, i, b, i, vmask);
1191             }
1192         }
1193 
1194         assertScatterArraysEquals(r, a, b, mask);
1195     }
1196 
1197 #if[short]
1198     static void assertGatherArraysEquals(char[] r, char[] a, int[] indexMap) {
1199         int i = 0;
1200         int j = 0;
1201         try {
1202             for (; i < a.length; i += SPECIES.length()) {
1203                 j = i;
1204                 for (; j < i + SPECIES.length(); j++) {
1205                     Assert.assertEquals(r[j], a[i + indexMap[j]]);
1206                 }
1207             }
1208         } catch (AssertionError e) {
1209             Assert.assertEquals(r[j], a[i + indexMap[j]], "at index #" + j);
1210         }
1211     }
1212 
1213     static void assertGatherArraysEquals(char[] r, char[] a, int[] indexMap, boolean[] mask) {
1214         int i = 0;
1215         int j = 0;
1216         try {
1217             for (; i < a.length; i += SPECIES.length()) {
1218                 j = i;
1219                 for (; j < i + SPECIES.length(); j++) {
1220                     Assert.assertEquals(r[j], mask[j % SPECIES.length()] ? a[i + indexMap[j]]: (char) 0);
1221                 }
1222             }
1223         } catch (AssertionError e) {
1224             Assert.assertEquals(r[i], mask[j % SPECIES.length()] ? a[i + indexMap[j]]: (char) 0, "at index #" + j);
1225         }
1226     }
1227 
1228     static void assertScatterArraysEquals(char[] r, char[] a, int[] indexMap, boolean[] mask) {
1229         char[] expected = new char[r.length];
1230 
1231         // Store before checking, since the same location may be stored to more than once
1232         for (int i = 0; i < a.length; i += SPECIES.length()) {
1233             for (int j = i; j < i + SPECIES.length(); j++) {
1234                 if (mask[j % SPECIES.length()]) {
1235                     expected[i + indexMap[j]] = a[j];
1236                 }
1237             }
1238         }
1239 
1240         Assert.assertEquals(r, expected);
1241     }
1242 
1243     static void assertScatterArraysEquals(char[] r, char[] a, int[] indexMap) {
1244         char[] expected = new char[r.length];
1245 
1246         // Store before checking, since the same location may be stored to more than once
1247         for (int i = 0; i < a.length; i += SPECIES.length()) {
1248             for (int j = i; j < i + SPECIES.length(); j++) {
1249                 expected[i + indexMap[j]] = a[j];
1250             }
1251         }
1252 
1253         Assert.assertEquals(r, expected);
1254     }
1255 
1256     @DataProvider
1257     public Object[][] charGatherScatterProvider() {
1258         return INT_INDEX_GENERATORS.stream().
1259                 flatMap(fs -> CHAR_GENERATORS.stream().map(fa -> {
1260                     return new Object[] {fa, fs};
1261                 })).
1262                 toArray(Object[][]::new);
1263     }
1264 
1265     @DataProvider
1266     public Object[][] charGatherScatterMaskProvider() {
1267         return BOOLEAN_MASK_GENERATORS.stream().
1268           flatMap(fs -> INT_INDEX_GENERATORS.stream().flatMap(fm ->
1269             CHAR_GENERATORS.stream().map(fa -> {
1270                     return new Object[] {fa, fm, fs};
1271             }))).
1272             toArray(Object[][]::new);
1273     }
1274 
1275 
1276     @Test(dataProvider = "charGatherScatterProvider")
1277     static void charGather(IntFunction<char[]> fa, BiFunction<Integer,Integer,int[]> fs) {
1278         char[] a = fa.apply(SPECIES.length());
1279         int[] b = fs.apply(a.length, SPECIES.length());
1280         char[] r = new char[a.length];
1281 
1282         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1283             for (int i = 0; i < a.length; i += SPECIES.length()) {
1284                 $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, i, b, i);
1285                 av.intoCharArray(r, i);
1286             }
1287         }
1288 
1289         assertGatherArraysEquals(r, a, b);
1290     }
1291 
1292     @Test(dataProvider = "charGatherScatterMaskProvider")
1293     static void charGatherMask(IntFunction<char[]> fa, BiFunction<Integer,Integer,int[]> fs, IntFunction<boolean[]> fm) {
1294         char[] a = fa.apply(SPECIES.length());
1295         int[] b = fs.apply(a.length, SPECIES.length());
1296         char[] r = new char[a.length];
1297         boolean[] mask = fm.apply(SPECIES.length());
1298         VectorMask<$Wideboxtype$> vmask = VectorMask.fromArray(SPECIES, mask, 0);
1299 
1300         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1301             for (int i = 0; i < a.length; i += SPECIES.length()) {
1302                 $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, i, b, i, vmask);
1303                 av.intoCharArray(r, i);
1304             }
1305         }
1306 
1307         assertGatherArraysEquals(r, a, b, mask);
1308     }
1309 
1310     @Test(dataProvider = "charGatherScatterProvider")
1311     static void charScatter(IntFunction<char[]> fa, BiFunction<Integer,Integer,int[]> fs) {
1312         char[] a = fa.apply(SPECIES.length());
1313         int[] b = fs.apply(a.length, SPECIES.length());
1314         char[] r = new char[a.length];
1315 
1316         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1317             for (int i = 0; i < a.length; i += SPECIES.length()) {
1318                 $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, i);
1319                 av.intoCharArray(r, i, b, i);
1320             }
1321         }
1322 
1323         assertScatterArraysEquals(r, a, b);
1324     }
1325 
1326     @Test(dataProvider = "charGatherScatterMaskProvider")
1327     static void charScatterMask(IntFunction<char[]> fa, BiFunction<Integer,Integer,int[]> fs, IntFunction<boolean[]> fm) {
1328         char[] a = fa.apply(SPECIES.length());
1329         int[] b = fs.apply(a.length, SPECIES.length());
1330         char[] r = new char[a.length];
1331         boolean[] mask = fm.apply(SPECIES.length());
1332         VectorMask<$Wideboxtype$> vmask = VectorMask.fromArray(SPECIES, mask, 0);
1333 
1334         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1335             for (int i = 0; i < a.length; i += SPECIES.length()) {
1336                 $abstractvectortype$ av = $abstractvectortype$.fromCharArray(SPECIES, a, i);
1337                 av.intoCharArray(r, i, b, i, vmask);
1338             }
1339         }
1340 
1341         assertScatterArraysEquals(r, a, b, mask);
1342     }
1343 #end[short]
1344 
1345 #if[byte]
1346     static void assertGatherArraysEquals(boolean[] r, boolean[] a, int[] indexMap) {
1347         int i = 0;
1348         int j = 0;
1349         try {
1350             for (; i < a.length; i += SPECIES.length()) {
1351                 j = i;
1352                 for (; j < i + SPECIES.length(); j++) {
1353                     Assert.assertEquals(r[j], a[i + indexMap[j]]);
1354                 }
1355             }
1356         } catch (AssertionError e) {
1357             Assert.assertEquals(r[j], a[i + indexMap[j]], "at index #" + j);
1358         }
1359     }
1360 
1361     static void assertGatherArraysEquals(boolean[] r, boolean[] a, int[] indexMap, boolean[] mask) {
1362         int i = 0;
1363         int j = 0;
1364         try {
1365             for (; i < a.length; i += SPECIES.length()) {
1366                 j = i;
1367                 for (; j < i + SPECIES.length(); j++) {
1368                     Assert.assertEquals(r[j], mask[j % SPECIES.length()] ? a[i + indexMap[j]]: false);
1369                 }
1370             }
1371         } catch (AssertionError e) {
1372             Assert.assertEquals(r[i], mask[j % SPECIES.length()] ? a[i + indexMap[j]]: false, "at index #" + j);
1373         }
1374     }
1375 
1376     static void assertScatterArraysEquals(boolean[] r, boolean[] a, int[] indexMap, boolean[] mask) {
1377         boolean[] expected = new boolean[r.length];
1378 
1379         // Store before checking, since the same location may be stored to more than once
1380         for (int i = 0; i < a.length; i += SPECIES.length()) {
1381             for (int j = i; j < i + SPECIES.length(); j++) {
1382                 if (mask[j % SPECIES.length()]) {
1383                     expected[i + indexMap[j]] = a[j];
1384                 }
1385             }
1386         }
1387 
1388         Assert.assertEquals(r, expected);
1389     }
1390 
1391     static void assertScatterArraysEquals(boolean[] r, boolean[] a, int[] indexMap) {
1392         boolean[] expected = new boolean[r.length];
1393 
1394         // Store before checking, since the same location may be stored to more than once
1395         for (int i = 0; i < a.length; i += SPECIES.length()) {
1396             for (int j = i; j < i + SPECIES.length(); j++) {
1397                 expected[i + indexMap[j]] = a[j];
1398             }
1399         }
1400 
1401         Assert.assertEquals(r, expected);
1402     }
1403 
1404     @Test(dataProvider = "gatherScatterProvider")
1405     static void booleanGather(IntFunction<byte[]> fa, BiFunction<Integer,Integer,int[]> fs) {
1406         boolean[] a = convertToBooleanArray(fa.apply(SPECIES.length()));
1407         int[] b = fs.apply(a.length, SPECIES.length());
1408         boolean[] r = new boolean[a.length];
1409 
1410         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1411             for (int i = 0; i < a.length; i += SPECIES.length()) {
1412                 $abstractvectortype$ av = $abstractvectortype$.fromBooleanArray(SPECIES, a, i, b, i);
1413                 av.intoBooleanArray(r, i);
1414             }
1415         }
1416 
1417         assertGatherArraysEquals(r, a, b);
1418     }
1419 
1420     @Test(dataProvider = "gatherScatterMaskProvider")
1421     static void booleanGatherMask(IntFunction<byte[]> fa, BiFunction<Integer,Integer,int[]> fs, IntFunction<boolean[]> fm) {
1422         boolean[] a = convertToBooleanArray(fa.apply(SPECIES.length()));
1423         int[] b = fs.apply(a.length, SPECIES.length());
1424         boolean[] r = new boolean[a.length];
1425         boolean[] mask = fm.apply(SPECIES.length());
1426         VectorMask<$Wideboxtype$> vmask = VectorMask.fromArray(SPECIES, mask, 0);
1427 
1428         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1429             for (int i = 0; i < a.length; i += SPECIES.length()) {
1430                 $abstractvectortype$ av = $abstractvectortype$.fromBooleanArray(SPECIES, a, i, b, i, vmask);
1431                 av.intoBooleanArray(r, i);
1432             }
1433         }
1434 
1435         assertGatherArraysEquals(r, a, b, mask);
1436     }
1437 
1438     @Test(dataProvider = "gatherScatterProvider")
1439     static void booleanScatter(IntFunction<byte[]> fa, BiFunction<Integer,Integer,int[]> fs) {
1440         boolean[] a = convertToBooleanArray(fa.apply(SPECIES.length()));
1441         int[] b = fs.apply(a.length, SPECIES.length());
1442         boolean[] r = new boolean[a.length];
1443 
1444         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1445             for (int i = 0; i < a.length; i += SPECIES.length()) {
1446                 $abstractvectortype$ av = $abstractvectortype$.fromBooleanArray(SPECIES, a, i);
1447                 av.intoBooleanArray(r, i, b, i);
1448             }
1449         }
1450 
1451         assertScatterArraysEquals(r, a, b);
1452     }
1453 
1454     @Test(dataProvider = "gatherScatterMaskProvider")
1455     static void booleanScatterMask(IntFunction<byte[]> fa, BiFunction<Integer,Integer,int[]> fs, IntFunction<boolean[]> fm) {
1456         boolean[] a = convertToBooleanArray(fa.apply(SPECIES.length()));
1457         int[] b = fs.apply(a.length, SPECIES.length());
1458         boolean[] r = new boolean[a.length];
1459         boolean[] mask = fm.apply(SPECIES.length());
1460         VectorMask<$Wideboxtype$> vmask = VectorMask.fromArray(SPECIES, mask, 0);
1461 
1462         for (int ic = 0; ic < INVOC_COUNT; ic++) {
1463             for (int i = 0; i < a.length; i += SPECIES.length()) {
1464                 $abstractvectortype$ av = $abstractvectortype$.fromBooleanArray(SPECIES, a, i);
1465                 av.intoBooleanArray(r, i, b, i, vmask);
1466             }
1467         }
1468 
1469         assertScatterArraysEquals(r, a, b, mask);
1470     }
1471 #end[byte]
1472 
1473 }