1 /*
   2  * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 /*
  25  * @test
  26  * @modules jdk.incubator.vector
  27  * @run testng/othervm -ea -esa -Xbatch -XX:-TieredCompilation $vectorteststype$
  28  */
  29 
  30 #warn This file is preprocessed before being compiled
  31 
  32 import jdk.incubator.vector.VectorShape;
  33 import jdk.incubator.vector.VectorSpecies;
  34 import jdk.incubator.vector.VectorShuffle;
  35 import jdk.incubator.vector.VectorMask;
  36 import jdk.incubator.vector.VectorOperators;
  37 import jdk.incubator.vector.Vector;
  38 
  39 #if[Byte]
  40 import jdk.incubator.vector.ByteVector;
  41 #end[Byte]
  42 #if[Float]
  43 import jdk.incubator.vector.FloatVector;
  44 #end[Float]
  45 #if[Int]
  46 import jdk.incubator.vector.IntVector;
  47 #end[Int]
  48 #if[Double]
  49 import jdk.incubator.vector.DoubleVector;
  50 #end[Double]
  51 #if[Short]
  52 import jdk.incubator.vector.ShortVector;
  53 #end[Short]
  54 #if[Long]
  55 import jdk.incubator.vector.LongVector;
  56 #end[Long]
  57 
  58 import org.testng.Assert;
  59 import org.testng.annotations.DataProvider;
  60 import org.testng.annotations.Test;
  61 
  62 import java.lang.Integer;
  63 import java.util.List;
  64 import java.util.Arrays;
  65 import java.util.function.BiFunction;
  66 import java.util.function.IntFunction;
  67 import java.util.Objects;
  68 import java.util.stream.Collectors;
  69 import java.util.stream.Stream;
  70 
  71 @Test
  72 public class $vectorteststype$ extends AbstractVectorTest {
  73 
  74 #if[MaxBit]
  75     static final VectorSpecies<$Wideboxtype$> SPECIES =
  76                 $Type$Vector.SPECIES_MAX;
  77 #else[MaxBit]
  78     static final VectorSpecies<$Wideboxtype$> SPECIES =
  79                 $Type$Vector.SPECIES_$bits$;
  80 #end[MaxBit]
  81 
  82     static final int INVOC_COUNT = Integer.getInteger("jdk.incubator.vector.test.loop-iterations", 100);
  83 
  84 #if[MaxBit]
  85     static VectorShape getMaxBit() {
  86         return VectorShape.S_Max_BIT;
  87     }
  88 
  89     private static final int Max = 256;  // juts so we can do N/$bits$
  90 #end[MaxBit]
  91 
  92     static final int BUFFER_REPS = Integer.getInteger("jdk.incubator.vector.test.buffer-vectors", 25000 / $bits$);
  93 
  94     interface FUnOp {
  95         $type$ apply($type$ a);
  96     }
  97 
  98     static void assertArraysEquals($type$[] r, $type$[] a, FUnOp f) {
  99         int i = 0;
 100         try {
 101             for (; i < a.length; i++) {
 102                 Assert.assertEquals(r[i], f.apply(a[i]));
 103             }
 104         } catch (AssertionError e) {
 105             Assert.assertEquals(r[i], f.apply(a[i]), "at index #" + i + ", input = " + a[i]);
 106         }
 107     }
 108 
 109     interface FUnArrayOp {
 110         $type$[] apply($type$ a);
 111     }
 112 
 113     static void assertArraysEquals($type$[] r, $type$[] a, FUnArrayOp f) {
 114         int i = 0;
 115         try {
 116             for (; i < a.length; i += SPECIES.length()) {
 117                 Assert.assertEquals(Arrays.copyOfRange(r, i, i+SPECIES.length()),
 118                   f.apply(a[i]));
 119             }
 120         } catch (AssertionError e) {
 121             $type$[] ref = f.apply(a[i]);
 122             $type$[] res = Arrays.copyOfRange(r, i, i+SPECIES.length());
 123             Assert.assertEquals(res, ref, "(ref: " + Arrays.toString(ref)
 124               + ", res: " + Arrays.toString(res)
 125               + "), at index #" + i);
 126         }
 127     }
 128 
 129     static void assertArraysEquals($type$[] r, $type$[] a, boolean[] mask, FUnOp f) {
 130         int i = 0;
 131         try {
 132             for (; i < a.length; i++) {
 133                 Assert.assertEquals(r[i], mask[i % SPECIES.length()] ? f.apply(a[i]) : a[i]);
 134             }
 135         } catch (AssertionError e) {
 136             Assert.assertEquals(r[i], mask[i % SPECIES.length()] ? f.apply(a[i]) : a[i], "at index #" + i + ", input = " + a[i] + ", mask = " + mask[i % SPECIES.length()]);
 137         }
 138     }
 139 
 140     interface FReductionOp {
 141         $type$ apply($type$[] a, int idx);
 142     }
 143 
 144     interface FReductionAllOp {
 145         $type$ apply($type$[] a);
 146     }
 147 
 148     static void assertReductionArraysEquals($type$[] r, $type$ rc, $type$[] a,
 149                                             FReductionOp f, FReductionAllOp fa) {
 150         int i = 0;
 151         try {
 152             Assert.assertEquals(rc, fa.apply(a));
 153             for (; i < a.length; i += SPECIES.length()) {
 154                 Assert.assertEquals(r[i], f.apply(a, i));
 155             }
 156         } catch (AssertionError e) {
 157             Assert.assertEquals(rc, fa.apply(a), "Final result is incorrect!");
 158             Assert.assertEquals(r[i], f.apply(a, i), "at index #" + i);
 159         }
 160     }
 161 
 162     interface FReductionMaskedOp {
 163         $type$ apply($type$[] a, int idx, boolean[] mask);
 164     }
 165 
 166     interface FReductionAllMaskedOp {
 167         $type$ apply($type$[] a, boolean[] mask);
 168     }
 169 
 170     static void assertReductionArraysEqualsMasked($type$[] r, $type$ rc, $type$[] a, boolean[] mask,
 171                                             FReductionMaskedOp f, FReductionAllMaskedOp fa) {
 172         int i = 0;
 173         try {
 174             Assert.assertEquals(rc, fa.apply(a, mask));
 175             for (; i < a.length; i += SPECIES.length()) {
 176                 Assert.assertEquals(r[i], f.apply(a, i, mask));
 177             }
 178         } catch (AssertionError e) {
 179             Assert.assertEquals(rc, fa.apply(a, mask), "Final result is incorrect!");
 180             Assert.assertEquals(r[i], f.apply(a, i, mask), "at index #" + i);
 181         }
 182     }
 183 
 184 #if[!Long]
 185     interface FReductionOpLong {
 186         long apply($type$[] a, int idx);
 187     }
 188 
 189     interface FReductionAllOpLong {
 190         long apply($type$[] a);
 191     }
 192 
 193     static void assertReductionLongArraysEquals(long[] r, long rc, $type$[] a,
 194                                             FReductionOpLong f, FReductionAllOpLong fa) {
 195         int i = 0;
 196         try {
 197             Assert.assertEquals(rc, fa.apply(a));
 198             for (; i < a.length; i += SPECIES.length()) {
 199                 Assert.assertEquals(r[i], f.apply(a, i));
 200             }
 201         } catch (AssertionError e) {
 202             Assert.assertEquals(rc, fa.apply(a), "Final result is incorrect!");
 203             Assert.assertEquals(r[i], f.apply(a, i), "at index #" + i);
 204         }
 205     }
 206 
 207     interface FReductionMaskedOpLong {
 208         long apply($type$[] a, int idx, boolean[] mask);
 209     }
 210 
 211     interface FReductionAllMaskedOpLong {
 212         long apply($type$[] a, boolean[] mask);
 213     }
 214 
 215     static void assertReductionLongArraysEqualsMasked(long[] r, long rc, $type$[] a, boolean[] mask,
 216                                             FReductionMaskedOpLong f, FReductionAllMaskedOpLong fa) {
 217         int i = 0;
 218         try {
 219             Assert.assertEquals(rc, fa.apply(a, mask));
 220             for (; i < a.length; i += SPECIES.length()) {
 221                 Assert.assertEquals(r[i], f.apply(a, i, mask));
 222             }
 223         } catch (AssertionError e) {
 224             Assert.assertEquals(rc, fa.apply(a, mask), "Final result is incorrect!");
 225             Assert.assertEquals(r[i], f.apply(a, i, mask), "at index #" + i);
 226         }
 227     }
 228 #end[!Long]
 229 
 230     interface FBoolReductionOp {
 231         boolean apply(boolean[] a, int idx);
 232     }
 233 
 234     static void assertReductionBoolArraysEquals(boolean[] r, boolean[] a, FBoolReductionOp f) {
 235         int i = 0;
 236         try {
 237             for (; i < a.length; i += SPECIES.length()) {
 238                 Assert.assertEquals(r[i], f.apply(a, i));
 239             }
 240         } catch (AssertionError e) {
 241             Assert.assertEquals(r[i], f.apply(a, i), "at index #" + i);
 242         }
 243     }
 244 
 245     interface FMaskReductionOp {
 246         int apply(boolean[] a, int idx);
 247     }
 248 
 249     static void assertMaskReductionArraysEquals(int[] r, boolean[] a, FMaskReductionOp f) {
 250         int i = 0;
 251         try {
 252             for (; i < a.length; i += SPECIES.length()) {
 253                 Assert.assertEquals(r[i], f.apply(a, i));
 254             }
 255         } catch (AssertionError e) {
 256             Assert.assertEquals(r[i], f.apply(a, i), "at index #" + i);
 257         }
 258     }
 259 
 260     static void assertInsertArraysEquals($type$[] r, $type$[] a, $type$ element, int index) {
 261         int i = 0;
 262         try {
 263             for (; i < a.length; i += 1) {
 264                 if(i%SPECIES.length() == index) {
 265                     Assert.assertEquals(r[i], element);
 266                 } else {
 267                     Assert.assertEquals(r[i], a[i]);
 268                 }
 269             }
 270         } catch (AssertionError e) {
 271             if (i%SPECIES.length() == index) {
 272                 Assert.assertEquals(r[i], element, "at index #" + i);
 273             } else {
 274                 Assert.assertEquals(r[i], a[i], "at index #" + i);
 275             }
 276         }
 277     }
 278 
 279     static void assertRearrangeArraysEquals($type$[] r, $type$[] a, int[] order, int vector_len) {
 280         int i = 0, j = 0;
 281         try {
 282             for (; i < a.length; i += vector_len) {
 283                 for (j = 0; j < vector_len; j++) {
 284                     Assert.assertEquals(r[i+j], a[i+order[i+j]]);
 285                 }
 286             }
 287         } catch (AssertionError e) {
 288             int idx = i + j;
 289             Assert.assertEquals(r[i+j], a[i+order[i+j]], "at index #" + idx + ", input = " + a[i+order[i+j]]);
 290         }
 291     }
 292 
 293     static void assertSelectFromArraysEquals($type$[] r, $type$[] a, $type$[] order, int vector_len) {
 294         int i = 0, j = 0;
 295         try {
 296             for (; i < a.length; i += vector_len) {
 297                 for (j = 0; j < vector_len; j++) {
 298                     Assert.assertEquals(r[i+j], a[i+(int)order[i+j]]);
 299                 }
 300             }
 301         } catch (AssertionError e) {
 302             int idx = i + j;
 303             Assert.assertEquals(r[i+j], a[i+(int)order[i+j]], "at index #" + idx + ", input = " + a[i+(int)order[i+j]]);
 304         }
 305     }
 306 
 307     static void assertRearrangeArraysEquals($type$[] r, $type$[] a, int[] order, boolean[] mask, int vector_len) {
 308         int i = 0, j = 0;
 309         try {
 310             for (; i < a.length; i += vector_len) {
 311                 for (j = 0; j < vector_len; j++) {
 312                     if (mask[j % SPECIES.length()])
 313                          Assert.assertEquals(r[i+j], a[i+order[i+j]]);
 314                     else
 315                          Assert.assertEquals(r[i+j], ($type$)0);
 316                 }
 317             }
 318         } catch (AssertionError e) {
 319             int idx = i + j;
 320             if (mask[j % SPECIES.length()])
 321                 Assert.assertEquals(r[i+j], a[i+order[i+j]], "at index #" + idx + ", input = " + a[i+order[i+j]] + ", mask = " + mask[j % SPECIES.length()]);
 322             else
 323                 Assert.assertEquals(r[i+j], ($type$)0, "at index #" + idx + ", input = " + a[i+order[i+j]] + ", mask = " + mask[j % SPECIES.length()]);
 324         }
 325     }
 326 
 327     static void assertSelectFromArraysEquals($type$[] r, $type$[] a, $type$[] order, boolean[] mask, int vector_len) {
 328         int i = 0, j = 0;
 329         try {
 330             for (; i < a.length; i += vector_len) {
 331                 for (j = 0; j < vector_len; j++) {
 332                     if (mask[j % SPECIES.length()])
 333                          Assert.assertEquals(r[i+j], a[i+(int)order[i+j]]);
 334                     else
 335                          Assert.assertEquals(r[i+j], ($type$)0);
 336                 }
 337             }
 338         } catch (AssertionError e) {
 339             int idx = i + j;
 340             if (mask[j % SPECIES.length()])
 341                 Assert.assertEquals(r[i+j], a[i+(int)order[i+j]], "at index #" + idx + ", input = " + a[i+(int)order[i+j]] + ", mask = " + mask[j % SPECIES.length()]);
 342             else
 343                 Assert.assertEquals(r[i+j], ($type$)0, "at index #" + idx + ", input = " + a[i+(int)order[i+j]] + ", mask = " + mask[j % SPECIES.length()]);
 344         }
 345     }
 346 
 347     static void assertBroadcastArraysEquals($type$[] r, $type$[] a) {
 348         int i = 0;
 349         for (; i < a.length; i += SPECIES.length()) {
 350             int idx = i;
 351             for (int j = idx; j < (idx + SPECIES.length()); j++)
 352                 a[j]=a[idx];
 353         }
 354 
 355         try {
 356             for (i = 0; i < a.length; i++) {
 357                 Assert.assertEquals(r[i], a[i]);
 358             }
 359         } catch (AssertionError e) {
 360             Assert.assertEquals(r[i], a[i], "at index #" + i + ", input = " + a[i]);
 361         }
 362     }
 363 
 364     interface FBinOp {
 365         $type$ apply($type$ a, $type$ b);
 366     }
 367 
 368     interface FBinMaskOp {
 369         $type$ apply($type$ a, $type$ b, boolean m);
 370 
 371         static FBinMaskOp lift(FBinOp f) {
 372             return (a, b, m) -> m ? f.apply(a, b) : a;
 373         }
 374     }
 375 
 376     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, FBinOp f) {
 377         int i = 0;
 378         try {
 379             for (; i < a.length; i++) {
 380                 Assert.assertEquals(r[i], f.apply(a[i], b[i]));
 381             }
 382         } catch (AssertionError e) {
 383             Assert.assertEquals(r[i], f.apply(a[i], b[i]), "(" + a[i] + ", " + b[i] + ") at index #" + i);
 384         }
 385     }
 386 
 387     static void assertBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, FBinOp f) {
 388         int i = 0;
 389         try {
 390             for (; i < a.length; i++) {
 391                 Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()]));
 392             }
 393         } catch (AssertionError e) {
 394             Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()]),
 395                                 "(" + a[i] + ", " + b[(i / SPECIES.length()) * SPECIES.length()] + ") at index #" + i);
 396         }
 397     }
 398 
 399     static void assertBroadcastLongArraysEquals($type$[] r, $type$[] a, $type$[] b, FBinOp f) {
 400         int i = 0;
 401         try {
 402             for (; i < a.length; i++) {
 403                 Assert.assertEquals(r[i], f.apply(a[i], ($type$)((long)b[(i / SPECIES.length()) * SPECIES.length()])));
 404             }
 405         } catch (AssertionError e) {
 406             Assert.assertEquals(r[i], f.apply(a[i], ($type$)((long)b[(i / SPECIES.length()) * SPECIES.length()])),
 407                                 "(" + a[i] + ", " + b[(i / SPECIES.length()) * SPECIES.length()] + ") at index #" + i);
 408         }
 409     }
 410 
 411     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, boolean[] mask, FBinOp f) {
 412         assertArraysEquals(r, a, b, mask, FBinMaskOp.lift(f));
 413     }
 414 
 415     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, boolean[] mask, FBinMaskOp f) {
 416         int i = 0;
 417         try {
 418             for (; i < a.length; i++) {
 419                 Assert.assertEquals(r[i], f.apply(a[i], b[i], mask[i % SPECIES.length()]));
 420             }
 421         } catch (AssertionError err) {
 422             Assert.assertEquals(r[i], f.apply(a[i], b[i], mask[i % SPECIES.length()]), "at index #" + i + ", input1 = " + a[i] + ", input2 = " + b[i] + ", mask = " + mask[i % SPECIES.length()]);
 423         }
 424     }
 425 
 426     static void assertBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, boolean[] mask, FBinOp f) {
 427         assertBroadcastArraysEquals(r, a, b, mask, FBinMaskOp.lift(f));
 428     }
 429 
 430     static void assertBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, boolean[] mask, FBinMaskOp f) {
 431         int i = 0;
 432         try {
 433             for (; i < a.length; i++) {
 434                 Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()], mask[i % SPECIES.length()]));
 435             }
 436         } catch (AssertionError err) {
 437             Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()],
 438                                 mask[i % SPECIES.length()]), "at index #" + i + ", input1 = " + a[i] +
 439                                 ", input2 = " + b[(i / SPECIES.length()) * SPECIES.length()] + ", mask = " +
 440                                 mask[i % SPECIES.length()]);
 441         }
 442     }
 443 
 444     static void assertBroadcastLongArraysEquals($type$[] r, $type$[] a, $type$[] b, boolean[] mask, FBinOp f) {
 445         assertBroadcastLongArraysEquals(r, a, b, mask, FBinMaskOp.lift(f));
 446     }
 447 
 448     static void assertBroadcastLongArraysEquals($type$[] r, $type$[] a, $type$[] b, boolean[] mask, FBinMaskOp f) {
 449         int i = 0;
 450         try {
 451             for (; i < a.length; i++) {
 452                 Assert.assertEquals(r[i], f.apply(a[i], ($type$)((long)b[(i / SPECIES.length()) * SPECIES.length()]), mask[i % SPECIES.length()]));
 453             }
 454         } catch (AssertionError err) {
 455             Assert.assertEquals(r[i], f.apply(a[i], ($type$)((long)b[(i / SPECIES.length()) * SPECIES.length()]),
 456                                 mask[i % SPECIES.length()]), "at index #" + i + ", input1 = " + a[i] +
 457                                 ", input2 = " + b[(i / SPECIES.length()) * SPECIES.length()] + ", mask = " +
 458                                 mask[i % SPECIES.length()]);
 459         }
 460     }
 461 
 462     static void assertShiftArraysEquals($type$[] r, $type$[] a, $type$[] b, FBinOp f) {
 463         int i = 0;
 464         int j = 0;
 465         try {
 466             for (; j < a.length; j += SPECIES.length()) {
 467                 for (i = 0; i < SPECIES.length(); i++) {
 468                     Assert.assertEquals(r[i+j], f.apply(a[i+j], b[j]));
 469                 }
 470             }
 471         } catch (AssertionError e) {
 472             Assert.assertEquals(r[i+j], f.apply(a[i+j], b[j]), "at index #" + i + ", " + j);
 473         }
 474     }
 475 
 476     static void assertShiftArraysEquals($type$[] r, $type$[] a, $type$[] b, boolean[] mask, FBinOp f) {
 477         assertShiftArraysEquals(r, a, b, mask, FBinMaskOp.lift(f));
 478     }
 479 
 480     static void assertShiftArraysEquals($type$[] r, $type$[] a, $type$[] b, boolean[] mask, FBinMaskOp f) {
 481         int i = 0;
 482         int j = 0;
 483         try {
 484             for (; j < a.length; j += SPECIES.length()) {
 485                 for (i = 0; i < SPECIES.length(); i++) {
 486                     Assert.assertEquals(r[i+j], f.apply(a[i+j], b[j], mask[i]));
 487                 }
 488             }
 489         } catch (AssertionError err) {
 490             Assert.assertEquals(r[i+j], f.apply(a[i+j], b[j], mask[i]), "at index #" + i + ", input1 = " + a[i+j] + ", input2 = " + b[j] + ", mask = " + mask[i]);
 491         }
 492     }
 493 
 494     interface FTernOp {
 495         $type$ apply($type$ a, $type$ b, $type$ c);
 496     }
 497 
 498     interface FTernMaskOp {
 499         $type$ apply($type$ a, $type$ b, $type$ c, boolean m);
 500 
 501         static FTernMaskOp lift(FTernOp f) {
 502             return (a, b, c, m) -> m ? f.apply(a, b, c) : a;
 503         }
 504     }
 505 
 506     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, FTernOp f) {
 507         int i = 0;
 508         try {
 509             for (; i < a.length; i++) {
 510                 Assert.assertEquals(r[i], f.apply(a[i], b[i], c[i]));
 511             }
 512         } catch (AssertionError e) {
 513             Assert.assertEquals(r[i], f.apply(a[i], b[i], c[i]), "at index #" + i + ", input1 = " + a[i] + ", input2 = " + b[i] + ", input3 = " + c[i]);
 514         }
 515     }
 516 
 517     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, boolean[] mask, FTernOp f) {
 518         assertArraysEquals(r, a, b, c, mask, FTernMaskOp.lift(f));
 519     }
 520 
 521     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, boolean[] mask, FTernMaskOp f) {
 522         int i = 0;
 523         try {
 524             for (; i < a.length; i++) {
 525                 Assert.assertEquals(r[i], f.apply(a[i], b[i], c[i], mask[i % SPECIES.length()]));
 526             }
 527         } catch (AssertionError err) {
 528             Assert.assertEquals(r[i], f.apply(a[i], b[i], c[i], mask[i % SPECIES.length()]), "at index #" + i + ", input1 = " + a[i] + ", input2 = "
 529               + b[i] + ", input3 = " + c[i] + ", mask = " + mask[i % SPECIES.length()]);
 530         }
 531     }
 532 
 533     static void assertBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, FTernOp f) {
 534         int i = 0;
 535         try {
 536             for (; i < a.length; i++) {
 537                 Assert.assertEquals(r[i], f.apply(a[i], b[i], c[(i / SPECIES.length()) * SPECIES.length()]));
 538             }
 539         } catch (AssertionError e) {
 540             Assert.assertEquals(r[i], f.apply(a[i], b[i], c[(i / SPECIES.length()) * SPECIES.length()]), "at index #" +
 541                                 i + ", input1 = " + a[i] + ", input2 = " + b[i] + ", input3 = " +
 542                                 c[(i / SPECIES.length()) * SPECIES.length()]);
 543         }
 544     }
 545 
 546     static void assertAltBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, FTernOp f) {
 547         int i = 0;
 548         try {
 549             for (; i < a.length; i++) {
 550                 Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()], c[i]));
 551             }
 552         } catch (AssertionError e) {
 553             Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()], c[i]), "at index #" +
 554                                 i + ", input1 = " + a[i] + ", input2 = " +
 555                                 b[(i / SPECIES.length()) * SPECIES.length()] + ",  input3 = " + c[i]);
 556         }
 557     }
 558 
 559     static void assertBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, boolean[] mask,
 560                                             FTernOp f) {
 561         assertBroadcastArraysEquals(r, a, b, c, mask, FTernMaskOp.lift(f));
 562     }
 563 
 564     static void assertBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, boolean[] mask,
 565                                             FTernMaskOp f) {
 566         int i = 0;
 567         try {
 568             for (; i < a.length; i++) {
 569                 Assert.assertEquals(r[i], f.apply(a[i], b[i], c[(i / SPECIES.length()) * SPECIES.length()],
 570                                     mask[i % SPECIES.length()]));
 571             }
 572         } catch (AssertionError err) {
 573             Assert.assertEquals(r[i], f.apply(a[i], b[i], c[(i / SPECIES.length()) * SPECIES.length()],
 574                                 mask[i % SPECIES.length()]), "at index #" + i + ", input1 = " + a[i] + ", input2 = " +
 575                                 b[i] + ", input3 = " + c[(i / SPECIES.length()) * SPECIES.length()] + ", mask = " +
 576                                 mask[i % SPECIES.length()]);
 577         }
 578     }
 579 
 580     static void assertAltBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, boolean[] mask,
 581                                             FTernOp f) {
 582         assertAltBroadcastArraysEquals(r, a, b, c, mask, FTernMaskOp.lift(f));
 583     }
 584 
 585     static void assertAltBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, boolean[] mask,
 586                                             FTernMaskOp f) {
 587         int i = 0;
 588         try {
 589             for (; i < a.length; i++) {
 590                 Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()], c[i],
 591                                     mask[i % SPECIES.length()]));
 592             }
 593         } catch (AssertionError err) {
 594             Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()], c[i],
 595                                 mask[i % SPECIES.length()]), "at index #" + i + ", input1 = " + a[i] +
 596                                 ", input2 = " + b[(i / SPECIES.length()) * SPECIES.length()] +
 597                                 ", input3 = " + c[i] + ", mask = " + mask[i % SPECIES.length()]);
 598         }
 599     }
 600 
 601     static void assertDoubleBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, FTernOp f) {
 602         int i = 0;
 603         try {
 604             for (; i < a.length; i++) {
 605                 Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()],
 606                                     c[(i / SPECIES.length()) * SPECIES.length()]));
 607             }
 608         } catch (AssertionError e) {
 609             Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()],
 610                                 c[(i / SPECIES.length()) * SPECIES.length()]), "at index #" + i + ", input1 = " + a[i]
 611                                 + ", input2 = " + b[(i / SPECIES.length()) * SPECIES.length()] + ", input3 = " +
 612                                 c[(i / SPECIES.length()) * SPECIES.length()]);
 613         }
 614     }
 615 
 616     static void assertDoubleBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, boolean[] mask,
 617                                                   FTernOp f) {
 618         assertDoubleBroadcastArraysEquals(r, a, b, c, mask, FTernMaskOp.lift(f));
 619     }
 620 
 621     static void assertDoubleBroadcastArraysEquals($type$[] r, $type$[] a, $type$[] b, $type$[] c, boolean[] mask,
 622                                                   FTernMaskOp f) {
 623         int i = 0;
 624         try {
 625             for (; i < a.length; i++) {
 626                 Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()],
 627                                     c[(i / SPECIES.length()) * SPECIES.length()], mask[i % SPECIES.length()]));
 628             }
 629         } catch (AssertionError err) {
 630             Assert.assertEquals(r[i], f.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()],
 631                                 c[(i / SPECIES.length()) * SPECIES.length()], mask[i % SPECIES.length()]), "at index #"
 632                                 + i + ", input1 = " + a[i] + ", input2 = " + b[(i / SPECIES.length()) * SPECIES.length()] +
 633                                 ", input3 = " + c[(i / SPECIES.length()) * SPECIES.length()] + ", mask = " +
 634                                 mask[i % SPECIES.length()]);
 635         }
 636     }
 637 
 638 
 639 #if[FP]
 640     static boolean isWithin1Ulp($type$ actual, $type$ expected) {
 641         if ($Type$.isNaN(expected) && !$Type$.isNaN(actual)) {
 642             return false;
 643         } else if (!$Type$.isNaN(expected) && $Type$.isNaN(actual)) {
 644             return false;
 645         }
 646 
 647         $type$ low = Math.nextDown(expected);
 648         $type$ high = Math.nextUp(expected);
 649 
 650         if ($Type$.compare(low, expected) > 0) {
 651             return false;
 652         }
 653 
 654         if ($Type$.compare(high, expected) < 0) {
 655             return false;
 656         }
 657 
 658         return true;
 659     }
 660 
 661     static void assertArraysEqualsWithinOneUlp($type$[] r, $type$[] a, FUnOp mathf, FUnOp strictmathf) {
 662         int i = 0;
 663         try {
 664             // Check that result is within 1 ulp of strict math or equivalent to math implementation.
 665             for (; i < a.length; i++) {
 666                 Assert.assertTrue($Type$.compare(r[i], mathf.apply(a[i])) == 0 ||
 667                                     isWithin1Ulp(r[i], strictmathf.apply(a[i])));
 668             }
 669         } catch (AssertionError e) {
 670             Assert.assertTrue($Type$.compare(r[i], mathf.apply(a[i])) == 0, "at index #" + i + ", input = " + a[i] + ", actual = " + r[i] + ", expected = " + mathf.apply(a[i]));
 671             Assert.assertTrue(isWithin1Ulp(r[i], strictmathf.apply(a[i])), "at index #" + i + ", input = " + a[i] + ", actual = " + r[i] + ", expected (within 1 ulp) = " + strictmathf.apply(a[i]));
 672         }
 673     }
 674 
 675     static void assertArraysEqualsWithinOneUlp($type$[] r, $type$[] a, $type$[] b, FBinOp mathf, FBinOp strictmathf) {
 676         int i = 0;
 677         try {
 678             // Check that result is within 1 ulp of strict math or equivalent to math implementation.
 679             for (; i < a.length; i++) {
 680                 Assert.assertTrue($Type$.compare(r[i], mathf.apply(a[i], b[i])) == 0 ||
 681                                     isWithin1Ulp(r[i], strictmathf.apply(a[i], b[i])));
 682             }
 683         } catch (AssertionError e) {
 684             Assert.assertTrue($Type$.compare(r[i], mathf.apply(a[i], b[i])) == 0, "at index #" + i + ", input1 = " + a[i] + ", input2 = " + b[i] + ", actual = " + r[i] + ", expected = " + mathf.apply(a[i], b[i]));
 685             Assert.assertTrue(isWithin1Ulp(r[i], strictmathf.apply(a[i], b[i])), "at index #" + i + ", input1 = " + a[i] + ", input2 = " + b[i] + ", actual = " + r[i] + ", expected (within 1 ulp) = " + strictmathf.apply(a[i], b[i]));
 686         }
 687     }
 688 
 689     static void assertBroadcastArraysEqualsWithinOneUlp($type$[] r, $type$[] a, $type$[] b,
 690                                                         FBinOp mathf, FBinOp strictmathf) {
 691         int i = 0;
 692         try {
 693             // Check that result is within 1 ulp of strict math or equivalent to math implementation.
 694             for (; i < a.length; i++) {
 695                 Assert.assertTrue($Type$.compare(r[i],
 696                                   mathf.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()])) == 0 ||
 697                                   isWithin1Ulp(r[i],
 698                                   strictmathf.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()])));
 699             }
 700         } catch (AssertionError e) {
 701             Assert.assertTrue($Type$.compare(r[i],
 702                               mathf.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()])) == 0,
 703                               "at index #" + i + ", input1 = " + a[i] + ", input2 = " +
 704                               b[(i / SPECIES.length()) * SPECIES.length()] + ", actual = " + r[i] +
 705                               ", expected = " + mathf.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()]));
 706             Assert.assertTrue(isWithin1Ulp(r[i],
 707                               strictmathf.apply(a[i], b[(i / SPECIES.length()) * SPECIES.length()])),
 708                              "at index #" + i + ", input1 = " + a[i] + ", input2 = " +
 709                              b[(i / SPECIES.length()) * SPECIES.length()] + ", actual = " + r[i] +
 710                              ", expected (within 1 ulp) = " + strictmathf.apply(a[i],
 711                              b[(i / SPECIES.length()) * SPECIES.length()]));
 712         }
 713     }
 714 #end[FP]
 715 
 716     interface FBinArrayOp {
 717         $type$ apply($type$[] a, int b);
 718     }
 719 
 720     static void assertArraysEquals($type$[] r, $type$[] a, FBinArrayOp f) {
 721         int i = 0;
 722         try {
 723             for (; i < a.length; i++) {
 724                 Assert.assertEquals(r[i], f.apply(a, i));
 725             }
 726         } catch (AssertionError e) {
 727             Assert.assertEquals(r[i], f.apply(a,i), "at index #" + i);
 728         }
 729     }
 730 
 731     interface FGatherScatterOp {
 732         $type$[] apply($type$[] a, int ix, int[] b, int iy);
 733     }
 734 
 735     static void assertArraysEquals($type$[] r, $type$[] a, int[] b, FGatherScatterOp f) {
 736         int i = 0;
 737         try {
 738             for (; i < a.length; i += SPECIES.length()) {
 739                 Assert.assertEquals(Arrays.copyOfRange(r, i, i+SPECIES.length()),
 740                   f.apply(a, i, b, i));
 741             }
 742         } catch (AssertionError e) {
 743             $type$[] ref = f.apply(a, i, b, i);
 744             $type$[] res = Arrays.copyOfRange(r, i, i+SPECIES.length());
 745             Assert.assertEquals(res, ref,
 746               "(ref: " + Arrays.toString(ref) + ", res: " + Arrays.toString(res) + ", a: "
 747               + Arrays.toString(Arrays.copyOfRange(a, i, i+SPECIES.length()))
 748               + ", b: "
 749               + Arrays.toString(Arrays.copyOfRange(b, i, i+SPECIES.length()))
 750               + " at index #" + i);
 751         }
 752     }
 753 
 754     interface FGatherMaskedOp {
 755         $type$[] apply($type$[] a, int ix, boolean[] mask, int[] b, int iy);
 756     }
 757 
 758     interface FScatterMaskedOp {
 759         $type$[] apply($type$[] r, $type$[] a, int ix, boolean[] mask, int[] b, int iy);
 760     }
 761 
 762     static void assertArraysEquals($type$[] r, $type$[] a, int[] b, boolean[] mask, FGatherMaskedOp f) {
 763         int i = 0;
 764         try {
 765             for (; i < a.length; i += SPECIES.length()) {
 766                 Assert.assertEquals(Arrays.copyOfRange(r, i, i+SPECIES.length()),
 767                   f.apply(a, i, mask, b, i));
 768             }
 769         } catch (AssertionError e) {
 770             $type$[] ref = f.apply(a, i, mask, b, i);
 771             $type$[] res = Arrays.copyOfRange(r, i, i+SPECIES.length());
 772             Assert.assertEquals(res, ref,
 773               "(ref: " + Arrays.toString(ref) + ", res: " + Arrays.toString(res) + ", a: "
 774               + Arrays.toString(Arrays.copyOfRange(a, i, i+SPECIES.length()))
 775               + ", b: "
 776               + Arrays.toString(Arrays.copyOfRange(b, i, i+SPECIES.length()))
 777               + ", mask: "
 778               + Arrays.toString(mask)
 779               + " at index #" + i);
 780         }
 781     }
 782 
 783     static void assertArraysEquals($type$[] r, $type$[] a, int[] b, boolean[] mask, FScatterMaskedOp f) {
 784         int i = 0;
 785         try {
 786             for (; i < a.length; i += SPECIES.length()) {
 787                 Assert.assertEquals(Arrays.copyOfRange(r, i, i+SPECIES.length()),
 788                   f.apply(r, a, i, mask, b, i));
 789             }
 790         } catch (AssertionError e) {
 791             $type$[] ref = f.apply(r, a, i, mask, b, i);
 792             $type$[] res = Arrays.copyOfRange(r, i, i+SPECIES.length());
 793             Assert.assertEquals(res, ref,
 794               "(ref: " + Arrays.toString(ref) + ", res: " + Arrays.toString(res) + ", a: "
 795               + Arrays.toString(Arrays.copyOfRange(a, i, i+SPECIES.length()))
 796               + ", b: "
 797               + Arrays.toString(Arrays.copyOfRange(b, i, i+SPECIES.length()))
 798               + ", r: "
 799               + Arrays.toString(Arrays.copyOfRange(r, i, i+SPECIES.length()))
 800               + ", mask: "
 801               + Arrays.toString(mask)
 802               + " at index #" + i);
 803         }
 804     }
 805 
 806     interface FLaneOp {
 807         $type$[] apply($type$[] a, int origin, int idx);
 808     }
 809 
 810     static void assertArraysEquals($type$[] r, $type$[] a, int origin, FLaneOp f) {
 811         int i = 0;
 812         try {
 813             for (; i < a.length; i += SPECIES.length()) {
 814                 Assert.assertEquals(Arrays.copyOfRange(r, i, i+SPECIES.length()),
 815                   f.apply(a, origin, i));
 816             }
 817         } catch (AssertionError e) {
 818             $type$[] ref = f.apply(a, origin, i);
 819             $type$[] res = Arrays.copyOfRange(r, i, i+SPECIES.length());
 820             Assert.assertEquals(res, ref, "(ref: " + Arrays.toString(ref)
 821               + ", res: " + Arrays.toString(res)
 822               + "), at index #" + i);
 823         }
 824     }
 825 
 826     interface FLaneBop {
 827         $type$[] apply($type$[] a, $type$[] b, int origin, int idx);
 828     }
 829 
 830     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, int origin, FLaneBop f) {
 831         int i = 0;
 832         try {
 833             for (; i < a.length; i += SPECIES.length()) {
 834                 Assert.assertEquals(Arrays.copyOfRange(r, i, i+SPECIES.length()),
 835                   f.apply(a, b, origin, i));
 836             }
 837         } catch (AssertionError e) {
 838             $type$[] ref = f.apply(a, b, origin, i);
 839             $type$[] res = Arrays.copyOfRange(r, i, i+SPECIES.length());
 840             Assert.assertEquals(res, ref, "(ref: " + Arrays.toString(ref)
 841               + ", res: " + Arrays.toString(res)
 842               + "), at index #" + i
 843               + ", at origin #" + origin);
 844         }
 845     }
 846 
 847     interface FLaneMaskedBop {
 848         $type$[] apply($type$[] a, $type$[] b, int origin, boolean[] mask, int idx);
 849     }
 850 
 851     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, int origin, boolean[] mask, FLaneMaskedBop f) {
 852         int i = 0;
 853         try {
 854             for (; i < a.length; i += SPECIES.length()) {
 855                 Assert.assertEquals(Arrays.copyOfRange(r, i, i+SPECIES.length()),
 856                   f.apply(a, b, origin, mask, i));
 857             }
 858         } catch (AssertionError e) {
 859             $type$[] ref = f.apply(a, b, origin, mask, i);
 860             $type$[] res = Arrays.copyOfRange(r, i, i+SPECIES.length());
 861             Assert.assertEquals(res, ref, "(ref: " + Arrays.toString(ref)
 862               + ", res: " + Arrays.toString(res)
 863               + "), at index #" + i
 864               + ", at origin #" + origin);
 865         }
 866     }
 867 
 868     interface FLanePartBop {
 869         $type$[] apply($type$[] a, $type$[] b, int origin, int part, int idx);
 870     }
 871 
 872     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, int origin, int part, FLanePartBop f) {
 873         int i = 0;
 874         try {
 875             for (; i < a.length; i += SPECIES.length()) {
 876                 Assert.assertEquals(Arrays.copyOfRange(r, i, i+SPECIES.length()),
 877                   f.apply(a, b, origin, part, i));
 878             }
 879         } catch (AssertionError e) {
 880             $type$[] ref = f.apply(a, b, origin, part, i);
 881             $type$[] res = Arrays.copyOfRange(r, i, i+SPECIES.length());
 882             Assert.assertEquals(res, ref, "(ref: " + Arrays.toString(ref)
 883               + ", res: " + Arrays.toString(res)
 884               + "), at index #" + i
 885               + ", at origin #" + origin
 886               + ", with part #" + part);
 887         }
 888     }
 889 
 890     interface FLanePartMaskedBop {
 891         $type$[] apply($type$[] a, $type$[] b, int origin, int part, boolean[] mask, int idx);
 892     }
 893 
 894     static void assertArraysEquals($type$[] r, $type$[] a, $type$[] b, int origin, int part, boolean[] mask, FLanePartMaskedBop f) {
 895         int i = 0;
 896         try {
 897             for (; i < a.length; i += SPECIES.length()) {
 898                 Assert.assertEquals(Arrays.copyOfRange(r, i, i+SPECIES.length()),
 899                   f.apply(a, b, origin, part, mask, i));
 900             }
 901         } catch (AssertionError e) {
 902             $type$[] ref = f.apply(a, b, origin, part, mask, i);
 903             $type$[] res = Arrays.copyOfRange(r, i, i+SPECIES.length());
 904             Assert.assertEquals(res, ref, "(ref: " + Arrays.toString(ref)
 905               + ", res: " + Arrays.toString(res)
 906               + "), at index #" + i
 907               + ", at origin #" + origin
 908               + ", with part #" + part);
 909         }
 910     }
 911 
 912 #if[!Int]
 913 #if[!byteOrShort]
 914     static int intCornerCaseValue(int i) {
 915         switch(i % 5) {
 916             case 0:
 917                 return Integer.MAX_VALUE;
 918             case 1:
 919                 return Integer.MIN_VALUE;
 920             case 2:
 921                 return Integer.MIN_VALUE;
 922             case 3:
 923                 return Integer.MAX_VALUE;
 924             default:
 925                 return (int)0;
 926         }
 927     }
 928 
 929     static final List<IntFunction<$type$[]>> INT_$TYPE$_GENERATORS = List.of(
 930             withToString("$type$[-i * 5]", (int s) -> {
 931                 return fill(s * BUFFER_REPS,
 932                             i -> ($type$)(-i * 5));
 933             }),
 934             withToString("$type$[i * 5]", (int s) -> {
 935                 return fill(s * BUFFER_REPS,
 936                             i -> ($type$)(i * 5));
 937             }),
 938             withToString("$type$[i + 1]", (int s) -> {
 939                 return fill(s * BUFFER_REPS,
 940                             i -> ((($type$)(i + 1) == 0) ? 1 : ($type$)(i + 1)));
 941             }),
 942             withToString("$type$[intCornerCaseValue(i)]", (int s) -> {
 943                 return fill(s * BUFFER_REPS,
 944                             i -> ($type$)intCornerCaseValue(i));
 945             })
 946     );
 947 #end[!byteOrShort]
 948 #end[!Int]
 949 
 950     static void assertArraysEquals(int[] r, $type$[] a, int offs) {
 951         int i = 0;
 952         try {
 953             for (; i < r.length; i++) {
 954                 Assert.assertEquals(r[i], (int)(a[i+offs]));
 955             }
 956         } catch (AssertionError e) {
 957             Assert.assertEquals(r[i], (int)(a[i+offs]), "at index #" + i + ", input = " + a[i+offs]);
 958         }
 959     }
 960 
 961 #if[!Long]
 962 #if[FP]
 963     static long longCornerCaseValue(int i) {
 964         switch(i % 5) {
 965             case 0:
 966                 return Long.MAX_VALUE;
 967             case 1:
 968                 return Long.MIN_VALUE;
 969             case 2:
 970                 return Long.MIN_VALUE;
 971             case 3:
 972                 return Long.MAX_VALUE;
 973             default:
 974                 return (long)0;
 975         }
 976     }
 977 
 978     static final List<IntFunction<$type$[]>> LONG_$TYPE$_GENERATORS = List.of(
 979             withToString("$type$[-i * 5]", (int s) -> {
 980                 return fill(s * BUFFER_REPS,
 981                             i -> ($type$)(-i * 5));
 982             }),
 983             withToString("$type$[i * 5]", (int s) -> {
 984                 return fill(s * BUFFER_REPS,
 985                             i -> ($type$)(i * 5));
 986             }),
 987             withToString("$type$[i + 1]", (int s) -> {
 988                 return fill(s * BUFFER_REPS,
 989                             i -> ((($type$)(i + 1) == 0) ? 1 : ($type$)(i + 1)));
 990             }),
 991             withToString("$type$[cornerCaseValue(i)]", (int s) -> {
 992                 return fill(s * BUFFER_REPS,
 993                             i -> ($type$)longCornerCaseValue(i));
 994             })
 995     );
 996 #end[FP]
 997 #end[!Long]
 998 
 999 #if[byte]
1000     static void assertArraysEquals($type$[] r, $type$[] a, int offs) {
1001         int i = 0;
1002         try {
1003             for (; i < r.length; i++) {
1004                 Assert.assertEquals(r[i], (long)(a[i+offs]));
1005             }
1006         } catch (AssertionError e) {
1007             Assert.assertEquals(r[i], (long)(a[i+offs]), "at index #" + i + ", input = " + a[i+offs]);
1008         }
1009     }
1010 #end[byte]
1011 
1012     static void assertArraysEquals(long[] r, $type$[] a, int offs) {
1013         int i = 0;
1014         try {
1015             for (; i < r.length; i++) {
1016                 Assert.assertEquals(r[i], (long)(a[i+offs]));
1017             }
1018         } catch (AssertionError e) {
1019             Assert.assertEquals(r[i], (long)(a[i+offs]), "at index #" + i + ", input = " + a[i+offs]);
1020         }
1021     }
1022 
1023 #if[!Double]
1024     static void assertArraysEquals(double[] r, $type$[] a, int offs) {
1025         int i = 0;
1026         try {
1027             for (; i < r.length; i++) {
1028                 Assert.assertEquals(r[i], (double)(a[i+offs]));
1029             }
1030         } catch (AssertionError e) {
1031             Assert.assertEquals(r[i], (double)(a[i+offs]), "at index #" + i + ", input = " + a[i+offs]);
1032         }
1033     }
1034 #end[!Double]
1035 
1036 
1037     static $bitstype$ bits($type$ e) {
1038         return {#if[FP]? $Type$.$type$To$Bitstype$Bits(e): e};
1039     }
1040 
1041     static final List<IntFunction<$type$[]>> $TYPE$_GENERATORS = List.of(
1042             withToString("$type$[-i * 5]", (int s) -> {
1043                 return fill(s * BUFFER_REPS,
1044                             i -> ($type$)(-i * 5));
1045             }),
1046             withToString("$type$[i * 5]", (int s) -> {
1047                 return fill(s * BUFFER_REPS,
1048                             i -> ($type$)(i * 5));
1049             }),
1050             withToString("$type$[i + 1]", (int s) -> {
1051                 return fill(s * BUFFER_REPS,
1052                             i -> ((($type$)(i + 1) == 0) ? 1 : ($type$)(i + 1)));
1053             }),
1054             withToString("$type$[cornerCaseValue(i)]", (int s) -> {
1055                 return fill(s * BUFFER_REPS,
1056                             i -> cornerCaseValue(i));
1057             })
1058     );
1059 
1060     // Create combinations of pairs
1061     // @@@ Might be sensitive to order e.g. div by 0
1062     static final List<List<IntFunction<$type$[]>>> $TYPE$_GENERATOR_PAIRS =
1063         Stream.of($TYPE$_GENERATORS.get(0)).
1064                 flatMap(fa -> $TYPE$_GENERATORS.stream().skip(1).map(fb -> List.of(fa, fb))).
1065                 collect(Collectors.toList());
1066 
1067     @DataProvider
1068     public Object[][] boolUnaryOpProvider() {
1069         return BOOL_ARRAY_GENERATORS.stream().
1070                 map(f -> new Object[]{f}).
1071                 toArray(Object[][]::new);
1072     }
1073 
1074     static final List<List<IntFunction<$type$[]>>> $TYPE$_GENERATOR_TRIPLES =
1075         $TYPE$_GENERATOR_PAIRS.stream().
1076                 flatMap(pair -> $TYPE$_GENERATORS.stream().map(f -> List.of(pair.get(0), pair.get(1), f))).
1077                 collect(Collectors.toList());
1078 
1079     @DataProvider
1080     public Object[][] $type$BinaryOpProvider() {
1081         return $TYPE$_GENERATOR_PAIRS.stream().map(List::toArray).
1082                 toArray(Object[][]::new);
1083     }
1084 
1085     @DataProvider
1086     public Object[][] $type$IndexedOpProvider() {
1087         return $TYPE$_GENERATOR_PAIRS.stream().map(List::toArray).
1088                 toArray(Object[][]::new);
1089     }
1090 
1091     @DataProvider
1092     public Object[][] $type$BinaryOpMaskProvider() {
1093         return BOOLEAN_MASK_GENERATORS.stream().
1094                 flatMap(fm -> $TYPE$_GENERATOR_PAIRS.stream().map(lfa -> {
1095                     return Stream.concat(lfa.stream(), Stream.of(fm)).toArray();
1096                 })).
1097                 toArray(Object[][]::new);
1098     }
1099 
1100     @DataProvider
1101     public Object[][] $type$TernaryOpProvider() {
1102         return $TYPE$_GENERATOR_TRIPLES.stream().map(List::toArray).
1103                 toArray(Object[][]::new);
1104     }
1105 
1106     @DataProvider
1107     public Object[][] $type$TernaryOpMaskProvider() {
1108         return BOOLEAN_MASK_GENERATORS.stream().
1109                 flatMap(fm -> $TYPE$_GENERATOR_TRIPLES.stream().map(lfa -> {
1110                     return Stream.concat(lfa.stream(), Stream.of(fm)).toArray();
1111                 })).
1112                 toArray(Object[][]::new);
1113     }
1114 
1115     @DataProvider
1116     public Object[][] $type$UnaryOpProvider() {
1117         return $TYPE$_GENERATORS.stream().
1118                 map(f -> new Object[]{f}).
1119                 toArray(Object[][]::new);
1120     }
1121 
1122     @DataProvider
1123     public Object[][] $type$UnaryOpMaskProvider() {
1124         return BOOLEAN_MASK_GENERATORS.stream().
1125                 flatMap(fm -> $TYPE$_GENERATORS.stream().map(fa -> {
1126                     return new Object[] {fa, fm};
1127                 })).
1128                 toArray(Object[][]::new);
1129     }
1130 
1131 #if[!Int]
1132 #if[!byteOrShort]
1133     @DataProvider
1134     public Object[][] $type$toIntUnaryOpProvider() {
1135         return INT_$TYPE$_GENERATORS.stream().
1136                 map(f -> new Object[]{f}).
1137                 toArray(Object[][]::new);
1138     }
1139 #end[!byteOrShort]
1140 #end[!Int]
1141 
1142 #if[FP]
1143     @DataProvider
1144     public Object[][] $type$toLongUnaryOpProvider() {
1145         return LONG_$TYPE$_GENERATORS.stream().
1146                 map(f -> new Object[]{f}).
1147                 toArray(Object[][]::new);
1148     }
1149 #end[FP]
1150 
1151     @DataProvider
1152     public Object[][] maskProvider() {
1153         return BOOLEAN_MASK_GENERATORS.stream().
1154                 map(f -> new Object[]{f}).
1155                 toArray(Object[][]::new);
1156     }
1157 
1158     @DataProvider
1159     public Object[][] maskCompareOpProvider() {
1160         return BOOLEAN_MASK_COMPARE_GENERATOR_PAIRS.stream().map(List::toArray).
1161                 toArray(Object[][]::new);
1162     }
1163 
1164     @DataProvider
1165     public Object[][] shuffleProvider() {
1166         return INT_SHUFFLE_GENERATORS.stream().
1167                 map(f -> new Object[]{f}).
1168                 toArray(Object[][]::new);
1169     }
1170 
1171     @DataProvider
1172     public Object[][] shuffleCompareOpProvider() {
1173         return INT_SHUFFLE_COMPARE_GENERATOR_PAIRS.stream().map(List::toArray).
1174                 toArray(Object[][]::new);
1175     }
1176 
1177     @DataProvider
1178     public Object[][] $type$UnaryOpShuffleProvider() {
1179         return INT_SHUFFLE_GENERATORS.stream().
1180                 flatMap(fs -> $TYPE$_GENERATORS.stream().map(fa -> {
1181                     return new Object[] {fa, fs};
1182                 })).
1183                 toArray(Object[][]::new);
1184     }
1185 
1186     @DataProvider
1187     public Object[][] $type$UnaryOpShuffleMaskProvider() {
1188         return BOOLEAN_MASK_GENERATORS.stream().
1189                 flatMap(fm -> INT_SHUFFLE_GENERATORS.stream().
1190                     flatMap(fs -> $TYPE$_GENERATORS.stream().map(fa -> {
1191                         return new Object[] {fa, fs, fm};
1192                 }))).
1193                 toArray(Object[][]::new);
1194     }
1195 
1196 #if[!Int]
1197     static final List<BiFunction<Integer,Integer,$type$[]>> $TYPE$_SHUFFLE_GENERATORS = List.of(
1198             withToStringBi("shuffle[random]", (Integer l, Integer m) -> {
1199                 $type$[] a = new $type$[l];
1200 #if[ByteMax]
1201                 int upper = Math.min(Byte.MAX_VALUE + 1, m);
1202 #else[ByteMax]
1203                 int upper = m;
1204 #end[ByteMax]
1205                 for (int i = 0; i < 1; i++) {
1206                     a[i] = ($type$)RAND.nextInt(upper);
1207                 }
1208                 return a;
1209             })
1210     );
1211 
1212     @DataProvider
1213     public Object[][] $type$UnaryOpSelectFromProvider() {
1214         return $TYPE$_SHUFFLE_GENERATORS.stream().
1215                 flatMap(fs -> $TYPE$_GENERATORS.stream().map(fa -> {
1216                     return new Object[] {fa, fs};
1217                 })).
1218                 toArray(Object[][]::new);
1219     }
1220 
1221     @DataProvider
1222     public Object[][] $type$UnaryOpSelectFromMaskProvider() {
1223         return BOOLEAN_MASK_GENERATORS.stream().
1224                 flatMap(fm -> $TYPE$_SHUFFLE_GENERATORS.stream().
1225                     flatMap(fs -> $TYPE$_GENERATORS.stream().map(fa -> {
1226                         return new Object[] {fa, fs, fm};
1227                 }))).
1228                 toArray(Object[][]::new);
1229     }
1230 
1231 #end[!Int]
1232 
1233     static final List<IntFunction<$type$[]>> $TYPE$_COMPARE_GENERATORS = List.of(
1234             withToString("$type$[i]", (int s) -> {
1235                 return fill(s * BUFFER_REPS,
1236                             i -> ($type$)i);
1237             }),
1238             withToString("$type$[i - length / 2]", (int s) -> {
1239                 return fill(s * BUFFER_REPS,
1240                             i -> ($type$)(i - (s * BUFFER_REPS / 2)));
1241             }),
1242             withToString("$type$[i + 1]", (int s) -> {
1243                 return fill(s * BUFFER_REPS,
1244                             i -> ($type$)(i + 1));
1245             }),
1246             withToString("$type$[i - 2]", (int s) -> {
1247                 return fill(s * BUFFER_REPS,
1248                             i -> ($type$)(i - 2));
1249             }),
1250             withToString("$type$[zigZag(i)]", (int s) -> {
1251                 return fill(s * BUFFER_REPS,
1252                             i -> i%3 == 0 ? ($type$)i : (i%3 == 1 ? ($type$)(i + 1) : ($type$)(i - 2)));
1253             }),
1254             withToString("$type$[cornerCaseValue(i)]", (int s) -> {
1255                 return fill(s * BUFFER_REPS,
1256                             i -> cornerCaseValue(i));
1257             })
1258     );
1259 
1260     static final List<List<IntFunction<$type$[]>>> $TYPE$_TEST_GENERATOR_ARGS =
1261         $TYPE$_COMPARE_GENERATORS.stream().
1262                 map(fa -> List.of(fa)).
1263                 collect(Collectors.toList());
1264 
1265     @DataProvider
1266     public Object[][] $type$TestOpProvider() {
1267         return $TYPE$_TEST_GENERATOR_ARGS.stream().map(List::toArray).
1268                 toArray(Object[][]::new);
1269     }
1270 
1271     @DataProvider
1272     public Object[][] $type$TestOpMaskProvider() {
1273         return BOOLEAN_MASK_GENERATORS.stream().
1274                 flatMap(fm -> $TYPE$_TEST_GENERATOR_ARGS.stream().map(lfa -> {
1275                     return Stream.concat(lfa.stream(), Stream.of(fm)).toArray();
1276                 })).
1277                 toArray(Object[][]::new);
1278     }
1279 
1280     static final List<List<IntFunction<$type$[]>>> $TYPE$_COMPARE_GENERATOR_PAIRS =
1281         $TYPE$_COMPARE_GENERATORS.stream().
1282                 flatMap(fa -> $TYPE$_COMPARE_GENERATORS.stream().map(fb -> List.of(fa, fb))).
1283                 collect(Collectors.toList());
1284 
1285     @DataProvider
1286     public Object[][] $type$CompareOpProvider() {
1287         return $TYPE$_COMPARE_GENERATOR_PAIRS.stream().map(List::toArray).
1288                 toArray(Object[][]::new);
1289     }
1290 
1291     @DataProvider
1292     public Object[][] $type$CompareOpMaskProvider() {
1293         return BOOLEAN_MASK_GENERATORS.stream().
1294                 flatMap(fm -> $TYPE$_COMPARE_GENERATOR_PAIRS.stream().map(lfa -> {
1295                     return Stream.concat(lfa.stream(), Stream.of(fm)).toArray();
1296                 })).
1297                 toArray(Object[][]::new);
1298     }
1299 
1300     interface To$Type$F {
1301         $type$ apply(int i);
1302     }
1303 
1304     static $type$[] fill(int s , To$Type$F f) {
1305         return fill(new $type$[s], f);
1306     }
1307 
1308     static $type$[] fill($type$[] a, To$Type$F f) {
1309         for (int i = 0; i < a.length; i++) {
1310             a[i] = f.apply(i);
1311         }
1312         return a;
1313     }
1314 
1315     static $type$ cornerCaseValue(int i) {
1316 #if[FP]
1317         switch(i % 7) {
1318             case 0:
1319                 return $Wideboxtype$.MAX_VALUE;
1320             case 1:
1321                 return $Wideboxtype$.MIN_VALUE;
1322             case 2:
1323                 return $Wideboxtype$.NEGATIVE_INFINITY;
1324             case 3:
1325                 return $Wideboxtype$.POSITIVE_INFINITY;
1326             case 4:
1327                 return $Wideboxtype$.NaN;
1328             case 5:
1329                 return ($type$)0.0;
1330             default:
1331                 return ($type$)-0.0;
1332         }
1333 #else[FP]
1334         switch(i % 5) {
1335             case 0:
1336                 return $Wideboxtype$.MAX_VALUE;
1337             case 1:
1338                 return $Wideboxtype$.MIN_VALUE;
1339             case 2:
1340                 return $Wideboxtype$.MIN_VALUE;
1341             case 3:
1342                 return $Wideboxtype$.MAX_VALUE;
1343             default:
1344                 return ($type$)0;
1345         }
1346 #end[FP]
1347     }
1348 
1349     static $type$ get($type$[] a, int i) {
1350         return ($type$) a[i];
1351     }
1352 
1353     static final IntFunction<$type$[]> fr = (vl) -> {
1354         int length = BUFFER_REPS * vl;
1355         return new $type$[length];
1356     };
1357 
1358     static final IntFunction<boolean[]> fmr = (vl) -> {
1359         int length = BUFFER_REPS * vl;
1360         return new boolean[length];
1361     };
1362 
1363 #if[!Long]
1364     static final IntFunction<long[]> lfr = (vl) -> {
1365         int length = BUFFER_REPS * vl;
1366         return new long[length];
1367     };
1368 #end[!Long]
1369 
1370 #if[BITWISE]
1371     static void replaceZero($type$[] a, $type$ v) {
1372         for (int i = 0; i < a.length; i++) {
1373             if (a[i] == 0) {
1374                 a[i] = v;
1375             }
1376         }
1377     }
1378 
1379     static void replaceZero($type$[] a, boolean[] mask, $type$ v) {
1380         for (int i = 0; i < a.length; i++) {
1381             if (mask[i % mask.length] && a[i] == 0) {
1382                 a[i] = v;
1383             }
1384         }
1385     }
1386 
1387     static $type$ ROL_scalar($type$ a, $type$ b) {
1388 #if[intOrLong]
1389         return $Wideboxtype$.rotateLeft(a, ((int)b));
1390 #else[intOrLong]
1391 #if[short]
1392         return (short)(((((short)a) & 0xFFFF) << (b & 15)) | ((((short)a) & 0xFFFF) >>> (16 - (b & 15))));
1393 #else[short]
1394         return (byte)(((((byte)a) & 0xFF) << (b & 7)) | ((((byte)a) & 0xFF) >>> (8 - (b & 7))));
1395 #end[short]
1396 #end[intOrLong]
1397     }
1398 
1399     static $type$ ROR_scalar($type$ a, $type$ b) {
1400 #if[intOrLong]
1401         return $Wideboxtype$.rotateRight(a, ((int)b));
1402 #else[intOrLong]
1403 #if[short]
1404         return (short)(((((short)a) & 0xFFFF) >>> (b & 15)) | ((((short)a) & 0xFFFF) << (16 - (b & 15))));
1405 #else[short]
1406         return (byte)(((((byte)a) & 0xFF) >>> (b & 7)) | ((((byte)a) & 0xFF) << (8 - (b & 7))));
1407 #end[short]
1408 #end[intOrLong]
1409     }
1410 #end[BITWISE]
1411 
1412     static boolean eq($type$ a, $type$ b) {
1413         return a == b;
1414     }
1415 
1416     static boolean neq($type$ a, $type$ b) {
1417         return a != b;
1418     }
1419 
1420     static boolean lt($type$ a, $type$ b) {
1421         return a < b;
1422     }
1423 
1424     static boolean le($type$ a, $type$ b) {
1425         return a <= b;
1426     }
1427 
1428     static boolean gt($type$ a, $type$ b) {
1429         return a > b;
1430     }
1431 
1432     static boolean ge($type$ a, $type$ b) {
1433         return a >= b;
1434     }
1435 
1436 #if[!FP]
1437     static boolean ult($type$ a, $type$ b) {
1438         return $Boxtype$.compareUnsigned(a, b) < 0;
1439     }
1440 
1441     static boolean ule($type$ a, $type$ b) {
1442         return $Boxtype$.compareUnsigned(a, b) <= 0;
1443     }
1444 
1445     static boolean ugt($type$ a, $type$ b) {
1446         return $Boxtype$.compareUnsigned(a, b) > 0;
1447     }
1448 
1449     static boolean uge($type$ a, $type$ b) {
1450         return $Boxtype$.compareUnsigned(a, b) >= 0;
1451     }
1452 #end[!FP]
1453 
1454     @Test
1455     static void smokeTest1() {
1456         $abstractvectortype$ three = $abstractvectortype$.broadcast(SPECIES, (byte)-3);
1457         $abstractvectortype$ three2 = ($abstractvectortype$) SPECIES.broadcast(-3);
1458         assert(three.eq(three2).allTrue());
1459         $abstractvectortype$ three3 = three2.broadcast(1).broadcast(-3);
1460         assert(three.eq(three3).allTrue());
1461         int scale = 2;
1462         Class<?> ETYPE = $type$.class;
1463         if (ETYPE == double.class || ETYPE == long.class)
1464             scale = 1000000;
1465         else if (ETYPE == byte.class && SPECIES.length() >= 64)
1466             scale = 1;
1467         $abstractvectortype$ higher = three.addIndex(scale);
1468         VectorMask<$Boxtype$> m = three.compare(VectorOperators.LE, higher);
1469         assert(m.allTrue());
1470         m = higher.min(($type$)-1).test(VectorOperators.IS_NEGATIVE);
1471         assert(m.allTrue());
1472 #if[FP]
1473         m = higher.test(VectorOperators.IS_FINITE);
1474         assert(m.allTrue());
1475 #end[FP]
1476         $type$ max = higher.reduceLanes(VectorOperators.MAX);
1477         assert(max == -3 + scale * (SPECIES.length()-1));
1478     }
1479 
1480     private static $type$[]
1481     bothToArray($abstractvectortype$ a, $abstractvectortype$ b) {
1482         $type$[] r = new $type$[a.length() + b.length()];
1483         a.intoArray(r, 0);
1484         b.intoArray(r, a.length());
1485         return r;
1486     }
1487 
1488     @Test
1489     static void smokeTest2() {
1490         // Do some zipping and shuffling.
1491         $abstractvectortype$ io = ($abstractvectortype$) SPECIES.broadcast(0).addIndex(1);
1492         $abstractvectortype$ io2 = ($abstractvectortype$) VectorShuffle.iota(SPECIES,0,1,false).toVector();
1493         Assert.assertEquals(io, io2);
1494         $abstractvectortype$ a = io.add(($type$)1); //[1,2]
1495         $abstractvectortype$ b = a.neg();  //[-1,-2]
1496         $type$[] abValues = bothToArray(a,b); //[1,2,-1,-2]
1497         VectorShuffle<$Boxtype$> zip0 = VectorShuffle.makeZip(SPECIES, 0);
1498         VectorShuffle<$Boxtype$> zip1 = VectorShuffle.makeZip(SPECIES, 1);
1499         $abstractvectortype$ zab0 = a.rearrange(zip0,b); //[1,-1]
1500         $abstractvectortype$ zab1 = a.rearrange(zip1,b); //[2,-2]
1501         $type$[] zabValues = bothToArray(zab0, zab1); //[1,-1,2,-2]
1502         // manually zip
1503         $type$[] manual = new $type$[zabValues.length];
1504         for (int i = 0; i < manual.length; i += 2) {
1505             manual[i+0] = abValues[i/2];
1506             manual[i+1] = abValues[a.length() + i/2];
1507         }
1508         Assert.assertEquals(Arrays.toString(zabValues), Arrays.toString(manual));
1509         VectorShuffle<$Boxtype$> unz0 = VectorShuffle.makeUnzip(SPECIES, 0);
1510         VectorShuffle<$Boxtype$> unz1 = VectorShuffle.makeUnzip(SPECIES, 1);
1511         $abstractvectortype$ uab0 = zab0.rearrange(unz0,zab1);
1512         $abstractvectortype$ uab1 = zab0.rearrange(unz1,zab1);
1513         $type$[] abValues1 = bothToArray(uab0, uab1);
1514         Assert.assertEquals(Arrays.toString(abValues), Arrays.toString(abValues1));
1515     }
1516 
1517     static void iotaShuffle() {
1518         $abstractvectortype$ io = ($abstractvectortype$) SPECIES.broadcast(0).addIndex(1);
1519         $abstractvectortype$ io2 = ($abstractvectortype$) VectorShuffle.iota(SPECIES, 0 , 1, false).toVector();
1520         Assert.assertEquals(io, io2);
1521     }
1522 
1523     @Test
1524     // Test all shuffle related operations.
1525     static void shuffleTest() {
1526         // To test backend instructions, make sure that C2 is used.
1527         for (int loop = 0; loop < INVOC_COUNT * INVOC_COUNT; loop++) {
1528             iotaShuffle();
1529         }
1530     }
1531 
1532     @Test
1533     void viewAsIntegeralLanesTest() {
1534 #if[FP]
1535         Vector<?> asIntegral = SPECIES.zero().viewAsIntegralLanes();
1536         VectorSpecies<?> asIntegralSpecies = asIntegral.species();
1537         Assert.assertNotEquals(asIntegralSpecies.elementType(), SPECIES.elementType());
1538         Assert.assertEquals(asIntegralSpecies.vectorShape(), SPECIES.vectorShape());
1539         Assert.assertEquals(asIntegralSpecies.length(), SPECIES.length());
1540         Assert.assertEquals(asIntegral.viewAsFloatingLanes().species(), SPECIES);
1541 #else[FP]
1542         Vector<?> asIntegral = SPECIES.zero().viewAsIntegralLanes();
1543         Assert.assertEquals(asIntegral.species(), SPECIES);
1544 #end[FP]
1545     }
1546 
1547 #if[FP]
1548     @Test
1549     void viewAsFloatingLanesTest() {
1550         Vector<?> asFloating = SPECIES.zero().viewAsFloatingLanes();
1551         Assert.assertEquals(asFloating.species(), SPECIES);
1552     }
1553 #else[FP]
1554 #if[byteOrShort]
1555     @Test(expectedExceptions = UnsupportedOperationException.class)
1556     void viewAsFloatingLanesTest() {
1557         SPECIES.zero().viewAsFloatingLanes();
1558     }
1559 #else[byteOrShort]
1560     @Test
1561     void viewAsFloatingLanesTest() {
1562         Vector<?> asFloating = SPECIES.zero().viewAsFloatingLanes();
1563         VectorSpecies<?> asFloatingSpecies = asFloating.species();
1564         Assert.assertNotEquals(asFloatingSpecies.elementType(), SPECIES.elementType());
1565         Assert.assertEquals(asFloatingSpecies.vectorShape(), SPECIES.vectorShape());
1566         Assert.assertEquals(asFloatingSpecies.length(), SPECIES.length());
1567         Assert.assertEquals(asFloating.viewAsIntegralLanes().species(), SPECIES);
1568     }
1569 #end[byteOrShort]
1570 #end[FP]
1571 
1572 #if[BITWISE]
1573     @Test
1574     // Test div by 0.
1575     static void bitwiseDivByZeroSmokeTest() {
1576         try {
1577             $abstractvectortype$ a = ($abstractvectortype$) SPECIES.broadcast(0).addIndex(1);
1578             $abstractvectortype$ b = ($abstractvectortype$) SPECIES.broadcast(0);
1579             a.div(b);
1580             Assert.fail();
1581         } catch (ArithmeticException e) {
1582         }
1583 
1584         try {
1585             $abstractvectortype$ a = ($abstractvectortype$) SPECIES.broadcast(0).addIndex(1);
1586             $abstractvectortype$ b = ($abstractvectortype$) SPECIES.broadcast(0);
1587             VectorMask<$Boxtype$> m = a.lt(($type$) 1);
1588             a.div(b, m);
1589             Assert.fail();
1590         } catch (ArithmeticException e) {
1591         }
1592     }
1593 #end[BITWISE]