1 /*
   2  * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.util.function.IntFunction;
  28 import java.util.HashMap;
  29 import java.util.ArrayList;
  30 
  31 import jdk.internal.vm.annotation.ForceInline;
  32 import jdk.internal.vm.annotation.Stable;
  33 
  34 import jdk.internal.vm.vector.VectorSupport;
  35 
  36 /**
  37  * This class consists solely of static constants
  38  * that describe lane-wise vector operations, plus nested interfaces
  39  * which classify them.
  40  * The static constants serve as tokens denoting specifically
  41  * requested lane operations in vector expressions, such
  42  * as the token {@code ADD} in
  43  * {@code w = v0.}{@link
  44  * Vector#lanewise(VectorOperators.Binary,Vector)
  45  * lanewise}{@code (ADD, v1)}.
  46  *
  47  * <p>
  48  *
  49  * The documentation for each individual operator token is very brief,
  50  * giving a symbolic Java expression for the operation that the token
  51  * requests.  Those symbolic expressions use the following conventional
  52  * elements:
  53  * <ul>
  54  * <li>{@code a}, {@code b}, {@code c} &mdash; names of lane values
  55  *
  56  * <li>Java operators like {@code +}, {@code ?:}, etc. &mdash;
  57  * expression operators
  58  *
  59  * <li>Java method names like {@code max}, {@code sin}, etc. &mdash;
  60  * methods in standard classes like {@code Math}, {@code Double}, etc.
  61  * Unqualified method names should be read as if in the context of a
  62  * static import, and with resolution of overloading.
  63  *
  64  * <li>{@code bits(x)} &mdash; a function call which produces the
  65  * underlying bits of the value {@code x}.  If {@code x} is a floating
  66  * point value, this is either {@code doubleToLongBits(x)} or
  67  * {@code floatToIntBits(x)}.  Otherwise, the value is just {@code x}.
  68  *
  69  * <li>{@code ESIZE} &mdash; the size in bytes of the operand type
  70  *
  71  * <li>{@code intVal}, {@code byteVal}, etc. &mdash; the operand of a
  72  * conversion, with the indicated type
  73  * </ul>
  74  *
  75  * <h2>Operations on floating point vectors</h2>
  76  * <ul>
  77  * <li>Lane-wise vector operations that apply to floating point vectors
  78  * follow the accuracy and monotonicity specifications of the equivalent
  79  * Java operation or method mentioned in its documentation unless specified otherwise.
  80  * If the vector element type is {@code float} and the Java operation or
  81  * method only accepts and returns {@code double} values, then the scalar operation
  82  * on each lane is adapted to cast operands and the result, specifically widening
  83  * {@code float} operands to {@code double} operands and narrowing the {@code double}
  84  * result to a {@code float}.
  85  *
  86  * <li id="fp_assoc">Certain associative operations that apply to floating point
  87  * vectors are not truly associative on the floating point lane values.
  88  * Specifically, {@link #ADD} and {@link #MUL} used with cross-lane reduction
  89  * operations, such as {@link FloatVector#reduceLanes(Associative)}.
  90  * The result of such an operation is a function both of the input
  91  * values (vector and mask) as well as the order of the scalar operations
  92  * applied to combine lane values.
  93  * In such cases the order is intentionally not defined.
  94  * This allows the JVM to generate optimal machine code for the underlying
  95  * platform at runtime.  If the platform supports a vector instruction
  96  * to add or multiply all values in the vector, or if there is some
  97  * other efficient machine code sequence, then the JVM has the option of
  98  * generating this machine code. Otherwise, the default
  99  * implementation is applied, which adds vector elements
 100  * sequentially from beginning to end.  For this reason, the
 101  * result of such an operation may vary for the same input values.
 102  * </ul>
 103  *
 104  * <p> Note that a particular operator token may apply to several
 105  * different lane types.  Thus, these tokens behave like overloaded
 106  * operators or methods, not like type-specific method handles or
 107  * lambdas.  Also unlike method handles or lambdas, these operators do
 108  * not possess operational semantics; they have no {@code apply} or
 109  * {@code invoke} method.  They are used only to request lane
 110  * operations from vector objects, and cannot (by themselves) perform
 111  * operations on individual lane values.
 112  *
 113  */
 114 public abstract class VectorOperators {
 115     private VectorOperators() { }
 116 
 117     /**
 118      * Root type for all operator tokens, providing queries for common
 119      * properties such as arity, argument and return types, symbolic
 120      * name, and operator name.
 121      *
 122      * @see VectorOperators.Unary Unary
 123      * @see VectorOperators.Binary Binary
 124      * @see VectorOperators.Ternary Ternary
 125      * @see VectorOperators.Associative Associative
 126      * @see VectorOperators.Comparison Comparison
 127      * @see VectorOperators.Test Test
 128      * @see VectorOperators.Conversion Conversion
 129      *
 130      * @apiNote
 131      * User code should not implement this interface.  A future release of
 132      * this type may restrict implementations to be members of the same
 133      * package.
 134      */
 135     public interface Operator {
 136         /**
 137          * Returns the symbolic name of this operator,
 138          * as a constant in {@link VectorOperators}.
 139          *
 140          * The operator symbol, Java method name,
 141          * or example expression,
 142          * such as {@code "+"}, {@code "max"} or {@code "-a"},
 143          * is also available as {@link #operatorName()}.
 144          *
 145          * @return the symbolic name of this operator,
 146          *         such as {@code "ADD"}
 147          */
 148         String name();
 149 
 150         /**
 151          * Returns the Java operator symbol or method
 152          * name corresponding to this operator.
 153          * If there is no symbol or method, return a
 154          * string containing a representative expression
 155          * for the operator, using operand names
 156          * {@code a}, {@code b} (for non-unary operators),
 157          * and {@code c} (for ternary operators).
 158          *
 159          * The symbolic name of the constant,
 160          * such as {@code "ADD"},
 161          * is also available as {@link #name()}.
 162          *
 163          * @return an operator token, such as {@code "+"},
 164          *         or a method name, such as {@code "max"},
 165          *         or a representative expression, such as {@code "-a"}
 166          */
 167         String operatorName();
 168 
 169         /**
 170          * Returns the arity of this operator (1, 2, or 3).
 171          * @return the arity of this operator (1, 2, or 3)
 172          */
 173         int arity();
 174 
 175         /**
 176          * Reports whether this operator returns a boolean (a mask).
 177          * A boolean operator also reports {@code boolean} as the
 178          * {@code rangeType}.
 179          * @return whether this operator returns a boolean
 180          */
 181         boolean isBoolean();
 182 
 183         /**
 184          * Reports the special return type of this operator.
 185          * If this operator is a boolean, returns {@code boolean.class}.
 186          * If this operator is a {@code Conversion},
 187          * returns its {@linkplain Conversion#rangeType range type}.
 188          *
 189          * Otherwise, the operator's return value always has
 190          * whatever type was given as an input, and this method
 191          * returns {@code Object.class} to denote that fact.
 192          * @return the special return type, or {@code Object.class} if none
 193          */
 194         Class<?> rangeType();
 195 
 196         /**
 197          * Returns the associativity of this operator.
 198          * Only binary operators can be associative.
 199          * @return the associativity of this operator
 200          */
 201         boolean isAssociative();
 202 
 203         /**
 204          * Reports whether this operator is compatible with
 205          * the proposed element type.
 206          *
 207          * First, unrestricted operators are compatible with all element
 208          * types.
 209          *
 210          * Next, if the element type is {@code double} or {@code float}
 211          * and the operator is restricted to floating point types, it is
 212          * compatible.
 213          *
 214          * Otherwise, if the element type is neither {@code double} nor
 215          * {@code float} and the operator is restricted to integral
 216          * types, it is compatible.  Otherwise, the operator is not
 217          * compatible.
 218          *
 219          * @param elementType the proposed operand type for the operator
 220          * @return whether the proposed type is compatible with this operator
 221          */
 222         boolean compatibleWith(Class<?> elementType);
 223 
 224         // FIXME: Maybe add a query about architectural support.
 225     }
 226 
 227     /**
 228      * Type for all
 229      * <a href="Vector.html#lane-wise">lane-wise</a>
 230      * unary (one-argument) operators,
 231      * usable in expressions like {@code w = v0.}{@link
 232      * Vector#lanewise(VectorOperators.Unary)
 233      * lanewise}{@code (NEG)}.
 234      *
 235      * @apiNote
 236      * User code should not implement this interface.  A future release of
 237      * this type may restrict implementations to be members of the same
 238      * package.
 239      */
 240     public interface Unary extends Operator {
 241     }
 242 
 243     /**
 244      * Type for all
 245      * <a href="Vector.html#lane-wise">lane-wise</a>
 246      * binary (two-argument) operators,
 247      * usable in expressions like {@code w = v0.}{@link
 248      * Vector#lanewise(VectorOperators.Binary,Vector)
 249      * lanewise}{@code (ADD, v1)}.
 250      *
 251      * @apiNote
 252      * User code should not implement this interface.  A future release of
 253      * this type may restrict implementations to be members of the same
 254      * package.
 255      */
 256     public interface Binary extends Operator {
 257     }
 258 
 259     /**
 260      * Type for all
 261      * <a href="Vector.html#lane-wise">lane-wise</a>
 262      * ternary (three-argument) operators,
 263      * usable in expressions like {@code w = v0.}{@link
 264      * Vector#lanewise(VectorOperators.Ternary,Vector,Vector)
 265      * lanewise}{@code (FMA, v1, v2)}.
 266      *
 267      * @apiNote
 268      * User code should not implement this interface.  A future release of
 269      * this type may restrict implementations to be members of the same
 270      * package.
 271      */
 272     public interface Ternary extends Operator {
 273     }
 274 
 275     /**
 276      * Type for all reassociating
 277      * <a href="Vector.html#lane-wise">lane-wise</a>
 278      * binary operators,
 279      * usable in expressions like {@code e = v0.}{@link
 280      * IntVector#reduceLanes(VectorOperators.Associative)
 281      * reduceLanes}{@code (ADD)}.
 282      *
 283      * @apiNote
 284      * User code should not implement this interface.  A future release of
 285      * this type may restrict implementations to be members of the same
 286      * package.
 287      */
 288     public interface Associative extends Binary {
 289     }
 290 
 291     /**
 292      * Type for all unary
 293      * <a href="Vector.html#lane-wise">lane-wise</a>
 294      * boolean tests on lane values,
 295      * usable in expressions like {@code m = v0.}{@link
 296      * FloatVector#test(VectorOperators.Test)
 297      * test}{@code (IS_FINITE)}.
 298      *
 299      * @apiNote
 300      * User code should not implement this interface.  A future release of
 301      * this type may restrict implementations to be members of the same
 302      * package.
 303      */
 304     public interface Test extends Operator {
 305     }
 306 
 307     /**
 308      * Type for all binary
 309      * <a href="Vector.html#lane-wise">lane-wise</a>
 310      * boolean comparisons on lane values,
 311      * usable in expressions like {@code m = v0.}{@link
 312      * Vector#compare(VectorOperators.Comparison,Vector)
 313      * compare}{@code (LT, v1)}.
 314      *
 315      * @apiNote
 316      * User code should not implement this interface.  A future release of
 317      * this type may restrict implementations to be members of the same
 318      * package.
 319      */
 320     public interface Comparison extends Operator {
 321     }
 322 
 323     /**
 324      * Type for all
 325      * <a href="Vector.html#lane-wise">lane-wise</a>
 326      * conversions on lane values,
 327      * usable in expressions like {@code w1 = v0.}{@link
 328      * Vector#convert(VectorOperators.Conversion,int)
 329      * convert}{@code (I2D, 1)}.
 330      *
 331      * @param <E> the boxed element type for the conversion
 332      *        domain type (the input lane type)
 333      * @param <F> the boxed element type for the conversion
 334      *        range type (the output lane type)
 335      *
 336      * @apiNote
 337      * User code should not implement this interface.  A future release of
 338      * this type may restrict implementations to be members of the same
 339      * package.
 340      */
 341     public interface Conversion<E,F> extends Operator {
 342         /**
 343          * The domain of this conversion, a primitive type.
 344          * @return the domain of this conversion
 345          */
 346         Class<E> domainType();
 347 
 348         /**
 349          * The range of this conversion, a primitive type.
 350          * @return the range of this conversion
 351          */
 352         @Override
 353         Class<F> rangeType();
 354 
 355         /**
 356          * Ensures that this conversion has the
 357          * desired domain and range types.
 358          * @param from the desired domain type
 359          * @param to the desired range type
 360          * @param <E> the desired domain type
 361          * @param <F> the desired range type
 362          * @return this conversion object, with validated domain and range
 363          */
 364         <E,F> Conversion<E,F> check(Class<E> from, Class<F> to);
 365 
 366         /// Factories.
 367 
 368         /**
 369          * The Java language assignment or casting conversion between two types.
 370          * @param <E> the domain type (boxed version of a lane type)
 371          * @param <F> the range type (boxed version of a lane type)
 372          * @param from the type of the value to convert
 373          * @param to the desired type after conversion
 374          * @return a Java assignment or casting conversion
 375          */
 376         @ForceInline
 377         static <E,F> Conversion<E,F> ofCast(Class<E> from, Class<F> to) {
 378             LaneType dom = LaneType.of(from);
 379             LaneType ran = LaneType.of(to);
 380             return ConversionImpl.ofCast(dom, ran).check(from, to);
 381         }
 382 
 383         /**
 384          * The bitwise reinterpretation between two types.
 385          * @param <E> the domain type (boxed version of a lane type)
 386          * @param <F> the range type (boxed version of a lane type)
 387          * @param from the type of the value to reinterpret
 388          * @param to the desired type after reinterpretation
 389          * @return a bitwise reinterpretation conversion
 390          */
 391         @ForceInline
 392         static <E,F> Conversion<E,F> ofReinterpret(Class<E> from, Class<F> to) {
 393             LaneType dom = LaneType.of(from);
 394             LaneType ran = LaneType.of(to);
 395             return ConversionImpl.ofReinterpret(dom, ran).check(from, to);
 396         }
 397 
 398     }
 399 
 400     /*package-private*/
 401     @ForceInline
 402     static int opCode(Operator op, int requireKind, int forbidKind) {
 403         return ((OperatorImpl)op).opCode(requireKind, forbidKind);
 404     }
 405 
 406     /*package-private*/
 407     @ForceInline
 408     static boolean opKind(Operator op, int bit) {
 409         return ((OperatorImpl)op).opKind(bit);
 410     }
 411 
 412     /*package-private*/
 413     static final int
 414         VO_ALL                     = 0,
 415         VO_UNARY                   = 0x001,
 416         VO_BINARY                  = 0x002,
 417         VO_TERNARY                 = 0x003,
 418         VO_ARITY_MASK              = (VO_UNARY|VO_BINARY|VO_TERNARY),
 419         VO_ASSOC                   = 0x004,
 420         VO_SHIFT                   = 0x008,
 421         VO_BOOL                    = 0x010,
 422         VO_CONV                    = 0x020,
 423         VO_PRIVATE                 = 0x040, // not public, invisible
 424         VO_SPECIAL                 = 0x080, // random special handling
 425         VO_NOFP                    = 0x100,
 426         VO_ONLYFP                  = 0x200,
 427         VO_OPCODE_VALID            = 0x800,
 428         VO_OPCODE_SHIFT            = 12,
 429         VO_OPCODE_LIMIT            = 0x400,
 430         VO_RAN_SHIFT               = 0,
 431         VO_DOM_SHIFT               = 4,
 432         VO_DOM_RAN_MASK            = 0x0FF,
 433         VO_KIND_CAST               = 0x000,
 434         VO_KIND_BITWISE            = 0x100;
 435 
 436     private static final HashMap<Integer, String> OPC_NAME
 437         = new HashMap<>();
 438     private static final HashMap<Integer, String> CMP_OPC_NAME
 439         = new HashMap<>();
 440     private static final HashMap<Integer, String> CONV_OPC_NAME
 441         = new HashMap<>();
 442 
 443     // Unary operators
 444 
 445     /** Produce {@code ~a}.  Integral only. */
 446     public static final /*bitwise*/ Unary NOT = unary("NOT", "~", -1 /*VectorSupport.VECTOR_OP_NOT*/, VO_NOFP | VO_SPECIAL);
 447     /** Produce {@code a==0?0:-1} (zero or minus one).  Integral only. */
 448     public static final /*bitwise*/ Unary ZOMO = unary("ZOMO", "a==0?0:-1", -1 /*VectorSupport.VECTOR_OP_ZOMO*/, VO_NOFP);
 449     /** Produce {@code abs(a)}. */
 450     public static final Unary ABS = unary("ABS", "abs", VectorSupport.VECTOR_OP_ABS, VO_ALL);
 451     /** Produce {@code -a}. */
 452     public static final Unary NEG = unary("NEG", "-a", VectorSupport.VECTOR_OP_NEG, VO_ALL|VO_SPECIAL);
 453     /** Produce {@code bitCount(a)}
 454      * @since 19
 455      */
 456     public static final Unary BIT_COUNT = unary("BIT_COUNT", "bitCount", VectorSupport.VECTOR_OP_BIT_COUNT, VO_NOFP);
 457     /** Produce {@code numberOfTrailingZeros(a)}
 458      * @since 19
 459      */
 460     public static final Unary TRAILING_ZEROS_COUNT = unary("TRAILING_ZEROS_COUNT", "numberOfTrailingZeros", VectorSupport.VECTOR_OP_TZ_COUNT, VO_NOFP);
 461     /** Produce {@code numberOfLeadingZeros(a)}
 462      * @since 19
 463      */
 464     public static final Unary LEADING_ZEROS_COUNT = unary("LEADING_ZEROS_COUNT", "numberOfLeadingZeros", VectorSupport.VECTOR_OP_LZ_COUNT, VO_NOFP);
 465     /** Produce {@code reverse(a)}
 466      * @since 19
 467      */
 468     public static final Unary REVERSE = unary("REVERSE", "reverse", VectorSupport.VECTOR_OP_REVERSE, VO_NOFP);
 469     /** Produce {@code reverseBytes(a)}
 470      * @since 19
 471      */
 472     public static final Unary REVERSE_BYTES = unary("REVERSE_BYTES", "reverseBytes", VectorSupport.VECTOR_OP_REVERSE_BYTES, VO_NOFP);
 473 
 474     /** Produce {@code sin(a)}.  Floating only.
 475      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 476      */
 477     public static final /*float*/ Unary SIN = unary("SIN", "sin", VectorSupport.VECTOR_OP_SIN, VO_ONLYFP);
 478     /** Produce {@code cos(a)}.  Floating only.
 479      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 480      */
 481     public static final /*float*/ Unary COS = unary("COS", "cos", VectorSupport.VECTOR_OP_COS, VO_ONLYFP);
 482     /** Produce {@code tan(a)}.  Floating only.
 483      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 484      */
 485     public static final /*float*/ Unary TAN = unary("TAN", "tan", VectorSupport.VECTOR_OP_TAN, VO_ONLYFP);
 486     /** Produce {@code asin(a)}.  Floating only.
 487      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 488      */
 489     public static final /*float*/ Unary ASIN = unary("ASIN", "asin", VectorSupport.VECTOR_OP_ASIN, VO_ONLYFP);
 490     /** Produce {@code acos(a)}.  Floating only.
 491      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 492      */
 493     public static final /*float*/ Unary ACOS = unary("ACOS", "acos", VectorSupport.VECTOR_OP_ACOS, VO_ONLYFP);
 494     /** Produce {@code atan(a)}.  Floating only.
 495      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 496      */
 497     public static final /*float*/ Unary ATAN = unary("ATAN", "atan", VectorSupport.VECTOR_OP_ATAN, VO_ONLYFP);
 498 
 499     /** Produce {@code exp(a)}.  Floating only.
 500      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 501      */
 502     public static final /*float*/ Unary EXP = unary("EXP", "exp", VectorSupport.VECTOR_OP_EXP, VO_ONLYFP);
 503     /** Produce {@code log(a)}.  Floating only.
 504      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 505      */
 506     public static final /*float*/ Unary LOG = unary("LOG", "log", VectorSupport.VECTOR_OP_LOG, VO_ONLYFP);
 507     /** Produce {@code log10(a)}.  Floating only.
 508      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 509      */
 510     public static final /*float*/ Unary LOG10 = unary("LOG10", "log10", VectorSupport.VECTOR_OP_LOG10, VO_ONLYFP);
 511     /** Produce {@code sqrt(a)}.  Floating only.  See section "Operations on floating point vectors" above */
 512     public static final /*float*/ Unary SQRT = unary("SQRT", "sqrt", VectorSupport.VECTOR_OP_SQRT, VO_ONLYFP);
 513     /** Produce {@code cbrt(a)}.  Floating only.
 514      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 515      */
 516     public static final /*float*/ Unary CBRT = unary("CBRT", "cbrt", VectorSupport.VECTOR_OP_CBRT, VO_ONLYFP);
 517 
 518     /** Produce {@code sinh(a)}.  Floating only.
 519      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 520      */
 521     public static final /*float*/ Unary SINH = unary("SINH", "sinh", VectorSupport.VECTOR_OP_SINH, VO_ONLYFP);
 522     /** Produce {@code cosh(a)}.  Floating only.
 523      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 524      */
 525     public static final /*float*/ Unary COSH = unary("COSH", "cosh", VectorSupport.VECTOR_OP_COSH, VO_ONLYFP);
 526     /** Produce {@code tanh(a)}.  Floating only.
 527      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 528      */
 529     public static final /*float*/ Unary TANH = unary("TANH", "tanh", VectorSupport.VECTOR_OP_TANH, VO_ONLYFP);
 530     /** Produce {@code expm1(a)}.  Floating only.
 531      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 532      */
 533     public static final /*float*/ Unary EXPM1 = unary("EXPM1", "expm1", VectorSupport.VECTOR_OP_EXPM1, VO_ONLYFP);
 534     /** Produce {@code log1p(a)}.  Floating only.
 535      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 536      */
 537     public static final /*float*/ Unary LOG1P = unary("LOG1P", "log1p", VectorSupport.VECTOR_OP_LOG1P, VO_ONLYFP);
 538 
 539     // Binary operators
 540 
 541     /** Produce {@code a+b}. */
 542     public static final Associative ADD = assoc("ADD", "+", VectorSupport.VECTOR_OP_ADD, VO_ALL+VO_ASSOC);
 543     /** Produce {@code a-b}. */
 544     public static final Binary SUB = binary("SUB", "-", VectorSupport.VECTOR_OP_SUB, VO_ALL);
 545     /** Produce {@code a*b}. */
 546     public static final Associative MUL = assoc("MUL", "*", VectorSupport.VECTOR_OP_MUL, VO_ALL+VO_ASSOC);
 547     /** Produce {@code a/b}. Floating only. */
 548     public static final Binary DIV = binary("DIV", "/", VectorSupport.VECTOR_OP_DIV, VO_ALL| VO_SPECIAL);
 549     /** Produce {@code min(a,b)}. */
 550     public static final Associative MIN = assoc("MIN", "min", VectorSupport.VECTOR_OP_MIN, VO_ALL+VO_ASSOC);
 551     /** Produce {@code max(a,b)}. */
 552     public static final Associative MAX = assoc("MAX", "max", VectorSupport.VECTOR_OP_MAX, VO_ALL+VO_ASSOC);
 553     /** Produce {@code bits(a)!=0?a:b}. */
 554     public static final Associative FIRST_NONZERO = assoc("FIRST_NONZERO", "a!=0?a:b", -1 /*VectorSupport.VECTOR_OP_FIRST_NONZERO*/, VO_ALL+VO_ASSOC);
 555 
 556     /** Produce {@code a&b}.  Integral only. */
 557     public static final Associative AND = assoc("AND", "&", VectorSupport.VECTOR_OP_AND, VO_NOFP+VO_ASSOC);
 558     /** Produce {@code a&~b}.  Integral only. */
 559     public static final /*bitwise*/ Binary AND_NOT = binary("AND_NOT", "&~", -1 /*VectorSupport.VECTOR_OP_AND_NOT*/, VO_NOFP); // FIXME
 560     /** Produce {@code a|b}.  Integral only. */
 561     public static final /*bitwise*/ Associative OR = assoc("OR", "|", VectorSupport.VECTOR_OP_OR, VO_NOFP+VO_ASSOC);
 562     /*package-private*/ /** Version of OR which works on float and double too. */
 563     static final Associative OR_UNCHECKED = assoc("OR_UNCHECKED", "|", VectorSupport.VECTOR_OP_OR, VO_ASSOC+VO_PRIVATE);
 564     /** Produce {@code a^b}.  Integral only. */
 565     public static final /*bitwise*/ Associative XOR = assoc("XOR", "^", VectorSupport.VECTOR_OP_XOR, VO_NOFP+VO_ASSOC);
 566 
 567     /** Produce {@code a<<(n&(ESIZE*8-1))}.  Integral only. */
 568     public static final /*bitwise*/ Binary LSHL = binary("LSHL", "<<", VectorSupport.VECTOR_OP_LSHIFT, VO_SHIFT);
 569     /** Produce {@code a>>(n&(ESIZE*8-1))}.  Integral only. */
 570     public static final /*bitwise*/ Binary ASHR = binary("ASHR", ">>", VectorSupport.VECTOR_OP_RSHIFT, VO_SHIFT);
 571     /** Produce {@code a>>>(n&(ESIZE*8-1))}.  Integral only. */
 572     public static final /*bitwise*/ Binary LSHR = binary("LSHR", ">>>", VectorSupport.VECTOR_OP_URSHIFT, VO_SHIFT);
 573     /** Produce {@code rotateLeft(a,n)}.  Integral only. */
 574     public static final /*bitwise*/ Binary ROL = binary("ROL", "rotateLeft", VectorSupport.VECTOR_OP_LROTATE, VO_SHIFT);
 575     /** Produce {@code rotateRight(a,n)}.  Integral only. */
 576     public static final /*bitwise*/ Binary ROR = binary("ROR", "rotateRight", VectorSupport.VECTOR_OP_RROTATE, VO_SHIFT);
 577     /** Produce {@code compress(a,n)}. Integral, {@code int} and {@code long}, only.
 578      * @since 19
 579      */
 580     public static final /*bitwise*/ Binary COMPRESS_BITS = binary("COMPRESS_BITS", "compressBits", VectorSupport.VECTOR_OP_COMPRESS_BITS, VO_NOFP);
 581     /** Produce {@code expand(a,n)}. Integral, {@code int} and {@code long}, only.
 582      * @since 19
 583      */
 584     public static final /*bitwise*/ Binary EXPAND_BITS = binary("EXPAND_BITS", "expandBits", VectorSupport.VECTOR_OP_EXPAND_BITS, VO_NOFP);
 585 
 586     /** Produce {@code atan2(a,b)}. See  Floating only.
 587      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 588      */
 589     public static final /*float*/ Binary ATAN2 = binary("ATAN2", "atan2", VectorSupport.VECTOR_OP_ATAN2, VO_ONLYFP);
 590     /** Produce {@code pow(a,b)}.  Floating only.
 591      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 592      */
 593     public static final /*float*/ Binary POW = binary("POW", "pow", VectorSupport.VECTOR_OP_POW, VO_ONLYFP);
 594     /** Produce {@code hypot(a,b)}.  Floating only.
 595      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 596      */
 597     public static final /*float*/ Binary HYPOT = binary("HYPOT", "hypot", VectorSupport.VECTOR_OP_HYPOT, VO_ONLYFP);
 598 
 599     // Ternary operators
 600 
 601     /** Produce {@code a^((a^b)&c)}.  (Bitwise {@code (c(i)?b(i):a(i))}.)  Integral only. */
 602     public static final /*float*/ Ternary BITWISE_BLEND = ternary("BITWISE_BLEND", "a^((a^b)&c)", -1 /*VectorSupport.VECTOR_OP_BITWISE_BLEND*/, VO_NOFP);
 603     /** Produce {@code fma(a,b,c)}.  Floating only. */
 604     public static final /*float*/ Ternary FMA = ternary("FMA", "fma", VectorSupport.VECTOR_OP_FMA, VO_ONLYFP);
 605 
 606     // Unary boolean operators
 607     /** Test {@code bits(a)==0}.  (Not true of {@code -0.0}.) */
 608     public static final Test IS_DEFAULT = predicate("IS_DEFAULT", "bits(a)==0", -1 /*VectorSupport.VECTOR_OP_TEST_DEFAULT*/, VO_ALL);
 609     /** Test {@code bits(a)<0}.  (True of {@code -0.0}.) */
 610     public static final Test IS_NEGATIVE = predicate("IS_NEGATIVE", "bits(a)<0", -1 /*VectorSupport.VECTOR_OP_TEST_NEGATIVE*/, VO_ALL);
 611     /** Test {@code isFinite(a)}.  Floating only. */
 612     public static final Test IS_FINITE = predicate("IS_FINITE", "isFinite", -1 /*VectorSupport.VECTOR_OP_TEST_FINITE*/, VO_ONLYFP);
 613     /** Test {@code isNaN(a)}.  Floating only. */
 614     public static final Test IS_NAN = predicate("IS_NAN", "isNaN", -1 /*VectorSupport.VECTOR_OP_TEST_NAN*/, VO_ONLYFP);
 615     /** Test {@code isInfinite(a)}.  Floating only. */
 616     public static final Test IS_INFINITE = predicate("IS_INFINITE", "isInfinite", -1 /*VectorSupport.VECTOR_OP_TEST_INFINITE*/, VO_ONLYFP);
 617 
 618     // Binary boolean operators
 619 
 620     /** Compare {@code a==b}. */
 621     public static final Comparison EQ = compare("EQ", "==", VectorSupport.BT_eq, VO_ALL);
 622     /** Compare {@code a!=b}. */
 623     public static final Comparison NE = compare("NE", "!=", VectorSupport.BT_ne, VO_ALL);
 624     /** Compare {@code a<b}. */
 625     public static final Comparison LT = compare("LT", "<",  VectorSupport.BT_lt, VO_ALL);
 626     /** Compare {@code a<=b}. */
 627     public static final Comparison LE = compare("LE", "<=", VectorSupport.BT_le, VO_ALL);
 628     /** Compare {@code a>b}. */
 629     public static final Comparison GT = compare("GT", ">",  VectorSupport.BT_gt, VO_ALL);
 630     /** Compare {@code a>=b}. */
 631     public static final Comparison GE = compare("GE", ">=", VectorSupport.BT_ge, VO_ALL);
 632     /** Unsigned compare {@code a<b}.  Integral only.
 633      * @see java.lang.Integer#compareUnsigned
 634      * @see java.lang.Long#compareUnsigned
 635      */
 636     public static final Comparison UNSIGNED_LT = compare("UNSIGNED_LT", "<",  VectorSupport.BT_ult, VO_NOFP);
 637     /** Unsigned compare {@code a<=b}.  Integral only.
 638      * @see java.lang.Integer#compareUnsigned
 639      * @see java.lang.Long#compareUnsigned
 640      */
 641     public static final Comparison UNSIGNED_LE = compare("UNSIGNED_LE", "<=", VectorSupport.BT_ule, VO_NOFP);
 642     /** Unsigned compare {@code a>b}.  Integral only.
 643      * @see java.lang.Integer#compareUnsigned
 644      * @see java.lang.Long#compareUnsigned
 645      */
 646     public static final Comparison UNSIGNED_GT = compare("UNSIGNED_GT", ">",  VectorSupport.BT_ugt, VO_NOFP);
 647     /** Unsigned compare {@code a>=b}.  Integral only.
 648      * @see java.lang.Integer#compareUnsigned
 649      * @see java.lang.Long#compareUnsigned
 650      */
 651     public static final Comparison UNSIGNED_GE = compare("UNSIGNED_GE", ">=", VectorSupport.BT_uge, VO_NOFP);
 652 
 653     // Conversion operators
 654 
 655     /** Convert {@code byteVal} to {@code (double)byteVal}. */
 656     public static final Conversion<Byte,Double> B2D = convert("B2D", 'C', byte.class, double.class, VO_KIND_CAST, VO_ALL);
 657     /** Convert {@code byteVal} to {@code (float)byteVal}. */
 658     public static final Conversion<Byte,Float> B2F = convert("B2F", 'C', byte.class, float.class, VO_KIND_CAST, VO_ALL);
 659     /** Convert {@code byteVal} to {@code (int)byteVal}. */
 660     public static final Conversion<Byte,Integer> B2I = convert("B2I", 'C', byte.class, int.class, VO_KIND_CAST, VO_ALL);
 661     /** Convert {@code byteVal} to {@code (long)byteVal}. */
 662     public static final Conversion<Byte,Long> B2L = convert("B2L", 'C', byte.class, long.class, VO_KIND_CAST, VO_ALL);
 663     /** Convert {@code byteVal} to {@code (short)byteVal}. */
 664     public static final Conversion<Byte,Short> B2S = convert("B2S", 'C', byte.class, short.class, VO_KIND_CAST, VO_ALL);
 665     /** Convert {@code doubleVal} to {@code (byte)doubleVal}. */
 666     public static final Conversion<Double,Byte> D2B = convert("D2B", 'C', double.class, byte.class, VO_KIND_CAST, VO_ALL);
 667     /** Convert {@code doubleVal} to {@code (float)doubleVal}. */
 668     public static final Conversion<Double,Float> D2F = convert("D2F", 'C', double.class, float.class, VO_KIND_CAST, VO_ALL);
 669     /** Convert {@code doubleVal} to {@code (int)doubleVal}. */
 670     public static final Conversion<Double,Integer> D2I = convert("D2I", 'C', double.class, int.class, VO_KIND_CAST, VO_ALL);
 671     /** Convert {@code doubleVal} to {@code (long)doubleVal}. */
 672     public static final Conversion<Double,Long> D2L = convert("D2L", 'C', double.class, long.class, VO_KIND_CAST, VO_ALL);
 673     /** Convert {@code doubleVal} to {@code (short)doubleVal}. */
 674     public static final Conversion<Double,Short> D2S = convert("D2S", 'C', double.class, short.class, VO_KIND_CAST, VO_ALL);
 675     /** Convert {@code floatVal} to {@code (byte)floatVal}. */
 676     public static final Conversion<Float,Byte> F2B = convert("F2B", 'C', float.class, byte.class, VO_KIND_CAST, VO_ALL);
 677     /** Convert {@code floatVal} to {@code (double)floatVal}. */
 678     public static final Conversion<Float,Double> F2D = convert("F2D", 'C', float.class, double.class, VO_KIND_CAST, VO_ALL);
 679     /** Convert {@code floatVal} to {@code (int)floatVal}. */
 680     public static final Conversion<Float,Integer> F2I = convert("F2I", 'C', float.class, int.class, VO_KIND_CAST, VO_ALL);
 681     /** Convert {@code floatVal} to {@code (long)floatVal}. */
 682     public static final Conversion<Float,Long> F2L = convert("F2L", 'C', float.class, long.class, VO_KIND_CAST, VO_ALL);
 683     /** Convert {@code floatVal} to {@code (short)floatVal}. */
 684     public static final Conversion<Float,Short> F2S = convert("F2S", 'C', float.class, short.class, VO_KIND_CAST, VO_ALL);
 685     /** Convert {@code intVal} to {@code (byte)intVal}. */
 686     public static final Conversion<Integer,Byte> I2B = convert("I2B", 'C', int.class, byte.class, VO_KIND_CAST, VO_ALL);
 687     /** Convert {@code intVal} to {@code (double)intVal}. */
 688     public static final Conversion<Integer,Double> I2D = convert("I2D", 'C', int.class, double.class, VO_KIND_CAST, VO_ALL);
 689     /** Convert {@code intVal} to {@code (float)intVal}. */
 690     public static final Conversion<Integer,Float> I2F = convert("I2F", 'C', int.class, float.class, VO_KIND_CAST, VO_ALL);
 691     /** Convert {@code intVal} to {@code (long)intVal}. */
 692     public static final Conversion<Integer,Long> I2L = convert("I2L", 'C', int.class, long.class, VO_KIND_CAST, VO_ALL);
 693     /** Convert {@code intVal} to {@code (short)intVal}. */
 694     public static final Conversion<Integer,Short> I2S = convert("I2S", 'C', int.class, short.class, VO_KIND_CAST, VO_ALL);
 695     /** Convert {@code longVal} to {@code (byte)longVal}. */
 696     public static final Conversion<Long,Byte> L2B = convert("L2B", 'C', long.class, byte.class, VO_KIND_CAST, VO_ALL);
 697     /** Convert {@code longVal} to {@code (double)longVal}. */
 698     public static final Conversion<Long,Double> L2D = convert("L2D", 'C', long.class, double.class, VO_KIND_CAST, VO_ALL);
 699     /** Convert {@code longVal} to {@code (float)longVal}. */
 700     public static final Conversion<Long,Float> L2F = convert("L2F", 'C', long.class, float.class, VO_KIND_CAST, VO_ALL);
 701     /** Convert {@code longVal} to {@code (int)longVal}. */
 702     public static final Conversion<Long,Integer> L2I = convert("L2I", 'C', long.class, int.class, VO_KIND_CAST, VO_ALL);
 703     /** Convert {@code longVal} to {@code (short)longVal}. */
 704     public static final Conversion<Long,Short> L2S = convert("L2S", 'C', long.class, short.class, VO_KIND_CAST, VO_ALL);
 705     /** Convert {@code shortVal} to {@code (byte)shortVal}. */
 706     public static final Conversion<Short,Byte> S2B = convert("S2B", 'C', short.class, byte.class, VO_KIND_CAST, VO_ALL);
 707     /** Convert {@code shortVal} to {@code (double)shortVal}. */
 708     public static final Conversion<Short,Double> S2D = convert("S2D", 'C', short.class, double.class, VO_KIND_CAST, VO_ALL);
 709     /** Convert {@code shortVal} to {@code (float)shortVal}. */
 710     public static final Conversion<Short,Float> S2F = convert("S2F", 'C', short.class, float.class, VO_KIND_CAST, VO_ALL);
 711     /** Convert {@code shortVal} to {@code (int)shortVal}. */
 712     public static final Conversion<Short,Integer> S2I = convert("S2I", 'C', short.class, int.class, VO_KIND_CAST, VO_ALL);
 713     /** Convert {@code shortVal} to {@code (long)shortVal}. */
 714     public static final Conversion<Short,Long> S2L = convert("S2L", 'C', short.class, long.class, VO_KIND_CAST, VO_ALL);
 715     /** Reinterpret bits of {@code doubleVal} as {@code long}. As if by {@link Double#doubleToRawLongBits(double)} */
 716     public static final Conversion<Double,Long> REINTERPRET_D2L = convert("REINTERPRET_D2L", 'R', double.class, long.class, VO_KIND_BITWISE, VO_ALL);
 717     /** Reinterpret bits of {@code floatVal} as {@code int}. As if by {@link Float#floatToRawIntBits(float)} */
 718     public static final Conversion<Float,Integer> REINTERPRET_F2I = convert("REINTERPRET_F2I", 'R', float.class, int.class, VO_KIND_BITWISE, VO_ALL);
 719     /** Reinterpret bits of {@code intVal} as {@code float}. As if by {@link Float#intBitsToFloat(int)} */
 720     public static final Conversion<Integer,Float> REINTERPRET_I2F = convert("REINTERPRET_I2F", 'R', int.class, float.class, VO_KIND_BITWISE, VO_ALL);
 721     /** Reinterpret bits of {@code longVal} as {@code double}. As if by {@link Double#longBitsToDouble(long)} */
 722     public static final Conversion<Long,Double> REINTERPRET_L2D = convert("REINTERPRET_L2D", 'R', long.class, double.class, VO_KIND_BITWISE, VO_ALL);
 723     /** Zero-extend {@code byteVal} to {@code int}. */
 724     public static final Conversion<Byte,Integer> ZERO_EXTEND_B2I = convert("ZERO_EXTEND_B2I", 'Z', byte.class, int.class, VO_KIND_BITWISE, VO_ALL);
 725     /** Zero-extend {@code byteVal} to {@code long}. */
 726     public static final Conversion<Byte,Long> ZERO_EXTEND_B2L = convert("ZERO_EXTEND_B2L", 'Z', byte.class, long.class, VO_KIND_BITWISE, VO_ALL);
 727     /** Zero-extend {@code byteVal} to {@code short}. */
 728     public static final Conversion<Byte,Short> ZERO_EXTEND_B2S = convert("ZERO_EXTEND_B2S", 'Z', byte.class, short.class, VO_KIND_BITWISE, VO_ALL);
 729     /** Zero-extend {@code intVal} to {@code long}. */
 730     public static final Conversion<Integer,Long> ZERO_EXTEND_I2L = convert("ZERO_EXTEND_I2L", 'Z', int.class, long.class, VO_KIND_BITWISE, VO_ALL);
 731     /** Zero-extend {@code shortVal} to {@code int}. */
 732     public static final Conversion<Short,Integer> ZERO_EXTEND_S2I = convert("ZERO_EXTEND_S2I", 'Z', short.class, int.class, VO_KIND_BITWISE, VO_ALL);
 733     /** Zero-extend {@code shortVal} to {@code long}. */
 734     public static final Conversion<Short,Long> ZERO_EXTEND_S2L = convert("ZERO_EXTEND_S2L", 'Z', short.class, long.class, VO_KIND_BITWISE, VO_ALL);
 735 
 736     // (End of conversion operators)
 737 
 738     private static int opInfo(int opCode, int bits) {
 739         if (opCode >= 0) {
 740             bits |= VO_OPCODE_VALID;
 741         } else {
 742             opCode &= (VO_OPCODE_LIMIT - 1);  // not a valid op
 743             bits |= VO_SPECIAL;  // mark for special handling
 744         }
 745         assert((bits >> VO_OPCODE_SHIFT) == 0);
 746         assert(opCode >= 0 && opCode < VO_OPCODE_LIMIT);
 747         return (opCode << VO_OPCODE_SHIFT) + bits;
 748     }
 749 
 750     private static Unary unary(String name, String opName, int opCode, int flags) {
 751         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 752             OPC_NAME.put(opCode, name);
 753         return new UnaryImpl(name, opName, opInfo(opCode, flags | VO_UNARY));
 754     }
 755 
 756     private static Binary binary(String name, String opName, int opCode, int flags) {
 757         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 758             OPC_NAME.put(opCode, name);
 759         return new BinaryImpl(name, opName, opInfo(opCode, flags | VO_BINARY));
 760     }
 761 
 762     private static Ternary ternary(String name, String opName, int opCode, int flags) {
 763         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 764             OPC_NAME.put(opCode, name);
 765         return new TernaryImpl(name, opName, opInfo(opCode, flags | VO_TERNARY));
 766     }
 767 
 768     private static Associative assoc(String name, String opName, int opCode, int flags) {
 769         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 770             OPC_NAME.put(opCode, name);
 771         return new AssociativeImpl(name, opName, opInfo(opCode, flags | VO_BINARY | VO_ASSOC));
 772     }
 773 
 774     private static Test predicate(String name, String opName, int opCode, int flags) {
 775         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 776             CMP_OPC_NAME.put(opCode, name);
 777         return new TestImpl(name, opName, opInfo(opCode, flags | VO_UNARY | VO_BOOL));
 778     }
 779 
 780     private static Comparison compare(String name, String opName, int opCode, int flags) {
 781         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 782             CMP_OPC_NAME.put(opCode, name);
 783         return new ComparisonImpl(name, opName, opInfo(opCode, flags | VO_BINARY | VO_BOOL));
 784     }
 785 
 786     private static <E,F> ConversionImpl<E,F>
 787     convert(String name, char kind, Class<E> dom, Class<F> ran, int opCode, int flags) {
 788         int domran = ((LaneType.of(dom).basicType << VO_DOM_SHIFT) +
 789                       (LaneType.of(ran).basicType << VO_RAN_SHIFT));
 790         if (opCode >= 0) {
 791             if ((opCode & VO_DOM_RAN_MASK) == 0) {
 792                 opCode += domran;
 793             }
 794             if ((flags & VO_PRIVATE) == 0)
 795                 CONV_OPC_NAME.put(opCode, name);
 796         }
 797         String opName = dom+"-"+kind+"-"+ran; //??
 798         return new ConversionImpl<>(name, opName, opInfo(opCode, flags | VO_UNARY | VO_CONV),
 799                                     kind, dom, ran);
 800     }
 801 
 802     private static abstract class OperatorImpl implements Operator {
 803         private final String symName;
 804         private final String opName;
 805         private final int opInfo;
 806 
 807         OperatorImpl(String symName, String opName, int opInfo) {
 808             this.symName = symName;
 809             this.opName = opName;
 810             this.opInfo = opInfo;
 811             assert(opInfo != 0);
 812         }
 813 
 814         @Override
 815         public final String name() {
 816             return symName;
 817         }
 818         @Override
 819         public final String operatorName() {
 820             return opName;
 821         }
 822         @Override
 823         public final String toString() {
 824             return name();
 825         }
 826         @Override
 827         public final int arity() {
 828             return opInfo & VO_ARITY_MASK;
 829         }
 830         @Override
 831         public final boolean isBoolean() {
 832             return opKind(VO_BOOL);
 833         }
 834         @Override
 835         public Class<?> rangeType() {
 836             return Object.class;
 837         }
 838         @Override
 839         public final boolean isAssociative() {
 840             return opKind(VO_ASSOC);
 841         }
 842 
 843         @ForceInline
 844         public boolean compatibleWith(Class<?> elementType) {
 845             return compatibleWith(LaneType.of(elementType));
 846         }
 847 
 848         /*package-private*/
 849         @ForceInline
 850         int opInfo() {
 851             return opInfo;
 852         }
 853 
 854         /*package-private*/
 855         @ForceInline
 856         int opCode(int requireKind, int forbidKind) {
 857             int opc = opCodeRaw();
 858             if ((opInfo & requireKind) != requireKind ||
 859                 (forbidKind != 0 &&
 860                  (opInfo & forbidKind)  == forbidKind)) {
 861                 throw illegalOperation(requireKind, forbidKind);
 862             }
 863             return opc;
 864         }
 865 
 866         /*package-private*/
 867         @ForceInline
 868         int opCodeRaw() {
 869             return (opInfo >> VO_OPCODE_SHIFT);
 870         }
 871 
 872         /*package-private*/
 873         UnsupportedOperationException
 874         illegalOperation(int requireKind, int forbidKind) {
 875             String msg1 = "";
 876             requireKind &=~ VO_OPCODE_VALID;
 877             switch (requireKind) {
 878             case VO_ONLYFP:  msg1 = "floating point operator required here"; break;
 879             case VO_NOFP:    msg1 = "integral/bitwise operator required here"; break;
 880             case VO_ASSOC:   msg1 = "associative operator required here"; break;
 881             }
 882             String msg2 = "";
 883             switch (forbidKind) {
 884             case VO_ONLYFP:  msg2 = "inapplicable floating point operator"; break;
 885             case VO_NOFP:    msg2 = "inapplicable integral/bitwise operator"; break;
 886             }
 887             if ((opInfo & VO_OPCODE_VALID) == 0) {
 888                 msg2 = "operator is not implemented";
 889             }
 890             return illegalOperation(msg1, msg2);
 891         }
 892 
 893         /*package-private*/
 894         UnsupportedOperationException
 895         illegalOperation(String msg1, String msg2) {
 896             String dot = "";
 897             if (!msg1.isEmpty() && !msg2.isEmpty()) {
 898                 dot = "; ";
 899             } else if (msg1.isEmpty() && msg2.isEmpty()) {
 900                 // Couldn't decode the *kind bits.
 901                 msg1 = "illegal operator";
 902             }
 903             String msg = String.format("%s: %s%s%s", this, msg1, dot, msg2);
 904             return new UnsupportedOperationException(msg);
 905         }
 906 
 907 
 908         /*package-private*/
 909         @ForceInline
 910         boolean opKind(int kindBit) {
 911             return (opInfo & kindBit) != 0;
 912         }
 913 
 914         @ForceInline
 915         /*package-private*/
 916         boolean compatibleWith(LaneType laneType) {
 917             if (laneType.elementKind == 'F') {
 918                 return !opKind(VO_NOFP);
 919             } else if (laneType.elementKind == 'I') {
 920                 return !opKind(VO_ONLYFP);
 921             } else {
 922                 throw new AssertionError();
 923             }
 924         }
 925     }
 926 
 927     private static class UnaryImpl extends OperatorImpl implements Unary {
 928         private UnaryImpl(String symName, String opName, int opInfo) {
 929             super(symName, opName, opInfo);
 930             assert((opInfo & VO_ARITY_MASK) == VO_UNARY);
 931         }
 932     }
 933 
 934     private static class BinaryImpl extends OperatorImpl implements Binary {
 935         private BinaryImpl(String symName, String opName, int opInfo) {
 936             super(symName, opName, opInfo);
 937             assert((opInfo & VO_ARITY_MASK) == VO_BINARY);
 938         }
 939     }
 940 
 941     private static class TernaryImpl extends OperatorImpl implements Ternary {
 942         private TernaryImpl(String symName, String opName, int opInfo) {
 943             super(symName, opName, opInfo);
 944             assert((opInfo & VO_ARITY_MASK) == VO_TERNARY);
 945         }
 946     }
 947 
 948     private static class AssociativeImpl extends BinaryImpl implements Associative {
 949         private AssociativeImpl(String symName, String opName, int opInfo) {
 950             super(symName, opName, opInfo);
 951         }
 952     }
 953 
 954     /*package-private*/
 955     static
 956     class ConversionImpl<E,F> extends OperatorImpl
 957                               implements Conversion<E,F> {
 958         private ConversionImpl(String symName, String opName, int opInfo,
 959                                char kind, Class<E> dom, Class<F> ran) {
 960             super(symName, opName, opInfo);
 961             assert((opInfo & VO_ARITY_MASK) == VO_UNARY);
 962             this.kind = kind;
 963             this.dom = LaneType.of(dom);
 964             this.ran = LaneType.of(ran);
 965             check(dom, ran);  // make sure it all lines up
 966         }
 967         private final char kind;
 968         private final LaneType dom;
 969         private final LaneType ran;
 970 
 971         // non-overrides are all package-private
 972 
 973         char kind()  { return kind; }
 974         LaneType domain() { return dom; }
 975         LaneType range()  { return ran; }
 976 
 977         int sizeChangeLog2() {
 978             return ran.elementSizeLog2 - dom.elementSizeLog2;
 979         }
 980 
 981         @SuppressWarnings("unchecked")
 982         @Override
 983         public Class<E> domainType() {
 984             return (Class<E>) dom.elementType;
 985         }
 986         @SuppressWarnings("unchecked")
 987         @Override
 988         public Class<F> rangeType() {
 989             return (Class<F>) ran.elementType;
 990         }
 991 
 992         @SuppressWarnings("unchecked")
 993         @ForceInline
 994         public
 995         <E,F> Conversion<E,F>
 996         check(Class<E> dom, Class<F> ran) {
 997             if (this.dom.elementType != dom ||
 998                 this.ran.elementType != ran)
 999                 throw checkFailed(dom, ran);
1000             return (Conversion<E,F>) this;
1001         }
1002         private RuntimeException checkFailed(Class<?> dom, Class<?> ran) {
1003             return new ClassCastException(toString()+": not "+dom+" -> "+ran);
1004         }
1005 
1006         static ConversionImpl<?,?> ofCopy(LaneType dom) {
1007             return findConv('I', dom, dom);
1008         }
1009         static ConversionImpl<?,?> ofCast(LaneType dom, LaneType ran) {
1010             if (dom == ran)  return ofCopy(dom);
1011             return findConv('C', dom, ran);
1012         }
1013         static ConversionImpl<?,?> ofReinterpret(LaneType dom, LaneType ran) {
1014             if (dom == ran)  return ofCopy(dom);
1015             if (dom.elementKind == 'I' &&
1016                 ran.elementKind == 'I' &&
1017                 dom.elementSize < ran.elementSize) {
1018                 // Zero extension of field (unsigned semantics).
1019                 return findConv('Z', dom, ran);
1020             }
1021             // if (dom.elementSize != ran.elementSize) {
1022             //     throw new IllegalArgumentException("bad reinterpret");
1023             // }
1024             return findConv('R', dom, ran);
1025         }
1026 
1027         @ForceInline
1028         private static ConversionImpl<?,?>
1029         findConv(char kind, LaneType dom, LaneType ran) {
1030             ConversionImpl<?,?>[] cache = cacheOf(kind, dom);
1031             int ranKey = ran.switchKey;
1032             ConversionImpl<?,?> conv = cache[ranKey];
1033             if (conv != null) {
1034                 return conv;
1035             }
1036             return makeConv(kind, dom, ran);
1037         }
1038 
1039         static String a2b(LaneType dom, LaneType ran) {
1040             return dom.typeChar + "2" + ran.typeChar;
1041         }
1042 
1043         static ConversionImpl<?,?>
1044         makeConv(char kind, LaneType dom, LaneType ran) {
1045             String name;
1046             Class<?> domType = dom.elementType;
1047             Class<?> ranType = ran.elementType;
1048             int domCode = (dom.basicType << VO_DOM_SHIFT);
1049             int ranCode = (ran.basicType << VO_RAN_SHIFT);
1050             int opCode = domCode + ranCode;
1051             switch (kind) {
1052             case 'I':
1053                 assert(dom == ran);
1054                 name = "COPY_"+a2b(dom, ran);
1055                 opCode = VO_KIND_CAST;
1056                 break;
1057             case 'C':
1058                 name = ""+a2b(dom, ran);
1059                 opCode = VO_KIND_CAST;
1060                 break;
1061             case 'R':
1062                 name = "REINTERPRET_"+a2b(dom, ran);
1063                 opCode = VO_KIND_BITWISE;
1064                 break;
1065             case 'Z':
1066                 name = "ZERO_EXTEND_"+a2b(dom, ran);
1067                 opCode = VO_KIND_BITWISE;
1068                 break;
1069             default:  throw new AssertionError();
1070             }
1071             ConversionImpl<?,?> conv = convert(name, kind, domType, ranType, opCode, VO_ALL);
1072             // Put into the cache for next time.
1073             // The JIT can see into this cache
1074             // when kind/dom/ran are all constant.
1075             ConversionImpl<?,?>[] cache = cacheOf(kind, dom);
1076             int ranKey = ran.switchKey;
1077             // The extra "check" calls help validate that
1078             // we aren't cross-wiring the cache.
1079             conv.check(domType, ranType);
1080             synchronized (ConversionImpl.class) {
1081                 if (cache[ranKey] == null) {
1082                     cache[ranKey] = conv;
1083                 } else {
1084                     conv = cache[ranKey];
1085                     conv.check(domType, ranType);
1086                 }
1087             }
1088             return conv;
1089         }
1090         private final void check(char kind, LaneType dom, LaneType ran) {
1091             if (this.kind != kind || this.dom != dom || this.ran != ran) {
1092                 throw new AssertionError(this + " != " + dom + kind + ran);
1093             }
1094         }
1095 
1096         /** Helper for cache probes. */
1097         @ForceInline
1098         private static ConversionImpl<?,?>[]
1099         cacheOf(char kind, LaneType dom) {
1100             assert("CIRZWN".indexOf(kind) >= 0);
1101             int k = (kind <= 'I' ? KIND_CI :
1102                      (kind == 'R' || kind == 'Z') ? KIND_RZ :
1103                      KIND_WN);
1104             return CACHES[k][dom.switchKey];
1105         }
1106         private static final int
1107             LINE_LIMIT = LaneType.SK_LIMIT,
1108             KIND_CI = 0, KIND_RZ = 1, KIND_WN = 2, KIND_LIMIT = 3;
1109         private static final @Stable ConversionImpl<?,?>[][][]
1110             CACHES = new ConversionImpl<?,?>[KIND_LIMIT][LINE_LIMIT][LINE_LIMIT];
1111 
1112         private synchronized static void initCaches() {
1113             for (var f : VectorOperators.class.getFields()) {
1114                 if (f.getType() != Conversion.class)  continue;
1115                 ConversionImpl<?,?> conv;
1116                 try {
1117                     conv = (ConversionImpl) f.get(null);
1118                 } catch (ReflectiveOperationException ex) {
1119                     throw new AssertionError(ex);
1120                 }
1121                 LaneType dom = conv.dom;
1122                 LaneType ran = conv.ran;
1123                 int opc = conv.opCodeRaw();
1124                 switch (conv.kind) {
1125                 case 'W':
1126                     int domCode = (opc >> VO_DOM_SHIFT) & 0xF;
1127                     dom = LaneType.ofBasicType(domCode);
1128                     break;
1129                 case 'N':
1130                     int ranCode = (opc >> VO_RAN_SHIFT) & 0xF;
1131                     ran = LaneType.ofBasicType(ranCode);
1132                     break;
1133                 }
1134                 assert((opc & VO_DOM_RAN_MASK) ==
1135                        ((dom.basicType << VO_DOM_SHIFT) +
1136                         (ran.basicType << VO_RAN_SHIFT)));
1137                 ConversionImpl<?,?>[] cache = cacheOf(conv.kind, dom);
1138                 int ranKey = ran.switchKey;
1139                 if (cache[ranKey] != conv) {
1140                     assert(cache[ranKey] == null ||
1141                            cache[ranKey].name().equals(conv.name()))
1142                         : conv + " vs. " + cache[ranKey];
1143                     cache[ranKey] = conv;
1144                 }
1145             }
1146         }
1147 
1148         // hack for generating static field defs
1149         static { assert(genCode()); }
1150         private static boolean genCode() {
1151             if (true)  return true;  // remove to enable code
1152             ArrayList<String> defs = new ArrayList<>();
1153             for (LaneType l1 : LaneType.values()) {
1154                 for (LaneType l2 : LaneType.values()) {
1155                     for (int i = 0; i <= 1; i++) {
1156                         ConversionImpl<?,?> c;
1157                         try {
1158                             c = ((i == 0) ? ofCast(l1, l2) : ofReinterpret(l1, l2));
1159                         } catch (IllegalArgumentException ex) {
1160                             assert((i == 1 && l1.elementSize != l2.elementSize) ||
1161                                    (i == 2 && l1.elementSize == l2.elementSize));
1162                             continue;  // ignore this combo
1163                         }
1164                         if (c.kind == 'C' ||
1165                             c.kind == 'Z' ||
1166                             (c.kind == 'R' &&
1167                              l1.elementKind+l2.elementKind == 'F'+'I' &&
1168                              l1.elementSize == l2.elementSize) ||
1169                             (c.kind == 'N' || c.kind == 'W')) {
1170                             int opc = c.opCodeRaw();
1171                             String opcs;
1172                             switch (opc & ~VO_DOM_RAN_MASK) {
1173                             case VO_KIND_CAST: opcs = "VO_KIND_CAST"; break;
1174                             case VO_KIND_BITWISE: opcs = "VO_KIND_BITWISE"; break;
1175                             default: opcs = Integer.toHexString(opc);
1176                             }
1177                             String code = c.genCode(opcs);
1178                             if (!defs.contains(code))  defs.add(code);
1179                         }
1180                     }
1181                 }
1182             }
1183             java.util.Collections.sort(defs);
1184             for (String def : defs)  System.out.println(def);
1185             return true;
1186         }
1187         private String genCode(String opcs) {
1188             if (true)  return null;  // remove to enable code
1189             int domran = opCodeRaw() & VO_DOM_RAN_MASK;
1190             switch (kind()) {
1191             case 'W': case 'N':
1192                 opcs += " + 0x" + Integer.toHexString(domran);
1193             }
1194             String doc;
1195             switch (kind()) {
1196             case 'R':
1197                 doc = "Reinterpret bits of {@code _domVal} as {@code _ran}";
1198                 break;
1199             case 'Z':
1200                 doc = "Zero-extend {@code _domVal} to {@code _ran}";
1201                 break;
1202             case 'W':
1203                 doc = "In-place widen {@code _domVal} inside _ran to {@code (_ran)_domVal}";
1204                 LaneType logdom = LaneType.ofBasicType(domran >> VO_DOM_SHIFT & 0xF);
1205                 doc = doc.replace("_dom", logdom.elementType.getSimpleName());
1206                 break;
1207             case 'N':
1208                 doc = "In-place narrow {@code _domVal} to {@code (_ran)_domVal} inside _dom";
1209                 LaneType logran = LaneType.ofBasicType(domran >> VO_RAN_SHIFT & 0xF);
1210                 doc = doc.replace("_ran", logran.elementType.getSimpleName());
1211                 break;
1212             default:
1213                 doc = "Convert {@code _domVal} to {@code (_ran)_domVal}";
1214             }
1215             String code = (
1216                     "    /** _Doc. */" + "\n" +
1217                     "    public static final Conversion<_Dom,_Ran> _N" +
1218                     " = convert(\"_N\", '_K', _dom.class, _ran.class, _opc, VO_ALL);");
1219             return code
1220                 .replace("_Doc", doc)
1221                 .replace("_dom", dom.elementType.getSimpleName())
1222                 .replace("_ran", ran.elementType.getSimpleName())
1223                 .replace("_Dom", dom.genericElementType.getSimpleName())
1224                 .replace("_Ran", ran.genericElementType.getSimpleName())
1225                 .replace("_N", name())
1226                 .replace("_K", ""+kind())
1227                 .replace("_opc", ""+opcs);
1228         }
1229     }
1230 
1231     private static class TestImpl extends OperatorImpl implements Test {
1232         private TestImpl(String symName, String opName, int opInfo) {
1233             super(symName, opName, opInfo);
1234             assert((opInfo & VO_ARITY_MASK) == VO_UNARY);
1235             assert((opInfo & VO_BOOL) == VO_BOOL);
1236         }
1237         @Override
1238         public Class<?> rangeType() {
1239             return boolean.class;
1240         }
1241     }
1242 
1243     private static class ComparisonImpl extends OperatorImpl implements Comparison {
1244         private ComparisonImpl(String symName, String opName, int opInfo) {
1245             super(symName, opName, opInfo);
1246             assert((opInfo & VO_ARITY_MASK) == VO_BINARY);
1247             assert((opInfo & VO_BOOL) == VO_BOOL);
1248         }
1249         @Override
1250         public Class<?> rangeType() {
1251             return boolean.class;
1252         }
1253         /* --- *
1254         boolean test(long a, long b) {
1255             switch (opInfo() >> VO_OPCODE_SHIFT) {
1256             case VectorSupport.BT_eq:  return a == b;
1257             case VectorSupport.BT_ne:  return a != b;
1258             case VectorSupport.BT_lt:  return a <  b;
1259             case VectorSupport.BT_le:  return a <= b;
1260             case VectorSupport.BT_gt:  return a >  b;
1261             case VectorSupport.BT_ge:  return a >= b;
1262             }
1263             throw new AssertionError();
1264         }
1265         * --- */
1266     }
1267 
1268     static {
1269         ConversionImpl.initCaches();
1270         assert(checkConstants());
1271     }
1272 
1273     private static boolean checkConstants() {
1274         // Check uniqueness of opcodes, to prevent dumb aliasing errors.
1275         OperatorImpl[] ops = new OperatorImpl[VO_OPCODE_LIMIT << VO_OPCODE_SHIFT];
1276         for (var f : VectorOperators.class.getFields()) {
1277             Class<?> ft = f.getType();
1278             OperatorImpl op;
1279             try {
1280                 op = (OperatorImpl) f.get(null);
1281             } catch (ReflectiveOperationException ex) {
1282                 throw new AssertionError(ex);
1283             }
1284             assert(op.name().equals(f.getName())) : op;
1285             assert(op.isAssociative() == (ft == Associative.class)) : op;
1286             if (op.isBoolean()) {
1287                 assert(ft == (op.arity() == 2 ? Comparison.class : Test.class)) : op;
1288             }
1289             if (ft == Unary.class || ft == Conversion.class || ft == Test.class) {
1290                 assert(op.arity() == 1) : op;
1291             } else if (ft == Ternary.class) {
1292                 assert(op.arity() == 3) : op;
1293             } else {
1294                 assert(op.arity() == 2) : op;
1295                 if (ft != Associative.class &&
1296                     ft != Comparison.class) {
1297                     assert(ft == Binary.class) : op;
1298                 }
1299             }
1300             if (op.opKind(VO_OPCODE_VALID)) {
1301                 int opsMask = (((VO_OPCODE_LIMIT-1) << VO_OPCODE_SHIFT)
1302                                | VO_BOOL | VO_CONV
1303                                | VO_ARITY_MASK);
1304                 int opsIndex = op.opInfo & opsMask;
1305                 OperatorImpl op0 = ops[opsIndex];
1306                 assert(op0 == null)
1307                     : java.util.Arrays.asList(op0, Integer.toHexString(op0.opInfo), op, Integer.toHexString(op.opInfo));
1308                 ops[opsIndex] = op;
1309             } else {
1310                 // These are all the "-1" opcode guys we know about:
1311                 assert(op == ZOMO ||
1312                        op == FIRST_NONZERO ||
1313                        op == AND_NOT || op == NOT ||
1314                        op == ROL ||
1315                        op == ROR ||
1316                        op == IS_DEFAULT || op == IS_NEGATIVE ||
1317                        op == IS_FINITE || op == IS_NAN || op == IS_INFINITE ||
1318                        op == BITWISE_BLEND) : op;
1319             }
1320         }
1321         return true;
1322     }
1323 
1324     // Managing behavioral information on slow paths:
1325     /*package-private*/
1326     static class ImplCache<OP extends Operator,T> {
1327         public ImplCache(Class<OP> whatKind,
1328                          Class<? extends Vector<?>> whatVec) {
1329             this.whatKind = whatKind;
1330             this.whatVec = whatVec;
1331         }
1332 
1333         // These are used only for forming diagnostics:
1334         private final Class<OP> whatKind;
1335         private final Class<? extends Vector<?>> whatVec;
1336 
1337         private final @Stable
1338         Object[] cache = new Object[VO_OPCODE_LIMIT];
1339 
1340         @ForceInline
1341         public T find(OP op, int opc, IntFunction<T> supplier) {
1342             @SuppressWarnings("unchecked")
1343             T fn = (T) cache[opc];
1344             if (fn != null)  return fn;
1345             fn = supplier.apply(opc);
1346             if (fn == null)  throw badOp(op);
1347             assert(VectorSupport.isNonCapturingLambda(fn)) : fn;
1348             // The JIT can see into this cache:
1349             cache[opc] = fn;
1350             return fn;
1351         }
1352 
1353         private UnsupportedOperationException badOp(Operator op) {
1354             String msg = String.format("%s: illegal %s in %s",
1355                                        op,
1356                                        whatKind.getSimpleName().toLowerCase(),
1357                                        whatVec.getSimpleName());
1358             return new UnsupportedOperationException(msg);
1359         }
1360 
1361         @Override public String toString() {
1362             ArrayList<String> entries = new ArrayList<>();
1363             for (int i = 0; i < cache.length; i++) {
1364                 Object fn = cache[i];
1365                 if (fn != null)  entries.add(i+": "+fn);
1366             }
1367             return String.format("ImplCache<%s,%s>[%s]",
1368                                  whatKind.getSimpleName(),
1369                                  whatVec.getSimpleName(),
1370                                  String.join(", ", entries));
1371         }
1372     }
1373 }
--- EOF ---