1 /*
   2  * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.util.function.IntFunction;
  28 import java.util.HashMap;
  29 import java.util.ArrayList;
  30 
  31 import jdk.internal.vm.annotation.ForceInline;
  32 import jdk.internal.vm.annotation.Stable;
  33 
  34 import jdk.internal.vm.vector.VectorSupport;
  35 
  36 /**
  37  * This class consists solely of static constants
  38  * that describe lane-wise vector operations, plus nested interfaces
  39  * which classify them.
  40  * The static constants serve as tokens denoting specifically
  41  * requested lane operations in vector expressions, such
  42  * as the token {@code ADD} in
  43  * {@code w = v0.}{@link
  44  * Vector#lanewise(VectorOperators.Binary,Vector)
  45  * lanewise}{@code (ADD, v1)}.
  46  *
  47  * <p>
  48  *
  49  * The documentation for each individual operator token is very brief,
  50  * giving a symbolic Java expression for the operation that the token
  51  * requests.  Those symbolic expressions use the following conventional
  52  * elements:
  53  * <ul>
  54  * <li>{@code a}, {@code b}, {@code c} &mdash; names of lane values
  55  *
  56  * <li>Java operators like {@code +}, {@code ?:}, etc. &mdash;
  57  * expression operators
  58  *
  59  * <li>Java method names like {@code max}, {@code sin}, etc. &mdash;
  60  * methods in standard classes like {@code Math}, {@code Double}, etc.
  61  * Unqualified method names should be read as if in the context of a
  62  * static import, and with resolution of overloading.
  63  *
  64  * <li>{@code bits(x)} &mdash; a function call which produces the
  65  * underlying bits of the value {@code x}.  If {@code x} is a floating
  66  * point value, this is either {@code doubleToLongBits(x)} or
  67  * {@code floatToIntBits(x)}.  Otherwise, the value is just {@code x}.
  68  *
  69  * <li>{@code ESIZE} &mdash; the size in bytes of the operand type
  70  *
  71  * <li>{@code intVal}, {@code byteVal}, etc. &mdash; the operand of a
  72  * conversion, with the indicated type
  73  * </ul>
  74  *
  75  * <h2>Operations on floating point vectors</h2>
  76  * <ul>
  77  * <li>Lane-wise vector operations that apply to floating point vectors
  78  * follow the accuracy and monotonicity specifications of the equivalent
  79  * Java operation or method mentioned in its documentation unless specified otherwise.
  80  * If the vector element type is {@code float} and the Java operation or
  81  * method only accepts and returns {@code double} values, then the scalar operation
  82  * on each lane is adapted to cast operands and the result, specifically widening
  83  * {@code float} operands to {@code double} operands and narrowing the {@code double}
  84  * result to a {@code float}.
  85  *
  86  * <li id="fp_assoc">Certain associative operations that apply to floating point
  87  * vectors are not truly associative on the floating point lane values.
  88  * Specifically, {@link #ADD} and {@link #MUL} used with cross-lane reduction
  89  * operations, such as {@link FloatVector#reduceLanes(Associative)}.
  90  * The result of such an operation is a function both of the input
  91  * values (vector and mask) as well as the order of the scalar operations
  92  * applied to combine lane values.
  93  * In such cases the order is intentionally not defined.
  94  * This allows the JVM to generate optimal machine code for the underlying
  95  * platform at runtime.  If the platform supports a vector instruction
  96  * to add or multiply all values in the vector, or if there is some
  97  * other efficient machine code sequence, then the JVM has the option of
  98  * generating this machine code. Otherwise, the default
  99  * implementation is applied, which adds vector elements
 100  * sequentially from beginning to end.  For this reason, the
 101  * result of such an operation may vary for the same input values.
 102  * </ul>
 103  *
 104  * <p> Note that a particular operator token may apply to several
 105  * different lane types.  Thus, these tokens behave like overloaded
 106  * operators or methods, not like type-specific method handles or
 107  * lambdas.  Also unlike method handles or lambdas, these operators do
 108  * not possess operational semantics; they have no {@code apply} or
 109  * {@code invoke} method.  They are used only to request lane
 110  * operations from vector objects, and cannot (by themselves) perform
 111  * operations on individual lane values.
 112  *
 113  */
 114 public abstract class VectorOperators {
 115     private VectorOperators() { }
 116 
 117     /**
 118      * Root type for all operator tokens, providing queries for common
 119      * properties such as arity, argument and return types, symbolic
 120      * name, and operator name.
 121      *
 122      * @see VectorOperators.Unary Unary
 123      * @see VectorOperators.Binary Binary
 124      * @see VectorOperators.Ternary Ternary
 125      * @see VectorOperators.Associative Associative
 126      * @see VectorOperators.Comparison Comparison
 127      * @see VectorOperators.Test Test
 128      * @see VectorOperators.Conversion Conversion
 129      *
 130      * @apiNote
 131      * User code should not implement this interface.  A future release of
 132      * this type may restrict implementations to be members of the same
 133      * package.
 134      */
 135     public interface Operator {
 136         /**
 137          * Returns the symbolic name of this operator,
 138          * as a constant in {@link VectorOperators}.
 139          *
 140          * The operator symbol, Java method name,
 141          * or example expression,
 142          * such as {@code "+"}, {@code "max"} or {@code "-a"},
 143          * is also available as {@link #operatorName()}.
 144          *
 145          * @return the symbolic name of this operator,
 146          *         such as {@code "ADD"}
 147          */
 148         String name();
 149 
 150         /**
 151          * Returns the Java operator symbol or method
 152          * name corresponding to this operator.
 153          * If there is no symbol or method, return a
 154          * string containing a representative expression
 155          * for the operator, using operand names
 156          * {@code a}, {@code b} (for non-unary operators),
 157          * and {@code c} (for ternary operators).
 158          *
 159          * The symbolic name of the constant,
 160          * such as {@code "ADD"},
 161          * is also available as {@link #name()}.
 162          *
 163          * @return an operator token, such as {@code "+"},
 164          *         or a method name, such as {@code "max"},
 165          *         or a representative expression, such as {@code "-a"}
 166          */
 167         String operatorName();
 168 
 169         /**
 170          * Returns the arity of this operator (1, 2, or 3).
 171          * @return the arity of this operator (1, 2, or 3)
 172          */
 173         int arity();
 174 
 175         /**
 176          * Reports whether this operator returns a boolean (a mask).
 177          * A boolean operator also reports {@code boolean} as the
 178          * {@code rangeType}.
 179          * @return whether this operator returns a boolean
 180          */
 181         boolean isBoolean();
 182 
 183         /**
 184          * Reports the special return type of this operator.
 185          * If this operator is a boolean, returns {@code boolean.class}.
 186          * If this operator is a {@code Conversion},
 187          * returns its {@linkplain Conversion#rangeType range type}.
 188          *
 189          * Otherwise, the operator's return value always has
 190          * whatever type was given as an input, and this method
 191          * returns {@code Object.class} to denote that fact.
 192          * @return the special return type, or {@code Object.class} if none
 193          */
 194         Class<?> rangeType();
 195 
 196         /**
 197          * Returns the associativity of this operator.
 198          * Only binary operators can be associative.
 199          * @return the associativity of this operator
 200          */
 201         boolean isAssociative();
 202 
 203         /**
 204          * Reports whether this operator is compatible with
 205          * the proposed element type.
 206          *
 207          * First, unrestricted operators are compatible with all element
 208          * types.
 209          *
 210          * Next, if the element type is {@code double} or {@code float}
 211          * and the operator is restricted to floating point types, it is
 212          * compatible.
 213          *
 214          * Otherwise, if the element type is neither {@code double} nor
 215          * {@code float} and the operator is restricted to integral
 216          * types, it is compatible.  Otherwise, the operator is not
 217          * compatible.
 218          *
 219          * @param elementType the proposed operand type for the operator
 220          * @return whether the proposed type is compatible with this operator
 221          */
 222         boolean compatibleWith(Class<?> elementType);
 223 
 224         // FIXME: Maybe add a query about architectural support.
 225     }
 226 
 227     /**
 228      * Type for all
 229      * <a href="Vector.html#lane-wise">lane-wise</a>
 230      * unary (one-argument) operators,
 231      * usable in expressions like {@code w = v0.}{@link
 232      * Vector#lanewise(VectorOperators.Unary)
 233      * lanewise}{@code (NEG)}.
 234      *
 235      * @apiNote
 236      * User code should not implement this interface.  A future release of
 237      * this type may restrict implementations to be members of the same
 238      * package.
 239      */
 240     public interface Unary extends Operator {
 241     }
 242 
 243     /**
 244      * Type for all
 245      * <a href="Vector.html#lane-wise">lane-wise</a>
 246      * binary (two-argument) operators,
 247      * usable in expressions like {@code w = v0.}{@link
 248      * Vector#lanewise(VectorOperators.Binary,Vector)
 249      * lanewise}{@code (ADD, v1)}.
 250      *
 251      * @apiNote
 252      * User code should not implement this interface.  A future release of
 253      * this type may restrict implementations to be members of the same
 254      * package.
 255      */
 256     public interface Binary extends Operator {
 257     }
 258 
 259     /**
 260      * Type for all
 261      * <a href="Vector.html#lane-wise">lane-wise</a>
 262      * ternary (three-argument) operators,
 263      * usable in expressions like {@code w = v0.}{@link
 264      * Vector#lanewise(VectorOperators.Ternary,Vector,Vector)
 265      * lanewise}{@code (FMA, v1, v2)}.
 266      *
 267      * @apiNote
 268      * User code should not implement this interface.  A future release of
 269      * this type may restrict implementations to be members of the same
 270      * package.
 271      */
 272     public interface Ternary extends Operator {
 273     }
 274 
 275     /**
 276      * Type for all reassociating
 277      * <a href="Vector.html#lane-wise">lane-wise</a>
 278      * binary operators,
 279      * usable in expressions like {@code e = v0.}{@link
 280      * IntVector#reduceLanes(VectorOperators.Associative)
 281      * reduceLanes}{@code (ADD)}.
 282      *
 283      * @apiNote
 284      * User code should not implement this interface.  A future release of
 285      * this type may restrict implementations to be members of the same
 286      * package.
 287      */
 288     public interface Associative extends Binary {
 289     }
 290 
 291     /**
 292      * Type for all unary
 293      * <a href="Vector.html#lane-wise">lane-wise</a>
 294      * boolean tests on lane values,
 295      * usable in expressions like {@code m = v0.}{@link
 296      * FloatVector#test(VectorOperators.Test)
 297      * test}{@code (IS_FINITE)}.
 298      *
 299      * @apiNote
 300      * User code should not implement this interface.  A future release of
 301      * this type may restrict implementations to be members of the same
 302      * package.
 303      */
 304     public interface Test extends Operator {
 305     }
 306 
 307     /**
 308      * Type for all binary
 309      * <a href="Vector.html#lane-wise">lane-wise</a>
 310      * boolean comparisons on lane values,
 311      * usable in expressions like {@code m = v0.}{@link
 312      * Vector#compare(VectorOperators.Comparison,Vector)
 313      * compare}{@code (LT, v1)}.
 314      *
 315      * @apiNote
 316      * User code should not implement this interface.  A future release of
 317      * this type may restrict implementations to be members of the same
 318      * package.
 319      */
 320     public interface Comparison extends Operator {
 321     }
 322 
 323     /**
 324      * Type for all
 325      * <a href="Vector.html#lane-wise">lane-wise</a>
 326      * conversions on lane values,
 327      * usable in expressions like {@code w1 = v0.}{@link
 328      * Vector#convert(VectorOperators.Conversion,int)
 329      * convert}{@code (I2D, 1)}.
 330      *
 331      * @param <E> the boxed element type for the conversion
 332      *        domain type (the input lane type)
 333      * @param <F> the boxed element type for the conversion
 334      *        range type (the output lane type)
 335      *
 336      * @apiNote
 337      * User code should not implement this interface.  A future release of
 338      * this type may restrict implementations to be members of the same
 339      * package.
 340      */
 341     public interface Conversion<E,F> extends Operator {
 342         /**
 343          * The domain of this conversion, a primitive type.
 344          * @return the domain of this conversion
 345          */
 346         Class<E> domainType();
 347 
 348         /**
 349          * The range of this conversion, a primitive type.
 350          * @return the range of this conversion
 351          */
 352         @Override
 353         Class<F> rangeType();
 354 
 355         /**
 356          * Ensures that this conversion has the
 357          * desired domain and range types.
 358          * @param from the desired domain type
 359          * @param to the desired range type
 360          * @param <E> the desired domain type
 361          * @param <F> the desired range type
 362          * @return this conversion object, with validated domain and range
 363          */
 364         <E,F> Conversion<E,F> check(Class<E> from, Class<F> to);
 365 
 366         /// Factories.
 367 
 368         /**
 369          * The Java language assignment or casting conversion between two types.
 370          * @param <E> the domain type (boxed version of a lane type)
 371          * @param <F> the range type (boxed version of a lane type)
 372          * @param from the type of the value to convert
 373          * @param to the desired type after conversion
 374          * @return a Java assignment or casting conversion
 375          */
 376         @ForceInline
 377         static <E,F> Conversion<E,F> ofCast(Class<E> from, Class<F> to) {
 378             LaneType dom = LaneType.of(from);
 379             LaneType ran = LaneType.of(to);
 380             return ConversionImpl.ofCast(dom, ran).check(from, to);
 381         }
 382 
 383         /**
 384          * The bitwise reinterpretation between two types.
 385          * @param <E> the domain type (boxed version of a lane type)
 386          * @param <F> the range type (boxed version of a lane type)
 387          * @param from the type of the value to reinterpret
 388          * @param to the desired type after reinterpretation
 389          * @return a bitwise reinterpretation conversion
 390          */
 391         @ForceInline
 392         static <E,F> Conversion<E,F> ofReinterpret(Class<E> from, Class<F> to) {
 393             LaneType dom = LaneType.of(from);
 394             LaneType ran = LaneType.of(to);
 395             return ConversionImpl.ofReinterpret(dom, ran).check(from, to);
 396         }
 397 
 398     }
 399 
 400     /*package-private*/
 401     @ForceInline
 402     static int opCode(Operator op, int requireKind, int forbidKind) {
 403         return ((OperatorImpl)op).opCode(requireKind, forbidKind);
 404     }
 405 
 406     /*package-private*/
 407     @ForceInline
 408     static boolean opKind(Operator op, int bit) {
 409         return ((OperatorImpl)op).opKind(bit);
 410     }
 411 
 412     /*package-private*/
 413     static final int
 414         VO_ALL                     = 0,
 415         VO_UNARY                   = 0x001,
 416         VO_BINARY                  = 0x002,
 417         VO_TERNARY                 = 0x003,
 418         VO_ARITY_MASK              = (VO_UNARY|VO_BINARY|VO_TERNARY),
 419         VO_ASSOC                   = 0x004,
 420         VO_SHIFT                   = 0x008,
 421         VO_BOOL                    = 0x010,
 422         VO_CONV                    = 0x020,
 423         VO_PRIVATE                 = 0x040, // not public, invisible
 424         VO_SPECIAL                 = 0x080, // random special handling
 425         VO_NOFP                    = 0x100,
 426         VO_ONLYFP                  = 0x200,
 427         VO_OPCODE_VALID            = 0x800,
 428         VO_OPCODE_SHIFT            = 12,
 429         VO_OPCODE_LIMIT            = 0x400,
 430         VO_RAN_SHIFT               = 0,
 431         VO_DOM_SHIFT               = 4,
 432         VO_DOM_RAN_MASK            = 0x0FF,
 433         VO_KIND_CAST               = 0x000,
 434         VO_KIND_BITWISE            = 0x100;
 435 
 436     private static final HashMap<Integer, String> OPC_NAME
 437         = new HashMap<>();
 438     private static final HashMap<Integer, String> CMP_OPC_NAME
 439         = new HashMap<>();
 440     private static final HashMap<Integer, String> CONV_OPC_NAME
 441         = new HashMap<>();
 442 
 443     // Unary operators
 444 
 445     /** Produce {@code ~a}.  Integral only. */
 446     public static final /*bitwise*/ Unary NOT = unary("NOT", "~", -1 /*VectorSupport.VECTOR_OP_NOT*/, VO_NOFP | VO_SPECIAL);
 447     /** Produce {@code a==0?0:-1} (zero or minus one).  Integral only. */
 448     public static final /*bitwise*/ Unary ZOMO = unary("ZOMO", "a==0?0:-1", -1 /*VectorSupport.VECTOR_OP_ZOMO*/, VO_NOFP);
 449     /** Produce {@code abs(a)}. */
 450     public static final Unary ABS = unary("ABS", "abs", VectorSupport.VECTOR_OP_ABS, VO_ALL);
 451     /** Produce {@code -a}. */
 452     public static final Unary NEG = unary("NEG", "-a", VectorSupport.VECTOR_OP_NEG, VO_ALL|VO_SPECIAL);




















 453 
 454     /** Produce {@code sin(a)}.  Floating only.
 455      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 456      */
 457     public static final /*float*/ Unary SIN = unary("SIN", "sin", VectorSupport.VECTOR_OP_SIN, VO_ONLYFP);
 458     /** Produce {@code cos(a)}.  Floating only.
 459      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 460      */
 461     public static final /*float*/ Unary COS = unary("COS", "cos", VectorSupport.VECTOR_OP_COS, VO_ONLYFP);
 462     /** Produce {@code tan(a)}.  Floating only.
 463      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 464      */
 465     public static final /*float*/ Unary TAN = unary("TAN", "tan", VectorSupport.VECTOR_OP_TAN, VO_ONLYFP);
 466     /** Produce {@code asin(a)}.  Floating only.
 467      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 468      */
 469     public static final /*float*/ Unary ASIN = unary("ASIN", "asin", VectorSupport.VECTOR_OP_ASIN, VO_ONLYFP);
 470     /** Produce {@code acos(a)}.  Floating only.
 471      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 472      */
 473     public static final /*float*/ Unary ACOS = unary("ACOS", "acos", VectorSupport.VECTOR_OP_ACOS, VO_ONLYFP);
 474     /** Produce {@code atan(a)}.  Floating only.
 475      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 476      */
 477     public static final /*float*/ Unary ATAN = unary("ATAN", "atan", VectorSupport.VECTOR_OP_ATAN, VO_ONLYFP);
 478 
 479     /** Produce {@code exp(a)}.  Floating only.
 480      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 481      */
 482     public static final /*float*/ Unary EXP = unary("EXP", "exp", VectorSupport.VECTOR_OP_EXP, VO_ONLYFP);
 483     /** Produce {@code log(a)}.  Floating only.
 484      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 485      */
 486     public static final /*float*/ Unary LOG = unary("LOG", "log", VectorSupport.VECTOR_OP_LOG, VO_ONLYFP);
 487     /** Produce {@code log10(a)}.  Floating only.
 488      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 489      */
 490     public static final /*float*/ Unary LOG10 = unary("LOG10", "log10", VectorSupport.VECTOR_OP_LOG10, VO_ONLYFP);
 491     /** Produce {@code sqrt(a)}.  Floating only.  See section "Operations on floating point vectors" above */
 492     public static final /*float*/ Unary SQRT = unary("SQRT", "sqrt", VectorSupport.VECTOR_OP_SQRT, VO_ONLYFP);
 493     /** Produce {@code cbrt(a)}.  Floating only.
 494      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 495      */
 496     public static final /*float*/ Unary CBRT = unary("CBRT", "cbrt", VectorSupport.VECTOR_OP_CBRT, VO_ONLYFP);
 497 
 498     /** Produce {@code sinh(a)}.  Floating only.
 499      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 500      */
 501     public static final /*float*/ Unary SINH = unary("SINH", "sinh", VectorSupport.VECTOR_OP_SINH, VO_ONLYFP);
 502     /** Produce {@code cosh(a)}.  Floating only.
 503      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 504      */
 505     public static final /*float*/ Unary COSH = unary("COSH", "cosh", VectorSupport.VECTOR_OP_COSH, VO_ONLYFP);
 506     /** Produce {@code tanh(a)}.  Floating only.
 507      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 508      */
 509     public static final /*float*/ Unary TANH = unary("TANH", "tanh", VectorSupport.VECTOR_OP_TANH, VO_ONLYFP);
 510     /** Produce {@code expm1(a)}.  Floating only.
 511      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 512      */
 513     public static final /*float*/ Unary EXPM1 = unary("EXPM1", "expm1", VectorSupport.VECTOR_OP_EXPM1, VO_ONLYFP);
 514     /** Produce {@code log1p(a)}.  Floating only.
 515      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 516      */
 517     public static final /*float*/ Unary LOG1P = unary("LOG1P", "log1p", VectorSupport.VECTOR_OP_LOG1P, VO_ONLYFP);
 518 
 519     // Binary operators
 520 
 521     /** Produce {@code a+b}. */
 522     public static final Associative ADD = assoc("ADD", "+", VectorSupport.VECTOR_OP_ADD, VO_ALL+VO_ASSOC);
 523     /** Produce {@code a-b}. */
 524     public static final Binary SUB = binary("SUB", "-", VectorSupport.VECTOR_OP_SUB, VO_ALL);
 525     /** Produce {@code a*b}. */
 526     public static final Associative MUL = assoc("MUL", "*", VectorSupport.VECTOR_OP_MUL, VO_ALL+VO_ASSOC);
 527     /** Produce {@code a/b}. Floating only. */
 528     public static final Binary DIV = binary("DIV", "/", VectorSupport.VECTOR_OP_DIV, VO_ALL| VO_SPECIAL);
 529     /** Produce {@code min(a,b)}. */
 530     public static final Associative MIN = assoc("MIN", "min", VectorSupport.VECTOR_OP_MIN, VO_ALL+VO_ASSOC);
 531     /** Produce {@code max(a,b)}. */
 532     public static final Associative MAX = assoc("MAX", "max", VectorSupport.VECTOR_OP_MAX, VO_ALL+VO_ASSOC);
 533     /** Produce {@code bits(a)!=0?a:b}. */
 534     public static final Associative FIRST_NONZERO = assoc("FIRST_NONZERO", "a!=0?a:b", -1 /*VectorSupport.VECTOR_OP_FIRST_NONZERO*/, VO_ALL+VO_ASSOC);
 535 
 536     /** Produce {@code a&b}.  Integral only. */
 537     public static final Associative AND = assoc("AND", "&", VectorSupport.VECTOR_OP_AND, VO_NOFP+VO_ASSOC);
 538     /** Produce {@code a&~b}.  Integral only. */
 539     public static final /*bitwise*/ Binary AND_NOT = binary("AND_NOT", "&~", -1 /*VectorSupport.VECTOR_OP_AND_NOT*/, VO_NOFP); // FIXME
 540     /** Produce {@code a|b}.  Integral only. */
 541     public static final /*bitwise*/ Associative OR = assoc("OR", "|", VectorSupport.VECTOR_OP_OR, VO_NOFP+VO_ASSOC);
 542     /*package-private*/ /** Version of OR which works on float and double too. */
 543     static final Associative OR_UNCHECKED = assoc("OR_UNCHECKED", "|", VectorSupport.VECTOR_OP_OR, VO_ASSOC+VO_PRIVATE);
 544     /** Produce {@code a^b}.  Integral only. */
 545     public static final /*bitwise*/ Associative XOR = assoc("XOR", "^", VectorSupport.VECTOR_OP_XOR, VO_NOFP+VO_ASSOC);
 546 
 547     /** Produce {@code a<<(n&(ESIZE*8-1))}.  Integral only. */
 548     public static final /*bitwise*/ Binary LSHL = binary("LSHL", "<<", VectorSupport.VECTOR_OP_LSHIFT, VO_SHIFT);
 549     /** Produce {@code a>>(n&(ESIZE*8-1))}.  Integral only. */
 550     public static final /*bitwise*/ Binary ASHR = binary("ASHR", ">>", VectorSupport.VECTOR_OP_RSHIFT, VO_SHIFT);
 551     /** Produce {@code a>>>(n&(ESIZE*8-1))}.  Integral only. */
 552     public static final /*bitwise*/ Binary LSHR = binary("LSHR", ">>>", VectorSupport.VECTOR_OP_URSHIFT, VO_SHIFT);
 553     /** Produce {@code rotateLeft(a,n)}.  Integral only. */
 554     public static final /*bitwise*/ Binary ROL = binary("ROL", "rotateLeft", VectorSupport.VECTOR_OP_LROTATE, VO_SHIFT);
 555     /** Produce {@code rotateRight(a,n)}.  Integral only. */
 556     public static final /*bitwise*/ Binary ROR = binary("ROR", "rotateRight", VectorSupport.VECTOR_OP_RROTATE, VO_SHIFT);








 557 
 558     /** Produce {@code atan2(a,b)}. See  Floating only.
 559      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 560      */
 561     public static final /*float*/ Binary ATAN2 = binary("ATAN2", "atan2", VectorSupport.VECTOR_OP_ATAN2, VO_ONLYFP);
 562     /** Produce {@code pow(a,b)}.  Floating only.
 563      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 564      */
 565     public static final /*float*/ Binary POW = binary("POW", "pow", VectorSupport.VECTOR_OP_POW, VO_ONLYFP);
 566     /** Produce {@code hypot(a,b)}.  Floating only.
 567      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
 568      */
 569     public static final /*float*/ Binary HYPOT = binary("HYPOT", "hypot", VectorSupport.VECTOR_OP_HYPOT, VO_ONLYFP);
 570 
 571     // Ternary operators
 572 
 573     /** Produce {@code a^((a^b)&c)}.  (Bitwise {@code (c(i)?b(i):a(i))}.)  Integral only. */
 574     public static final /*float*/ Ternary BITWISE_BLEND = ternary("BITWISE_BLEND", "a^((a^b)&c)", -1 /*VectorSupport.VECTOR_OP_BITWISE_BLEND*/, VO_NOFP);
 575     /** Produce {@code fma(a,b,c)}.  Floating only. */
 576     public static final /*float*/ Ternary FMA = ternary("FMA", "fma", VectorSupport.VECTOR_OP_FMA, VO_ONLYFP);
 577 
 578     // Unary boolean operators
 579     /** Test {@code bits(a)==0}.  (Not true of {@code -0.0}.) */
 580     public static final Test IS_DEFAULT = predicate("IS_DEFAULT", "bits(a)==0", -1 /*VectorSupport.VECTOR_OP_TEST_DEFAULT*/, VO_ALL);
 581     /** Test {@code bits(a)<0}.  (True of {@code -0.0}.) */
 582     public static final Test IS_NEGATIVE = predicate("IS_NEGATIVE", "bits(a)<0", -1 /*VectorSupport.VECTOR_OP_TEST_NEGATIVE*/, VO_ALL);
 583     /** Test {@code isFinite(a)}.  Floating only. */
 584     public static final Test IS_FINITE = predicate("IS_FINITE", "isFinite", -1 /*VectorSupport.VECTOR_OP_TEST_FINITE*/, VO_ONLYFP);
 585     /** Test {@code isNaN(a)}.  Floating only. */
 586     public static final Test IS_NAN = predicate("IS_NAN", "isNaN", -1 /*VectorSupport.VECTOR_OP_TEST_NAN*/, VO_ONLYFP);
 587     /** Test {@code isInfinite(a)}.  Floating only. */
 588     public static final Test IS_INFINITE = predicate("IS_INFINITE", "isInfinite", -1 /*VectorSupport.VECTOR_OP_TEST_INFINITE*/, VO_ONLYFP);
 589 
 590     // Binary boolean operators
 591 
 592     /** Compare {@code a==b}. */
 593     public static final Comparison EQ = compare("EQ", "==", VectorSupport.BT_eq, VO_ALL);
 594     /** Compare {@code a!=b}. */
 595     public static final Comparison NE = compare("NE", "!=", VectorSupport.BT_ne, VO_ALL);
 596     /** Compare {@code a<b}. */
 597     public static final Comparison LT = compare("LT", "<",  VectorSupport.BT_lt, VO_ALL);
 598     /** Compare {@code a<=b}. */
 599     public static final Comparison LE = compare("LE", "<=", VectorSupport.BT_le, VO_ALL);
 600     /** Compare {@code a>b}. */
 601     public static final Comparison GT = compare("GT", ">",  VectorSupport.BT_gt, VO_ALL);
 602     /** Compare {@code a>=b}. */
 603     public static final Comparison GE = compare("GE", ">=", VectorSupport.BT_ge, VO_ALL);
 604     /** Unsigned compare {@code a<b}.  Integral only.
 605      * @see java.lang.Integer#compareUnsigned
 606      * @see java.lang.Long#compareUnsigned
 607      */
 608     public static final Comparison UNSIGNED_LT = compare("UNSIGNED_LT", "<",  VectorSupport.BT_ult, VO_NOFP);
 609     /** Unsigned compare {@code a<=b}.  Integral only.
 610      * @see java.lang.Integer#compareUnsigned
 611      * @see java.lang.Long#compareUnsigned
 612      */
 613     public static final Comparison UNSIGNED_LE = compare("UNSIGNED_LE", "<=", VectorSupport.BT_ule, VO_NOFP);
 614     /** Unsigned compare {@code a>b}.  Integral only.
 615      * @see java.lang.Integer#compareUnsigned
 616      * @see java.lang.Long#compareUnsigned
 617      */
 618     public static final Comparison UNSIGNED_GT = compare("UNSIGNED_GT", ">",  VectorSupport.BT_ugt, VO_NOFP);
 619     /** Unsigned compare {@code a>=b}.  Integral only.
 620      * @see java.lang.Integer#compareUnsigned
 621      * @see java.lang.Long#compareUnsigned
 622      */
 623     public static final Comparison UNSIGNED_GE = compare("UNSIGNED_GE", ">=", VectorSupport.BT_uge, VO_NOFP);
 624 
 625     // Conversion operators
 626 
 627     /** Convert {@code byteVal} to {@code (double)byteVal}. */
 628     public static final Conversion<Byte,Double> B2D = convert("B2D", 'C', byte.class, double.class, VO_KIND_CAST, VO_ALL);
 629     /** Convert {@code byteVal} to {@code (float)byteVal}. */
 630     public static final Conversion<Byte,Float> B2F = convert("B2F", 'C', byte.class, float.class, VO_KIND_CAST, VO_ALL);
 631     /** Convert {@code byteVal} to {@code (int)byteVal}. */
 632     public static final Conversion<Byte,Integer> B2I = convert("B2I", 'C', byte.class, int.class, VO_KIND_CAST, VO_ALL);
 633     /** Convert {@code byteVal} to {@code (long)byteVal}. */
 634     public static final Conversion<Byte,Long> B2L = convert("B2L", 'C', byte.class, long.class, VO_KIND_CAST, VO_ALL);
 635     /** Convert {@code byteVal} to {@code (short)byteVal}. */
 636     public static final Conversion<Byte,Short> B2S = convert("B2S", 'C', byte.class, short.class, VO_KIND_CAST, VO_ALL);
 637     /** Convert {@code doubleVal} to {@code (byte)doubleVal}. */
 638     public static final Conversion<Double,Byte> D2B = convert("D2B", 'C', double.class, byte.class, VO_KIND_CAST, VO_ALL);
 639     /** Convert {@code doubleVal} to {@code (float)doubleVal}. */
 640     public static final Conversion<Double,Float> D2F = convert("D2F", 'C', double.class, float.class, VO_KIND_CAST, VO_ALL);
 641     /** Convert {@code doubleVal} to {@code (int)doubleVal}. */
 642     public static final Conversion<Double,Integer> D2I = convert("D2I", 'C', double.class, int.class, VO_KIND_CAST, VO_ALL);
 643     /** Convert {@code doubleVal} to {@code (long)doubleVal}. */
 644     public static final Conversion<Double,Long> D2L = convert("D2L", 'C', double.class, long.class, VO_KIND_CAST, VO_ALL);
 645     /** Convert {@code doubleVal} to {@code (short)doubleVal}. */
 646     public static final Conversion<Double,Short> D2S = convert("D2S", 'C', double.class, short.class, VO_KIND_CAST, VO_ALL);
 647     /** Convert {@code floatVal} to {@code (byte)floatVal}. */
 648     public static final Conversion<Float,Byte> F2B = convert("F2B", 'C', float.class, byte.class, VO_KIND_CAST, VO_ALL);
 649     /** Convert {@code floatVal} to {@code (double)floatVal}. */
 650     public static final Conversion<Float,Double> F2D = convert("F2D", 'C', float.class, double.class, VO_KIND_CAST, VO_ALL);
 651     /** Convert {@code floatVal} to {@code (int)floatVal}. */
 652     public static final Conversion<Float,Integer> F2I = convert("F2I", 'C', float.class, int.class, VO_KIND_CAST, VO_ALL);
 653     /** Convert {@code floatVal} to {@code (long)floatVal}. */
 654     public static final Conversion<Float,Long> F2L = convert("F2L", 'C', float.class, long.class, VO_KIND_CAST, VO_ALL);
 655     /** Convert {@code floatVal} to {@code (short)floatVal}. */
 656     public static final Conversion<Float,Short> F2S = convert("F2S", 'C', float.class, short.class, VO_KIND_CAST, VO_ALL);
 657     /** Convert {@code intVal} to {@code (byte)intVal}. */
 658     public static final Conversion<Integer,Byte> I2B = convert("I2B", 'C', int.class, byte.class, VO_KIND_CAST, VO_ALL);
 659     /** Convert {@code intVal} to {@code (double)intVal}. */
 660     public static final Conversion<Integer,Double> I2D = convert("I2D", 'C', int.class, double.class, VO_KIND_CAST, VO_ALL);
 661     /** Convert {@code intVal} to {@code (float)intVal}. */
 662     public static final Conversion<Integer,Float> I2F = convert("I2F", 'C', int.class, float.class, VO_KIND_CAST, VO_ALL);
 663     /** Convert {@code intVal} to {@code (long)intVal}. */
 664     public static final Conversion<Integer,Long> I2L = convert("I2L", 'C', int.class, long.class, VO_KIND_CAST, VO_ALL);
 665     /** Convert {@code intVal} to {@code (short)intVal}. */
 666     public static final Conversion<Integer,Short> I2S = convert("I2S", 'C', int.class, short.class, VO_KIND_CAST, VO_ALL);
 667     /** Convert {@code longVal} to {@code (byte)longVal}. */
 668     public static final Conversion<Long,Byte> L2B = convert("L2B", 'C', long.class, byte.class, VO_KIND_CAST, VO_ALL);
 669     /** Convert {@code longVal} to {@code (double)longVal}. */
 670     public static final Conversion<Long,Double> L2D = convert("L2D", 'C', long.class, double.class, VO_KIND_CAST, VO_ALL);
 671     /** Convert {@code longVal} to {@code (float)longVal}. */
 672     public static final Conversion<Long,Float> L2F = convert("L2F", 'C', long.class, float.class, VO_KIND_CAST, VO_ALL);
 673     /** Convert {@code longVal} to {@code (int)longVal}. */
 674     public static final Conversion<Long,Integer> L2I = convert("L2I", 'C', long.class, int.class, VO_KIND_CAST, VO_ALL);
 675     /** Convert {@code longVal} to {@code (short)longVal}. */
 676     public static final Conversion<Long,Short> L2S = convert("L2S", 'C', long.class, short.class, VO_KIND_CAST, VO_ALL);
 677     /** Convert {@code shortVal} to {@code (byte)shortVal}. */
 678     public static final Conversion<Short,Byte> S2B = convert("S2B", 'C', short.class, byte.class, VO_KIND_CAST, VO_ALL);
 679     /** Convert {@code shortVal} to {@code (double)shortVal}. */
 680     public static final Conversion<Short,Double> S2D = convert("S2D", 'C', short.class, double.class, VO_KIND_CAST, VO_ALL);
 681     /** Convert {@code shortVal} to {@code (float)shortVal}. */
 682     public static final Conversion<Short,Float> S2F = convert("S2F", 'C', short.class, float.class, VO_KIND_CAST, VO_ALL);
 683     /** Convert {@code shortVal} to {@code (int)shortVal}. */
 684     public static final Conversion<Short,Integer> S2I = convert("S2I", 'C', short.class, int.class, VO_KIND_CAST, VO_ALL);
 685     /** Convert {@code shortVal} to {@code (long)shortVal}. */
 686     public static final Conversion<Short,Long> S2L = convert("S2L", 'C', short.class, long.class, VO_KIND_CAST, VO_ALL);
 687     /** Reinterpret bits of {@code doubleVal} as {@code long}. As if by {@link Double#doubleToRawLongBits(double)} */
 688     public static final Conversion<Double,Long> REINTERPRET_D2L = convert("REINTERPRET_D2L", 'R', double.class, long.class, VO_KIND_BITWISE, VO_ALL);
 689     /** Reinterpret bits of {@code floatVal} as {@code int}. As if by {@link Float#floatToRawIntBits(float)} */
 690     public static final Conversion<Float,Integer> REINTERPRET_F2I = convert("REINTERPRET_F2I", 'R', float.class, int.class, VO_KIND_BITWISE, VO_ALL);
 691     /** Reinterpret bits of {@code intVal} as {@code float}. As if by {@link Float#intBitsToFloat(int)} */
 692     public static final Conversion<Integer,Float> REINTERPRET_I2F = convert("REINTERPRET_I2F", 'R', int.class, float.class, VO_KIND_BITWISE, VO_ALL);
 693     /** Reinterpret bits of {@code longVal} as {@code double}. As if by {@link Double#longBitsToDouble(long)} */
 694     public static final Conversion<Long,Double> REINTERPRET_L2D = convert("REINTERPRET_L2D", 'R', long.class, double.class, VO_KIND_BITWISE, VO_ALL);
 695     /** Zero-extend {@code byteVal} to {@code int}. */
 696     public static final Conversion<Byte,Integer> ZERO_EXTEND_B2I = convert("ZERO_EXTEND_B2I", 'Z', byte.class, int.class, VO_KIND_BITWISE, VO_ALL);
 697     /** Zero-extend {@code byteVal} to {@code long}. */
 698     public static final Conversion<Byte,Long> ZERO_EXTEND_B2L = convert("ZERO_EXTEND_B2L", 'Z', byte.class, long.class, VO_KIND_BITWISE, VO_ALL);
 699     /** Zero-extend {@code byteVal} to {@code short}. */
 700     public static final Conversion<Byte,Short> ZERO_EXTEND_B2S = convert("ZERO_EXTEND_B2S", 'Z', byte.class, short.class, VO_KIND_BITWISE, VO_ALL);
 701     /** Zero-extend {@code intVal} to {@code long}. */
 702     public static final Conversion<Integer,Long> ZERO_EXTEND_I2L = convert("ZERO_EXTEND_I2L", 'Z', int.class, long.class, VO_KIND_BITWISE, VO_ALL);
 703     /** Zero-extend {@code shortVal} to {@code int}. */
 704     public static final Conversion<Short,Integer> ZERO_EXTEND_S2I = convert("ZERO_EXTEND_S2I", 'Z', short.class, int.class, VO_KIND_BITWISE, VO_ALL);
 705     /** Zero-extend {@code shortVal} to {@code long}. */
 706     public static final Conversion<Short,Long> ZERO_EXTEND_S2L = convert("ZERO_EXTEND_S2L", 'Z', short.class, long.class, VO_KIND_BITWISE, VO_ALL);
 707 
 708     // (End of conversion operators)
 709 
 710     private static int opInfo(int opCode, int bits) {
 711         if (opCode >= 0) {
 712             bits |= VO_OPCODE_VALID;
 713         } else {
 714             opCode &= (VO_OPCODE_LIMIT - 1);  // not a valid op
 715             bits |= VO_SPECIAL;  // mark for special handling
 716         }
 717         assert((bits >> VO_OPCODE_SHIFT) == 0);
 718         assert(opCode >= 0 && opCode < VO_OPCODE_LIMIT);
 719         return (opCode << VO_OPCODE_SHIFT) + bits;
 720     }
 721 
 722     private static Unary unary(String name, String opName, int opCode, int flags) {
 723         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 724             OPC_NAME.put(opCode, name);
 725         return new UnaryImpl(name, opName, opInfo(opCode, flags | VO_UNARY));
 726     }
 727 
 728     private static Binary binary(String name, String opName, int opCode, int flags) {
 729         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 730             OPC_NAME.put(opCode, name);
 731         return new BinaryImpl(name, opName, opInfo(opCode, flags | VO_BINARY));
 732     }
 733 
 734     private static Ternary ternary(String name, String opName, int opCode, int flags) {
 735         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 736             OPC_NAME.put(opCode, name);
 737         return new TernaryImpl(name, opName, opInfo(opCode, flags | VO_TERNARY));
 738     }
 739 
 740     private static Associative assoc(String name, String opName, int opCode, int flags) {
 741         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 742             OPC_NAME.put(opCode, name);
 743         return new AssociativeImpl(name, opName, opInfo(opCode, flags | VO_BINARY | VO_ASSOC));
 744     }
 745 
 746     private static Test predicate(String name, String opName, int opCode, int flags) {
 747         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 748             CMP_OPC_NAME.put(opCode, name);
 749         return new TestImpl(name, opName, opInfo(opCode, flags | VO_UNARY | VO_BOOL));
 750     }
 751 
 752     private static Comparison compare(String name, String opName, int opCode, int flags) {
 753         if (opCode >= 0 && (flags & VO_PRIVATE) == 0)
 754             CMP_OPC_NAME.put(opCode, name);
 755         return new ComparisonImpl(name, opName, opInfo(opCode, flags | VO_BINARY | VO_BOOL));
 756     }
 757 
 758     private static <E,F> ConversionImpl<E,F>
 759     convert(String name, char kind, Class<E> dom, Class<F> ran, int opCode, int flags) {
 760         int domran = ((LaneType.of(dom).basicType << VO_DOM_SHIFT) +
 761                       (LaneType.of(ran).basicType << VO_RAN_SHIFT));
 762         if (opCode >= 0) {
 763             if ((opCode & VO_DOM_RAN_MASK) == 0) {
 764                 opCode += domran;
 765             }
 766             if ((flags & VO_PRIVATE) == 0)
 767                 CONV_OPC_NAME.put(opCode, name);
 768         }
 769         String opName = dom+"-"+kind+"-"+ran; //??
 770         return new ConversionImpl<>(name, opName, opInfo(opCode, flags | VO_UNARY | VO_CONV),
 771                                     kind, dom, ran);
 772     }
 773 
 774     private static abstract class OperatorImpl implements Operator {
 775         private final String symName;
 776         private final String opName;
 777         private final int opInfo;
 778 
 779         OperatorImpl(String symName, String opName, int opInfo) {
 780             this.symName = symName;
 781             this.opName = opName;
 782             this.opInfo = opInfo;
 783             assert(opInfo != 0);
 784         }
 785 
 786         @Override
 787         public final String name() {
 788             return symName;
 789         }
 790         @Override
 791         public final String operatorName() {
 792             return opName;
 793         }
 794         @Override
 795         public final String toString() {
 796             return name();
 797         }
 798         @Override
 799         public final int arity() {
 800             return opInfo & VO_ARITY_MASK;
 801         }
 802         @Override
 803         public final boolean isBoolean() {
 804             return opKind(VO_BOOL);
 805         }
 806         @Override
 807         public Class<?> rangeType() {
 808             return Object.class;
 809         }
 810         @Override
 811         public final boolean isAssociative() {
 812             return opKind(VO_ASSOC);
 813         }
 814 
 815         @ForceInline
 816         public boolean compatibleWith(Class<?> elementType) {
 817             return compatibleWith(LaneType.of(elementType));
 818         }
 819 
 820         /*package-private*/
 821         @ForceInline
 822         int opInfo() {
 823             return opInfo;
 824         }
 825 
 826         /*package-private*/
 827         @ForceInline
 828         int opCode(int requireKind, int forbidKind) {
 829             int opc = opCodeRaw();
 830             if ((opInfo & requireKind) != requireKind ||
 831                 (forbidKind != 0 &&
 832                  (opInfo & forbidKind)  == forbidKind)) {
 833                 throw illegalOperation(requireKind, forbidKind);
 834             }
 835             return opc;
 836         }
 837 
 838         /*package-private*/
 839         @ForceInline
 840         int opCodeRaw() {
 841             return (opInfo >> VO_OPCODE_SHIFT);
 842         }
 843 
 844         /*package-private*/
 845         UnsupportedOperationException
 846         illegalOperation(int requireKind, int forbidKind) {
 847             String msg1 = "";
 848             requireKind &=~ VO_OPCODE_VALID;
 849             switch (requireKind) {
 850             case VO_ONLYFP:  msg1 = "floating point operator required here"; break;
 851             case VO_NOFP:    msg1 = "integral/bitwise operator required here"; break;
 852             case VO_ASSOC:   msg1 = "associative operator required here"; break;
 853             }
 854             String msg2 = "";
 855             switch (forbidKind) {
 856             case VO_ONLYFP:  msg2 = "inapplicable floating point operator"; break;
 857             case VO_NOFP:    msg2 = "inapplicable integral/bitwise operator"; break;
 858             }
 859             if ((opInfo & VO_OPCODE_VALID) == 0) {
 860                 msg2 = "operator is not implemented";
 861             }
 862             return illegalOperation(msg1, msg2);
 863         }
 864 
 865         /*package-private*/
 866         UnsupportedOperationException
 867         illegalOperation(String msg1, String msg2) {
 868             String dot = "";
 869             if (!msg1.isEmpty() && !msg2.isEmpty()) {
 870                 dot = "; ";
 871             } else if (msg1.isEmpty() && msg2.isEmpty()) {
 872                 // Couldn't decode the *kind bits.
 873                 msg1 = "illegal operator";
 874             }
 875             String msg = String.format("%s: %s%s%s", this, msg1, dot, msg2);
 876             return new UnsupportedOperationException(msg);
 877         }
 878 
 879 
 880         /*package-private*/
 881         @ForceInline
 882         boolean opKind(int kindBit) {
 883             return (opInfo & kindBit) != 0;
 884         }
 885 
 886         @ForceInline
 887         /*package-private*/
 888         boolean compatibleWith(LaneType laneType) {
 889             if (laneType.elementKind == 'F') {
 890                 return !opKind(VO_NOFP);
 891             } else if (laneType.elementKind == 'I') {
 892                 return !opKind(VO_ONLYFP);
 893             } else {
 894                 throw new AssertionError();
 895             }
 896         }
 897     }
 898 
 899     private static class UnaryImpl extends OperatorImpl implements Unary {
 900         private UnaryImpl(String symName, String opName, int opInfo) {
 901             super(symName, opName, opInfo);
 902             assert((opInfo & VO_ARITY_MASK) == VO_UNARY);
 903         }
 904     }
 905 
 906     private static class BinaryImpl extends OperatorImpl implements Binary {
 907         private BinaryImpl(String symName, String opName, int opInfo) {
 908             super(symName, opName, opInfo);
 909             assert((opInfo & VO_ARITY_MASK) == VO_BINARY);
 910         }
 911     }
 912 
 913     private static class TernaryImpl extends OperatorImpl implements Ternary {
 914         private TernaryImpl(String symName, String opName, int opInfo) {
 915             super(symName, opName, opInfo);
 916             assert((opInfo & VO_ARITY_MASK) == VO_TERNARY);
 917         }
 918     }
 919 
 920     private static class AssociativeImpl extends BinaryImpl implements Associative {
 921         private AssociativeImpl(String symName, String opName, int opInfo) {
 922             super(symName, opName, opInfo);
 923         }
 924     }
 925 
 926     /*package-private*/
 927     static
 928     class ConversionImpl<E,F> extends OperatorImpl
 929                               implements Conversion<E,F> {
 930         private ConversionImpl(String symName, String opName, int opInfo,
 931                                char kind, Class<E> dom, Class<F> ran) {
 932             super(symName, opName, opInfo);
 933             assert((opInfo & VO_ARITY_MASK) == VO_UNARY);
 934             this.kind = kind;
 935             this.dom = LaneType.of(dom);
 936             this.ran = LaneType.of(ran);
 937             check(dom, ran);  // make sure it all lines up
 938         }
 939         private final char kind;
 940         private final LaneType dom;
 941         private final LaneType ran;
 942 
 943         // non-overrides are all package-private
 944 
 945         char kind()  { return kind; }
 946         LaneType domain() { return dom; }
 947         LaneType range()  { return ran; }
 948 
 949         int sizeChangeLog2() {
 950             return ran.elementSizeLog2 - dom.elementSizeLog2;
 951         }
 952 
 953         @SuppressWarnings("unchecked")
 954         @Override
 955         public Class<E> domainType() {
 956             return (Class<E>) dom.elementType;
 957         }
 958         @SuppressWarnings("unchecked")
 959         @Override
 960         public Class<F> rangeType() {
 961             return (Class<F>) ran.elementType;
 962         }
 963 
 964         @SuppressWarnings("unchecked")
 965         @ForceInline
 966         public
 967         <E,F> Conversion<E,F>
 968         check(Class<E> dom, Class<F> ran) {
 969             if (this.dom.elementType != dom ||
 970                 this.ran.elementType != ran)
 971                 throw checkFailed(dom, ran);
 972             return (Conversion<E,F>) this;
 973         }
 974         private RuntimeException checkFailed(Class<?> dom, Class<?> ran) {
 975             return new ClassCastException(toString()+": not "+dom+" -> "+ran);
 976         }
 977 
 978         static ConversionImpl<?,?> ofCopy(LaneType dom) {
 979             return findConv('I', dom, dom);
 980         }
 981         static ConversionImpl<?,?> ofCast(LaneType dom, LaneType ran) {
 982             if (dom == ran)  return ofCopy(dom);
 983             return findConv('C', dom, ran);
 984         }
 985         static ConversionImpl<?,?> ofReinterpret(LaneType dom, LaneType ran) {
 986             if (dom == ran)  return ofCopy(dom);
 987             if (dom.elementKind == 'I' &&
 988                 ran.elementKind == 'I' &&
 989                 dom.elementSize < ran.elementSize) {
 990                 // Zero extension of field (unsigned semantics).
 991                 return findConv('Z', dom, ran);
 992             }
 993             // if (dom.elementSize != ran.elementSize) {
 994             //     throw new IllegalArgumentException("bad reinterpret");
 995             // }
 996             return findConv('R', dom, ran);
 997         }
 998 
 999         @ForceInline
1000         private static ConversionImpl<?,?>
1001         findConv(char kind, LaneType dom, LaneType ran) {
1002             ConversionImpl<?,?>[] cache = cacheOf(kind, dom);
1003             int ranKey = ran.switchKey;
1004             ConversionImpl<?,?> conv = cache[ranKey];
1005             if (conv != null) {
1006                 return conv;
1007             }
1008             return makeConv(kind, dom, ran);
1009         }
1010 
1011         static String a2b(LaneType dom, LaneType ran) {
1012             return dom.typeChar + "2" + ran.typeChar;
1013         }
1014 
1015         static ConversionImpl<?,?>
1016         makeConv(char kind, LaneType dom, LaneType ran) {
1017             String name;
1018             Class<?> domType = dom.elementType;
1019             Class<?> ranType = ran.elementType;
1020             int domCode = (dom.basicType << VO_DOM_SHIFT);
1021             int ranCode = (ran.basicType << VO_RAN_SHIFT);
1022             int opCode = domCode + ranCode;
1023             switch (kind) {
1024             case 'I':
1025                 assert(dom == ran);
1026                 name = "COPY_"+a2b(dom, ran);
1027                 opCode = VO_KIND_CAST;
1028                 break;
1029             case 'C':
1030                 name = ""+a2b(dom, ran);
1031                 opCode = VO_KIND_CAST;
1032                 break;
1033             case 'R':
1034                 name = "REINTERPRET_"+a2b(dom, ran);
1035                 opCode = VO_KIND_BITWISE;
1036                 break;
1037             case 'Z':
1038                 name = "ZERO_EXTEND_"+a2b(dom, ran);
1039                 opCode = VO_KIND_BITWISE;
1040                 break;
1041             default:  throw new AssertionError();
1042             }
1043             ConversionImpl<?,?> conv = convert(name, kind, domType, ranType, opCode, VO_ALL);
1044             // Put into the cache for next time.
1045             // The JIT can see into this cache
1046             // when kind/dom/ran are all constant.
1047             ConversionImpl<?,?>[] cache = cacheOf(kind, dom);
1048             int ranKey = ran.switchKey;
1049             // The extra "check" calls help validate that
1050             // we aren't cross-wiring the cache.
1051             conv.check(domType, ranType);
1052             synchronized (ConversionImpl.class) {
1053                 if (cache[ranKey] == null) {
1054                     cache[ranKey] = conv;
1055                 } else {
1056                     conv = cache[ranKey];
1057                     conv.check(domType, ranType);
1058                 }
1059             }
1060             return conv;
1061         }
1062         private final void check(char kind, LaneType dom, LaneType ran) {
1063             if (this.kind != kind || this.dom != dom || this.ran != ran) {
1064                 throw new AssertionError(this + " != " + dom + kind + ran);
1065             }
1066         }
1067 
1068         /** Helper for cache probes. */
1069         @ForceInline
1070         private static ConversionImpl<?,?>[]
1071         cacheOf(char kind, LaneType dom) {
1072             assert("CIRZWN".indexOf(kind) >= 0);
1073             int k = (kind <= 'I' ? KIND_CI :
1074                      (kind == 'R' || kind == 'Z') ? KIND_RZ :
1075                      KIND_WN);
1076             return CACHES[k][dom.switchKey];
1077         }
1078         private static final int
1079             LINE_LIMIT = LaneType.SK_LIMIT,
1080             KIND_CI = 0, KIND_RZ = 1, KIND_WN = 2, KIND_LIMIT = 3;
1081         private static final @Stable ConversionImpl<?,?>[][][]
1082             CACHES = new ConversionImpl<?,?>[KIND_LIMIT][LINE_LIMIT][LINE_LIMIT];
1083 
1084         private synchronized static void initCaches() {
1085             for (var f : VectorOperators.class.getFields()) {
1086                 if (f.getType() != Conversion.class)  continue;
1087                 ConversionImpl<?,?> conv;
1088                 try {
1089                     conv = (ConversionImpl) f.get(null);
1090                 } catch (ReflectiveOperationException ex) {
1091                     throw new AssertionError(ex);
1092                 }
1093                 LaneType dom = conv.dom;
1094                 LaneType ran = conv.ran;
1095                 int opc = conv.opCodeRaw();
1096                 switch (conv.kind) {
1097                 case 'W':
1098                     int domCode = (opc >> VO_DOM_SHIFT) & 0xF;
1099                     dom = LaneType.ofBasicType(domCode);
1100                     break;
1101                 case 'N':
1102                     int ranCode = (opc >> VO_RAN_SHIFT) & 0xF;
1103                     ran = LaneType.ofBasicType(ranCode);
1104                     break;
1105                 }
1106                 assert((opc & VO_DOM_RAN_MASK) ==
1107                        ((dom.basicType << VO_DOM_SHIFT) +
1108                         (ran.basicType << VO_RAN_SHIFT)));
1109                 ConversionImpl<?,?>[] cache = cacheOf(conv.kind, dom);
1110                 int ranKey = ran.switchKey;
1111                 if (cache[ranKey] != conv) {
1112                     assert(cache[ranKey] == null ||
1113                            cache[ranKey].name().equals(conv.name()))
1114                         : conv + " vs. " + cache[ranKey];
1115                     cache[ranKey] = conv;
1116                 }
1117             }
1118         }
1119 
1120         // hack for generating static field defs
1121         static { assert(genCode()); }
1122         private static boolean genCode() {
1123             if (true)  return true;  // remove to enable code
1124             ArrayList<String> defs = new ArrayList<>();
1125             for (LaneType l1 : LaneType.values()) {
1126                 for (LaneType l2 : LaneType.values()) {
1127                     for (int i = 0; i <= 1; i++) {
1128                         ConversionImpl<?,?> c;
1129                         try {
1130                             c = ((i == 0) ? ofCast(l1, l2) : ofReinterpret(l1, l2));
1131                         } catch (IllegalArgumentException ex) {
1132                             assert((i == 1 && l1.elementSize != l2.elementSize) ||
1133                                    (i == 2 && l1.elementSize == l2.elementSize));
1134                             continue;  // ignore this combo
1135                         }
1136                         if (c.kind == 'C' ||
1137                             c.kind == 'Z' ||
1138                             (c.kind == 'R' &&
1139                              l1.elementKind+l2.elementKind == 'F'+'I' &&
1140                              l1.elementSize == l2.elementSize) ||
1141                             (c.kind == 'N' || c.kind == 'W')) {
1142                             int opc = c.opCodeRaw();
1143                             String opcs;
1144                             switch (opc & ~VO_DOM_RAN_MASK) {
1145                             case VO_KIND_CAST: opcs = "VO_KIND_CAST"; break;
1146                             case VO_KIND_BITWISE: opcs = "VO_KIND_BITWISE"; break;
1147                             default: opcs = Integer.toHexString(opc);
1148                             }
1149                             String code = c.genCode(opcs);
1150                             if (!defs.contains(code))  defs.add(code);
1151                         }
1152                     }
1153                 }
1154             }
1155             java.util.Collections.sort(defs);
1156             for (String def : defs)  System.out.println(def);
1157             return true;
1158         }
1159         private String genCode(String opcs) {
1160             if (true)  return null;  // remove to enable code
1161             int domran = opCodeRaw() & VO_DOM_RAN_MASK;
1162             switch (kind()) {
1163             case 'W': case 'N':
1164                 opcs += " + 0x" + Integer.toHexString(domran);
1165             }
1166             String doc;
1167             switch (kind()) {
1168             case 'R':
1169                 doc = "Reinterpret bits of {@code _domVal} as {@code _ran}";
1170                 break;
1171             case 'Z':
1172                 doc = "Zero-extend {@code _domVal} to {@code _ran}";
1173                 break;
1174             case 'W':
1175                 doc = "In-place widen {@code _domVal} inside _ran to {@code (_ran)_domVal}";
1176                 LaneType logdom = LaneType.ofBasicType(domran >> VO_DOM_SHIFT & 0xF);
1177                 doc = doc.replace("_dom", logdom.elementType.getSimpleName());
1178                 break;
1179             case 'N':
1180                 doc = "In-place narrow {@code _domVal} to {@code (_ran)_domVal} inside _dom";
1181                 LaneType logran = LaneType.ofBasicType(domran >> VO_RAN_SHIFT & 0xF);
1182                 doc = doc.replace("_ran", logran.elementType.getSimpleName());
1183                 break;
1184             default:
1185                 doc = "Convert {@code _domVal} to {@code (_ran)_domVal}";
1186             }
1187             String code = (
1188                     "    /** _Doc. */" + "\n" +
1189                     "    public static final Conversion<_Dom,_Ran> _N" +
1190                     " = convert(\"_N\", '_K', _dom.class, _ran.class, _opc, VO_ALL);");
1191             return code
1192                 .replace("_Doc", doc)
1193                 .replace("_dom", dom.elementType.getSimpleName())
1194                 .replace("_ran", ran.elementType.getSimpleName())
1195                 .replace("_Dom", dom.genericElementType.getSimpleName())
1196                 .replace("_Ran", ran.genericElementType.getSimpleName())
1197                 .replace("_N", name())
1198                 .replace("_K", ""+kind())
1199                 .replace("_opc", ""+opcs);
1200         }
1201     }
1202 
1203     private static class TestImpl extends OperatorImpl implements Test {
1204         private TestImpl(String symName, String opName, int opInfo) {
1205             super(symName, opName, opInfo);
1206             assert((opInfo & VO_ARITY_MASK) == VO_UNARY);
1207             assert((opInfo & VO_BOOL) == VO_BOOL);
1208         }
1209         @Override
1210         public Class<?> rangeType() {
1211             return boolean.class;
1212         }
1213     }
1214 
1215     private static class ComparisonImpl extends OperatorImpl implements Comparison {
1216         private ComparisonImpl(String symName, String opName, int opInfo) {
1217             super(symName, opName, opInfo);
1218             assert((opInfo & VO_ARITY_MASK) == VO_BINARY);
1219             assert((opInfo & VO_BOOL) == VO_BOOL);
1220         }
1221         @Override
1222         public Class<?> rangeType() {
1223             return boolean.class;
1224         }
1225         /* --- *
1226         boolean test(long a, long b) {
1227             switch (opInfo() >> VO_OPCODE_SHIFT) {
1228             case VectorSupport.BT_eq:  return a == b;
1229             case VectorSupport.BT_ne:  return a != b;
1230             case VectorSupport.BT_lt:  return a <  b;
1231             case VectorSupport.BT_le:  return a <= b;
1232             case VectorSupport.BT_gt:  return a >  b;
1233             case VectorSupport.BT_ge:  return a >= b;
1234             }
1235             throw new AssertionError();
1236         }
1237         * --- */
1238     }
1239 
1240     static {
1241         ConversionImpl.initCaches();
1242         assert(checkConstants());
1243     }
1244 
1245     private static boolean checkConstants() {
1246         // Check uniqueness of opcodes, to prevent dumb aliasing errors.
1247         OperatorImpl[] ops = new OperatorImpl[VO_OPCODE_LIMIT << VO_OPCODE_SHIFT];
1248         for (var f : VectorOperators.class.getFields()) {
1249             Class<?> ft = f.getType();
1250             OperatorImpl op;
1251             try {
1252                 op = (OperatorImpl) f.get(null);
1253             } catch (ReflectiveOperationException ex) {
1254                 throw new AssertionError(ex);
1255             }
1256             assert(op.name().equals(f.getName())) : op;
1257             assert(op.isAssociative() == (ft == Associative.class)) : op;
1258             if (op.isBoolean()) {
1259                 assert(ft == (op.arity() == 2 ? Comparison.class : Test.class)) : op;
1260             }
1261             if (ft == Unary.class || ft == Conversion.class || ft == Test.class) {
1262                 assert(op.arity() == 1) : op;
1263             } else if (ft == Ternary.class) {
1264                 assert(op.arity() == 3) : op;
1265             } else {
1266                 assert(op.arity() == 2) : op;
1267                 if (ft != Associative.class &&
1268                     ft != Comparison.class) {
1269                     assert(ft == Binary.class) : op;
1270                 }
1271             }
1272             if (op.opKind(VO_OPCODE_VALID)) {
1273                 int opsMask = (((VO_OPCODE_LIMIT-1) << VO_OPCODE_SHIFT)
1274                                | VO_BOOL | VO_CONV
1275                                | VO_ARITY_MASK);
1276                 int opsIndex = op.opInfo & opsMask;
1277                 OperatorImpl op0 = ops[opsIndex];
1278                 assert(op0 == null)
1279                     : java.util.Arrays.asList(op0, Integer.toHexString(op0.opInfo), op, Integer.toHexString(op.opInfo));
1280                 ops[opsIndex] = op;
1281             } else {
1282                 // These are all the "-1" opcode guys we know about:
1283                 assert(op == ZOMO ||
1284                        op == FIRST_NONZERO ||
1285                        op == AND_NOT || op == NOT ||
1286                        op == ROL ||
1287                        op == ROR ||
1288                        op == IS_DEFAULT || op == IS_NEGATIVE ||
1289                        op == IS_FINITE || op == IS_NAN || op == IS_INFINITE ||
1290                        op == BITWISE_BLEND) : op;
1291             }
1292         }
1293         return true;
1294     }
1295 
1296     // Managing behavioral information on slow paths:
1297     /*package-private*/
1298     static class ImplCache<OP extends Operator,T> {
1299         public ImplCache(Class<OP> whatKind,
1300                          Class<? extends Vector<?>> whatVec) {
1301             this.whatKind = whatKind;
1302             this.whatVec = whatVec;
1303         }
1304 
1305         // These are used only for forming diagnostics:
1306         private final Class<OP> whatKind;
1307         private final Class<? extends Vector<?>> whatVec;
1308 
1309         private final @Stable
1310         Object[] cache = new Object[VO_OPCODE_LIMIT];
1311 
1312         @ForceInline
1313         public T find(OP op, int opc, IntFunction<T> supplier) {
1314             @SuppressWarnings("unchecked")
1315             T fn = (T) cache[opc];
1316             if (fn != null)  return fn;
1317             fn = supplier.apply(opc);
1318             if (fn == null)  throw badOp(op);
1319             assert(VectorSupport.isNonCapturingLambda(fn)) : fn;
1320             // The JIT can see into this cache:
1321             cache[opc] = fn;
1322             return fn;
1323         }
1324 
1325         private UnsupportedOperationException badOp(Operator op) {
1326             String msg = String.format("%s: illegal %s in %s",
1327                                        op,
1328                                        whatKind.getSimpleName().toLowerCase(),
1329                                        whatVec.getSimpleName());
1330             return new UnsupportedOperationException(msg);
1331         }
1332 
1333         @Override public String toString() {
1334             ArrayList<String> entries = new ArrayList<>();
1335             for (int i = 0; i < cache.length; i++) {
1336                 Object fn = cache[i];
1337                 if (fn != null)  entries.add(i+": "+fn);
1338             }
1339             return String.format("ImplCache<%s,%s>[%s]",
1340                                  whatKind.getSimpleName(),
1341                                  whatVec.getSimpleName(),
1342                                  String.join(", ", entries));
1343         }
1344     }
1345 }
--- EOF ---