1 /* 2 * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26 package jdk.internal.vm.vector; 27 28 import jdk.internal.vm.annotation.IntrinsicCandidate; 29 import jdk.internal.misc.Unsafe; 30 31 import java.util.function.*; 32 33 public class VectorSupport { 34 static { 35 registerNatives(); 36 } 37 38 private static final Unsafe U = Unsafe.getUnsafe(); 39 40 // Unary 41 public static final int VECTOR_OP_ABS = 0; 42 public static final int VECTOR_OP_NEG = 1; 43 public static final int VECTOR_OP_SQRT = 2; 44 45 // Binary 46 public static final int VECTOR_OP_ADD = 4; 47 public static final int VECTOR_OP_SUB = 5; 48 public static final int VECTOR_OP_MUL = 6; 49 public static final int VECTOR_OP_DIV = 7; 50 public static final int VECTOR_OP_MIN = 8; 51 public static final int VECTOR_OP_MAX = 9; 52 53 public static final int VECTOR_OP_AND = 10; 54 public static final int VECTOR_OP_OR = 11; 55 public static final int VECTOR_OP_XOR = 12; 56 57 // Ternary 58 public static final int VECTOR_OP_FMA = 13; 59 60 // Broadcast int 61 public static final int VECTOR_OP_LSHIFT = 14; 62 public static final int VECTOR_OP_RSHIFT = 15; 63 public static final int VECTOR_OP_URSHIFT = 16; 64 65 public static final int VECTOR_OP_CAST = 17; 66 public static final int VECTOR_OP_UCAST = 18; 67 public static final int VECTOR_OP_REINTERPRET = 19; 68 69 // Mask manipulation operations 70 public static final int VECTOR_OP_MASK_TRUECOUNT = 20; 71 public static final int VECTOR_OP_MASK_FIRSTTRUE = 21; 72 public static final int VECTOR_OP_MASK_LASTTRUE = 22; 73 public static final int VECTOR_OP_MASK_TOLONG = 23; 74 75 // Rotate operations 76 public static final int VECTOR_OP_LROTATE = 24; 77 public static final int VECTOR_OP_RROTATE = 25; 78 79 // Math routines 80 public static final int VECTOR_OP_TAN = 101; 81 public static final int VECTOR_OP_TANH = 102; 82 public static final int VECTOR_OP_SIN = 103; 83 public static final int VECTOR_OP_SINH = 104; 84 public static final int VECTOR_OP_COS = 105; 85 public static final int VECTOR_OP_COSH = 106; 86 public static final int VECTOR_OP_ASIN = 107; 87 public static final int VECTOR_OP_ACOS = 108; 88 public static final int VECTOR_OP_ATAN = 109; 89 public static final int VECTOR_OP_ATAN2 = 110; 90 public static final int VECTOR_OP_CBRT = 111; 91 public static final int VECTOR_OP_LOG = 112; 92 public static final int VECTOR_OP_LOG10 = 113; 93 public static final int VECTOR_OP_LOG1P = 114; 94 public static final int VECTOR_OP_POW = 115; 95 public static final int VECTOR_OP_EXP = 116; 96 public static final int VECTOR_OP_EXPM1 = 117; 97 public static final int VECTOR_OP_HYPOT = 118; 98 99 // See src/hotspot/share/opto/subnode.hpp 100 // struct BoolTest, and enclosed enum mask 101 public static final int BT_eq = 0; // 0000 102 public static final int BT_ne = 4; // 0100 103 public static final int BT_le = 5; // 0101 104 public static final int BT_ge = 7; // 0111 105 public static final int BT_lt = 3; // 0011 106 public static final int BT_gt = 1; // 0001 107 public static final int BT_overflow = 2; // 0010 108 public static final int BT_no_overflow = 6; // 0110 109 // never = 8 1000 110 // illegal = 9 1001 111 // Unsigned comparisons apply to BT_le, BT_ge, BT_lt, BT_gt for integral types 112 public static final int BT_unsigned_compare = 0b10000; 113 public static final int BT_ule = BT_le | BT_unsigned_compare; 114 public static final int BT_uge = BT_ge | BT_unsigned_compare; 115 public static final int BT_ult = BT_lt | BT_unsigned_compare; 116 public static final int BT_ugt = BT_gt | BT_unsigned_compare; 117 118 // Various broadcasting modes. 119 public static final int MODE_BROADCAST = 0; 120 public static final int MODE_BITS_COERCED_LONG_TO_MASK = 1; 121 122 // BasicType codes, for primitives only: 123 public static final int 124 T_FLOAT = 6, 125 T_DOUBLE = 7, 126 T_BYTE = 8, 127 T_SHORT = 9, 128 T_INT = 10, 129 T_LONG = 11; 130 131 /* ============================================================================ */ 132 133 public static class VectorSpecies<E> {} 134 135 public static class VectorPayload { 136 private final Object payload; // array of primitives 137 138 public VectorPayload(Object payload) { 139 this.payload = payload; 140 } 141 142 protected final Object getPayload() { 143 return VectorSupport.maybeRebox(this).payload; 144 } 145 } 146 147 public static class Vector<E> extends VectorPayload { 148 public Vector(Object payload) { 149 super(payload); 150 } 151 } 152 153 public static class VectorShuffle<E> extends VectorPayload { 154 public VectorShuffle(Object payload) { 155 super(payload); 156 } 157 } 158 public static class VectorMask<E> extends VectorPayload { 159 public VectorMask(Object payload) { 160 super(payload); 161 } 162 } 163 164 /* ============================================================================ */ 165 public interface FromBitsCoercedOperation<VM extends VectorPayload, 166 S extends VectorSpecies<?>> { 167 VM fromBits(long l, S s); 168 } 169 170 @IntrinsicCandidate 171 public static 172 <VM extends VectorPayload, 173 S extends VectorSpecies<E>, 174 E> 175 VM fromBitsCoerced(Class<? extends VM> vmClass, Class<E> eClass, 176 int length, 177 long bits, int mode, S s, 178 FromBitsCoercedOperation<VM, S> defaultImpl) { 179 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 180 return defaultImpl.fromBits(bits, s); 181 } 182 183 /* ============================================================================ */ 184 public interface ShuffleIotaOperation<S extends VectorSpecies<?>, 185 SH extends VectorShuffle<?>> { 186 SH apply(int length, int start, int step, S s); 187 } 188 189 @IntrinsicCandidate 190 public static 191 <E, 192 S extends VectorSpecies<E>, 193 SH extends VectorShuffle<E>> 194 SH shuffleIota(Class<E> eClass, Class<? extends SH> shClass, S s, 195 int length, 196 int start, int step, int wrap, 197 ShuffleIotaOperation<S, SH> defaultImpl) { 198 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 199 return defaultImpl.apply(length, start, step, s); 200 } 201 202 public interface ShuffleToVectorOperation<V extends Vector<?>, 203 SH extends VectorShuffle<?>> { 204 V apply(SH sh); 205 } 206 207 @IntrinsicCandidate 208 public static 209 <V extends Vector<E>, 210 SH extends VectorShuffle<E>, 211 E> 212 V shuffleToVector(Class<? extends Vector<E>> vClass, Class<E> eClass, Class<? extends SH> shClass, SH sh, 213 int length, 214 ShuffleToVectorOperation<V, SH> defaultImpl) { 215 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 216 return defaultImpl.apply(sh); 217 } 218 219 /* ============================================================================ */ 220 public interface IndexOperation<V extends Vector<?>, 221 S extends VectorSpecies<?>> { 222 V index(V v, int step, S s); 223 } 224 225 //FIXME @IntrinsicCandidate 226 public static 227 <V extends Vector<E>, 228 E, 229 S extends VectorSpecies<E>> 230 V indexVector(Class<? extends V> vClass, Class<E> eClass, 231 int length, 232 V v, int step, S s, 233 IndexOperation<V, S> defaultImpl) { 234 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 235 return defaultImpl.index(v, step, s); 236 } 237 238 /* ============================================================================ */ 239 240 public interface ReductionOperation<V extends Vector<?>, 241 M extends VectorMask<?>> { 242 long apply(V v, M m); 243 } 244 245 @IntrinsicCandidate 246 public static 247 <V extends Vector<E>, 248 M extends VectorMask<E>, 249 E> 250 long reductionCoerced(int oprId, 251 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass, 252 int length, 253 V v, M m, 254 ReductionOperation<V, M> defaultImpl) { 255 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 256 return defaultImpl.apply(v, m); 257 } 258 259 260 /* ============================================================================ */ 261 262 public interface VecExtractOp<V extends Vector<?>> { 263 long apply(V v, int i); 264 } 265 266 @IntrinsicCandidate 267 public static 268 <V extends Vector<E>, 269 E> 270 long extract(Class<? extends V> vClass, Class<E> eClass, 271 int length, 272 V v, int i, 273 VecExtractOp<V> defaultImpl) { 274 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 275 return defaultImpl.apply(v, i); 276 } 277 278 /* ============================================================================ */ 279 280 public interface VecInsertOp<V extends Vector<?>> { 281 V apply(V v, int i, long val); 282 } 283 284 @IntrinsicCandidate 285 public static 286 <V extends Vector<E>, 287 E> 288 V insert(Class<? extends V> vClass, Class<E> eClass, 289 int length, 290 V v, int i, long val, 291 VecInsertOp<V> defaultImpl) { 292 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 293 return defaultImpl.apply(v, i, val); 294 } 295 296 /* ============================================================================ */ 297 298 public interface UnaryOperation<V extends Vector<?>, 299 M extends VectorMask<?>> { 300 V apply(V v, M m); 301 } 302 303 @IntrinsicCandidate 304 public static 305 <V extends Vector<E>, 306 M extends VectorMask<E>, 307 E> 308 V unaryOp(int oprId, 309 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass, 310 int length, 311 V v, M m, 312 UnaryOperation<V, M> defaultImpl) { 313 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 314 return defaultImpl.apply(v, m); 315 } 316 317 /* ============================================================================ */ 318 319 public interface BinaryOperation<VM extends VectorPayload, 320 M extends VectorMask<?>> { 321 VM apply(VM v1, VM v2, M m); 322 } 323 324 @IntrinsicCandidate 325 public static 326 <VM extends VectorPayload, 327 M extends VectorMask<E>, 328 E> 329 VM binaryOp(int oprId, 330 Class<? extends VM> vmClass, Class<? extends M> mClass, Class<E> eClass, 331 int length, 332 VM v1, VM v2, M m, 333 BinaryOperation<VM, M> defaultImpl) { 334 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 335 return defaultImpl.apply(v1, v2, m); 336 } 337 338 /* ============================================================================ */ 339 340 public interface TernaryOperation<V extends Vector<?>, 341 M extends VectorMask<?>> { 342 V apply(V v1, V v2, V v3, M m); 343 } 344 345 @IntrinsicCandidate 346 public static 347 <V extends Vector<E>, 348 M extends VectorMask<E>, 349 E> 350 V ternaryOp(int oprId, 351 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass, 352 int length, 353 V v1, V v2, V v3, M m, 354 TernaryOperation<V, M> defaultImpl) { 355 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 356 return defaultImpl.apply(v1, v2, v3, m); 357 } 358 359 /* ============================================================================ */ 360 361 // Memory operations 362 363 public interface LoadOperation<C, 364 VM extends VectorPayload, 365 S extends VectorSpecies<?>> { 366 VM load(C container, int index, S s); 367 } 368 369 @IntrinsicCandidate 370 public static 371 <C, 372 VM extends VectorPayload, 373 E, 374 S extends VectorSpecies<E>> 375 VM load(Class<? extends VM> vmClass, Class<E> eClass, 376 int length, 377 Object base, long offset, 378 C container, int index, S s, 379 LoadOperation<C, VM, S> defaultImpl) { 380 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 381 return defaultImpl.load(container, index, s); 382 } 383 384 /* ============================================================================ */ 385 386 public interface LoadVectorMaskedOperation<C, 387 V extends Vector<?>, 388 S extends VectorSpecies<?>, 389 M extends VectorMask<?>> { 390 V load(C container, int index, S s, M m); 391 } 392 393 @IntrinsicCandidate 394 public static 395 <C, 396 V extends Vector<?>, 397 E, 398 S extends VectorSpecies<E>, 399 M extends VectorMask<E>> 400 V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, 401 int length, 402 Object base, long offset, 403 M m, C container, int index, S s, 404 LoadVectorMaskedOperation<C, V, S, M> defaultImpl) { 405 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 406 return defaultImpl.load(container, index, s, m); 407 } 408 409 /* ============================================================================ */ 410 411 public interface LoadVectorOperationWithMap<C, 412 V extends Vector<?>, 413 S extends VectorSpecies<?>, 414 M extends VectorMask<?>> { 415 V loadWithMap(C container, int index, int[] indexMap, int indexM, S s, M m); 416 } 417 418 @IntrinsicCandidate 419 public static 420 <C, 421 V extends Vector<?>, 422 W extends Vector<Integer>, 423 S extends VectorSpecies<E>, 424 M extends VectorMask<E>, 425 E> 426 V loadWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, 427 int length, 428 Class<? extends Vector<Integer>> vectorIndexClass, 429 Object base, long offset, 430 W index_vector, 431 M m, C container, int index, int[] indexMap, int indexM, S s, 432 LoadVectorOperationWithMap<C, V, S, M> defaultImpl) { 433 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 434 return defaultImpl.loadWithMap(container, index, indexMap, indexM, s, m); 435 } 436 437 /* ============================================================================ */ 438 439 public interface StoreVectorOperation<C, 440 V extends Vector<?>> { 441 void store(C container, int index, V v); 442 } 443 444 @IntrinsicCandidate 445 public static 446 <C, 447 V extends Vector<?>> 448 void store(Class<?> vClass, Class<?> eClass, 449 int length, 450 Object base, long offset, 451 V v, C container, int index, 452 StoreVectorOperation<C, V> defaultImpl) { 453 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 454 defaultImpl.store(container, index, v); 455 } 456 457 public interface StoreVectorMaskedOperation<C, 458 V extends Vector<?>, 459 M extends VectorMask<?>> { 460 void store(C container, int index, V v, M m); 461 } 462 463 @IntrinsicCandidate 464 public static 465 <C, 466 V extends Vector<E>, 467 M extends VectorMask<E>, 468 E> 469 void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, 470 int length, 471 Object base, long offset, 472 V v, M m, C container, int index, 473 StoreVectorMaskedOperation<C, V, M> defaultImpl) { 474 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 475 defaultImpl.store(container, index, v, m); 476 } 477 478 /* ============================================================================ */ 479 480 public interface StoreVectorOperationWithMap<C, 481 V extends Vector<?>, 482 M extends VectorMask<?>> { 483 void storeWithMap(C container, int index, V v, int[] indexMap, int indexM, M m); 484 } 485 486 @IntrinsicCandidate 487 public static 488 <C, 489 V extends Vector<E>, 490 W extends Vector<Integer>, 491 M extends VectorMask<E>, 492 E> 493 void storeWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, 494 int length, 495 Class<? extends Vector<Integer>> vectorIndexClass, 496 Object base, long offset, 497 W index_vector, 498 V v, M m, C container, int index, int[] indexMap, int indexM, 499 StoreVectorOperationWithMap<C, V, M> defaultImpl) { 500 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 501 defaultImpl.storeWithMap(container, index, v, indexMap, indexM, m); 502 } 503 504 /* ============================================================================ */ 505 506 @IntrinsicCandidate 507 public static 508 <M extends VectorMask<E>, 509 E> 510 boolean test(int cond, 511 Class<?> mClass, Class<?> eClass, 512 int length, 513 M m1, M m2, 514 BiFunction<M, M, Boolean> defaultImpl) { 515 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 516 return defaultImpl.apply(m1, m2); 517 } 518 519 /* ============================================================================ */ 520 521 public interface VectorCompareOp<V extends Vector<?>, 522 M extends VectorMask<?>> { 523 M apply(int cond, V v1, V v2, M m); 524 } 525 526 @IntrinsicCandidate 527 public static 528 <V extends Vector<E>, 529 M extends VectorMask<E>, 530 E> 531 M compare(int cond, 532 Class<? extends V> vectorClass, Class<M> mClass, Class<E> eClass, 533 int length, 534 V v1, V v2, M m, 535 VectorCompareOp<V, M> defaultImpl) { 536 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 537 return defaultImpl.apply(cond, v1, v2, m); 538 } 539 540 /* ============================================================================ */ 541 public interface VectorRearrangeOp<V extends Vector<?>, 542 SH extends VectorShuffle<?>, 543 M extends VectorMask<?>> { 544 V apply(V v, SH sh, M m); 545 } 546 547 @IntrinsicCandidate 548 public static 549 <V extends Vector<E>, 550 SH extends VectorShuffle<E>, 551 M extends VectorMask<E>, 552 E> 553 V rearrangeOp(Class<? extends V> vClass, Class<SH> shClass, Class<M> mClass, Class<E> eClass, 554 int length, 555 V v, SH sh, M m, 556 VectorRearrangeOp<V, SH, M> defaultImpl) { 557 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 558 return defaultImpl.apply(v, sh, m); 559 } 560 561 /* ============================================================================ */ 562 563 public interface VectorBlendOp<V extends Vector<?>, 564 M extends VectorMask<?>> { 565 V apply(V v1, V v2, M m); 566 } 567 568 @IntrinsicCandidate 569 public static 570 <V extends Vector<E>, 571 M extends VectorMask<E>, 572 E> 573 V blend(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, 574 int length, 575 V v1, V v2, M m, 576 VectorBlendOp<V, M> defaultImpl) { 577 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 578 return defaultImpl.apply(v1, v2, m); 579 } 580 581 /* ============================================================================ */ 582 583 public interface VectorBroadcastIntOp<V extends Vector<?>, 584 M extends VectorMask<?>> { 585 V apply(V v, int n, M m); 586 } 587 588 @IntrinsicCandidate 589 public static 590 <V extends Vector<E>, 591 M extends VectorMask<E>, 592 E> 593 V broadcastInt(int opr, 594 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass, 595 int length, 596 V v, int n, M m, 597 VectorBroadcastIntOp<V, M> defaultImpl) { 598 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 599 return defaultImpl.apply(v, n, m); 600 } 601 602 /* ============================================================================ */ 603 604 public interface VectorConvertOp<VOUT extends VectorPayload, 605 VIN extends VectorPayload, 606 S extends VectorSpecies<?>> { 607 VOUT apply(VIN v, S s); 608 } 609 610 // Users of this intrinsic assume that it respects 611 // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN. 612 // See javadoc for REGISTER_ENDIAN. 613 614 @IntrinsicCandidate 615 public static <VOUT extends VectorPayload, 616 VIN extends VectorPayload, 617 S extends VectorSpecies<?>> 618 VOUT convert(int oprId, 619 Class<?> fromVectorClass, Class<?> fromeClass, int fromVLen, 620 Class<?> toVectorClass, Class<?> toeClass, int toVLen, 621 VIN v, S s, 622 VectorConvertOp<VOUT, VIN, S> defaultImpl) { 623 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 624 return defaultImpl.apply(v, s); 625 } 626 627 /* ============================================================================ */ 628 629 @IntrinsicCandidate 630 public static 631 <VP extends VectorPayload> 632 VP maybeRebox(VP v) { 633 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses. 634 // TODO: move the fence generation into C2. Generate only when reboxing is taking place. 635 U.loadFence(); 636 return v; 637 } 638 639 /* ============================================================================ */ 640 public interface VectorMaskOp<M extends VectorMask<?>> { 641 long apply(M m); 642 } 643 644 @IntrinsicCandidate 645 public static 646 <M extends VectorMask<E>, 647 E> 648 long maskReductionCoerced(int oper, 649 Class<? extends M> mClass, Class<?> eClass, 650 int length, 651 M m, 652 VectorMaskOp<M> defaultImpl) { 653 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 654 return defaultImpl.apply(m); 655 } 656 657 /* ============================================================================ */ 658 659 // query the JVM's supported vector sizes and types 660 public static native int getMaxLaneCount(Class<?> etype); 661 662 /* ============================================================================ */ 663 664 public static boolean isNonCapturingLambda(Object o) { 665 return o.getClass().getDeclaredFields().length == 0; 666 } 667 668 /* ============================================================================ */ 669 670 private static native int registerNatives(); 671 }