24 */
25
26 package jdk.internal.vm.vector;
27
28 import jdk.internal.vm.annotation.IntrinsicCandidate;
29 import jdk.internal.misc.Unsafe;
30
31 import java.util.function.*;
32
33 public class VectorSupport {
34 static {
35 registerNatives();
36 }
37
38 private static final Unsafe U = Unsafe.getUnsafe();
39
40 // Unary
41 public static final int VECTOR_OP_ABS = 0;
42 public static final int VECTOR_OP_NEG = 1;
43 public static final int VECTOR_OP_SQRT = 2;
44
45 // Binary
46 public static final int VECTOR_OP_ADD = 4;
47 public static final int VECTOR_OP_SUB = 5;
48 public static final int VECTOR_OP_MUL = 6;
49 public static final int VECTOR_OP_DIV = 7;
50 public static final int VECTOR_OP_MIN = 8;
51 public static final int VECTOR_OP_MAX = 9;
52
53 public static final int VECTOR_OP_AND = 10;
54 public static final int VECTOR_OP_OR = 11;
55 public static final int VECTOR_OP_XOR = 12;
56
57 // Ternary
58 public static final int VECTOR_OP_FMA = 13;
59
60 // Broadcast int
61 public static final int VECTOR_OP_LSHIFT = 14;
62 public static final int VECTOR_OP_RSHIFT = 15;
63 public static final int VECTOR_OP_URSHIFT = 16;
64
65 public static final int VECTOR_OP_CAST = 17;
66 public static final int VECTOR_OP_UCAST = 18;
67 public static final int VECTOR_OP_REINTERPRET = 19;
68
69 // Mask manipulation operations
70 public static final int VECTOR_OP_MASK_TRUECOUNT = 20;
71 public static final int VECTOR_OP_MASK_FIRSTTRUE = 21;
72 public static final int VECTOR_OP_MASK_LASTTRUE = 22;
73 public static final int VECTOR_OP_MASK_TOLONG = 23;
74
75 // Rotate operations
76 public static final int VECTOR_OP_LROTATE = 24;
77 public static final int VECTOR_OP_RROTATE = 25;
78
79 // Math routines
80 public static final int VECTOR_OP_TAN = 101;
81 public static final int VECTOR_OP_TANH = 102;
82 public static final int VECTOR_OP_SIN = 103;
83 public static final int VECTOR_OP_SINH = 104;
84 public static final int VECTOR_OP_COS = 105;
85 public static final int VECTOR_OP_COSH = 106;
86 public static final int VECTOR_OP_ASIN = 107;
87 public static final int VECTOR_OP_ACOS = 108;
88 public static final int VECTOR_OP_ATAN = 109;
89 public static final int VECTOR_OP_ATAN2 = 110;
90 public static final int VECTOR_OP_CBRT = 111;
91 public static final int VECTOR_OP_LOG = 112;
92 public static final int VECTOR_OP_LOG10 = 113;
93 public static final int VECTOR_OP_LOG1P = 114;
94 public static final int VECTOR_OP_POW = 115;
95 public static final int VECTOR_OP_EXP = 116;
96 public static final int VECTOR_OP_EXPM1 = 117;
97 public static final int VECTOR_OP_HYPOT = 118;
98
346 public static
347 <V extends Vector<E>,
348 M extends VectorMask<E>,
349 E>
350 V ternaryOp(int oprId,
351 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
352 int length,
353 V v1, V v2, V v3, M m,
354 TernaryOperation<V, M> defaultImpl) {
355 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
356 return defaultImpl.apply(v1, v2, v3, m);
357 }
358
359 /* ============================================================================ */
360
361 // Memory operations
362
363 public interface LoadOperation<C,
364 VM extends VectorPayload,
365 S extends VectorSpecies<?>> {
366 VM load(C container, int index, S s);
367 }
368
369 @IntrinsicCandidate
370 public static
371 <C,
372 VM extends VectorPayload,
373 E,
374 S extends VectorSpecies<E>>
375 VM load(Class<? extends VM> vmClass, Class<E> eClass,
376 int length,
377 Object base, long offset,
378 C container, int index, S s,
379 LoadOperation<C, VM, S> defaultImpl) {
380 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
381 return defaultImpl.load(container, index, s);
382 }
383
384 /* ============================================================================ */
385
386 public interface LoadVectorMaskedOperation<C,
387 V extends Vector<?>,
388 S extends VectorSpecies<?>,
389 M extends VectorMask<?>> {
390 V load(C container, int index, S s, M m);
391 }
392
393 @IntrinsicCandidate
394 public static
395 <C,
396 V extends Vector<?>,
397 E,
398 S extends VectorSpecies<E>,
399 M extends VectorMask<E>>
400 V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
401 int length,
402 Object base, long offset,
403 M m, C container, int index, S s,
404 LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
405 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
406 return defaultImpl.load(container, index, s, m);
407 }
408
409 /* ============================================================================ */
410
411 public interface LoadVectorOperationWithMap<C,
412 V extends Vector<?>,
413 S extends VectorSpecies<?>,
414 M extends VectorMask<?>> {
415 V loadWithMap(C container, int index, int[] indexMap, int indexM, S s, M m);
416 }
417
418 @IntrinsicCandidate
419 public static
420 <C,
421 V extends Vector<?>,
422 W extends Vector<Integer>,
423 S extends VectorSpecies<E>,
424 M extends VectorMask<E>,
425 E>
426 V loadWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
427 int length,
428 Class<? extends Vector<Integer>> vectorIndexClass,
429 Object base, long offset,
430 W index_vector,
431 M m, C container, int index, int[] indexMap, int indexM, S s,
432 LoadVectorOperationWithMap<C, V, S, M> defaultImpl) {
433 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
434 return defaultImpl.loadWithMap(container, index, indexMap, indexM, s, m);
435 }
436
437 /* ============================================================================ */
438
439 public interface StoreVectorOperation<C,
440 V extends Vector<?>> {
441 void store(C container, int index, V v);
442 }
443
444 @IntrinsicCandidate
445 public static
446 <C,
447 V extends Vector<?>>
448 void store(Class<?> vClass, Class<?> eClass,
449 int length,
450 Object base, long offset,
451 V v, C container, int index,
452 StoreVectorOperation<C, V> defaultImpl) {
453 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
454 defaultImpl.store(container, index, v);
455 }
456
457 public interface StoreVectorMaskedOperation<C,
458 V extends Vector<?>,
459 M extends VectorMask<?>> {
460 void store(C container, int index, V v, M m);
461 }
462
463 @IntrinsicCandidate
464 public static
465 <C,
466 V extends Vector<E>,
467 M extends VectorMask<E>,
468 E>
469 void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
470 int length,
471 Object base, long offset,
472 V v, M m, C container, int index,
473 StoreVectorMaskedOperation<C, V, M> defaultImpl) {
474 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
475 defaultImpl.store(container, index, v, m);
476 }
477
478 /* ============================================================================ */
479
480 public interface StoreVectorOperationWithMap<C,
481 V extends Vector<?>,
482 M extends VectorMask<?>> {
483 void storeWithMap(C container, int index, V v, int[] indexMap, int indexM, M m);
484 }
485
486 @IntrinsicCandidate
487 public static
488 <C,
489 V extends Vector<E>,
490 W extends Vector<Integer>,
491 M extends VectorMask<E>,
492 E>
609
610 // Users of this intrinsic assume that it respects
611 // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN.
612 // See javadoc for REGISTER_ENDIAN.
613
614 @IntrinsicCandidate
615 public static <VOUT extends VectorPayload,
616 VIN extends VectorPayload,
617 S extends VectorSpecies<?>>
618 VOUT convert(int oprId,
619 Class<?> fromVectorClass, Class<?> fromeClass, int fromVLen,
620 Class<?> toVectorClass, Class<?> toeClass, int toVLen,
621 VIN v, S s,
622 VectorConvertOp<VOUT, VIN, S> defaultImpl) {
623 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
624 return defaultImpl.apply(v, s);
625 }
626
627 /* ============================================================================ */
628
629 @IntrinsicCandidate
630 public static
631 <VP extends VectorPayload>
632 VP maybeRebox(VP v) {
633 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
634 // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
635 U.loadFence();
636 return v;
637 }
638
639 /* ============================================================================ */
640 public interface VectorMaskOp<M extends VectorMask<?>> {
641 long apply(M m);
642 }
643
644 @IntrinsicCandidate
645 public static
646 <M extends VectorMask<E>,
647 E>
648 long maskReductionCoerced(int oper,
|
24 */
25
26 package jdk.internal.vm.vector;
27
28 import jdk.internal.vm.annotation.IntrinsicCandidate;
29 import jdk.internal.misc.Unsafe;
30
31 import java.util.function.*;
32
33 public class VectorSupport {
34 static {
35 registerNatives();
36 }
37
38 private static final Unsafe U = Unsafe.getUnsafe();
39
40 // Unary
41 public static final int VECTOR_OP_ABS = 0;
42 public static final int VECTOR_OP_NEG = 1;
43 public static final int VECTOR_OP_SQRT = 2;
44 public static final int VECTOR_OP_BIT_COUNT = 3;
45
46 // Binary
47 public static final int VECTOR_OP_ADD = 4;
48 public static final int VECTOR_OP_SUB = 5;
49 public static final int VECTOR_OP_MUL = 6;
50 public static final int VECTOR_OP_DIV = 7;
51 public static final int VECTOR_OP_MIN = 8;
52 public static final int VECTOR_OP_MAX = 9;
53
54 public static final int VECTOR_OP_AND = 10;
55 public static final int VECTOR_OP_OR = 11;
56 public static final int VECTOR_OP_XOR = 12;
57
58 // Ternary
59 public static final int VECTOR_OP_FMA = 13;
60
61 // Broadcast int
62 public static final int VECTOR_OP_LSHIFT = 14;
63 public static final int VECTOR_OP_RSHIFT = 15;
64 public static final int VECTOR_OP_URSHIFT = 16;
65
66 public static final int VECTOR_OP_CAST = 17;
67 public static final int VECTOR_OP_UCAST = 18;
68 public static final int VECTOR_OP_REINTERPRET = 19;
69
70 // Mask manipulation operations
71 public static final int VECTOR_OP_MASK_TRUECOUNT = 20;
72 public static final int VECTOR_OP_MASK_FIRSTTRUE = 21;
73 public static final int VECTOR_OP_MASK_LASTTRUE = 22;
74 public static final int VECTOR_OP_MASK_TOLONG = 23;
75
76 // Rotate operations
77 public static final int VECTOR_OP_LROTATE = 24;
78 public static final int VECTOR_OP_RROTATE = 25;
79
80 // Compression expansion operations
81 public static final int VECTOR_OP_COMPRESS = 26;
82 public static final int VECTOR_OP_EXPAND = 27;
83 public static final int VECTOR_OP_MASK_COMPRESS = 28;
84
85 // Leading/Trailing zeros count operations
86 public static final int VECTOR_OP_TZ_COUNT = 29;
87 public static final int VECTOR_OP_LZ_COUNT = 30;
88
89 // Reverse operation
90 public static final int VECTOR_OP_REVERSE = 31;
91 public static final int VECTOR_OP_REVERSE_BYTES = 32;
92
93 // Compress and Expand Bits operation
94 public static final int VECTOR_OP_COMPRESS_BITS = 33;
95 public static final int VECTOR_OP_EXPAND_BITS = 34;
96
97 // Math routines
98 public static final int VECTOR_OP_TAN = 101;
99 public static final int VECTOR_OP_TANH = 102;
100 public static final int VECTOR_OP_SIN = 103;
101 public static final int VECTOR_OP_SINH = 104;
102 public static final int VECTOR_OP_COS = 105;
103 public static final int VECTOR_OP_COSH = 106;
104 public static final int VECTOR_OP_ASIN = 107;
105 public static final int VECTOR_OP_ACOS = 108;
106 public static final int VECTOR_OP_ATAN = 109;
107 public static final int VECTOR_OP_ATAN2 = 110;
108 public static final int VECTOR_OP_CBRT = 111;
109 public static final int VECTOR_OP_LOG = 112;
110 public static final int VECTOR_OP_LOG10 = 113;
111 public static final int VECTOR_OP_LOG1P = 114;
112 public static final int VECTOR_OP_POW = 115;
113 public static final int VECTOR_OP_EXP = 116;
114 public static final int VECTOR_OP_EXPM1 = 117;
115 public static final int VECTOR_OP_HYPOT = 118;
116
364 public static
365 <V extends Vector<E>,
366 M extends VectorMask<E>,
367 E>
368 V ternaryOp(int oprId,
369 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
370 int length,
371 V v1, V v2, V v3, M m,
372 TernaryOperation<V, M> defaultImpl) {
373 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
374 return defaultImpl.apply(v1, v2, v3, m);
375 }
376
377 /* ============================================================================ */
378
379 // Memory operations
380
381 public interface LoadOperation<C,
382 VM extends VectorPayload,
383 S extends VectorSpecies<?>> {
384 VM load(C container, long index, S s);
385 }
386
387 @IntrinsicCandidate
388 public static
389 <C,
390 VM extends VectorPayload,
391 E,
392 S extends VectorSpecies<E>>
393 VM load(Class<? extends VM> vmClass, Class<E> eClass,
394 int length,
395 Object base, long offset,
396 C container, long index, S s,
397 LoadOperation<C, VM, S> defaultImpl) {
398 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
399 return defaultImpl.load(container, index, s);
400 }
401
402 /* ============================================================================ */
403
404 public interface LoadVectorMaskedOperation<C,
405 V extends Vector<?>,
406 S extends VectorSpecies<?>,
407 M extends VectorMask<?>> {
408 V load(C container, long index, S s, M m);
409 }
410
411 @IntrinsicCandidate
412 public static
413 <C,
414 V extends Vector<?>,
415 E,
416 S extends VectorSpecies<E>,
417 M extends VectorMask<E>>
418 V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
419 int length,
420 Object base, long offset,
421 M m, C container, long index, S s,
422 LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
423 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
424 return defaultImpl.load(container, index, s, m);
425 }
426
427 /* ============================================================================ */
428
429 public interface LoadVectorOperationWithMap<C,
430 V extends Vector<?>,
431 S extends VectorSpecies<?>,
432 M extends VectorMask<?>> {
433 V loadWithMap(C container, int index, int[] indexMap, int indexM, S s, M m);
434 }
435
436 @IntrinsicCandidate
437 public static
438 <C,
439 V extends Vector<?>,
440 W extends Vector<Integer>,
441 S extends VectorSpecies<E>,
442 M extends VectorMask<E>,
443 E>
444 V loadWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
445 int length,
446 Class<? extends Vector<Integer>> vectorIndexClass,
447 Object base, long offset,
448 W index_vector,
449 M m, C container, int index, int[] indexMap, int indexM, S s,
450 LoadVectorOperationWithMap<C, V, S, M> defaultImpl) {
451 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
452 return defaultImpl.loadWithMap(container, index, indexMap, indexM, s, m);
453 }
454
455 /* ============================================================================ */
456
457 public interface StoreVectorOperation<C,
458 V extends VectorPayload> {
459 void store(C container, long index, V v);
460 }
461
462 @IntrinsicCandidate
463 public static
464 <C,
465 V extends VectorPayload>
466 void store(Class<?> vClass, Class<?> eClass,
467 int length,
468 Object base, long offset,
469 V v, C container, long index,
470 StoreVectorOperation<C, V> defaultImpl) {
471 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
472 defaultImpl.store(container, index, v);
473 }
474
475 public interface StoreVectorMaskedOperation<C,
476 V extends Vector<?>,
477 M extends VectorMask<?>> {
478 void store(C container, long index, V v, M m);
479 }
480
481 @IntrinsicCandidate
482 public static
483 <C,
484 V extends Vector<E>,
485 M extends VectorMask<E>,
486 E>
487 void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
488 int length,
489 Object base, long offset,
490 V v, M m, C container, long index,
491 StoreVectorMaskedOperation<C, V, M> defaultImpl) {
492 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
493 defaultImpl.store(container, index, v, m);
494 }
495
496 /* ============================================================================ */
497
498 public interface StoreVectorOperationWithMap<C,
499 V extends Vector<?>,
500 M extends VectorMask<?>> {
501 void storeWithMap(C container, int index, V v, int[] indexMap, int indexM, M m);
502 }
503
504 @IntrinsicCandidate
505 public static
506 <C,
507 V extends Vector<E>,
508 W extends Vector<Integer>,
509 M extends VectorMask<E>,
510 E>
627
628 // Users of this intrinsic assume that it respects
629 // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN.
630 // See javadoc for REGISTER_ENDIAN.
631
632 @IntrinsicCandidate
633 public static <VOUT extends VectorPayload,
634 VIN extends VectorPayload,
635 S extends VectorSpecies<?>>
636 VOUT convert(int oprId,
637 Class<?> fromVectorClass, Class<?> fromeClass, int fromVLen,
638 Class<?> toVectorClass, Class<?> toeClass, int toVLen,
639 VIN v, S s,
640 VectorConvertOp<VOUT, VIN, S> defaultImpl) {
641 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
642 return defaultImpl.apply(v, s);
643 }
644
645 /* ============================================================================ */
646
647 public interface ComExpOperation<V extends Vector<?>,
648 M extends VectorMask<?>> {
649 VectorPayload apply(V v, M m);
650 }
651
652 @IntrinsicCandidate
653 public static
654 <V extends Vector<E>,
655 M extends VectorMask<E>,
656 E>
657 VectorPayload comExpOp(int opr,
658 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
659 int length, V v, M m,
660 ComExpOperation<V, M> defaultImpl) {
661 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
662 return defaultImpl.apply(v, m);
663 }
664
665 /* ============================================================================ */
666
667 @IntrinsicCandidate
668 public static
669 <VP extends VectorPayload>
670 VP maybeRebox(VP v) {
671 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
672 // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
673 U.loadFence();
674 return v;
675 }
676
677 /* ============================================================================ */
678 public interface VectorMaskOp<M extends VectorMask<?>> {
679 long apply(M m);
680 }
681
682 @IntrinsicCandidate
683 public static
684 <M extends VectorMask<E>,
685 E>
686 long maskReductionCoerced(int oper,
|