1 /*
2 * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package jdk.internal.vm.vector;
27
28 import jdk.internal.vm.annotation.IntrinsicCandidate;
29 import jdk.internal.misc.Unsafe;
30
31 import java.util.function.*;
32
33 public class VectorSupport {
34 static {
35 registerNatives();
36 }
37
38 private static final Unsafe U = Unsafe.getUnsafe();
39
40 // Unary
41 public static final int VECTOR_OP_ABS = 0;
42 public static final int VECTOR_OP_NEG = 1;
43 public static final int VECTOR_OP_SQRT = 2;
44
45 // Binary
46 public static final int VECTOR_OP_ADD = 4;
47 public static final int VECTOR_OP_SUB = 5;
48 public static final int VECTOR_OP_MUL = 6;
49 public static final int VECTOR_OP_DIV = 7;
50 public static final int VECTOR_OP_MIN = 8;
51 public static final int VECTOR_OP_MAX = 9;
52
53 public static final int VECTOR_OP_AND = 10;
54 public static final int VECTOR_OP_OR = 11;
55 public static final int VECTOR_OP_XOR = 12;
56
57 // Ternary
58 public static final int VECTOR_OP_FMA = 13;
59
60 // Broadcast int
61 public static final int VECTOR_OP_LSHIFT = 14;
62 public static final int VECTOR_OP_RSHIFT = 15;
63 public static final int VECTOR_OP_URSHIFT = 16;
64
65 public static final int VECTOR_OP_CAST = 17;
66 public static final int VECTOR_OP_UCAST = 18;
67 public static final int VECTOR_OP_REINTERPRET = 19;
68
69 // Mask manipulation operations
70 public static final int VECTOR_OP_MASK_TRUECOUNT = 20;
71 public static final int VECTOR_OP_MASK_FIRSTTRUE = 21;
72 public static final int VECTOR_OP_MASK_LASTTRUE = 22;
73 public static final int VECTOR_OP_MASK_TOLONG = 23;
74
75 // Rotate operations
76 public static final int VECTOR_OP_LROTATE = 24;
77 public static final int VECTOR_OP_RROTATE = 25;
78
79 // Math routines
80 public static final int VECTOR_OP_TAN = 101;
81 public static final int VECTOR_OP_TANH = 102;
82 public static final int VECTOR_OP_SIN = 103;
83 public static final int VECTOR_OP_SINH = 104;
84 public static final int VECTOR_OP_COS = 105;
85 public static final int VECTOR_OP_COSH = 106;
86 public static final int VECTOR_OP_ASIN = 107;
87 public static final int VECTOR_OP_ACOS = 108;
88 public static final int VECTOR_OP_ATAN = 109;
89 public static final int VECTOR_OP_ATAN2 = 110;
90 public static final int VECTOR_OP_CBRT = 111;
91 public static final int VECTOR_OP_LOG = 112;
92 public static final int VECTOR_OP_LOG10 = 113;
93 public static final int VECTOR_OP_LOG1P = 114;
94 public static final int VECTOR_OP_POW = 115;
95 public static final int VECTOR_OP_EXP = 116;
96 public static final int VECTOR_OP_EXPM1 = 117;
97 public static final int VECTOR_OP_HYPOT = 118;
98
99 // See src/hotspot/share/opto/subnode.hpp
100 // struct BoolTest, and enclosed enum mask
101 public static final int BT_eq = 0; // 0000
102 public static final int BT_ne = 4; // 0100
103 public static final int BT_le = 5; // 0101
104 public static final int BT_ge = 7; // 0111
105 public static final int BT_lt = 3; // 0011
106 public static final int BT_gt = 1; // 0001
107 public static final int BT_overflow = 2; // 0010
108 public static final int BT_no_overflow = 6; // 0110
109 // never = 8 1000
110 // illegal = 9 1001
111 // Unsigned comparisons apply to BT_le, BT_ge, BT_lt, BT_gt for integral types
112 public static final int BT_unsigned_compare = 0b10000;
113 public static final int BT_ule = BT_le | BT_unsigned_compare;
114 public static final int BT_uge = BT_ge | BT_unsigned_compare;
115 public static final int BT_ult = BT_lt | BT_unsigned_compare;
116 public static final int BT_ugt = BT_gt | BT_unsigned_compare;
117
118 // Various broadcasting modes.
119 public static final int MODE_BROADCAST = 0;
120 public static final int MODE_BITS_COERCED_LONG_TO_MASK = 1;
121
122 // BasicType codes, for primitives only:
123 public static final int
124 T_FLOAT = 6,
125 T_DOUBLE = 7,
126 T_BYTE = 8,
127 T_SHORT = 9,
128 T_INT = 10,
129 T_LONG = 11;
130
131 /* ============================================================================ */
132
133 public static class VectorSpecies<E> {}
134
135 public static class VectorPayload {
136 private final Object payload; // array of primitives
137
138 public VectorPayload(Object payload) {
139 this.payload = payload;
140 }
141
142 protected final Object getPayload() {
143 return VectorSupport.maybeRebox(this).payload;
144 }
145 }
146
147 public static class Vector<E> extends VectorPayload {
148 public Vector(Object payload) {
149 super(payload);
150 }
151 }
152
153 public static class VectorShuffle<E> extends VectorPayload {
154 public VectorShuffle(Object payload) {
155 super(payload);
156 }
157 }
158 public static class VectorMask<E> extends VectorPayload {
159 public VectorMask(Object payload) {
160 super(payload);
161 }
162 }
163
164 /* ============================================================================ */
165 public interface FromBitsCoercedOperation<VM extends VectorPayload,
166 S extends VectorSpecies<?>> {
167 VM fromBits(long l, S s);
168 }
169
170 @IntrinsicCandidate
171 public static
172 <VM extends VectorPayload,
173 S extends VectorSpecies<E>,
174 E>
175 VM fromBitsCoerced(Class<? extends VM> vmClass, Class<E> eClass,
176 int length,
177 long bits, int mode, S s,
178 FromBitsCoercedOperation<VM, S> defaultImpl) {
179 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
180 return defaultImpl.fromBits(bits, s);
181 }
182
183 /* ============================================================================ */
184 public interface ShuffleIotaOperation<S extends VectorSpecies<?>,
185 SH extends VectorShuffle<?>> {
186 SH apply(int length, int start, int step, S s);
187 }
188
189 @IntrinsicCandidate
190 public static
191 <E,
192 S extends VectorSpecies<E>,
193 SH extends VectorShuffle<E>>
194 SH shuffleIota(Class<E> eClass, Class<? extends SH> shClass, S s,
195 int length,
196 int start, int step, int wrap,
197 ShuffleIotaOperation<S, SH> defaultImpl) {
198 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
199 return defaultImpl.apply(length, start, step, s);
200 }
201
202 public interface ShuffleToVectorOperation<V extends Vector<?>,
203 SH extends VectorShuffle<?>> {
204 V apply(SH sh);
205 }
206
207 @IntrinsicCandidate
208 public static
209 <V extends Vector<E>,
210 SH extends VectorShuffle<E>,
211 E>
212 V shuffleToVector(Class<? extends Vector<E>> vClass, Class<E> eClass, Class<? extends SH> shClass, SH sh,
213 int length,
214 ShuffleToVectorOperation<V, SH> defaultImpl) {
215 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
216 return defaultImpl.apply(sh);
217 }
218
219 /* ============================================================================ */
220 public interface IndexOperation<V extends Vector<?>,
221 S extends VectorSpecies<?>> {
222 V index(V v, int step, S s);
223 }
224
225 //FIXME @IntrinsicCandidate
226 public static
227 <V extends Vector<E>,
228 E,
229 S extends VectorSpecies<E>>
230 V indexVector(Class<? extends V> vClass, Class<E> eClass,
231 int length,
232 V v, int step, S s,
233 IndexOperation<V, S> defaultImpl) {
234 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
235 return defaultImpl.index(v, step, s);
236 }
237
238 /* ============================================================================ */
239
240 public interface ReductionOperation<V extends Vector<?>,
241 M extends VectorMask<?>> {
242 long apply(V v, M m);
243 }
244
245 @IntrinsicCandidate
246 public static
247 <V extends Vector<E>,
248 M extends VectorMask<E>,
249 E>
250 long reductionCoerced(int oprId,
251 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
252 int length,
253 V v, M m,
254 ReductionOperation<V, M> defaultImpl) {
255 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
256 return defaultImpl.apply(v, m);
257 }
258
259
260 /* ============================================================================ */
261
262 public interface VecExtractOp<V extends Vector<?>> {
263 long apply(V v, int i);
264 }
265
266 @IntrinsicCandidate
267 public static
268 <V extends Vector<E>,
269 E>
270 long extract(Class<? extends V> vClass, Class<E> eClass,
271 int length,
272 V v, int i,
273 VecExtractOp<V> defaultImpl) {
274 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
275 return defaultImpl.apply(v, i);
276 }
277
278 /* ============================================================================ */
279
280 public interface VecInsertOp<V extends Vector<?>> {
281 V apply(V v, int i, long val);
282 }
283
284 @IntrinsicCandidate
285 public static
286 <V extends Vector<E>,
287 E>
288 V insert(Class<? extends V> vClass, Class<E> eClass,
289 int length,
290 V v, int i, long val,
291 VecInsertOp<V> defaultImpl) {
292 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
293 return defaultImpl.apply(v, i, val);
294 }
295
296 /* ============================================================================ */
297
298 public interface UnaryOperation<V extends Vector<?>,
299 M extends VectorMask<?>> {
300 V apply(V v, M m);
301 }
302
303 @IntrinsicCandidate
304 public static
305 <V extends Vector<E>,
306 M extends VectorMask<E>,
307 E>
308 V unaryOp(int oprId,
309 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
310 int length,
311 V v, M m,
312 UnaryOperation<V, M> defaultImpl) {
313 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
314 return defaultImpl.apply(v, m);
315 }
316
317 /* ============================================================================ */
318
319 public interface BinaryOperation<VM extends VectorPayload,
320 M extends VectorMask<?>> {
321 VM apply(VM v1, VM v2, M m);
322 }
323
324 @IntrinsicCandidate
325 public static
326 <VM extends VectorPayload,
327 M extends VectorMask<E>,
328 E>
329 VM binaryOp(int oprId,
330 Class<? extends VM> vmClass, Class<? extends M> mClass, Class<E> eClass,
331 int length,
332 VM v1, VM v2, M m,
333 BinaryOperation<VM, M> defaultImpl) {
334 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
335 return defaultImpl.apply(v1, v2, m);
336 }
337
338 /* ============================================================================ */
339
340 public interface TernaryOperation<V extends Vector<?>,
341 M extends VectorMask<?>> {
342 V apply(V v1, V v2, V v3, M m);
343 }
344
345 @IntrinsicCandidate
346 public static
347 <V extends Vector<E>,
348 M extends VectorMask<E>,
349 E>
350 V ternaryOp(int oprId,
351 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
352 int length,
353 V v1, V v2, V v3, M m,
354 TernaryOperation<V, M> defaultImpl) {
355 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
356 return defaultImpl.apply(v1, v2, v3, m);
357 }
358
359 /* ============================================================================ */
360
361 // Memory operations
362
363 public interface LoadOperation<C,
364 VM extends VectorPayload,
365 S extends VectorSpecies<?>> {
366 VM load(C container, int index, S s);
367 }
368
369 @IntrinsicCandidate
370 public static
371 <C,
372 VM extends VectorPayload,
373 E,
374 S extends VectorSpecies<E>>
375 VM load(Class<? extends VM> vmClass, Class<E> eClass,
376 int length,
377 Object base, long offset,
378 C container, int index, S s,
379 LoadOperation<C, VM, S> defaultImpl) {
380 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
381 return defaultImpl.load(container, index, s);
382 }
383
384 /* ============================================================================ */
385
386 public interface LoadVectorMaskedOperation<C,
387 V extends Vector<?>,
388 S extends VectorSpecies<?>,
389 M extends VectorMask<?>> {
390 V load(C container, int index, S s, M m);
391 }
392
393 @IntrinsicCandidate
394 public static
395 <C,
396 V extends Vector<?>,
397 E,
398 S extends VectorSpecies<E>,
399 M extends VectorMask<E>>
400 V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
401 int length,
402 Object base, long offset,
403 M m, C container, int index, S s,
404 LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
405 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
406 return defaultImpl.load(container, index, s, m);
407 }
408
409 /* ============================================================================ */
410
411 public interface LoadVectorOperationWithMap<C,
412 V extends Vector<?>,
413 S extends VectorSpecies<?>,
414 M extends VectorMask<?>> {
415 V loadWithMap(C container, int index, int[] indexMap, int indexM, S s, M m);
416 }
417
418 @IntrinsicCandidate
419 public static
420 <C,
421 V extends Vector<?>,
422 W extends Vector<Integer>,
423 S extends VectorSpecies<E>,
424 M extends VectorMask<E>,
425 E>
426 V loadWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
427 int length,
428 Class<? extends Vector<Integer>> vectorIndexClass,
429 Object base, long offset,
430 W index_vector,
431 M m, C container, int index, int[] indexMap, int indexM, S s,
432 LoadVectorOperationWithMap<C, V, S, M> defaultImpl) {
433 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
434 return defaultImpl.loadWithMap(container, index, indexMap, indexM, s, m);
435 }
436
437 /* ============================================================================ */
438
439 public interface StoreVectorOperation<C,
440 V extends Vector<?>> {
441 void store(C container, int index, V v);
442 }
443
444 @IntrinsicCandidate
445 public static
446 <C,
447 V extends Vector<?>>
448 void store(Class<?> vClass, Class<?> eClass,
449 int length,
450 Object base, long offset,
451 V v, C container, int index,
452 StoreVectorOperation<C, V> defaultImpl) {
453 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
454 defaultImpl.store(container, index, v);
455 }
456
457 public interface StoreVectorMaskedOperation<C,
458 V extends Vector<?>,
459 M extends VectorMask<?>> {
460 void store(C container, int index, V v, M m);
461 }
462
463 @IntrinsicCandidate
464 public static
465 <C,
466 V extends Vector<E>,
467 M extends VectorMask<E>,
468 E>
469 void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
470 int length,
471 Object base, long offset,
472 V v, M m, C container, int index,
473 StoreVectorMaskedOperation<C, V, M> defaultImpl) {
474 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
475 defaultImpl.store(container, index, v, m);
476 }
477
478 /* ============================================================================ */
479
480 public interface StoreVectorOperationWithMap<C,
481 V extends Vector<?>,
482 M extends VectorMask<?>> {
483 void storeWithMap(C container, int index, V v, int[] indexMap, int indexM, M m);
484 }
485
486 @IntrinsicCandidate
487 public static
488 <C,
489 V extends Vector<E>,
490 W extends Vector<Integer>,
491 M extends VectorMask<E>,
492 E>
493 void storeWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
494 int length,
495 Class<? extends Vector<Integer>> vectorIndexClass,
496 Object base, long offset,
497 W index_vector,
498 V v, M m, C container, int index, int[] indexMap, int indexM,
499 StoreVectorOperationWithMap<C, V, M> defaultImpl) {
500 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
501 defaultImpl.storeWithMap(container, index, v, indexMap, indexM, m);
502 }
503
504 /* ============================================================================ */
505
506 @IntrinsicCandidate
507 public static
508 <M extends VectorMask<E>,
509 E>
510 boolean test(int cond,
511 Class<?> mClass, Class<?> eClass,
512 int length,
513 M m1, M m2,
514 BiFunction<M, M, Boolean> defaultImpl) {
515 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
516 return defaultImpl.apply(m1, m2);
517 }
518
519 /* ============================================================================ */
520
521 public interface VectorCompareOp<V extends Vector<?>,
522 M extends VectorMask<?>> {
523 M apply(int cond, V v1, V v2, M m);
524 }
525
526 @IntrinsicCandidate
527 public static
528 <V extends Vector<E>,
529 M extends VectorMask<E>,
530 E>
531 M compare(int cond,
532 Class<? extends V> vectorClass, Class<M> mClass, Class<E> eClass,
533 int length,
534 V v1, V v2, M m,
535 VectorCompareOp<V, M> defaultImpl) {
536 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
537 return defaultImpl.apply(cond, v1, v2, m);
538 }
539
540 /* ============================================================================ */
541 public interface VectorRearrangeOp<V extends Vector<?>,
542 SH extends VectorShuffle<?>,
543 M extends VectorMask<?>> {
544 V apply(V v, SH sh, M m);
545 }
546
547 @IntrinsicCandidate
548 public static
549 <V extends Vector<E>,
550 SH extends VectorShuffle<E>,
551 M extends VectorMask<E>,
552 E>
553 V rearrangeOp(Class<? extends V> vClass, Class<SH> shClass, Class<M> mClass, Class<E> eClass,
554 int length,
555 V v, SH sh, M m,
556 VectorRearrangeOp<V, SH, M> defaultImpl) {
557 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
558 return defaultImpl.apply(v, sh, m);
559 }
560
561 /* ============================================================================ */
562
563 public interface VectorBlendOp<V extends Vector<?>,
564 M extends VectorMask<?>> {
565 V apply(V v1, V v2, M m);
566 }
567
568 @IntrinsicCandidate
569 public static
570 <V extends Vector<E>,
571 M extends VectorMask<E>,
572 E>
573 V blend(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
574 int length,
575 V v1, V v2, M m,
576 VectorBlendOp<V, M> defaultImpl) {
577 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
578 return defaultImpl.apply(v1, v2, m);
579 }
580
581 /* ============================================================================ */
582
583 public interface VectorBroadcastIntOp<V extends Vector<?>,
584 M extends VectorMask<?>> {
585 V apply(V v, int n, M m);
586 }
587
588 @IntrinsicCandidate
589 public static
590 <V extends Vector<E>,
591 M extends VectorMask<E>,
592 E>
593 V broadcastInt(int opr,
594 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
595 int length,
596 V v, int n, M m,
597 VectorBroadcastIntOp<V, M> defaultImpl) {
598 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
599 return defaultImpl.apply(v, n, m);
600 }
601
602 /* ============================================================================ */
603
604 public interface VectorConvertOp<VOUT extends VectorPayload,
605 VIN extends VectorPayload,
606 S extends VectorSpecies<?>> {
607 VOUT apply(VIN v, S s);
608 }
609
610 // Users of this intrinsic assume that it respects
611 // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN.
612 // See javadoc for REGISTER_ENDIAN.
613
614 @IntrinsicCandidate
615 public static <VOUT extends VectorPayload,
616 VIN extends VectorPayload,
617 S extends VectorSpecies<?>>
618 VOUT convert(int oprId,
619 Class<?> fromVectorClass, Class<?> fromeClass, int fromVLen,
620 Class<?> toVectorClass, Class<?> toeClass, int toVLen,
621 VIN v, S s,
622 VectorConvertOp<VOUT, VIN, S> defaultImpl) {
623 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
624 return defaultImpl.apply(v, s);
625 }
626
627 /* ============================================================================ */
628
629 @IntrinsicCandidate
630 public static
631 <VP extends VectorPayload>
632 VP maybeRebox(VP v) {
633 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
634 // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
635 U.loadFence();
636 return v;
637 }
638
639 /* ============================================================================ */
640 public interface VectorMaskOp<M extends VectorMask<?>> {
641 long apply(M m);
642 }
643
644 @IntrinsicCandidate
645 public static
646 <M extends VectorMask<E>,
647 E>
648 long maskReductionCoerced(int oper,
649 Class<? extends M> mClass, Class<?> eClass,
650 int length,
651 M m,
652 VectorMaskOp<M> defaultImpl) {
653 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
654 return defaultImpl.apply(m);
655 }
656
657 /* ============================================================================ */
658
659 // query the JVM's supported vector sizes and types
660 public static native int getMaxLaneCount(Class<?> etype);
661
662 /* ============================================================================ */
663
664 public static boolean isNonCapturingLambda(Object o) {
665 return o.getClass().getDeclaredFields().length == 0;
666 }
667
668 /* ============================================================================ */
669
670 private static native int registerNatives();
671 }