1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.  Oracle designates this
  8  * particular file as subject to the "Classpath" exception as provided
  9  * by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  */
 25 
 26 package jdk.internal.vm.vector;
 27 
 28 import jdk.internal.vm.annotation.IntrinsicCandidate;
 29 import jdk.internal.misc.Unsafe;
 30 
 31 import java.util.function.*;
 32 
 33 public class VectorSupport {
 34     static {
 35         registerNatives();
 36     }
 37 
 38     private static final Unsafe U = Unsafe.getUnsafe();
 39 
 40     // Unary
 41     public static final int VECTOR_OP_ABS  = 0;
 42     public static final int VECTOR_OP_NEG  = 1;
 43     public static final int VECTOR_OP_SQRT = 2;
 44 
 45     // Binary
 46     public static final int VECTOR_OP_ADD  = 4;
 47     public static final int VECTOR_OP_SUB  = 5;
 48     public static final int VECTOR_OP_MUL  = 6;
 49     public static final int VECTOR_OP_DIV  = 7;
 50     public static final int VECTOR_OP_MIN  = 8;
 51     public static final int VECTOR_OP_MAX  = 9;
 52 
 53     public static final int VECTOR_OP_AND  = 10;
 54     public static final int VECTOR_OP_OR   = 11;
 55     public static final int VECTOR_OP_XOR  = 12;
 56 
 57     // Ternary
 58     public static final int VECTOR_OP_FMA  = 13;
 59 
 60     // Broadcast int
 61     public static final int VECTOR_OP_LSHIFT  = 14;
 62     public static final int VECTOR_OP_RSHIFT  = 15;
 63     public static final int VECTOR_OP_URSHIFT = 16;
 64 
 65     public static final int VECTOR_OP_CAST        = 17;
 66     public static final int VECTOR_OP_REINTERPRET = 18;
 67 
 68     // Mask manipulation operations
 69     public static final int VECTOR_OP_MASK_TRUECOUNT = 19;
 70     public static final int VECTOR_OP_MASK_FIRSTTRUE = 20;
 71     public static final int VECTOR_OP_MASK_LASTTRUE  = 21;
 72     public static final int VECTOR_OP_MASK_TOLONG    = 22;
 73 
 74     // Rotate operations
 75     public static final int VECTOR_OP_LROTATE = 23;
 76     public static final int VECTOR_OP_RROTATE = 24;
 77 





 78     // Math routines
 79     public static final int VECTOR_OP_TAN = 101;
 80     public static final int VECTOR_OP_TANH = 102;
 81     public static final int VECTOR_OP_SIN = 103;
 82     public static final int VECTOR_OP_SINH = 104;
 83     public static final int VECTOR_OP_COS = 105;
 84     public static final int VECTOR_OP_COSH = 106;
 85     public static final int VECTOR_OP_ASIN = 107;
 86     public static final int VECTOR_OP_ACOS = 108;
 87     public static final int VECTOR_OP_ATAN = 109;
 88     public static final int VECTOR_OP_ATAN2 = 110;
 89     public static final int VECTOR_OP_CBRT = 111;
 90     public static final int VECTOR_OP_LOG = 112;
 91     public static final int VECTOR_OP_LOG10 = 113;
 92     public static final int VECTOR_OP_LOG1P = 114;
 93     public static final int VECTOR_OP_POW = 115;
 94     public static final int VECTOR_OP_EXP = 116;
 95     public static final int VECTOR_OP_EXPM1 = 117;
 96     public static final int VECTOR_OP_HYPOT = 118;
 97 
 98     // See src/hotspot/share/opto/subnode.hpp
 99     //     struct BoolTest, and enclosed enum mask
100     public static final int BT_eq = 0;  // 0000
101     public static final int BT_ne = 4;  // 0100
102     public static final int BT_le = 5;  // 0101
103     public static final int BT_ge = 7;  // 0111
104     public static final int BT_lt = 3;  // 0011
105     public static final int BT_gt = 1;  // 0001
106     public static final int BT_overflow = 2;     // 0010
107     public static final int BT_no_overflow = 6;  // 0110
108     // never = 8    1000
109     // illegal = 9  1001
110     // Unsigned comparisons apply to BT_le, BT_ge, BT_lt, BT_gt for integral types
111     public static final int BT_unsigned_compare = 0b10000;
112     public static final int BT_ule = BT_le | BT_unsigned_compare;
113     public static final int BT_uge = BT_ge | BT_unsigned_compare;
114     public static final int BT_ult = BT_lt | BT_unsigned_compare;
115     public static final int BT_ugt = BT_gt | BT_unsigned_compare;
116 
117     // BasicType codes, for primitives only:
118     public static final int
119         T_FLOAT   = 6,
120         T_DOUBLE  = 7,
121         T_BYTE    = 8,
122         T_SHORT   = 9,
123         T_INT     = 10,
124         T_LONG    = 11;
125 
126     /* ============================================================================ */
127 
128     public static class VectorSpecies<E> {}
129 
130     public static class VectorPayload {
131         private final Object payload; // array of primitives
132 
133         public VectorPayload(Object payload) {
134             this.payload = payload;
135         }
136 
137         protected final Object getPayload() {
138             return VectorSupport.maybeRebox(this).payload;
139         }
140     }
141 
142     public static class Vector<E> extends VectorPayload {
143         public Vector(Object payload) {
144             super(payload);
145         }
146     }
147 
148     public static class VectorShuffle<E> extends VectorPayload {
149         public VectorShuffle(Object payload) {
150             super(payload);
151         }
152     }
153     public static class VectorMask<E> extends VectorPayload {
154         public VectorMask(Object payload) {
155             super(payload);
156         }
157     }
158 
159     /* ============================================================================ */
160     public interface BroadcastOperation<VM extends VectorPayload,
161                                         S extends VectorSpecies<?>> {
162         VM broadcast(long l, S s);
163     }
164 
165     @IntrinsicCandidate
166     public static
167     <VM extends VectorPayload,
168      S extends VectorSpecies<E>,
169      E>
170     VM broadcastCoerced(Class<? extends VM> vmClass, Class<E> eClass,
171                         int length,
172                         long bits, S s,
173                         BroadcastOperation<VM, S> defaultImpl) {
174         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
175         return defaultImpl.broadcast(bits, s);
176     }
177 
178     /* ============================================================================ */
179     public interface ShuffleIotaOperation<S extends VectorSpecies<?>,
180                                           SH extends VectorShuffle<?>> {
181         SH apply(int length, int start, int step, S s);
182     }
183 
184     @IntrinsicCandidate
185     public static
186     <E,
187      S extends VectorSpecies<E>,
188      SH extends VectorShuffle<E>>
189     SH shuffleIota(Class<E> eClass, Class<? extends SH> shClass, S s,
190                    int length,
191                    int start, int step, int wrap,
192                    ShuffleIotaOperation<S, SH> defaultImpl) {
193        assert isNonCapturingLambda(defaultImpl) : defaultImpl;
194        return defaultImpl.apply(length, start, step, s);
195     }
196 
197     public interface ShuffleToVectorOperation<V extends Vector<?>,
198                                               SH extends VectorShuffle<?>> {
199        V apply(SH sh);
200     }
201 
202     @IntrinsicCandidate
203     public static
204     <V extends Vector<E>,
205      SH extends VectorShuffle<E>,
206      E>
207     V shuffleToVector(Class<? extends Vector<E>> vClass, Class<E> eClass, Class<? extends SH> shClass, SH sh,
208                       int length,
209                       ShuffleToVectorOperation<V, SH> defaultImpl) {
210       assert isNonCapturingLambda(defaultImpl) : defaultImpl;
211       return defaultImpl.apply(sh);
212     }
213 
214     /* ============================================================================ */
215     public interface IndexOperation<V extends Vector<?>,
216                                     S extends VectorSpecies<?>> {
217         V index(V v, int step, S s);
218     }
219 
220     //FIXME @IntrinsicCandidate
221     public static
222     <V extends Vector<E>,
223      E,
224      S extends VectorSpecies<E>>
225     V indexVector(Class<? extends V> vClass, Class<E> eClass,
226                   int length,
227                   V v, int step, S s,
228                   IndexOperation<V, S> defaultImpl) {
229         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
230         return defaultImpl.index(v, step, s);
231     }
232 
233     /* ============================================================================ */
234 
235     public interface ReductionOperation<V extends Vector<?>,
236                                         M extends VectorMask<?>> {
237         long apply(V v, M m);
238     }
239 
240     @IntrinsicCandidate
241     public static
242     <V extends Vector<E>,
243      M extends VectorMask<E>,
244      E>
245     long reductionCoerced(int oprId,
246                           Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
247                           int length,
248                           V v, M m,
249                           ReductionOperation<V, M> defaultImpl) {
250         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
251         return defaultImpl.apply(v, m);
252     }
253 
254 
255     /* ============================================================================ */
256 
257     public interface VecExtractOp<V extends Vector<?>> {
258         long apply(V v, int i);
259     }
260 
261     @IntrinsicCandidate
262     public static
263     <V extends Vector<E>,
264      E>
265     long extract(Class<? extends V> vClass, Class<E> eClass,
266                  int length,
267                  V v, int i,
268                  VecExtractOp<V> defaultImpl) {
269         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
270         return defaultImpl.apply(v, i);
271     }
272 
273     /* ============================================================================ */
274 
275     public interface VecInsertOp<V extends Vector<?>> {
276         V apply(V v, int i, long val);
277     }
278 
279     @IntrinsicCandidate
280     public static
281     <V extends Vector<E>,
282      E>
283     V insert(Class<? extends V> vClass, Class<E> eClass,
284              int length,
285              V v, int i, long val,
286              VecInsertOp<V> defaultImpl) {
287         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
288         return defaultImpl.apply(v, i, val);
289     }
290 
291     /* ============================================================================ */
292 
293     public interface UnaryOperation<V extends Vector<?>,
294                                     M extends VectorMask<?>> {
295         V apply(V v, M m);
296     }
297 
298     @IntrinsicCandidate
299     public static
300     <V extends Vector<E>,
301      M extends VectorMask<E>,
302      E>
303     V unaryOp(int oprId,
304               Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
305               int length,
306               V v, M m,
307               UnaryOperation<V, M> defaultImpl) {
308         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
309         return defaultImpl.apply(v, m);
310     }
311 
312     /* ============================================================================ */
313 
314     public interface BinaryOperation<VM extends VectorPayload,
315                                      M extends VectorMask<?>> {
316         VM apply(VM v1, VM v2, M m);
317     }
318 
319     @IntrinsicCandidate
320     public static
321     <VM extends VectorPayload,
322      M extends VectorMask<E>,
323      E>
324     VM binaryOp(int oprId,
325                 Class<? extends VM> vmClass, Class<? extends M> mClass, Class<E> eClass,
326                 int length,
327                 VM v1, VM v2, M m,
328                 BinaryOperation<VM, M> defaultImpl) {
329         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
330         return defaultImpl.apply(v1, v2, m);
331     }
332 
333     /* ============================================================================ */
334 
335     public interface TernaryOperation<V extends Vector<?>,
336                                       M extends VectorMask<?>> {
337         V apply(V v1, V v2, V v3, M m);
338     }
339 
340     @IntrinsicCandidate
341     public static
342     <V extends Vector<E>,
343      M extends VectorMask<E>,
344      E>
345     V ternaryOp(int oprId,
346                 Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
347                 int length,
348                 V v1, V v2, V v3, M m,
349                 TernaryOperation<V, M> defaultImpl) {
350         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
351         return defaultImpl.apply(v1, v2, v3, m);
352     }
353 
354     /* ============================================================================ */
355 
356     // Memory operations
357 
358     public interface LoadOperation<C,
359                                    VM extends VectorPayload,
360                                    S extends VectorSpecies<?>> {
361         VM load(C container, int index, S s);
362     }
363 
364     @IntrinsicCandidate
365     public static
366     <C,
367      VM extends VectorPayload,
368      E,
369      S extends VectorSpecies<E>>
370     VM load(Class<? extends VM> vmClass, Class<E> eClass,
371             int length,
372             Object base, long offset,
373             C container, int index, S s,
374             LoadOperation<C, VM, S> defaultImpl) {
375         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
376         return defaultImpl.load(container, index, s);
377     }
378 
379     /* ============================================================================ */
380 
381     public interface LoadVectorMaskedOperation<C,
382                                                V extends Vector<?>,
383                                                S extends VectorSpecies<?>,
384                                                M extends VectorMask<?>> {
385         V load(C container, int index, S s, M m);
386     }
387 
388     @IntrinsicCandidate
389     public static
390     <C,
391      V extends Vector<?>,
392      E,
393      S extends VectorSpecies<E>,
394      M extends VectorMask<E>>
395     V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
396                  int length,
397                  Object base, long offset,
398                  M m, C container, int index, S s,
399                  LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
400         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
401         return defaultImpl.load(container, index, s, m);
402     }
403 
404     /* ============================================================================ */
405 
406     public interface LoadVectorOperationWithMap<C,
407                                                 V extends Vector<?>,
408                                                 S extends VectorSpecies<?>,
409                                                 M extends VectorMask<?>> {
410         V loadWithMap(C container, int index, int[] indexMap, int indexM, S s, M m);
411     }
412 
413     @IntrinsicCandidate
414     public static
415     <C,
416      V extends Vector<?>,
417      W extends Vector<Integer>,
418      S extends VectorSpecies<E>,
419      M extends VectorMask<E>,
420      E>
421     V loadWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
422                   int length,
423                   Class<? extends Vector<Integer>> vectorIndexClass,
424                   Object base, long offset,
425                   W index_vector,
426                   M m, C container, int index, int[] indexMap, int indexM, S s,
427                   LoadVectorOperationWithMap<C, V, S, M> defaultImpl) {
428         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
429         return defaultImpl.loadWithMap(container, index, indexMap, indexM, s, m);
430     }
431 
432     /* ============================================================================ */
433 
434     public interface StoreVectorOperation<C,
435                                           V extends Vector<?>> {
436         void store(C container, int index, V v);
437     }
438 
439     @IntrinsicCandidate
440     public static
441     <C,
442      V extends Vector<?>>
443     void store(Class<?> vClass, Class<?> eClass,
444                int length,
445                Object base, long offset,
446                V v, C container, int index,
447                StoreVectorOperation<C, V> defaultImpl) {
448         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
449         defaultImpl.store(container, index, v);
450     }
451 
452     public interface StoreVectorMaskedOperation<C,
453                                                 V extends Vector<?>,
454                                                 M extends VectorMask<?>> {
455         void store(C container, int index, V v, M m);
456     }
457 
458     @IntrinsicCandidate
459     public static
460     <C,
461      V extends Vector<E>,
462      M extends VectorMask<E>,
463      E>
464     void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
465                      int length,
466                      Object base, long offset,
467                      V v, M m, C container, int index,
468                      StoreVectorMaskedOperation<C, V, M> defaultImpl) {
469         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
470         defaultImpl.store(container, index, v, m);
471     }
472 
473     /* ============================================================================ */
474 
475     public interface StoreVectorOperationWithMap<C,
476                                                  V extends Vector<?>,
477                                                  M extends VectorMask<?>> {
478         void storeWithMap(C container, int index, V v, int[] indexMap, int indexM, M m);
479     }
480 
481     @IntrinsicCandidate
482     public static
483     <C,
484      V extends Vector<E>,
485      W extends Vector<Integer>,
486      M extends VectorMask<E>,
487      E>
488     void storeWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
489                       int length,
490                       Class<? extends Vector<Integer>> vectorIndexClass,
491                       Object base, long offset,
492                       W index_vector,
493                       V v, M m, C container, int index, int[] indexMap, int indexM,
494                       StoreVectorOperationWithMap<C, V, M> defaultImpl) {
495         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
496         defaultImpl.storeWithMap(container, index, v, indexMap, indexM, m);
497     }
498 
499     /* ============================================================================ */
500 
501     @IntrinsicCandidate
502     public static
503     <M extends VectorMask<E>,
504      E>
505     boolean test(int cond,
506                  Class<?> mClass, Class<?> eClass,
507                  int length,
508                  M m1, M m2,
509                  BiFunction<M, M, Boolean> defaultImpl) {
510         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
511         return defaultImpl.apply(m1, m2);
512     }
513 
514     /* ============================================================================ */
515 
516     public interface VectorCompareOp<V extends Vector<?>,
517                                      M extends VectorMask<?>> {
518         M apply(int cond, V v1, V v2, M m);
519     }
520 
521     @IntrinsicCandidate
522     public static
523     <V extends Vector<E>,
524      M extends VectorMask<E>,
525      E>
526     M compare(int cond,
527               Class<? extends V> vectorClass, Class<M> mClass, Class<E> eClass,
528               int length,
529               V v1, V v2, M m,
530               VectorCompareOp<V, M> defaultImpl) {
531         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
532         return defaultImpl.apply(cond, v1, v2, m);
533     }
534 
535     /* ============================================================================ */
536     public interface VectorRearrangeOp<V extends Vector<?>,
537                                        SH extends VectorShuffle<?>,
538                                        M extends VectorMask<?>> {
539         V apply(V v, SH sh, M m);
540     }
541 
542     @IntrinsicCandidate
543     public static
544     <V extends Vector<E>,
545      SH extends VectorShuffle<E>,
546      M  extends VectorMask<E>,
547      E>
548     V rearrangeOp(Class<? extends V> vClass, Class<SH> shClass, Class<M> mClass, Class<E> eClass,
549                   int length,
550                   V v, SH sh, M m,
551                   VectorRearrangeOp<V, SH, M> defaultImpl) {
552         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
553         return defaultImpl.apply(v, sh, m);
554     }
555 
556     /* ============================================================================ */
557 
558     public interface VectorBlendOp<V extends Vector<?>,
559                                    M extends VectorMask<?>> {
560         V apply(V v1, V v2, M m);
561     }
562 
563     @IntrinsicCandidate
564     public static
565     <V extends Vector<E>,
566      M extends VectorMask<E>,
567      E>
568     V blend(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
569             int length,
570             V v1, V v2, M m,
571             VectorBlendOp<V, M> defaultImpl) {
572         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
573         return defaultImpl.apply(v1, v2, m);
574     }
575 
576     /* ============================================================================ */
577 
578     public interface VectorBroadcastIntOp<V extends Vector<?>,
579                                           M extends VectorMask<?>> {
580         V apply(V v, int n, M m);
581     }
582 
583     @IntrinsicCandidate
584     public static
585     <V extends Vector<E>,
586      M extends VectorMask<E>,
587      E>
588     V broadcastInt(int opr,
589                    Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
590                    int length,
591                    V v, int n, M m,
592                    VectorBroadcastIntOp<V, M> defaultImpl) {
593         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
594         return defaultImpl.apply(v, n, m);
595     }
596 
597     /* ============================================================================ */
598 
599     public interface VectorConvertOp<VOUT extends VectorPayload,
600                                      VIN extends VectorPayload,
601                                      S extends VectorSpecies<?>> {
602         VOUT apply(VIN v, S s);
603     }
604 
605     // Users of this intrinsic assume that it respects
606     // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN.
607     // See javadoc for REGISTER_ENDIAN.
608 
609     @IntrinsicCandidate
610     public static <VOUT extends VectorPayload,
611                     VIN extends VectorPayload,
612                       S extends VectorSpecies<?>>
613     VOUT convert(int oprId,
614               Class<?> fromVectorClass, Class<?> fromeClass, int fromVLen,
615               Class<?>   toVectorClass, Class<?>   toeClass, int   toVLen,
616               VIN v, S s,
617               VectorConvertOp<VOUT, VIN, S> defaultImpl) {
618         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
619         return defaultImpl.apply(v, s);
620     }
621 
622     /* ============================================================================ */
623 




















624     @IntrinsicCandidate
625     public static
626     <VP extends VectorPayload>
627     VP maybeRebox(VP v) {
628         // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
629         // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
630         U.loadFence();
631         return v;
632     }
633 
634     /* ============================================================================ */
635     public interface VectorMaskOp<M extends VectorMask<?>> {
636         long apply(M m);
637     }
638 
639     @IntrinsicCandidate
640     public static
641     <M extends VectorMask<E>,
642      E>
643     long maskReductionCoerced(int oper,
644                               Class<? extends M> mClass, Class<?> eClass,
645                               int length,
646                               M m,
647                               VectorMaskOp<M> defaultImpl) {
648        assert isNonCapturingLambda(defaultImpl) : defaultImpl;
649        return defaultImpl.apply(m);
650     }
651 
652     /* ============================================================================ */
653 
654     // query the JVM's supported vector sizes and types
655     public static native int getMaxLaneCount(Class<?> etype);
656 
657     /* ============================================================================ */
658 
659     public static boolean isNonCapturingLambda(Object o) {
660         return o.getClass().getDeclaredFields().length == 0;
661     }
662 
663     /* ============================================================================ */
664 
665     private static native int registerNatives();
666 }
--- EOF ---