1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.  Oracle designates this
  8  * particular file as subject to the "Classpath" exception as provided
  9  * by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  */
 25 
 26 package jdk.internal.vm.vector;
 27 
 28 import jdk.internal.vm.annotation.IntrinsicCandidate;
 29 import jdk.internal.misc.Unsafe;
 30 
 31 import java.util.function.*;
 32 
 33 public class VectorSupport {
 34     static {
 35         registerNatives();
 36     }
 37 
 38     private static final Unsafe U = Unsafe.getUnsafe();
 39 
 40     // Unary
 41     public static final int VECTOR_OP_ABS  = 0;
 42     public static final int VECTOR_OP_NEG  = 1;
 43     public static final int VECTOR_OP_SQRT = 2;
 44 
 45     // Binary
 46     public static final int VECTOR_OP_ADD  = 4;
 47     public static final int VECTOR_OP_SUB  = 5;
 48     public static final int VECTOR_OP_MUL  = 6;
 49     public static final int VECTOR_OP_DIV  = 7;
 50     public static final int VECTOR_OP_MIN  = 8;
 51     public static final int VECTOR_OP_MAX  = 9;
 52 
 53     public static final int VECTOR_OP_AND  = 10;
 54     public static final int VECTOR_OP_OR   = 11;
 55     public static final int VECTOR_OP_XOR  = 12;
 56 
 57     // Ternary
 58     public static final int VECTOR_OP_FMA  = 13;
 59 
 60     // Broadcast int
 61     public static final int VECTOR_OP_LSHIFT  = 14;
 62     public static final int VECTOR_OP_RSHIFT  = 15;
 63     public static final int VECTOR_OP_URSHIFT = 16;
 64 
 65     public static final int VECTOR_OP_CAST        = 17;
 66     public static final int VECTOR_OP_REINTERPRET = 18;
 67 
 68     // Mask manipulation operations
 69     public static final int VECTOR_OP_MASK_TRUECOUNT = 19;
 70     public static final int VECTOR_OP_MASK_FIRSTTRUE = 20;
 71     public static final int VECTOR_OP_MASK_LASTTRUE  = 21;
 72 
 73     // Rotate operations
 74     public static final int VECTOR_OP_LROTATE = 22;
 75     public static final int VECTOR_OP_RROTATE = 23;
 76 
 77     // Math routines
 78     public static final int VECTOR_OP_TAN = 101;
 79     public static final int VECTOR_OP_TANH = 102;
 80     public static final int VECTOR_OP_SIN = 103;
 81     public static final int VECTOR_OP_SINH = 104;
 82     public static final int VECTOR_OP_COS = 105;
 83     public static final int VECTOR_OP_COSH = 106;
 84     public static final int VECTOR_OP_ASIN = 107;
 85     public static final int VECTOR_OP_ACOS = 108;
 86     public static final int VECTOR_OP_ATAN = 109;
 87     public static final int VECTOR_OP_ATAN2 = 110;
 88     public static final int VECTOR_OP_CBRT = 111;
 89     public static final int VECTOR_OP_LOG = 112;
 90     public static final int VECTOR_OP_LOG10 = 113;
 91     public static final int VECTOR_OP_LOG1P = 114;
 92     public static final int VECTOR_OP_POW = 115;
 93     public static final int VECTOR_OP_EXP = 116;
 94     public static final int VECTOR_OP_EXPM1 = 117;
 95     public static final int VECTOR_OP_HYPOT = 118;
 96 
 97     // See src/hotspot/share/opto/subnode.hpp
 98     //     struct BoolTest, and enclosed enum mask
 99     public static final int BT_eq = 0;  // 0000
100     public static final int BT_ne = 4;  // 0100
101     public static final int BT_le = 5;  // 0101
102     public static final int BT_ge = 7;  // 0111
103     public static final int BT_lt = 3;  // 0011
104     public static final int BT_gt = 1;  // 0001
105     public static final int BT_overflow = 2;     // 0010
106     public static final int BT_no_overflow = 6;  // 0110
107     // never = 8    1000
108     // illegal = 9  1001
109     // Unsigned comparisons apply to BT_le, BT_ge, BT_lt, BT_gt for integral types
110     public static final int BT_unsigned_compare = 0b10000;
111     public static final int BT_ule = BT_le | BT_unsigned_compare;
112     public static final int BT_uge = BT_ge | BT_unsigned_compare;
113     public static final int BT_ult = BT_lt | BT_unsigned_compare;
114     public static final int BT_ugt = BT_gt | BT_unsigned_compare;
115 
116     // BasicType codes, for primitives only:
117     public static final int
118         T_FLOAT   = 6,
119         T_DOUBLE  = 7,
120         T_BYTE    = 8,
121         T_SHORT   = 9,
122         T_INT     = 10,
123         T_LONG    = 11;
124 
125     /* ============================================================================ */
126 
127     public static class VectorSpecies<E> {}
128 
129     public static class VectorPayload {
130         private final Object payload; // array of primitives
131 
132         public VectorPayload(Object payload) {
133             this.payload = payload;
134         }
135 
136         protected final Object getPayload() {
137             return VectorSupport.maybeRebox(this).payload;
138         }
139     }
140 
141     public static class Vector<E> extends VectorPayload {
142         public Vector(Object payload) {
143             super(payload);
144         }
145     }
146 
147     public static class VectorShuffle<E> extends VectorPayload {
148         public VectorShuffle(Object payload) {
149             super(payload);
150         }
151     }
152     public static class VectorMask<E> extends VectorPayload {
153         public VectorMask(Object payload) {
154             super(payload);
155         }
156     }
157 
158     /* ============================================================================ */
159     public interface BroadcastOperation<VM, E, S extends VectorSpecies<E>> {
160         VM broadcast(long l, S s);
161     }
162 
163     @IntrinsicCandidate
164     public static
165     <VM, E, S extends VectorSpecies<E>>
166     VM broadcastCoerced(Class<? extends VM> vmClass, Class<E> E, int length,
167                                   long bits, S s,
168                                   BroadcastOperation<VM, E, S> defaultImpl) {
169         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
170         return defaultImpl.broadcast(bits, s);
171     }
172 
173     /* ============================================================================ */
174     public interface ShuffleIotaOperation<E, S extends VectorSpecies<E>> {
175         VectorShuffle<E> apply(int length, int start, int step, S s);
176     }
177 
178     @IntrinsicCandidate
179     public static
180     <E, S extends VectorSpecies<E>>
181     VectorShuffle<E> shuffleIota(Class<?> E, Class<?> ShuffleClass, S s, int length,
182                      int start, int step, int wrap, ShuffleIotaOperation<E, S> defaultImpl) {
183        assert isNonCapturingLambda(defaultImpl) : defaultImpl;
184        return defaultImpl.apply(length, start, step, s);
185     }
186 
187     public interface ShuffleToVectorOperation<VM, Sh, E> {
188        VM apply(Sh s);
189     }
190 
191     @IntrinsicCandidate
192     public static
193     <VM ,Sh extends VectorShuffle<E>, E>
194     VM shuffleToVector(Class<?> VM, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
195                        ShuffleToVectorOperation<VM,Sh,E> defaultImpl) {
196       assert isNonCapturingLambda(defaultImpl) : defaultImpl;
197       return defaultImpl.apply(s);
198     }
199 
200     /* ============================================================================ */
201     public interface IndexOperation<V extends Vector<E>, E, S extends VectorSpecies<E>> {
202         V index(V v, int step, S s);
203     }
204 
205     //FIXME @IntrinsicCandidate
206     public static
207     <V extends Vector<E>, E, S extends VectorSpecies<E>>
208     V indexVector(Class<? extends V> vClass, Class<E> E, int length,
209                   V v, int step, S s,
210                   IndexOperation<V, E, S> defaultImpl) {
211         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
212         return defaultImpl.index(v, step, s);
213     }
214 
215     /* ============================================================================ */
216 
217     @IntrinsicCandidate
218     public static
219     <V extends Vector<?>>
220     long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length,
221                           V v,
222                           Function<V,Long> defaultImpl) {
223         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
224         return defaultImpl.apply(v);
225     }
226 
227     /* ============================================================================ */
228 
229     public interface VecExtractOp<V> {
230         long apply(V v1, int idx);
231     }
232 
233     @IntrinsicCandidate
234     public static
235     <V extends Vector<?>>
236     long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
237                  V vec, int ix,
238                  VecExtractOp<V> defaultImpl) {
239         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
240         return defaultImpl.apply(vec, ix);
241     }
242 
243     /* ============================================================================ */
244 
245     public interface VecInsertOp<V> {
246         V apply(V v1, int idx, long val);
247     }
248 
249     @IntrinsicCandidate
250     public static
251     <V extends Vector<?>>
252     V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,
253              V vec, int ix, long val,
254              VecInsertOp<V> defaultImpl) {
255         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
256         return defaultImpl.apply(vec, ix, val);
257     }
258 
259     /* ============================================================================ */
260 
261     @IntrinsicCandidate
262     public static
263     <VM>
264     VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
265                VM vm,
266                Function<VM, VM> defaultImpl) {
267         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
268         return defaultImpl.apply(vm);
269     }
270 
271     /* ============================================================================ */
272 
273     @IntrinsicCandidate
274     public static
275     <VM>
276     VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
277                 VM vm1, VM vm2,
278                 BiFunction<VM, VM, VM> defaultImpl) {
279         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
280         return defaultImpl.apply(vm1, vm2);
281     }
282 
283     /* ============================================================================ */
284 
285     public interface TernaryOperation<V> {
286         V apply(V v1, V v2, V v3);
287     }
288 
289     @IntrinsicCandidate
290     public static
291     <VM>
292     VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
293                  VM vm1, VM vm2, VM vm3,
294                  TernaryOperation<VM> defaultImpl) {
295         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
296         return defaultImpl.apply(vm1, vm2, vm3);
297     }
298 
299     /* ============================================================================ */
300 
301     // Memory operations
302 
303     public interface LoadOperation<C, V, E, S extends VectorSpecies<E>> {
304         V load(C container, int index, S s);
305     }
306 
307     @IntrinsicCandidate
308     public static
309     <C, VM, E, S extends VectorSpecies<E>>
310     VM load(Class<? extends VM> vmClass, Class<E> E, int length,
311            Object base, long offset,    // Unsafe addressing
312            C container, int index, S s,     // Arguments for default implementation
313            LoadOperation<C, VM, E, S> defaultImpl) {
314         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
315         return defaultImpl.load(container, index, s);
316     }
317 
318     /* ============================================================================ */
319 
320     public interface LoadVectorOperationWithMap<C, V extends Vector<?>, E, S extends VectorSpecies<E>> {
321         V loadWithMap(C container, int index, int[] indexMap, int indexM, S s);
322     }
323 
324     @IntrinsicCandidate
325     public static
326     <C, V extends Vector<?>, W extends Vector<Integer>, E, S extends VectorSpecies<E>>
327     V loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass,
328                   Object base, long offset, // Unsafe addressing
329                   W index_vector,
330                   C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
331                   LoadVectorOperationWithMap<C, V, E, S> defaultImpl) {
332         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
333         return defaultImpl.loadWithMap(container, index, indexMap, indexM, s);
334     }
335 
336     /* ============================================================================ */
337 
338     public interface StoreVectorOperation<C, V extends Vector<?>> {
339         void store(C container, int index, V v);
340     }
341 
342     @IntrinsicCandidate
343     public static
344     <C, V extends Vector<?>>
345     void store(Class<?> vectorClass, Class<?> elementType, int length,
346                Object base, long offset,    // Unsafe addressing
347                V v,
348                C container, int index,      // Arguments for default implementation
349                StoreVectorOperation<C, V> defaultImpl) {
350         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
351         defaultImpl.store(container, index, v);
352     }
353 
354     /* ============================================================================ */
355 
356     public interface StoreVectorOperationWithMap<C, V extends Vector<?>> {
357         void storeWithMap(C container, int index, V v, int[] indexMap, int indexM);
358     }
359 
360     @IntrinsicCandidate
361     public static
362     <C, V extends Vector<?>, W extends Vector<Integer>>
363     void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
364                       Object base, long offset,    // Unsafe addressing
365                       W index_vector, V v,
366                       C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
367                       StoreVectorOperationWithMap<C, V> defaultImpl) {
368         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
369         defaultImpl.storeWithMap(container, index, v, indexMap, indexM);
370     }
371 
372     /* ============================================================================ */
373 
374     @IntrinsicCandidate
375     public static
376     <VM>
377     boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length,
378                  VM vm1, VM vm2,
379                  BiFunction<VM, VM, Boolean> defaultImpl) {
380         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
381         return defaultImpl.apply(vm1, vm2);
382     }
383 
384     /* ============================================================================ */
385 
386     public interface VectorCompareOp<V,M> {
387         M apply(int cond, V v1, V v2);
388     }
389 
390     @IntrinsicCandidate
391     public static <V extends Vector<E>,
392                    M extends VectorMask<E>,
393                    E>
394     M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
395               V v1, V v2,
396               VectorCompareOp<V,M> defaultImpl) {
397         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
398         return defaultImpl.apply(cond, v1, v2);
399     }
400 
401     /* ============================================================================ */
402 
403     public interface VectorRearrangeOp<V extends Vector<E>,
404             Sh extends VectorShuffle<E>,
405             E> {
406         V apply(V v1, Sh shuffle);
407     }
408 
409     @IntrinsicCandidate
410     public static
411     <V extends Vector<E>,
412             Sh extends VectorShuffle<E>,
413             E>
414     V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen,
415                   V v1, Sh sh,
416                   VectorRearrangeOp<V,Sh, E> defaultImpl) {
417         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
418         return defaultImpl.apply(v1, sh);
419     }
420 
421     /* ============================================================================ */
422 
423     public interface VectorBlendOp<V extends Vector<E>,
424             M extends VectorMask<E>,
425             E> {
426         V apply(V v1, V v2, M mask);
427     }
428 
429     @IntrinsicCandidate
430     public static
431     <V extends Vector<E>,
432      M extends VectorMask<E>,
433      E>
434     V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
435             V v1, V v2, M m,
436             VectorBlendOp<V,M, E> defaultImpl) {
437         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
438         return defaultImpl.apply(v1, v2, m);
439     }
440 
441     /* ============================================================================ */
442 
443     public interface VectorBroadcastIntOp<V extends Vector<?>> {
444         V apply(V v, int n);
445     }
446 
447     @IntrinsicCandidate
448     public static
449     <V extends Vector<?>>
450     V broadcastInt(int opr, Class<? extends V> vectorClass, Class<?> elementType, int length,
451                    V v, int n,
452                    VectorBroadcastIntOp<V> defaultImpl) {
453         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
454         return defaultImpl.apply(v, n);
455     }
456 
457     /* ============================================================================ */
458 
459     public interface VectorConvertOp<VOUT, VIN, S> {
460         VOUT apply(VIN v, S species);
461     }
462 
463     // Users of this intrinsic assume that it respects
464     // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN.
465     // See javadoc for REGISTER_ENDIAN.
466 
467     @IntrinsicCandidate
468     public static <VOUT extends VectorPayload,
469                     VIN extends VectorPayload,
470                       S extends VectorSpecies<?>>
471     VOUT convert(int oprId,
472               Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
473               Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
474               VIN v, S s,
475               VectorConvertOp<VOUT, VIN, S> defaultImpl) {
476         assert isNonCapturingLambda(defaultImpl) : defaultImpl;
477         return defaultImpl.apply(v, s);
478     }
479 
480     /* ============================================================================ */
481 
482     @IntrinsicCandidate
483     public static <V> V maybeRebox(V v) {
484         // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
485         // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
486         U.loadFence();
487         return v;
488     }
489 
490     /* ============================================================================ */
491     public interface VectorMaskOp<M> {
492         int apply(M m);
493     }
494 
495     @IntrinsicCandidate
496     public static
497     <E, M>
498     int maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass, int length, M m,
499                VectorMaskOp<M> defaultImpl) {
500        assert isNonCapturingLambda(defaultImpl) : defaultImpl;
501        return defaultImpl.apply(m);
502     }
503 
504     /* ============================================================================ */
505 
506     // query the JVM's supported vector sizes and types
507     public static native int getMaxLaneCount(Class<?> etype);
508 
509     /* ============================================================================ */
510 
511     public static boolean isNonCapturingLambda(Object o) {
512         return o.getClass().getDeclaredFields().length == 0;
513     }
514 
515     /* ============================================================================ */
516 
517     private static native int registerNatives();
518 }