1 /*
2 * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package jdk.internal.vm.vector;
27
28 import jdk.internal.vm.annotation.IntrinsicCandidate;
29 import jdk.internal.misc.Unsafe;
30
31 import java.util.function.*;
32
33 public class VectorSupport {
34 static {
35 registerNatives();
36 }
37
38 private static final Unsafe U = Unsafe.getUnsafe();
39
40 // Unary
41 public static final int VECTOR_OP_ABS = 0;
42 public static final int VECTOR_OP_NEG = 1;
43 public static final int VECTOR_OP_SQRT = 2;
44
45 // Binary
46 public static final int VECTOR_OP_ADD = 4;
47 public static final int VECTOR_OP_SUB = 5;
48 public static final int VECTOR_OP_MUL = 6;
49 public static final int VECTOR_OP_DIV = 7;
50 public static final int VECTOR_OP_MIN = 8;
51 public static final int VECTOR_OP_MAX = 9;
52
53 public static final int VECTOR_OP_AND = 10;
54 public static final int VECTOR_OP_OR = 11;
55 public static final int VECTOR_OP_XOR = 12;
56
57 // Ternary
58 public static final int VECTOR_OP_FMA = 13;
59
60 // Broadcast int
61 public static final int VECTOR_OP_LSHIFT = 14;
62 public static final int VECTOR_OP_RSHIFT = 15;
63 public static final int VECTOR_OP_URSHIFT = 16;
64
65 public static final int VECTOR_OP_CAST = 17;
66 public static final int VECTOR_OP_REINTERPRET = 18;
67
68 // Mask manipulation operations
69 public static final int VECTOR_OP_MASK_TRUECOUNT = 19;
70 public static final int VECTOR_OP_MASK_FIRSTTRUE = 20;
71 public static final int VECTOR_OP_MASK_LASTTRUE = 21;
72
73 // Rotate operations
74 public static final int VECTOR_OP_LROTATE = 22;
75 public static final int VECTOR_OP_RROTATE = 23;
76
77 // Math routines
78 public static final int VECTOR_OP_TAN = 101;
79 public static final int VECTOR_OP_TANH = 102;
80 public static final int VECTOR_OP_SIN = 103;
81 public static final int VECTOR_OP_SINH = 104;
82 public static final int VECTOR_OP_COS = 105;
83 public static final int VECTOR_OP_COSH = 106;
84 public static final int VECTOR_OP_ASIN = 107;
85 public static final int VECTOR_OP_ACOS = 108;
86 public static final int VECTOR_OP_ATAN = 109;
87 public static final int VECTOR_OP_ATAN2 = 110;
88 public static final int VECTOR_OP_CBRT = 111;
89 public static final int VECTOR_OP_LOG = 112;
90 public static final int VECTOR_OP_LOG10 = 113;
91 public static final int VECTOR_OP_LOG1P = 114;
92 public static final int VECTOR_OP_POW = 115;
93 public static final int VECTOR_OP_EXP = 116;
94 public static final int VECTOR_OP_EXPM1 = 117;
95 public static final int VECTOR_OP_HYPOT = 118;
96
97 // See src/hotspot/share/opto/subnode.hpp
98 // struct BoolTest, and enclosed enum mask
99 public static final int BT_eq = 0; // 0000
100 public static final int BT_ne = 4; // 0100
101 public static final int BT_le = 5; // 0101
102 public static final int BT_ge = 7; // 0111
103 public static final int BT_lt = 3; // 0011
104 public static final int BT_gt = 1; // 0001
105 public static final int BT_overflow = 2; // 0010
106 public static final int BT_no_overflow = 6; // 0110
107 // never = 8 1000
108 // illegal = 9 1001
109 // Unsigned comparisons apply to BT_le, BT_ge, BT_lt, BT_gt for integral types
110 public static final int BT_unsigned_compare = 0b10000;
111 public static final int BT_ule = BT_le | BT_unsigned_compare;
112 public static final int BT_uge = BT_ge | BT_unsigned_compare;
113 public static final int BT_ult = BT_lt | BT_unsigned_compare;
114 public static final int BT_ugt = BT_gt | BT_unsigned_compare;
115
116 // BasicType codes, for primitives only:
117 public static final int
118 T_FLOAT = 6,
119 T_DOUBLE = 7,
120 T_BYTE = 8,
121 T_SHORT = 9,
122 T_INT = 10,
123 T_LONG = 11;
124
125 /* ============================================================================ */
126
127 public static class VectorSpecies<E> {}
128
129 public static class VectorPayload {
130 private final Object payload; // array of primitives
131
132 public VectorPayload(Object payload) {
133 this.payload = payload;
134 }
135
136 protected final Object getPayload() {
137 return VectorSupport.maybeRebox(this).payload;
138 }
139 }
140
141 public static class Vector<E> extends VectorPayload {
142 public Vector(Object payload) {
143 super(payload);
144 }
145 }
146
147 public static class VectorShuffle<E> extends VectorPayload {
148 public VectorShuffle(Object payload) {
149 super(payload);
150 }
151 }
152 public static class VectorMask<E> extends VectorPayload {
153 public VectorMask(Object payload) {
154 super(payload);
155 }
156 }
157
158 /* ============================================================================ */
159 public interface BroadcastOperation<VM, E, S extends VectorSpecies<E>> {
160 VM broadcast(long l, S s);
161 }
162
163 @IntrinsicCandidate
164 public static
165 <VM, E, S extends VectorSpecies<E>>
166 VM broadcastCoerced(Class<? extends VM> vmClass, Class<E> E, int length,
167 long bits, S s,
168 BroadcastOperation<VM, E, S> defaultImpl) {
169 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
170 return defaultImpl.broadcast(bits, s);
171 }
172
173 /* ============================================================================ */
174 public interface ShuffleIotaOperation<E, S extends VectorSpecies<E>> {
175 VectorShuffle<E> apply(int length, int start, int step, S s);
176 }
177
178 @IntrinsicCandidate
179 public static
180 <E, S extends VectorSpecies<E>>
181 VectorShuffle<E> shuffleIota(Class<?> E, Class<?> ShuffleClass, S s, int length,
182 int start, int step, int wrap, ShuffleIotaOperation<E, S> defaultImpl) {
183 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
184 return defaultImpl.apply(length, start, step, s);
185 }
186
187 public interface ShuffleToVectorOperation<VM, Sh, E> {
188 VM apply(Sh s);
189 }
190
191 @IntrinsicCandidate
192 public static
193 <VM ,Sh extends VectorShuffle<E>, E>
194 VM shuffleToVector(Class<?> VM, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
195 ShuffleToVectorOperation<VM,Sh,E> defaultImpl) {
196 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
197 return defaultImpl.apply(s);
198 }
199
200 /* ============================================================================ */
201 public interface IndexOperation<V extends Vector<E>, E, S extends VectorSpecies<E>> {
202 V index(V v, int step, S s);
203 }
204
205 //FIXME @IntrinsicCandidate
206 public static
207 <V extends Vector<E>, E, S extends VectorSpecies<E>>
208 V indexVector(Class<? extends V> vClass, Class<E> E, int length,
209 V v, int step, S s,
210 IndexOperation<V, E, S> defaultImpl) {
211 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
212 return defaultImpl.index(v, step, s);
213 }
214
215 /* ============================================================================ */
216
217 @IntrinsicCandidate
218 public static
219 <V extends Vector<?>>
220 long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length,
221 V v,
222 Function<V,Long> defaultImpl) {
223 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
224 return defaultImpl.apply(v);
225 }
226
227 /* ============================================================================ */
228
229 public interface VecExtractOp<V> {
230 long apply(V v1, int idx);
231 }
232
233 @IntrinsicCandidate
234 public static
235 <V extends Vector<?>>
236 long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
237 V vec, int ix,
238 VecExtractOp<V> defaultImpl) {
239 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
240 return defaultImpl.apply(vec, ix);
241 }
242
243 /* ============================================================================ */
244
245 public interface VecInsertOp<V> {
246 V apply(V v1, int idx, long val);
247 }
248
249 @IntrinsicCandidate
250 public static
251 <V extends Vector<?>>
252 V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,
253 V vec, int ix, long val,
254 VecInsertOp<V> defaultImpl) {
255 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
256 return defaultImpl.apply(vec, ix, val);
257 }
258
259 /* ============================================================================ */
260
261 @IntrinsicCandidate
262 public static
263 <VM>
264 VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
265 VM vm,
266 Function<VM, VM> defaultImpl) {
267 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
268 return defaultImpl.apply(vm);
269 }
270
271 /* ============================================================================ */
272
273 @IntrinsicCandidate
274 public static
275 <VM>
276 VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
277 VM vm1, VM vm2,
278 BiFunction<VM, VM, VM> defaultImpl) {
279 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
280 return defaultImpl.apply(vm1, vm2);
281 }
282
283 /* ============================================================================ */
284
285 public interface TernaryOperation<V> {
286 V apply(V v1, V v2, V v3);
287 }
288
289 @IntrinsicCandidate
290 public static
291 <VM>
292 VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
293 VM vm1, VM vm2, VM vm3,
294 TernaryOperation<VM> defaultImpl) {
295 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
296 return defaultImpl.apply(vm1, vm2, vm3);
297 }
298
299 /* ============================================================================ */
300
301 // Memory operations
302
303 public interface LoadOperation<C, V, E, S extends VectorSpecies<E>> {
304 V load(C container, int index, S s);
305 }
306
307 @IntrinsicCandidate
308 public static
309 <C, VM, E, S extends VectorSpecies<E>>
310 VM load(Class<? extends VM> vmClass, Class<E> E, int length,
311 Object base, long offset, // Unsafe addressing
312 C container, int index, S s, // Arguments for default implementation
313 LoadOperation<C, VM, E, S> defaultImpl) {
314 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
315 return defaultImpl.load(container, index, s);
316 }
317
318 /* ============================================================================ */
319
320 public interface LoadVectorOperationWithMap<C, V extends Vector<?>, E, S extends VectorSpecies<E>> {
321 V loadWithMap(C container, int index, int[] indexMap, int indexM, S s);
322 }
323
324 @IntrinsicCandidate
325 public static
326 <C, V extends Vector<?>, W extends Vector<Integer>, E, S extends VectorSpecies<E>>
327 V loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass,
328 Object base, long offset, // Unsafe addressing
329 W index_vector,
330 C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
331 LoadVectorOperationWithMap<C, V, E, S> defaultImpl) {
332 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
333 return defaultImpl.loadWithMap(container, index, indexMap, indexM, s);
334 }
335
336 /* ============================================================================ */
337
338 public interface StoreVectorOperation<C, V extends Vector<?>> {
339 void store(C container, int index, V v);
340 }
341
342 @IntrinsicCandidate
343 public static
344 <C, V extends Vector<?>>
345 void store(Class<?> vectorClass, Class<?> elementType, int length,
346 Object base, long offset, // Unsafe addressing
347 V v,
348 C container, int index, // Arguments for default implementation
349 StoreVectorOperation<C, V> defaultImpl) {
350 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
351 defaultImpl.store(container, index, v);
352 }
353
354 /* ============================================================================ */
355
356 public interface StoreVectorOperationWithMap<C, V extends Vector<?>> {
357 void storeWithMap(C container, int index, V v, int[] indexMap, int indexM);
358 }
359
360 @IntrinsicCandidate
361 public static
362 <C, V extends Vector<?>, W extends Vector<Integer>>
363 void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
364 Object base, long offset, // Unsafe addressing
365 W index_vector, V v,
366 C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
367 StoreVectorOperationWithMap<C, V> defaultImpl) {
368 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
369 defaultImpl.storeWithMap(container, index, v, indexMap, indexM);
370 }
371
372 /* ============================================================================ */
373
374 @IntrinsicCandidate
375 public static
376 <VM>
377 boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length,
378 VM vm1, VM vm2,
379 BiFunction<VM, VM, Boolean> defaultImpl) {
380 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
381 return defaultImpl.apply(vm1, vm2);
382 }
383
384 /* ============================================================================ */
385
386 public interface VectorCompareOp<V,M> {
387 M apply(int cond, V v1, V v2);
388 }
389
390 @IntrinsicCandidate
391 public static <V extends Vector<E>,
392 M extends VectorMask<E>,
393 E>
394 M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
395 V v1, V v2,
396 VectorCompareOp<V,M> defaultImpl) {
397 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
398 return defaultImpl.apply(cond, v1, v2);
399 }
400
401 /* ============================================================================ */
402
403 public interface VectorRearrangeOp<V extends Vector<E>,
404 Sh extends VectorShuffle<E>,
405 E> {
406 V apply(V v1, Sh shuffle);
407 }
408
409 @IntrinsicCandidate
410 public static
411 <V extends Vector<E>,
412 Sh extends VectorShuffle<E>,
413 E>
414 V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen,
415 V v1, Sh sh,
416 VectorRearrangeOp<V,Sh, E> defaultImpl) {
417 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
418 return defaultImpl.apply(v1, sh);
419 }
420
421 /* ============================================================================ */
422
423 public interface VectorBlendOp<V extends Vector<E>,
424 M extends VectorMask<E>,
425 E> {
426 V apply(V v1, V v2, M mask);
427 }
428
429 @IntrinsicCandidate
430 public static
431 <V extends Vector<E>,
432 M extends VectorMask<E>,
433 E>
434 V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length,
435 V v1, V v2, M m,
436 VectorBlendOp<V,M, E> defaultImpl) {
437 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
438 return defaultImpl.apply(v1, v2, m);
439 }
440
441 /* ============================================================================ */
442
443 public interface VectorBroadcastIntOp<V extends Vector<?>> {
444 V apply(V v, int n);
445 }
446
447 @IntrinsicCandidate
448 public static
449 <V extends Vector<?>>
450 V broadcastInt(int opr, Class<? extends V> vectorClass, Class<?> elementType, int length,
451 V v, int n,
452 VectorBroadcastIntOp<V> defaultImpl) {
453 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
454 return defaultImpl.apply(v, n);
455 }
456
457 /* ============================================================================ */
458
459 public interface VectorConvertOp<VOUT, VIN, S> {
460 VOUT apply(VIN v, S species);
461 }
462
463 // Users of this intrinsic assume that it respects
464 // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN.
465 // See javadoc for REGISTER_ENDIAN.
466
467 @IntrinsicCandidate
468 public static <VOUT extends VectorPayload,
469 VIN extends VectorPayload,
470 S extends VectorSpecies<?>>
471 VOUT convert(int oprId,
472 Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
473 Class<?> toVectorClass, Class<?> toElementType, int toVLen,
474 VIN v, S s,
475 VectorConvertOp<VOUT, VIN, S> defaultImpl) {
476 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
477 return defaultImpl.apply(v, s);
478 }
479
480 /* ============================================================================ */
481
482 @IntrinsicCandidate
483 public static <V> V maybeRebox(V v) {
484 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses.
485 // TODO: move the fence generation into C2. Generate only when reboxing is taking place.
486 U.loadFence();
487 return v;
488 }
489
490 /* ============================================================================ */
491 public interface VectorMaskOp<M> {
492 int apply(M m);
493 }
494
495 @IntrinsicCandidate
496 public static
497 <E, M>
498 int maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass, int length, M m,
499 VectorMaskOp<M> defaultImpl) {
500 assert isNonCapturingLambda(defaultImpl) : defaultImpl;
501 return defaultImpl.apply(m);
502 }
503
504 /* ============================================================================ */
505
506 // query the JVM's supported vector sizes and types
507 public static native int getMaxLaneCount(Class<?> etype);
508
509 /* ============================================================================ */
510
511 public static boolean isNonCapturingLambda(Object o) {
512 return o.getClass().getDeclaredFields().length == 0;
513 }
514
515 /* ============================================================================ */
516
517 private static native int registerNatives();
518 }