12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25 package jdk.incubator.vector;
26
27 import java.nio.ByteBuffer;
28 import java.nio.ByteOrder;
29 import java.nio.ReadOnlyBufferException;
30 import java.util.Arrays;
31 import java.util.Objects;
32 import java.util.function.BinaryOperator;
33 import java.util.function.Function;
34 import java.util.function.UnaryOperator;
35
36 import jdk.internal.misc.ScopedMemoryAccess;
37 import jdk.internal.misc.Unsafe;
38 import jdk.internal.vm.annotation.ForceInline;
39 import jdk.internal.vm.vector.VectorSupport;
40
41 import static jdk.internal.vm.vector.VectorSupport.*;
42 import static jdk.incubator.vector.VectorIntrinsics.*;
43
44 import static jdk.incubator.vector.VectorOperators.*;
45
46 // -- This file was mechanically generated: Do not edit! -- //
47
48 /**
49 * A specialized {@link Vector} representing an ordered immutable sequence of
50 * {@code int} values.
51 */
52 @SuppressWarnings("cast") // warning: redundant cast
156 IntVector uOp(FUnOp f);
157 @ForceInline
158 final
159 IntVector uOpTemplate(FUnOp f) {
160 int[] vec = vec();
161 int[] res = new int[length()];
162 for (int i = 0; i < res.length; i++) {
163 res[i] = f.apply(i, vec[i]);
164 }
165 return vectorFactory(res);
166 }
167
168 /*package-private*/
169 abstract
170 IntVector uOp(VectorMask<Integer> m,
171 FUnOp f);
172 @ForceInline
173 final
174 IntVector uOpTemplate(VectorMask<Integer> m,
175 FUnOp f) {
176 int[] vec = vec();
177 int[] res = new int[length()];
178 boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
179 for (int i = 0; i < res.length; i++) {
180 res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
181 }
182 return vectorFactory(res);
183 }
184
185 // Binary operator
186
187 /*package-private*/
188 interface FBinOp {
189 int apply(int i, int a, int b);
190 }
191
192 /*package-private*/
193 abstract
194 IntVector bOp(Vector<Integer> o,
195 FBinOp f);
199 FBinOp f) {
200 int[] res = new int[length()];
201 int[] vec1 = this.vec();
202 int[] vec2 = ((IntVector)o).vec();
203 for (int i = 0; i < res.length; i++) {
204 res[i] = f.apply(i, vec1[i], vec2[i]);
205 }
206 return vectorFactory(res);
207 }
208
209 /*package-private*/
210 abstract
211 IntVector bOp(Vector<Integer> o,
212 VectorMask<Integer> m,
213 FBinOp f);
214 @ForceInline
215 final
216 IntVector bOpTemplate(Vector<Integer> o,
217 VectorMask<Integer> m,
218 FBinOp f) {
219 int[] res = new int[length()];
220 int[] vec1 = this.vec();
221 int[] vec2 = ((IntVector)o).vec();
222 boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
223 for (int i = 0; i < res.length; i++) {
224 res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
225 }
226 return vectorFactory(res);
227 }
228
229 // Ternary operator
230
231 /*package-private*/
232 interface FTriOp {
233 int apply(int i, int a, int b, int c);
234 }
235
236 /*package-private*/
237 abstract
238 IntVector tOp(Vector<Integer> o1,
248 int[] vec2 = ((IntVector)o1).vec();
249 int[] vec3 = ((IntVector)o2).vec();
250 for (int i = 0; i < res.length; i++) {
251 res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
252 }
253 return vectorFactory(res);
254 }
255
256 /*package-private*/
257 abstract
258 IntVector tOp(Vector<Integer> o1,
259 Vector<Integer> o2,
260 VectorMask<Integer> m,
261 FTriOp f);
262 @ForceInline
263 final
264 IntVector tOpTemplate(Vector<Integer> o1,
265 Vector<Integer> o2,
266 VectorMask<Integer> m,
267 FTriOp f) {
268 int[] res = new int[length()];
269 int[] vec1 = this.vec();
270 int[] vec2 = ((IntVector)o1).vec();
271 int[] vec3 = ((IntVector)o2).vec();
272 boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
273 for (int i = 0; i < res.length; i++) {
274 res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
275 }
276 return vectorFactory(res);
277 }
278
279 // Reduction operator
280
281 /*package-private*/
282 abstract
283 int rOp(int v, FBinOp f);
284 @ForceInline
285 final
286 int rOpTemplate(int v, FBinOp f) {
287 int[] vec = vec();
288 for (int i = 0; i < vec.length; i++) {
289 v = f.apply(i, v, vec[i]);
290 }
291 return v;
292 }
293
294 // Memory reference
295
296 /*package-private*/
297 interface FLdOp<M> {
298 int apply(M memory, int offset, int i);
299 }
300
301 /*package-private*/
302 @ForceInline
303 final
532 final IntVector broadcastTemplate(long e) {
533 return vspecies().broadcast(e);
534 }
535
536 // Unary lanewise support
537
538 /**
539 * {@inheritDoc} <!--workaround-->
540 */
541 public abstract
542 IntVector lanewise(VectorOperators.Unary op);
543
544 @ForceInline
545 final
546 IntVector lanewiseTemplate(VectorOperators.Unary op) {
547 if (opKind(op, VO_SPECIAL)) {
548 if (op == ZOMO) {
549 return blend(broadcast(-1), compare(NE, 0));
550 }
551 if (op == NOT) {
552 return broadcast(-1).lanewiseTemplate(XOR, this);
553 } else if (op == NEG) {
554 // FIXME: Support this in the JIT.
555 return broadcast(0).lanewiseTemplate(SUB, this);
556 }
557 }
558 int opc = opCode(op);
559 return VectorSupport.unaryOp(
560 opc, getClass(), int.class, length(),
561 this,
562 UN_IMPL.find(op, opc, (opc_) -> {
563 switch (opc_) {
564 case VECTOR_OP_NEG: return v0 ->
565 v0.uOp((i, a) -> (int) -a);
566 case VECTOR_OP_ABS: return v0 ->
567 v0.uOp((i, a) -> (int) Math.abs(a));
568 default: return null;
569 }}));
570 }
571 private static final
572 ImplCache<Unary,UnaryOperator<IntVector>> UN_IMPL
573 = new ImplCache<>(Unary.class, IntVector.class);
574
575 /**
576 * {@inheritDoc} <!--workaround-->
577 */
578 @ForceInline
579 public final
580 IntVector lanewise(VectorOperators.Unary op,
581 VectorMask<Integer> m) {
582 return blend(lanewise(op), m);
583 }
584
585 // Binary lanewise support
586
587 /**
588 * {@inheritDoc} <!--workaround-->
589 * @see #lanewise(VectorOperators.Binary,int)
590 * @see #lanewise(VectorOperators.Binary,int,VectorMask)
591 */
592 @Override
593 public abstract
594 IntVector lanewise(VectorOperators.Binary op,
595 Vector<Integer> v);
596 @ForceInline
597 final
598 IntVector lanewiseTemplate(VectorOperators.Binary op,
599 Vector<Integer> v) {
600 IntVector that = (IntVector) v;
601 that.check(this);
602 if (opKind(op, VO_SPECIAL | VO_SHIFT)) {
603 if (op == FIRST_NONZERO) {
604 // FIXME: Support this in the JIT.
605 VectorMask<Integer> thisNZ
606 = this.viewAsIntegralLanes().compare(NE, (int) 0);
607 that = that.blend((int) 0, thisNZ.cast(vspecies()));
608 op = OR_UNCHECKED;
609 }
610 if (opKind(op, VO_SHIFT)) {
611 // As per shift specification for Java, mask the shift count.
612 // This allows the JIT to ignore some ISA details.
613 that = that.lanewise(AND, SHIFT_MASK);
614 }
615 if (op == AND_NOT) {
616 // FIXME: Support this in the JIT.
617 that = that.lanewise(NOT);
618 op = AND;
619 } else if (op == DIV) {
620 VectorMask<Integer> eqz = that.eq((int)0);
621 if (eqz.anyTrue()) {
622 throw that.divZeroException();
623 }
624 }
625 }
626 int opc = opCode(op);
627 return VectorSupport.binaryOp(
628 opc, getClass(), int.class, length(),
629 this, that,
630 BIN_IMPL.find(op, opc, (opc_) -> {
631 switch (opc_) {
632 case VECTOR_OP_ADD: return (v0, v1) ->
633 v0.bOp(v1, (i, a, b) -> (int)(a + b));
634 case VECTOR_OP_SUB: return (v0, v1) ->
635 v0.bOp(v1, (i, a, b) -> (int)(a - b));
636 case VECTOR_OP_MUL: return (v0, v1) ->
637 v0.bOp(v1, (i, a, b) -> (int)(a * b));
638 case VECTOR_OP_DIV: return (v0, v1) ->
639 v0.bOp(v1, (i, a, b) -> (int)(a / b));
640 case VECTOR_OP_MAX: return (v0, v1) ->
641 v0.bOp(v1, (i, a, b) -> (int)Math.max(a, b));
642 case VECTOR_OP_MIN: return (v0, v1) ->
643 v0.bOp(v1, (i, a, b) -> (int)Math.min(a, b));
644 case VECTOR_OP_AND: return (v0, v1) ->
645 v0.bOp(v1, (i, a, b) -> (int)(a & b));
646 case VECTOR_OP_OR: return (v0, v1) ->
647 v0.bOp(v1, (i, a, b) -> (int)(a | b));
648 case VECTOR_OP_XOR: return (v0, v1) ->
649 v0.bOp(v1, (i, a, b) -> (int)(a ^ b));
650 case VECTOR_OP_LSHIFT: return (v0, v1) ->
651 v0.bOp(v1, (i, a, n) -> (int)(a << n));
652 case VECTOR_OP_RSHIFT: return (v0, v1) ->
653 v0.bOp(v1, (i, a, n) -> (int)(a >> n));
654 case VECTOR_OP_URSHIFT: return (v0, v1) ->
655 v0.bOp(v1, (i, a, n) -> (int)((a & LSHR_SETUP_MASK) >>> n));
656 case VECTOR_OP_LROTATE: return (v0, v1) ->
657 v0.bOp(v1, (i, a, n) -> rotateLeft(a, (int)n));
658 case VECTOR_OP_RROTATE: return (v0, v1) ->
659 v0.bOp(v1, (i, a, n) -> rotateRight(a, (int)n));
660 default: return null;
661 }}));
662 }
663 private static final
664 ImplCache<Binary,BinaryOperator<IntVector>> BIN_IMPL
665 = new ImplCache<>(Binary.class, IntVector.class);
666
667 /**
668 * {@inheritDoc} <!--workaround-->
669 * @see #lanewise(VectorOperators.Binary,int,VectorMask)
670 */
671 @ForceInline
672 public final
673 IntVector lanewise(VectorOperators.Binary op,
674 Vector<Integer> v,
675 VectorMask<Integer> m) {
676 IntVector that = (IntVector) v;
677 if (op == DIV) {
678 VectorMask<Integer> eqz = that.eq((int)0);
679 if (eqz.and(m).anyTrue()) {
680 throw that.divZeroException();
681 }
682 // suppress div/0 exceptions in unset lanes
683 that = that.lanewise(NOT, eqz);
684 return blend(lanewise(DIV, that), m);
685 }
686 return blend(lanewise(op, v), m);
687 }
688 // FIXME: Maybe all of the public final methods in this file (the
689 // simple ones that just call lanewise) should be pushed down to
690 // the X-VectorBits template. They can't optimize properly at
691 // this level, and must rely on inlining. Does it work?
692 // (If it works, of course keep the code here.)
693
694 /**
695 * Combines the lane values of this vector
696 * with the value of a broadcast scalar.
697 *
698 * This is a lane-wise binary operation which applies
699 * the selected operation to each lane.
700 * The return value will be equal to this expression:
701 * {@code this.lanewise(op, this.broadcast(e))}.
702 *
703 * @param op the operation used to process lane values
704 * @param e the input scalar
705 * @return the result of applying the operation lane-wise
706 * to the two input vectors
707 * @throws UnsupportedOperationException if this vector does
730 * This is a masked lane-wise binary operation which applies
731 * the selected operation to each lane.
732 * The return value will be equal to this expression:
733 * {@code this.lanewise(op, this.broadcast(e), m)}.
734 *
735 * @param op the operation used to process lane values
736 * @param e the input scalar
737 * @param m the mask controlling lane selection
738 * @return the result of applying the operation lane-wise
739 * to the input vector and the scalar
740 * @throws UnsupportedOperationException if this vector does
741 * not support the requested operation
742 * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
743 * @see #lanewise(VectorOperators.Binary,int)
744 */
745 @ForceInline
746 public final
747 IntVector lanewise(VectorOperators.Binary op,
748 int e,
749 VectorMask<Integer> m) {
750 return blend(lanewise(op, e), m);
751 }
752
753 /**
754 * {@inheritDoc} <!--workaround-->
755 * @apiNote
756 * When working with vector subtypes like {@code IntVector},
757 * {@linkplain #lanewise(VectorOperators.Binary,int)
758 * the more strongly typed method}
759 * is typically selected. It can be explicitly selected
760 * using a cast: {@code v.lanewise(op,(int)e)}.
761 * The two expressions will produce numerically identical results.
762 */
763 @ForceInline
764 public final
765 IntVector lanewise(VectorOperators.Binary op,
766 long e) {
767 int e1 = (int) e;
768 if ((long)e1 != e
769 // allow shift ops to clip down their int parameters
770 && !(opKind(op, VO_SHIFT) && (int)e1 == e)
771 ) {
772 vspecies().checkValue(e); // for exception
773 }
774 return lanewise(op, e1);
775 }
776
777 /**
778 * {@inheritDoc} <!--workaround-->
779 * @apiNote
780 * When working with vector subtypes like {@code IntVector},
781 * {@linkplain #lanewise(VectorOperators.Binary,int,VectorMask)
782 * the more strongly typed method}
783 * is typically selected. It can be explicitly selected
784 * using a cast: {@code v.lanewise(op,(int)e,m)}.
785 * The two expressions will produce numerically identical results.
786 */
787 @ForceInline
788 public final
789 IntVector lanewise(VectorOperators.Binary op,
790 long e, VectorMask<Integer> m) {
791 return blend(lanewise(op, e), m);
792 }
793
794 /*package-private*/
795 abstract IntVector
796 lanewiseShift(VectorOperators.Binary op, int e);
797
798 /*package-private*/
799 @ForceInline
800 final IntVector
801 lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
802 // Special handling for these. FIXME: Refactor?
803 assert(opKind(op, VO_SHIFT));
804 // As per shift specification for Java, mask the shift count.
805 e &= SHIFT_MASK;
806 int opc = opCode(op);
807 return VectorSupport.broadcastInt(
808 opc, getClass(), int.class, length(),
809 this, e,
810 BIN_INT_IMPL.find(op, opc, (opc_) -> {
811 switch (opc_) {
812 case VECTOR_OP_LSHIFT: return (v, n) ->
813 v.uOp((i, a) -> (int)(a << n));
814 case VECTOR_OP_RSHIFT: return (v, n) ->
815 v.uOp((i, a) -> (int)(a >> n));
816 case VECTOR_OP_URSHIFT: return (v, n) ->
817 v.uOp((i, a) -> (int)((a & LSHR_SETUP_MASK) >>> n));
818 case VECTOR_OP_LROTATE: return (v, n) ->
819 v.uOp((i, a) -> rotateLeft(a, (int)n));
820 case VECTOR_OP_RROTATE: return (v, n) ->
821 v.uOp((i, a) -> rotateRight(a, (int)n));
822 default: return null;
823 }}));
824 }
825 private static final
826 ImplCache<Binary,VectorBroadcastIntOp<IntVector>> BIN_INT_IMPL
827 = new ImplCache<>(Binary.class, IntVector.class);
828
829 // As per shift specification for Java, mask the shift count.
830 // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
831 // The latter two maskings go beyond the JLS, but seem reasonable
832 // since our lane types are first-class types, not just dressed
833 // up ints.
834 private static final int SHIFT_MASK = (Integer.SIZE - 1);
835 private static final int LSHR_SETUP_MASK = -1;
836
837 // Ternary lanewise support
838
839 // Ternary operators come in eight variations:
840 // lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
841 // lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
842
843 // It is annoying to support all of these variations of masking
844 // and broadcast, but it would be more surprising not to continue
845 // the obvious pattern started by unary and binary.
846
847 /**
848 * {@inheritDoc} <!--workaround-->
860 Vector<Integer> v2);
861 @ForceInline
862 final
863 IntVector lanewiseTemplate(VectorOperators.Ternary op,
864 Vector<Integer> v1,
865 Vector<Integer> v2) {
866 IntVector that = (IntVector) v1;
867 IntVector tother = (IntVector) v2;
868 // It's a word: https://www.dictionary.com/browse/tother
869 // See also Chapter 11 of Dickens, Our Mutual Friend:
870 // "Totherest Governor," replied Mr Riderhood...
871 that.check(this);
872 tother.check(this);
873 if (op == BITWISE_BLEND) {
874 // FIXME: Support this in the JIT.
875 that = this.lanewise(XOR, that).lanewise(AND, tother);
876 return this.lanewise(XOR, that);
877 }
878 int opc = opCode(op);
879 return VectorSupport.ternaryOp(
880 opc, getClass(), int.class, length(),
881 this, that, tother,
882 TERN_IMPL.find(op, opc, (opc_) -> {
883 switch (opc_) {
884 default: return null;
885 }}));
886 }
887 private static final
888 ImplCache<Ternary,TernaryOperation<IntVector>> TERN_IMPL
889 = new ImplCache<>(Ternary.class, IntVector.class);
890
891 /**
892 * {@inheritDoc} <!--workaround-->
893 * @see #lanewise(VectorOperators.Ternary,int,int,VectorMask)
894 * @see #lanewise(VectorOperators.Ternary,Vector,int,VectorMask)
895 * @see #lanewise(VectorOperators.Ternary,int,Vector,VectorMask)
896 */
897 @ForceInline
898 public final
899 IntVector lanewise(VectorOperators.Ternary op,
900 Vector<Integer> v1,
901 Vector<Integer> v2,
902 VectorMask<Integer> m) {
903 return blend(lanewise(op, v1, v2), m);
904 }
905
906 /**
907 * Combines the lane values of this vector
908 * with the values of two broadcast scalars.
909 *
910 * This is a lane-wise ternary operation which applies
911 * the selected operation to each lane.
912 * The return value will be equal to this expression:
913 * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
914 *
915 * @param op the operation used to combine lane values
916 * @param e1 the first input scalar
917 * @param e2 the second input scalar
918 * @return the result of applying the operation lane-wise
919 * to the input vector and the scalars
920 * @throws UnsupportedOperationException if this vector does
921 * not support the requested operation
922 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
923 * @see #lanewise(VectorOperators.Ternary,int,int,VectorMask)
940 * The return value will be equal to this expression:
941 * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
942 *
943 * @param op the operation used to combine lane values
944 * @param e1 the first input scalar
945 * @param e2 the second input scalar
946 * @param m the mask controlling lane selection
947 * @return the result of applying the operation lane-wise
948 * to the input vector and the scalars
949 * @throws UnsupportedOperationException if this vector does
950 * not support the requested operation
951 * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
952 * @see #lanewise(VectorOperators.Ternary,int,int)
953 */
954 @ForceInline
955 public final
956 IntVector lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
957 int e1,
958 int e2,
959 VectorMask<Integer> m) {
960 return blend(lanewise(op, e1, e2), m);
961 }
962
963 /**
964 * Combines the lane values of this vector
965 * with the values of another vector and a broadcast scalar.
966 *
967 * This is a lane-wise ternary operation which applies
968 * the selected operation to each lane.
969 * The return value will be equal to this expression:
970 * {@code this.lanewise(op, v1, this.broadcast(e2))}.
971 *
972 * @param op the operation used to combine lane values
973 * @param v1 the other input vector
974 * @param e2 the input scalar
975 * @return the result of applying the operation lane-wise
976 * to the input vectors and the scalar
977 * @throws UnsupportedOperationException if this vector does
978 * not support the requested operation
979 * @see #lanewise(VectorOperators.Ternary,int,int)
980 * @see #lanewise(VectorOperators.Ternary,Vector,int,VectorMask)
998 * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
999 *
1000 * @param op the operation used to combine lane values
1001 * @param v1 the other input vector
1002 * @param e2 the input scalar
1003 * @param m the mask controlling lane selection
1004 * @return the result of applying the operation lane-wise
1005 * to the input vectors and the scalar
1006 * @throws UnsupportedOperationException if this vector does
1007 * not support the requested operation
1008 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1009 * @see #lanewise(VectorOperators.Ternary,int,int,VectorMask)
1010 * @see #lanewise(VectorOperators.Ternary,Vector,int)
1011 */
1012 @ForceInline
1013 public final
1014 IntVector lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
1015 Vector<Integer> v1,
1016 int e2,
1017 VectorMask<Integer> m) {
1018 return blend(lanewise(op, v1, e2), m);
1019 }
1020
1021 /**
1022 * Combines the lane values of this vector
1023 * with the values of another vector and a broadcast scalar.
1024 *
1025 * This is a lane-wise ternary operation which applies
1026 * the selected operation to each lane.
1027 * The return value will be equal to this expression:
1028 * {@code this.lanewise(op, this.broadcast(e1), v2)}.
1029 *
1030 * @param op the operation used to combine lane values
1031 * @param e1 the input scalar
1032 * @param v2 the other input vector
1033 * @return the result of applying the operation lane-wise
1034 * to the input vectors and the scalar
1035 * @throws UnsupportedOperationException if this vector does
1036 * not support the requested operation
1037 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1038 * @see #lanewise(VectorOperators.Ternary,int,Vector,VectorMask)
1055 * The return value will be equal to this expression:
1056 * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
1057 *
1058 * @param op the operation used to combine lane values
1059 * @param e1 the input scalar
1060 * @param v2 the other input vector
1061 * @param m the mask controlling lane selection
1062 * @return the result of applying the operation lane-wise
1063 * to the input vectors and the scalar
1064 * @throws UnsupportedOperationException if this vector does
1065 * not support the requested operation
1066 * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1067 * @see #lanewise(VectorOperators.Ternary,int,Vector)
1068 */
1069 @ForceInline
1070 public final
1071 IntVector lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
1072 int e1,
1073 Vector<Integer> v2,
1074 VectorMask<Integer> m) {
1075 return blend(lanewise(op, e1, v2), m);
1076 }
1077
1078 // (Thus endeth the Great and Mighty Ternary Ogdoad.)
1079 // https://en.wikipedia.org/wiki/Ogdoad
1080
1081 /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1082 //
1083 // These include masked and non-masked versions.
1084 // This subclass adds broadcast (masked or not).
1085
1086 /**
1087 * {@inheritDoc} <!--workaround-->
1088 * @see #add(int)
1089 */
1090 @Override
1091 @ForceInline
1092 public final IntVector add(Vector<Integer> v) {
1093 return lanewise(ADD, v);
1094 }
1095
1727 @Override
1728 @ForceInline
1729 public final
1730 VectorMask<Integer> test(VectorOperators.Test op,
1731 VectorMask<Integer> m) {
1732 return test(op).and(m);
1733 }
1734
1735 /**
1736 * {@inheritDoc} <!--workaround-->
1737 */
1738 @Override
1739 public abstract
1740 VectorMask<Integer> compare(VectorOperators.Comparison op, Vector<Integer> v);
1741
1742 /*package-private*/
1743 @ForceInline
1744 final
1745 <M extends VectorMask<Integer>>
1746 M compareTemplate(Class<M> maskType, Comparison op, Vector<Integer> v) {
1747 Objects.requireNonNull(v);
1748 IntSpecies vsp = vspecies();
1749 IntVector that = (IntVector) v;
1750 that.check(this);
1751 int opc = opCode(op);
1752 return VectorSupport.compare(
1753 opc, getClass(), maskType, int.class, length(),
1754 this, that,
1755 (cond, v0, v1) -> {
1756 AbstractMask<Integer> m
1757 = v0.bTest(cond, v1, (cond_, i, a, b)
1758 -> compareWithOp(cond, a, b));
1759 @SuppressWarnings("unchecked")
1760 M m2 = (M) m;
1761 return m2;
1762 });
1763 }
1764
1765 @ForceInline
1766 private static boolean compareWithOp(int cond, int a, int b) {
1767 return switch (cond) {
1768 case BT_eq -> a == b;
1769 case BT_ne -> a != b;
1770 case BT_lt -> a < b;
1771 case BT_le -> a <= b;
1772 case BT_gt -> a > b;
1773 case BT_ge -> a >= b;
1774 case BT_ult -> Integer.compareUnsigned(a, b) < 0;
1775 case BT_ule -> Integer.compareUnsigned(a, b) <= 0;
1776 case BT_ugt -> Integer.compareUnsigned(a, b) > 0;
1777 case BT_uge -> Integer.compareUnsigned(a, b) >= 0;
1778 default -> throw new AssertionError();
1779 };
1780 }
1781
1782 /**
1783 * {@inheritDoc} <!--workaround-->
1784 */
1785 @Override
1786 @ForceInline
1787 public final
1788 VectorMask<Integer> compare(VectorOperators.Comparison op,
1789 Vector<Integer> v,
1790 VectorMask<Integer> m) {
1791 return compare(op, v).and(m);
1792 }
1793
1794 /**
1795 * Tests this vector by comparing it with an input scalar,
1796 * according to the given comparison operation.
1797 *
1798 * This is a lane-wise binary test operation which applies
1799 * the comparison operation to each lane.
1800 * <p>
1801 * The result is the same as
1802 * {@code compare(op, broadcast(species(), e))}.
1803 * That is, the scalar may be regarded as broadcast to
1804 * a vector of the same species, and then compared
1805 * against the original vector, using the selected
1806 * comparison operation.
1807 *
1808 * @param op the operation used to compare lane values
1809 * @param e the input scalar
1810 * @return the mask result of testing lane-wise if this vector
1811 * compares to the input, according to the selected
1812 * comparison operator
1813 * @see IntVector#compare(VectorOperators.Comparison,Vector)
1832 *
1833 * This is a masked lane-wise binary test operation which applies
1834 * to each pair of corresponding lane values.
1835 *
1836 * The returned result is equal to the expression
1837 * {@code compare(op,s).and(m)}.
1838 *
1839 * @param op the operation used to compare lane values
1840 * @param e the input scalar
1841 * @param m the mask controlling lane selection
1842 * @return the mask result of testing lane-wise if this vector
1843 * compares to the input, according to the selected
1844 * comparison operator,
1845 * and only in the lanes selected by the mask
1846 * @see IntVector#compare(VectorOperators.Comparison,Vector,VectorMask)
1847 */
1848 @ForceInline
1849 public final VectorMask<Integer> compare(VectorOperators.Comparison op,
1850 int e,
1851 VectorMask<Integer> m) {
1852 return compare(op, e).and(m);
1853 }
1854
1855 /**
1856 * {@inheritDoc} <!--workaround-->
1857 */
1858 @Override
1859 public abstract
1860 VectorMask<Integer> compare(Comparison op, long e);
1861
1862 /*package-private*/
1863 @ForceInline
1864 final
1865 <M extends VectorMask<Integer>>
1866 M compareTemplate(Class<M> maskType, Comparison op, long e) {
1867 return compareTemplate(maskType, op, broadcast(e));
1868 }
1869
1870 /**
1871 * {@inheritDoc} <!--workaround-->
1872 */
2083 wrongPartForSlice(int part) {
2084 String msg = String.format("bad part number %d for slice operation",
2085 part);
2086 return new ArrayIndexOutOfBoundsException(msg);
2087 }
2088
2089 /**
2090 * {@inheritDoc} <!--workaround-->
2091 */
2092 @Override
2093 public abstract
2094 IntVector rearrange(VectorShuffle<Integer> m);
2095
2096 /*package-private*/
2097 @ForceInline
2098 final
2099 <S extends VectorShuffle<Integer>>
2100 IntVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
2101 shuffle.checkIndexes();
2102 return VectorSupport.rearrangeOp(
2103 getClass(), shuffletype, int.class, length(),
2104 this, shuffle,
2105 (v1, s_) -> v1.uOp((i, a) -> {
2106 int ei = s_.laneSource(i);
2107 return v1.lane(ei);
2108 }));
2109 }
2110
2111 /**
2112 * {@inheritDoc} <!--workaround-->
2113 */
2114 @Override
2115 public abstract
2116 IntVector rearrange(VectorShuffle<Integer> s,
2117 VectorMask<Integer> m);
2118
2119 /*package-private*/
2120 @ForceInline
2121 final
2122 <S extends VectorShuffle<Integer>>
2123 IntVector rearrangeTemplate(Class<S> shuffletype,
2124 S shuffle,
2125 VectorMask<Integer> m) {
2126 IntVector unmasked =
2127 VectorSupport.rearrangeOp(
2128 getClass(), shuffletype, int.class, length(),
2129 this, shuffle,
2130 (v1, s_) -> v1.uOp((i, a) -> {
2131 int ei = s_.laneSource(i);
2132 return ei < 0 ? 0 : v1.lane(ei);
2133 }));
2134 VectorMask<Integer> valid = shuffle.laneIsValid();
2135 if (m.andNot(valid).anyTrue()) {
2136 shuffle.checkIndexes();
2137 throw new AssertionError();
2138 }
2139 return broadcast((int)0).blend(unmasked, m);
2140 }
2141
2142 /**
2143 * {@inheritDoc} <!--workaround-->
2144 */
2145 @Override
2146 public abstract
2147 IntVector rearrange(VectorShuffle<Integer> s,
2148 Vector<Integer> v);
2149
2150 /*package-private*/
2151 @ForceInline
2152 final
2153 <S extends VectorShuffle<Integer>>
2154 IntVector rearrangeTemplate(Class<S> shuffletype,
2155 S shuffle,
2156 IntVector v) {
2157 VectorMask<Integer> valid = shuffle.laneIsValid();
2158 @SuppressWarnings("unchecked")
2159 S ws = (S) shuffle.wrapIndexes();
2160 IntVector r0 =
2161 VectorSupport.rearrangeOp(
2162 getClass(), shuffletype, int.class, length(),
2163 this, ws,
2164 (v0, s_) -> v0.uOp((i, a) -> {
2165 int ei = s_.laneSource(i);
2166 return v0.lane(ei);
2167 }));
2168 IntVector r1 =
2169 VectorSupport.rearrangeOp(
2170 getClass(), shuffletype, int.class, length(),
2171 v, ws,
2172 (v1, s_) -> v1.uOp((i, a) -> {
2173 int ei = s_.laneSource(i);
2174 return v1.lane(ei);
2175 }));
2176 return r1.blend(r0, valid);
2177 }
2178
2179 @ForceInline
2180 private final
2181 VectorShuffle<Integer> toShuffle0(IntSpecies dsp) {
2182 int[] a = toArray();
2183 int[] sa = new int[a.length];
2184 for (int i = 0; i < a.length; i++) {
2185 sa[i] = (int) a[i];
2186 }
2187 return VectorShuffle.fromArray(dsp, sa, 0);
2188 }
2189
2190 /*package-private*/
2191 @ForceInline
2192 final
2415 * <li>
2416 * All other reduction operations are fully commutative and
2417 * associative. The implementation can choose any order of
2418 * processing, yet it will always produce the same result.
2419 * </ul>
2420 *
2421 * @param op the operation used to combine lane values
2422 * @param m the mask controlling lane selection
2423 * @return the reduced result accumulated from the selected lane values
2424 * @throws UnsupportedOperationException if this vector does
2425 * not support the requested operation
2426 * @see #reduceLanes(VectorOperators.Associative)
2427 */
2428 public abstract int reduceLanes(VectorOperators.Associative op,
2429 VectorMask<Integer> m);
2430
2431 /*package-private*/
2432 @ForceInline
2433 final
2434 int reduceLanesTemplate(VectorOperators.Associative op,
2435 VectorMask<Integer> m) {
2436 IntVector v = reduceIdentityVector(op).blend(this, m);
2437 return v.reduceLanesTemplate(op);
2438 }
2439
2440 /*package-private*/
2441 @ForceInline
2442 final
2443 int reduceLanesTemplate(VectorOperators.Associative op) {
2444 if (op == FIRST_NONZERO) {
2445 // FIXME: The JIT should handle this, and other scan ops alos.
2446 VectorMask<Integer> thisNZ
2447 = this.viewAsIntegralLanes().compare(NE, (int) 0);
2448 return this.lane(thisNZ.firstTrue());
2449 }
2450 int opc = opCode(op);
2451 return fromBits(VectorSupport.reductionCoerced(
2452 opc, getClass(), int.class, length(),
2453 this,
2454 REDUCE_IMPL.find(op, opc, (opc_) -> {
2455 switch (opc_) {
2456 case VECTOR_OP_ADD: return v ->
2457 toBits(v.rOp((int)0, (i, a, b) -> (int)(a + b)));
2458 case VECTOR_OP_MUL: return v ->
2459 toBits(v.rOp((int)1, (i, a, b) -> (int)(a * b)));
2460 case VECTOR_OP_MIN: return v ->
2461 toBits(v.rOp(MAX_OR_INF, (i, a, b) -> (int) Math.min(a, b)));
2462 case VECTOR_OP_MAX: return v ->
2463 toBits(v.rOp(MIN_OR_INF, (i, a, b) -> (int) Math.max(a, b)));
2464 case VECTOR_OP_AND: return v ->
2465 toBits(v.rOp((int)-1, (i, a, b) -> (int)(a & b)));
2466 case VECTOR_OP_OR: return v ->
2467 toBits(v.rOp((int)0, (i, a, b) -> (int)(a | b)));
2468 case VECTOR_OP_XOR: return v ->
2469 toBits(v.rOp((int)0, (i, a, b) -> (int)(a ^ b)));
2470 default: return null;
2471 }})));
2472 }
2473 private static final
2474 ImplCache<Associative,Function<IntVector,Long>> REDUCE_IMPL
2475 = new ImplCache<>(Associative.class, IntVector.class);
2476
2477 private
2478 @ForceInline
2479 IntVector reduceIdentityVector(VectorOperators.Associative op) {
2480 int opc = opCode(op);
2481 UnaryOperator<IntVector> fn
2482 = REDUCE_ID_IMPL.find(op, opc, (opc_) -> {
2483 switch (opc_) {
2484 case VECTOR_OP_ADD:
2485 case VECTOR_OP_OR:
2486 case VECTOR_OP_XOR:
2487 return v -> v.broadcast(0);
2488 case VECTOR_OP_MUL:
2489 return v -> v.broadcast(1);
2490 case VECTOR_OP_AND:
2491 return v -> v.broadcast(-1);
2492 case VECTOR_OP_MIN:
2493 return v -> v.broadcast(MAX_OR_INF);
2494 case VECTOR_OP_MAX:
2495 return v -> v.broadcast(MIN_OR_INF);
2674 * @param species species of desired vector
2675 * @param a the byte array
2676 * @param offset the offset into the array
2677 * @param bo the intended byte order
2678 * @param m the mask controlling lane selection
2679 * @return a vector loaded from a byte array
2680 * @throws IndexOutOfBoundsException
2681 * if {@code offset+N*ESIZE < 0}
2682 * or {@code offset+(N+1)*ESIZE > a.length}
2683 * for any lane {@code N} in the vector
2684 * where the mask is set
2685 */
2686 @ForceInline
2687 public static
2688 IntVector fromByteArray(VectorSpecies<Integer> species,
2689 byte[] a, int offset,
2690 ByteOrder bo,
2691 VectorMask<Integer> m) {
2692 IntSpecies vsp = (IntSpecies) species;
2693 if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
2694 IntVector zero = vsp.zero();
2695 IntVector v = zero.fromByteArray0(a, offset);
2696 return zero.blend(v.maybeSwap(bo), m);
2697 }
2698
2699 // FIXME: optimize
2700 checkMaskFromIndexSize(offset, vsp, m, 4, a.length);
2701 ByteBuffer wb = wrapper(a, bo);
2702 return vsp.ldOp(wb, offset, (AbstractMask<Integer>)m,
2703 (wb_, o, i) -> wb_.getInt(o + i * 4));
2704 }
2705
2706 /**
2707 * Loads a vector from an array of type {@code int[]}
2708 * starting at an offset.
2709 * For each vector lane, where {@code N} is the vector lane index, the
2710 * array element at index {@code offset + N} is placed into the
2711 * resulting vector at lane index {@code N}.
2712 *
2713 * @param species species of desired vector
2714 * @param a the array
2715 * @param offset the offset into the array
2716 * @return the vector loaded from an array
2738 * {@code N}, otherwise the default element value is placed into the
2739 * resulting vector at lane index {@code N}.
2740 *
2741 * @param species species of desired vector
2742 * @param a the array
2743 * @param offset the offset into the array
2744 * @param m the mask controlling lane selection
2745 * @return the vector loaded from an array
2746 * @throws IndexOutOfBoundsException
2747 * if {@code offset+N < 0} or {@code offset+N >= a.length}
2748 * for any lane {@code N} in the vector
2749 * where the mask is set
2750 */
2751 @ForceInline
2752 public static
2753 IntVector fromArray(VectorSpecies<Integer> species,
2754 int[] a, int offset,
2755 VectorMask<Integer> m) {
2756 IntSpecies vsp = (IntSpecies) species;
2757 if (offset >= 0 && offset <= (a.length - species.length())) {
2758 IntVector zero = vsp.zero();
2759 return zero.blend(zero.fromArray0(a, offset), m);
2760 }
2761
2762 // FIXME: optimize
2763 checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
2764 return vsp.vOp(m, i -> a[offset + i]);
2765 }
2766
2767 /**
2768 * Gathers a new vector composed of elements from an array of type
2769 * {@code int[]},
2770 * using indexes obtained by adding a fixed {@code offset} to a
2771 * series of secondary offsets from an <em>index map</em>.
2772 * The index map is a contiguous sequence of {@code VLENGTH}
2773 * elements in a second array of {@code int}s, starting at a given
2774 * {@code mapOffset}.
2775 * <p>
2776 * For each vector lane, where {@code N} is the vector lane index,
2777 * the lane is loaded from the array
2778 * element {@code a[f(N)]}, where {@code f(N)} is the
2779 * index mapping expression
2797 */
2798 @ForceInline
2799 public static
2800 IntVector fromArray(VectorSpecies<Integer> species,
2801 int[] a, int offset,
2802 int[] indexMap, int mapOffset) {
2803 IntSpecies vsp = (IntSpecies) species;
2804 IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
2805 Objects.requireNonNull(a);
2806 Objects.requireNonNull(indexMap);
2807 Class<? extends IntVector> vectorType = vsp.vectorType();
2808
2809 // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
2810 IntVector vix = IntVector
2811 .fromArray(isp, indexMap, mapOffset)
2812 .add(offset);
2813
2814 vix = VectorIntrinsics.checkIndex(vix, a.length);
2815
2816 return VectorSupport.loadWithMap(
2817 vectorType, int.class, vsp.laneCount(),
2818 IntVector.species(vsp.indexShape()).vectorType(),
2819 a, ARRAY_BASE, vix,
2820 a, offset, indexMap, mapOffset, vsp,
2821 (int[] c, int idx, int[] iMap, int idy, IntSpecies s) ->
2822 s.vOp(n -> c[idx + iMap[idy+n]]));
2823 }
2824
2825 /**
2826 * Gathers a new vector composed of elements from an array of type
2827 * {@code int[]},
2828 * under the control of a mask, and
2829 * using indexes obtained by adding a fixed {@code offset} to a
2830 * series of secondary offsets from an <em>index map</em>.
2831 * The index map is a contiguous sequence of {@code VLENGTH}
2832 * elements in a second array of {@code int}s, starting at a given
2833 * {@code mapOffset}.
2834 * <p>
2835 * For each vector lane, where {@code N} is the vector lane index,
2836 * if the lane is set in the mask,
2837 * the lane is loaded from the array
2838 * element {@code a[f(N)]}, where {@code f(N)} is the
2839 * index mapping expression
2840 * {@code offset + indexMap[mapOffset + N]]}.
2841 * Unset lanes in the resulting vector are set to zero.
2842 *
2843 * @param species species of desired vector
2851 * @return the vector loaded from the indexed elements of the array
2852 * @throws IndexOutOfBoundsException
2853 * if {@code mapOffset+N < 0}
2854 * or if {@code mapOffset+N >= indexMap.length},
2855 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
2856 * is an invalid index into {@code a},
2857 * for any lane {@code N} in the vector
2858 * where the mask is set
2859 * @see IntVector#toIntArray()
2860 */
2861 @ForceInline
2862 public static
2863 IntVector fromArray(VectorSpecies<Integer> species,
2864 int[] a, int offset,
2865 int[] indexMap, int mapOffset,
2866 VectorMask<Integer> m) {
2867 if (m.allTrue()) {
2868 return fromArray(species, a, offset, indexMap, mapOffset);
2869 }
2870 else {
2871 // FIXME: Cannot vectorize yet, if there's a mask.
2872 IntSpecies vsp = (IntSpecies) species;
2873 return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
2874 }
2875 }
2876
2877
2878
2879 /**
2880 * Loads a vector from a {@linkplain ByteBuffer byte buffer}
2881 * starting at an offset into the byte buffer.
2882 * Bytes are composed into primitive lane elements according
2883 * to the specified byte order.
2884 * The vector is arranged into lanes according to
2885 * <a href="Vector.html#lane-order">memory ordering</a>.
2886 * <p>
2887 * This method behaves as if it returns the result of calling
2888 * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
2889 * fromByteBuffer()} as follows:
2890 * <pre>{@code
2891 * var m = species.maskAll(true);
2892 * return fromByteBuffer(species, bb, offset, bo, m);
2893 * }</pre>
2947 * @param species species of desired vector
2948 * @param bb the byte buffer
2949 * @param offset the offset into the byte buffer
2950 * @param bo the intended byte order
2951 * @param m the mask controlling lane selection
2952 * @return a vector loaded from a byte buffer
2953 * @throws IndexOutOfBoundsException
2954 * if {@code offset+N*4 < 0}
2955 * or {@code offset+N*4 >= bb.limit()}
2956 * for any lane {@code N} in the vector
2957 * where the mask is set
2958 */
2959 @ForceInline
2960 public static
2961 IntVector fromByteBuffer(VectorSpecies<Integer> species,
2962 ByteBuffer bb, int offset,
2963 ByteOrder bo,
2964 VectorMask<Integer> m) {
2965 IntSpecies vsp = (IntSpecies) species;
2966 if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
2967 IntVector zero = vsp.zero();
2968 IntVector v = zero.fromByteBuffer0(bb, offset);
2969 return zero.blend(v.maybeSwap(bo), m);
2970 }
2971
2972 // FIXME: optimize
2973 checkMaskFromIndexSize(offset, vsp, m, 4, bb.limit());
2974 ByteBuffer wb = wrapper(bb, bo);
2975 return vsp.ldOp(wb, offset, (AbstractMask<Integer>)m,
2976 (wb_, o, i) -> wb_.getInt(o + i * 4));
2977 }
2978
2979 // Memory store operations
2980
2981 /**
2982 * Stores this vector into an array of type {@code int[]}
2983 * starting at an offset.
2984 * <p>
2985 * For each vector lane, where {@code N} is the vector lane index,
2986 * the lane element at index {@code N} is stored into the array
2987 * element {@code a[offset+N]}.
2988 *
2989 * @param a the array, of type {@code int[]}
3021 * Lanes where the mask is unset are not stored and do not need
3022 * to correspond to legitimate elements of {@code a}.
3023 * That is, unset lanes may correspond to array indexes less than
3024 * zero or beyond the end of the array.
3025 *
3026 * @param a the array, of type {@code int[]}
3027 * @param offset the offset into the array
3028 * @param m the mask controlling lane storage
3029 * @throws IndexOutOfBoundsException
3030 * if {@code offset+N < 0} or {@code offset+N >= a.length}
3031 * for any lane {@code N} in the vector
3032 * where the mask is set
3033 */
3034 @ForceInline
3035 public final
3036 void intoArray(int[] a, int offset,
3037 VectorMask<Integer> m) {
3038 if (m.allTrue()) {
3039 intoArray(a, offset);
3040 } else {
3041 // FIXME: optimize
3042 IntSpecies vsp = vspecies();
3043 checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3044 stOp(a, offset, m, (arr, off, i, v) -> arr[off+i] = v);
3045 }
3046 }
3047
3048 /**
3049 * Scatters this vector into an array of type {@code int[]}
3050 * using indexes obtained by adding a fixed {@code offset} to a
3051 * series of secondary offsets from an <em>index map</em>.
3052 * The index map is a contiguous sequence of {@code VLENGTH}
3053 * elements in a second array of {@code int}s, starting at a given
3054 * {@code mapOffset}.
3055 * <p>
3056 * For each vector lane, where {@code N} is the vector lane index,
3057 * the lane element at index {@code N} is stored into the array
3058 * element {@code a[f(N)]}, where {@code f(N)} is the
3059 * index mapping expression
3060 * {@code offset + indexMap[mapOffset + N]]}.
3061 *
3062 * @param a the array
3063 * @param offset an offset to combine with the index map offsets
3064 * @param indexMap the index map
3068 * or if {@code mapOffset+N >= indexMap.length},
3069 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
3070 * is an invalid index into {@code a},
3071 * for any lane {@code N} in the vector
3072 * @see IntVector#toIntArray()
3073 */
3074 @ForceInline
3075 public final
3076 void intoArray(int[] a, int offset,
3077 int[] indexMap, int mapOffset) {
3078 IntSpecies vsp = vspecies();
3079 IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3080 // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
3081 IntVector vix = IntVector
3082 .fromArray(isp, indexMap, mapOffset)
3083 .add(offset);
3084
3085 vix = VectorIntrinsics.checkIndex(vix, a.length);
3086
3087 VectorSupport.storeWithMap(
3088 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3089 isp.vectorType(),
3090 a, arrayAddress(a, 0), vix,
3091 this,
3092 a, offset, indexMap, mapOffset,
3093 (arr, off, v, map, mo)
3094 -> v.stOp(arr, off,
3095 (arr_, off_, i, e) -> {
3096 int j = map[mo + i];
3097 arr[off + j] = e;
3098 }));
3099 }
3100
3101 /**
3102 * Scatters this vector into an array of type {@code int[]},
3103 * under the control of a mask, and
3104 * using indexes obtained by adding a fixed {@code offset} to a
3105 * series of secondary offsets from an <em>index map</em>.
3106 * The index map is a contiguous sequence of {@code VLENGTH}
3107 * elements in a second array of {@code int}s, starting at a given
3108 * {@code mapOffset}.
3109 * <p>
3110 * For each vector lane, where {@code N} is the vector lane index,
3111 * if the mask lane at index {@code N} is set then
3112 * the lane element at index {@code N} is stored into the array
3113 * element {@code a[f(N)]}, where {@code f(N)} is the
3120 * @param mapOffset the offset into the index map
3121 * @param m the mask
3122 * @throws IndexOutOfBoundsException
3123 * if {@code mapOffset+N < 0}
3124 * or if {@code mapOffset+N >= indexMap.length},
3125 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
3126 * is an invalid index into {@code a},
3127 * for any lane {@code N} in the vector
3128 * where the mask is set
3129 * @see IntVector#toIntArray()
3130 */
3131 @ForceInline
3132 public final
3133 void intoArray(int[] a, int offset,
3134 int[] indexMap, int mapOffset,
3135 VectorMask<Integer> m) {
3136 if (m.allTrue()) {
3137 intoArray(a, offset, indexMap, mapOffset);
3138 }
3139 else {
3140 // FIXME: Cannot vectorize yet, if there's a mask.
3141 stOp(a, offset, m,
3142 (arr, off, i, e) -> {
3143 int j = indexMap[mapOffset + i];
3144 arr[off + j] = e;
3145 });
3146 }
3147 }
3148
3149
3150
3151 /**
3152 * {@inheritDoc} <!--workaround-->
3153 */
3154 @Override
3155 @ForceInline
3156 public final
3157 void intoByteArray(byte[] a, int offset,
3158 ByteOrder bo) {
3159 offset = checkFromIndexSize(offset, byteSize(), a.length);
3160 maybeSwap(bo).intoByteArray0(a, offset);
3161 }
3162
3163 /**
3164 * {@inheritDoc} <!--workaround-->
3165 */
3166 @Override
3167 @ForceInline
3168 public final
3169 void intoByteArray(byte[] a, int offset,
3170 ByteOrder bo,
3171 VectorMask<Integer> m) {
3172 if (m.allTrue()) {
3173 intoByteArray(a, offset, bo);
3174 } else {
3175 // FIXME: optimize
3176 IntSpecies vsp = vspecies();
3177 checkMaskFromIndexSize(offset, vsp, m, 4, a.length);
3178 ByteBuffer wb = wrapper(a, bo);
3179 this.stOp(wb, offset, m,
3180 (wb_, o, i, e) -> wb_.putInt(o + i * 4, e));
3181 }
3182 }
3183
3184 /**
3185 * {@inheritDoc} <!--workaround-->
3186 */
3187 @Override
3188 @ForceInline
3189 public final
3190 void intoByteBuffer(ByteBuffer bb, int offset,
3191 ByteOrder bo) {
3192 if (bb.isReadOnly()) {
3193 throw new ReadOnlyBufferException();
3194 }
3195 offset = checkFromIndexSize(offset, byteSize(), bb.limit());
3196 maybeSwap(bo).intoByteBuffer0(bb, offset);
3197 }
3198
3199 /**
3200 * {@inheritDoc} <!--workaround-->
3201 */
3202 @Override
3203 @ForceInline
3204 public final
3205 void intoByteBuffer(ByteBuffer bb, int offset,
3206 ByteOrder bo,
3207 VectorMask<Integer> m) {
3208 if (m.allTrue()) {
3209 intoByteBuffer(bb, offset, bo);
3210 } else {
3211 // FIXME: optimize
3212 if (bb.isReadOnly()) {
3213 throw new ReadOnlyBufferException();
3214 }
3215 IntSpecies vsp = vspecies();
3216 checkMaskFromIndexSize(offset, vsp, m, 4, bb.limit());
3217 ByteBuffer wb = wrapper(bb, bo);
3218 this.stOp(wb, offset, m,
3219 (wb_, o, i, e) -> wb_.putInt(o + i * 4, e));
3220 }
3221 }
3222
3223 // ================================================
3224
3225 // Low-level memory operations.
3226 //
3227 // Note that all of these operations *must* inline into a context
3228 // where the exact species of the involved vector is a
3229 // compile-time constant. Otherwise, the intrinsic generation
3230 // will fail and performance will suffer.
3231 //
3232 // In many cases this is achieved by re-deriving a version of the
3233 // method in each concrete subclass (per species). The re-derived
3234 // method simply calls one of these generic methods, with exact
3235 // parameters for the controlling metadata, which is either a
3236 // typed vector or constant species instance.
3237
3238 // Unchecked loading operations in native byte order.
3239 // Caller is responsible for applying index checks, masking, and
3240 // byte swapping.
3241
3242 /*package-private*/
3243 abstract
3244 IntVector fromArray0(int[] a, int offset);
3245 @ForceInline
3246 final
3247 IntVector fromArray0Template(int[] a, int offset) {
3248 IntSpecies vsp = vspecies();
3249 return VectorSupport.load(
3250 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3251 a, arrayAddress(a, offset),
3252 a, offset, vsp,
3253 (arr, off, s) -> s.ldOp(arr, off,
3254 (arr_, off_, i) -> arr_[off_ + i]));
3255 }
3256
3257
3258
3259 @Override
3260 abstract
3261 IntVector fromByteArray0(byte[] a, int offset);
3262 @ForceInline
3263 final
3264 IntVector fromByteArray0Template(byte[] a, int offset) {
3265 IntSpecies vsp = vspecies();
3266 return VectorSupport.load(
3267 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3268 a, byteArrayAddress(a, offset),
3269 a, offset, vsp,
3270 (arr, off, s) -> {
3271 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3272 return s.ldOp(wb, off,
3273 (wb_, o, i) -> wb_.getInt(o + i * 4));
3274 });
3275 }
3276
3277 abstract
3278 IntVector fromByteBuffer0(ByteBuffer bb, int offset);
3279 @ForceInline
3280 final
3281 IntVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
3282 IntSpecies vsp = vspecies();
3283 return ScopedMemoryAccess.loadFromByteBuffer(
3284 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3285 bb, offset, vsp,
3286 (buf, off, s) -> {
3287 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3288 return s.ldOp(wb, off,
3289 (wb_, o, i) -> wb_.getInt(o + i * 4));
3290 });
3291 }
3292
3293 // Unchecked storing operations in native byte order.
3294 // Caller is responsible for applying index checks, masking, and
3295 // byte swapping.
3296
3297 abstract
3298 void intoArray0(int[] a, int offset);
3299 @ForceInline
3300 final
3301 void intoArray0Template(int[] a, int offset) {
3302 IntSpecies vsp = vspecies();
3303 VectorSupport.store(
3304 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3305 a, arrayAddress(a, offset),
3306 this, a, offset,
3307 (arr, off, v)
3308 -> v.stOp(arr, off,
3309 (arr_, off_, i, e) -> arr_[off_+i] = e));
3310 }
3311
3312 abstract
3313 void intoByteArray0(byte[] a, int offset);
3314 @ForceInline
3315 final
3316 void intoByteArray0Template(byte[] a, int offset) {
3317 IntSpecies vsp = vspecies();
3318 VectorSupport.store(
3319 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3320 a, byteArrayAddress(a, offset),
3321 this, a, offset,
3322 (arr, off, v) -> {
3323 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3324 v.stOp(wb, off,
3325 (tb_, o, i, e) -> tb_.putInt(o + i * 4, e));
3326 });
3327 }
3328
3329 @ForceInline
3330 final
3331 void intoByteBuffer0(ByteBuffer bb, int offset) {
3332 IntSpecies vsp = vspecies();
3333 ScopedMemoryAccess.storeIntoByteBuffer(
3334 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3335 this, bb, offset,
3336 (buf, off, v) -> {
3337 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3338 v.stOp(wb, off,
3339 (wb_, o, i, e) -> wb_.putInt(o + i * 4, e));
3340 });
3341 }
3342
3343 // End of low-level memory operations.
3344
3345 private static
3346 void checkMaskFromIndexSize(int offset,
3347 IntSpecies vsp,
3348 VectorMask<Integer> m,
3349 int scale,
3350 int limit) {
3351 ((AbstractMask<Integer>)m)
3352 .checkIndexByLane(offset, limit, vsp.iota(), scale);
3353 }
3354
3355 @ForceInline
3356 private void conditionalStoreNYI(int offset,
3357 IntSpecies vsp,
3358 VectorMask<Integer> m,
3359 int scale,
3360 int limit) {
3361 if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
3362 String msg =
3640 int[] res = new int[laneCount()];
3641 boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
3642 for (int i = 0; i < res.length; i++) {
3643 if (mbits[i]) {
3644 res[i] = f.apply(i);
3645 }
3646 }
3647 return dummyVector().vectorFactory(res);
3648 }
3649
3650 /*package-private*/
3651 @ForceInline
3652 <M> IntVector ldOp(M memory, int offset,
3653 FLdOp<M> f) {
3654 return dummyVector().ldOp(memory, offset, f);
3655 }
3656
3657 /*package-private*/
3658 @ForceInline
3659 <M> IntVector ldOp(M memory, int offset,
3660 AbstractMask<Integer> m,
3661 FLdOp<M> f) {
3662 return dummyVector().ldOp(memory, offset, m, f);
3663 }
3664
3665 /*package-private*/
3666 @ForceInline
3667 <M> void stOp(M memory, int offset, FStOp<M> f) {
3668 dummyVector().stOp(memory, offset, f);
3669 }
3670
3671 /*package-private*/
3672 @ForceInline
3673 <M> void stOp(M memory, int offset,
3674 AbstractMask<Integer> m,
3675 FStOp<M> f) {
3676 dummyVector().stOp(memory, offset, m, f);
3677 }
3678
3679 // N.B. Make sure these constant vectors and
3680 // masks load up correctly into registers.
|
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25 package jdk.incubator.vector;
26
27 import java.nio.ByteBuffer;
28 import java.nio.ByteOrder;
29 import java.nio.ReadOnlyBufferException;
30 import java.util.Arrays;
31 import java.util.Objects;
32 import java.util.function.Function;
33 import java.util.function.UnaryOperator;
34
35 import jdk.internal.misc.ScopedMemoryAccess;
36 import jdk.internal.misc.Unsafe;
37 import jdk.internal.vm.annotation.ForceInline;
38 import jdk.internal.vm.vector.VectorSupport;
39
40 import static jdk.internal.vm.vector.VectorSupport.*;
41 import static jdk.incubator.vector.VectorIntrinsics.*;
42
43 import static jdk.incubator.vector.VectorOperators.*;
44
45 // -- This file was mechanically generated: Do not edit! -- //
46
47 /**
48 * A specialized {@link Vector} representing an ordered immutable sequence of
49 * {@code int} values.
50 */
51 @SuppressWarnings("cast") // warning: redundant cast
155 IntVector uOp(FUnOp f);
156 @ForceInline
157 final
158 IntVector uOpTemplate(FUnOp f) {
159 int[] vec = vec();
160 int[] res = new int[length()];
161 for (int i = 0; i < res.length; i++) {
162 res[i] = f.apply(i, vec[i]);
163 }
164 return vectorFactory(res);
165 }
166
167 /*package-private*/
168 abstract
169 IntVector uOp(VectorMask<Integer> m,
170 FUnOp f);
171 @ForceInline
172 final
173 IntVector uOpTemplate(VectorMask<Integer> m,
174 FUnOp f) {
175 if (m == null) {
176 return uOpTemplate(f);
177 }
178 int[] vec = vec();
179 int[] res = new int[length()];
180 boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
181 for (int i = 0; i < res.length; i++) {
182 res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
183 }
184 return vectorFactory(res);
185 }
186
187 // Binary operator
188
189 /*package-private*/
190 interface FBinOp {
191 int apply(int i, int a, int b);
192 }
193
194 /*package-private*/
195 abstract
196 IntVector bOp(Vector<Integer> o,
197 FBinOp f);
201 FBinOp f) {
202 int[] res = new int[length()];
203 int[] vec1 = this.vec();
204 int[] vec2 = ((IntVector)o).vec();
205 for (int i = 0; i < res.length; i++) {
206 res[i] = f.apply(i, vec1[i], vec2[i]);
207 }
208 return vectorFactory(res);
209 }
210
211 /*package-private*/
212 abstract
213 IntVector bOp(Vector<Integer> o,
214 VectorMask<Integer> m,
215 FBinOp f);
216 @ForceInline
217 final
218 IntVector bOpTemplate(Vector<Integer> o,
219 VectorMask<Integer> m,
220 FBinOp f) {
221 if (m == null) {
222 return bOpTemplate(o, f);
223 }
224 int[] res = new int[length()];
225 int[] vec1 = this.vec();
226 int[] vec2 = ((IntVector)o).vec();
227 boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
228 for (int i = 0; i < res.length; i++) {
229 res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
230 }
231 return vectorFactory(res);
232 }
233
234 // Ternary operator
235
236 /*package-private*/
237 interface FTriOp {
238 int apply(int i, int a, int b, int c);
239 }
240
241 /*package-private*/
242 abstract
243 IntVector tOp(Vector<Integer> o1,
253 int[] vec2 = ((IntVector)o1).vec();
254 int[] vec3 = ((IntVector)o2).vec();
255 for (int i = 0; i < res.length; i++) {
256 res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
257 }
258 return vectorFactory(res);
259 }
260
261 /*package-private*/
262 abstract
263 IntVector tOp(Vector<Integer> o1,
264 Vector<Integer> o2,
265 VectorMask<Integer> m,
266 FTriOp f);
267 @ForceInline
268 final
269 IntVector tOpTemplate(Vector<Integer> o1,
270 Vector<Integer> o2,
271 VectorMask<Integer> m,
272 FTriOp f) {
273 if (m == null) {
274 return tOpTemplate(o1, o2, f);
275 }
276 int[] res = new int[length()];
277 int[] vec1 = this.vec();
278 int[] vec2 = ((IntVector)o1).vec();
279 int[] vec3 = ((IntVector)o2).vec();
280 boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
281 for (int i = 0; i < res.length; i++) {
282 res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
283 }
284 return vectorFactory(res);
285 }
286
287 // Reduction operator
288
289 /*package-private*/
290 abstract
291 int rOp(int v, VectorMask<Integer> m, FBinOp f);
292
293 @ForceInline
294 final
295 int rOpTemplate(int v, VectorMask<Integer> m, FBinOp f) {
296 if (m == null) {
297 return rOpTemplate(v, f);
298 }
299 int[] vec = vec();
300 boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
301 for (int i = 0; i < vec.length; i++) {
302 v = mbits[i] ? f.apply(i, v, vec[i]) : v;
303 }
304 return v;
305 }
306
307 @ForceInline
308 final
309 int rOpTemplate(int v, FBinOp f) {
310 int[] vec = vec();
311 for (int i = 0; i < vec.length; i++) {
312 v = f.apply(i, v, vec[i]);
313 }
314 return v;
315 }
316
317 // Memory reference
318
319 /*package-private*/
320 interface FLdOp<M> {
321 int apply(M memory, int offset, int i);
322 }
323
324 /*package-private*/
325 @ForceInline
326 final
555 final IntVector broadcastTemplate(long e) {
556 return vspecies().broadcast(e);
557 }
558
559 // Unary lanewise support
560
561 /**
562 * {@inheritDoc} <!--workaround-->
563 */
564 public abstract
565 IntVector lanewise(VectorOperators.Unary op);
566
567 @ForceInline
568 final
569 IntVector lanewiseTemplate(VectorOperators.Unary op) {
570 if (opKind(op, VO_SPECIAL)) {
571 if (op == ZOMO) {
572 return blend(broadcast(-1), compare(NE, 0));
573 }
574 if (op == NOT) {
575 return broadcast(-1).lanewise(XOR, this);
576 } else if (op == NEG) {
577 // FIXME: Support this in the JIT.
578 return broadcast(0).lanewise(SUB, this);
579 }
580 }
581 int opc = opCode(op);
582 return VectorSupport.unaryOp(
583 opc, getClass(), null, int.class, length(),
584 this, null,
585 UN_IMPL.find(op, opc, IntVector::unaryOperations));
586 }
587
588 /**
589 * {@inheritDoc} <!--workaround-->
590 */
591 @Override
592 public abstract
593 IntVector lanewise(VectorOperators.Unary op,
594 VectorMask<Integer> m);
595 @ForceInline
596 final
597 IntVector lanewiseTemplate(VectorOperators.Unary op,
598 Class<? extends VectorMask<Integer>> maskClass,
599 VectorMask<Integer> m) {
600 m.check(maskClass, this);
601 if (opKind(op, VO_SPECIAL)) {
602 if (op == ZOMO) {
603 return blend(broadcast(-1), compare(NE, 0, m));
604 }
605 if (op == NOT) {
606 return lanewise(XOR, broadcast(-1), m);
607 } else if (op == NEG) {
608 return lanewise(NOT, m).lanewise(ADD, broadcast(1), m);
609 }
610 }
611 int opc = opCode(op);
612 return VectorSupport.unaryOp(
613 opc, getClass(), maskClass, int.class, length(),
614 this, m,
615 UN_IMPL.find(op, opc, IntVector::unaryOperations));
616 }
617
618 private static final
619 ImplCache<Unary, UnaryOperation<IntVector, VectorMask<Integer>>>
620 UN_IMPL = new ImplCache<>(Unary.class, IntVector.class);
621
622 private static UnaryOperation<IntVector, VectorMask<Integer>> unaryOperations(int opc_) {
623 switch (opc_) {
624 case VECTOR_OP_NEG: return (v0, m) ->
625 v0.uOp(m, (i, a) -> (int) -a);
626 case VECTOR_OP_ABS: return (v0, m) ->
627 v0.uOp(m, (i, a) -> (int) Math.abs(a));
628 default: return null;
629 }
630 }
631
632 // Binary lanewise support
633
634 /**
635 * {@inheritDoc} <!--workaround-->
636 * @see #lanewise(VectorOperators.Binary,int)
637 * @see #lanewise(VectorOperators.Binary,int,VectorMask)
638 */
639 @Override
640 public abstract
641 IntVector lanewise(VectorOperators.Binary op,
642 Vector<Integer> v);
643 @ForceInline
644 final
645 IntVector lanewiseTemplate(VectorOperators.Binary op,
646 Vector<Integer> v) {
647 IntVector that = (IntVector) v;
648 that.check(this);
649
650 if (opKind(op, VO_SPECIAL | VO_SHIFT)) {
651 if (op == FIRST_NONZERO) {
652 // FIXME: Support this in the JIT.
653 VectorMask<Integer> thisNZ
654 = this.viewAsIntegralLanes().compare(NE, (int) 0);
655 that = that.blend((int) 0, thisNZ.cast(vspecies()));
656 op = OR_UNCHECKED;
657 }
658 if (opKind(op, VO_SHIFT)) {
659 // As per shift specification for Java, mask the shift count.
660 // This allows the JIT to ignore some ISA details.
661 that = that.lanewise(AND, SHIFT_MASK);
662 }
663 if (op == AND_NOT) {
664 // FIXME: Support this in the JIT.
665 that = that.lanewise(NOT);
666 op = AND;
667 } else if (op == DIV) {
668 VectorMask<Integer> eqz = that.eq((int) 0);
669 if (eqz.anyTrue()) {
670 throw that.divZeroException();
671 }
672 }
673 }
674
675 int opc = opCode(op);
676 return VectorSupport.binaryOp(
677 opc, getClass(), null, int.class, length(),
678 this, that, null,
679 BIN_IMPL.find(op, opc, IntVector::binaryOperations));
680 }
681
682 /**
683 * {@inheritDoc} <!--workaround-->
684 * @see #lanewise(VectorOperators.Binary,int,VectorMask)
685 */
686 @Override
687 public abstract
688 IntVector lanewise(VectorOperators.Binary op,
689 Vector<Integer> v,
690 VectorMask<Integer> m);
691 @ForceInline
692 final
693 IntVector lanewiseTemplate(VectorOperators.Binary op,
694 Class<? extends VectorMask<Integer>> maskClass,
695 Vector<Integer> v, VectorMask<Integer> m) {
696 IntVector that = (IntVector) v;
697 that.check(this);
698 m.check(maskClass, this);
699
700 if (opKind(op, VO_SPECIAL | VO_SHIFT)) {
701 if (op == FIRST_NONZERO) {
702 // FIXME: Support this in the JIT.
703 VectorMask<Integer> thisNZ
704 = this.viewAsIntegralLanes().compare(NE, (int) 0);
705 that = that.blend((int) 0, thisNZ.cast(vspecies()));
706 op = OR_UNCHECKED;
707 }
708 if (opKind(op, VO_SHIFT)) {
709 // As per shift specification for Java, mask the shift count.
710 // This allows the JIT to ignore some ISA details.
711 that = that.lanewise(AND, SHIFT_MASK);
712 }
713 if (op == AND_NOT) {
714 // FIXME: Support this in the JIT.
715 that = that.lanewise(NOT);
716 op = AND;
717 } else if (op == DIV) {
718 VectorMask<Integer> eqz = that.eq((int)0);
719 if (eqz.and(m).anyTrue()) {
720 throw that.divZeroException();
721 }
722 // suppress div/0 exceptions in unset lanes
723 that = that.lanewise(NOT, eqz);
724 }
725 }
726
727 int opc = opCode(op);
728 return VectorSupport.binaryOp(
729 opc, getClass(), maskClass, int.class, length(),
730 this, that, m,
731 BIN_IMPL.find(op, opc, IntVector::binaryOperations));
732 }
733
734 private static final
735 ImplCache<Binary, BinaryOperation<IntVector, VectorMask<Integer>>>
736 BIN_IMPL = new ImplCache<>(Binary.class, IntVector.class);
737
738 private static BinaryOperation<IntVector, VectorMask<Integer>> binaryOperations(int opc_) {
739 switch (opc_) {
740 case VECTOR_OP_ADD: return (v0, v1, vm) ->
741 v0.bOp(v1, vm, (i, a, b) -> (int)(a + b));
742 case VECTOR_OP_SUB: return (v0, v1, vm) ->
743 v0.bOp(v1, vm, (i, a, b) -> (int)(a - b));
744 case VECTOR_OP_MUL: return (v0, v1, vm) ->
745 v0.bOp(v1, vm, (i, a, b) -> (int)(a * b));
746 case VECTOR_OP_DIV: return (v0, v1, vm) ->
747 v0.bOp(v1, vm, (i, a, b) -> (int)(a / b));
748 case VECTOR_OP_MAX: return (v0, v1, vm) ->
749 v0.bOp(v1, vm, (i, a, b) -> (int)Math.max(a, b));
750 case VECTOR_OP_MIN: return (v0, v1, vm) ->
751 v0.bOp(v1, vm, (i, a, b) -> (int)Math.min(a, b));
752 case VECTOR_OP_AND: return (v0, v1, vm) ->
753 v0.bOp(v1, vm, (i, a, b) -> (int)(a & b));
754 case VECTOR_OP_OR: return (v0, v1, vm) ->
755 v0.bOp(v1, vm, (i, a, b) -> (int)(a | b));
756 case VECTOR_OP_XOR: return (v0, v1, vm) ->
757 v0.bOp(v1, vm, (i, a, b) -> (int)(a ^ b));
758 case VECTOR_OP_LSHIFT: return (v0, v1, vm) ->
759 v0.bOp(v1, vm, (i, a, n) -> (int)(a << n));
760 case VECTOR_OP_RSHIFT: return (v0, v1, vm) ->
761 v0.bOp(v1, vm, (i, a, n) -> (int)(a >> n));
762 case VECTOR_OP_URSHIFT: return (v0, v1, vm) ->
763 v0.bOp(v1, vm, (i, a, n) -> (int)((a & LSHR_SETUP_MASK) >>> n));
764 case VECTOR_OP_LROTATE: return (v0, v1, vm) ->
765 v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
766 case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
767 v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
768 default: return null;
769 }
770 }
771
772 // FIXME: Maybe all of the public final methods in this file (the
773 // simple ones that just call lanewise) should be pushed down to
774 // the X-VectorBits template. They can't optimize properly at
775 // this level, and must rely on inlining. Does it work?
776 // (If it works, of course keep the code here.)
777
778 /**
779 * Combines the lane values of this vector
780 * with the value of a broadcast scalar.
781 *
782 * This is a lane-wise binary operation which applies
783 * the selected operation to each lane.
784 * The return value will be equal to this expression:
785 * {@code this.lanewise(op, this.broadcast(e))}.
786 *
787 * @param op the operation used to process lane values
788 * @param e the input scalar
789 * @return the result of applying the operation lane-wise
790 * to the two input vectors
791 * @throws UnsupportedOperationException if this vector does
814 * This is a masked lane-wise binary operation which applies
815 * the selected operation to each lane.
816 * The return value will be equal to this expression:
817 * {@code this.lanewise(op, this.broadcast(e), m)}.
818 *
819 * @param op the operation used to process lane values
820 * @param e the input scalar
821 * @param m the mask controlling lane selection
822 * @return the result of applying the operation lane-wise
823 * to the input vector and the scalar
824 * @throws UnsupportedOperationException if this vector does
825 * not support the requested operation
826 * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
827 * @see #lanewise(VectorOperators.Binary,int)
828 */
829 @ForceInline
830 public final
831 IntVector lanewise(VectorOperators.Binary op,
832 int e,
833 VectorMask<Integer> m) {
834 if (opKind(op, VO_SHIFT) && (int)(int)e == e) {
835 return lanewiseShift(op, (int) e, m);
836 }
837 if (op == AND_NOT) {
838 op = AND; e = (int) ~e;
839 }
840 return lanewise(op, broadcast(e), m);
841 }
842
843 /**
844 * {@inheritDoc} <!--workaround-->
845 * @apiNote
846 * When working with vector subtypes like {@code IntVector},
847 * {@linkplain #lanewise(VectorOperators.Binary,int)
848 * the more strongly typed method}
849 * is typically selected. It can be explicitly selected
850 * using a cast: {@code v.lanewise(op,(int)e)}.
851 * The two expressions will produce numerically identical results.
852 */
853 @ForceInline
854 public final
855 IntVector lanewise(VectorOperators.Binary op,
856 long e) {
857 int e1 = (int) e;
858 if ((long)e1 != e
859 // allow shift ops to clip down their int parameters
860 && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
861 vspecies().checkValue(e); // for exception
862 }
863 return lanewise(op, e1);
864 }
865
866 /**
867 * {@inheritDoc} <!--workaround-->
868 * @apiNote
869 * When working with vector subtypes like {@code IntVector},
870 * {@linkplain #lanewise(VectorOperators.Binary,int,VectorMask)
871 * the more strongly typed method}
872 * is typically selected. It can be explicitly selected
873 * using a cast: {@code v.lanewise(op,(int)e,m)}.
874 * The two expressions will produce numerically identical results.
875 */
876 @ForceInline
877 public final
878 IntVector lanewise(VectorOperators.Binary op,
879 long e, VectorMask<Integer> m) {
880 int e1 = (int) e;
881 if ((long)e1 != e
882 // allow shift ops to clip down their int parameters
883 && !(opKind(op, VO_SHIFT) && (int)e1 == e)) {
884 vspecies().checkValue(e); // for exception
885 }
886 return lanewise(op, e1, m);
887 }
888
889 /*package-private*/
890 abstract IntVector
891 lanewiseShift(VectorOperators.Binary op, int e);
892
893 /*package-private*/
894 @ForceInline
895 final IntVector
896 lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
897 // Special handling for these. FIXME: Refactor?
898 assert(opKind(op, VO_SHIFT));
899 // As per shift specification for Java, mask the shift count.
900 e &= SHIFT_MASK;
901 int opc = opCode(op);
902 return VectorSupport.broadcastInt(
903 opc, getClass(), null, int.class, length(),
904 this, e, null,
905 BIN_INT_IMPL.find(op, opc, IntVector::broadcastIntOperations));
906 }
907
908 /*package-private*/
909 abstract IntVector
910 lanewiseShift(VectorOperators.Binary op, int e, VectorMask<Integer> m);
911
912 /*package-private*/
913 @ForceInline
914 final IntVector
915 lanewiseShiftTemplate(VectorOperators.Binary op,
916 Class<? extends VectorMask<Integer>> maskClass,
917 int e, VectorMask<Integer> m) {
918 m.check(maskClass, this);
919 assert(opKind(op, VO_SHIFT));
920 // As per shift specification for Java, mask the shift count.
921 e &= SHIFT_MASK;
922 int opc = opCode(op);
923 return VectorSupport.broadcastInt(
924 opc, getClass(), maskClass, int.class, length(),
925 this, e, m,
926 BIN_INT_IMPL.find(op, opc, IntVector::broadcastIntOperations));
927 }
928
929 private static final
930 ImplCache<Binary,VectorBroadcastIntOp<IntVector, VectorMask<Integer>>> BIN_INT_IMPL
931 = new ImplCache<>(Binary.class, IntVector.class);
932
933 private static VectorBroadcastIntOp<IntVector, VectorMask<Integer>> broadcastIntOperations(int opc_) {
934 switch (opc_) {
935 case VECTOR_OP_LSHIFT: return (v, n, m) ->
936 v.uOp(m, (i, a) -> (int)(a << n));
937 case VECTOR_OP_RSHIFT: return (v, n, m) ->
938 v.uOp(m, (i, a) -> (int)(a >> n));
939 case VECTOR_OP_URSHIFT: return (v, n, m) ->
940 v.uOp(m, (i, a) -> (int)((a & LSHR_SETUP_MASK) >>> n));
941 case VECTOR_OP_LROTATE: return (v, n, m) ->
942 v.uOp(m, (i, a) -> rotateLeft(a, (int)n));
943 case VECTOR_OP_RROTATE: return (v, n, m) ->
944 v.uOp(m, (i, a) -> rotateRight(a, (int)n));
945 default: return null;
946 }
947 }
948
949 // As per shift specification for Java, mask the shift count.
950 // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
951 // The latter two maskings go beyond the JLS, but seem reasonable
952 // since our lane types are first-class types, not just dressed
953 // up ints.
954 private static final int SHIFT_MASK = (Integer.SIZE - 1);
955 private static final int LSHR_SETUP_MASK = -1;
956
957 // Ternary lanewise support
958
959 // Ternary operators come in eight variations:
960 // lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
961 // lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
962
963 // It is annoying to support all of these variations of masking
964 // and broadcast, but it would be more surprising not to continue
965 // the obvious pattern started by unary and binary.
966
967 /**
968 * {@inheritDoc} <!--workaround-->
980 Vector<Integer> v2);
981 @ForceInline
982 final
983 IntVector lanewiseTemplate(VectorOperators.Ternary op,
984 Vector<Integer> v1,
985 Vector<Integer> v2) {
986 IntVector that = (IntVector) v1;
987 IntVector tother = (IntVector) v2;
988 // It's a word: https://www.dictionary.com/browse/tother
989 // See also Chapter 11 of Dickens, Our Mutual Friend:
990 // "Totherest Governor," replied Mr Riderhood...
991 that.check(this);
992 tother.check(this);
993 if (op == BITWISE_BLEND) {
994 // FIXME: Support this in the JIT.
995 that = this.lanewise(XOR, that).lanewise(AND, tother);
996 return this.lanewise(XOR, that);
997 }
998 int opc = opCode(op);
999 return VectorSupport.ternaryOp(
1000 opc, getClass(), null, int.class, length(),
1001 this, that, tother, null,
1002 TERN_IMPL.find(op, opc, IntVector::ternaryOperations));
1003 }
1004
1005 /**
1006 * {@inheritDoc} <!--workaround-->
1007 * @see #lanewise(VectorOperators.Ternary,int,int,VectorMask)
1008 * @see #lanewise(VectorOperators.Ternary,Vector,int,VectorMask)
1009 * @see #lanewise(VectorOperators.Ternary,int,Vector,VectorMask)
1010 */
1011 @Override
1012 public abstract
1013 IntVector lanewise(VectorOperators.Ternary op,
1014 Vector<Integer> v1,
1015 Vector<Integer> v2,
1016 VectorMask<Integer> m);
1017 @ForceInline
1018 final
1019 IntVector lanewiseTemplate(VectorOperators.Ternary op,
1020 Class<? extends VectorMask<Integer>> maskClass,
1021 Vector<Integer> v1,
1022 Vector<Integer> v2,
1023 VectorMask<Integer> m) {
1024 IntVector that = (IntVector) v1;
1025 IntVector tother = (IntVector) v2;
1026 // It's a word: https://www.dictionary.com/browse/tother
1027 // See also Chapter 11 of Dickens, Our Mutual Friend:
1028 // "Totherest Governor," replied Mr Riderhood...
1029 that.check(this);
1030 tother.check(this);
1031 m.check(maskClass, this);
1032
1033 if (op == BITWISE_BLEND) {
1034 // FIXME: Support this in the JIT.
1035 that = this.lanewise(XOR, that).lanewise(AND, tother);
1036 return this.lanewise(XOR, that, m);
1037 }
1038 int opc = opCode(op);
1039 return VectorSupport.ternaryOp(
1040 opc, getClass(), maskClass, int.class, length(),
1041 this, that, tother, m,
1042 TERN_IMPL.find(op, opc, IntVector::ternaryOperations));
1043 }
1044
1045 private static final
1046 ImplCache<Ternary, TernaryOperation<IntVector, VectorMask<Integer>>>
1047 TERN_IMPL = new ImplCache<>(Ternary.class, IntVector.class);
1048
1049 private static TernaryOperation<IntVector, VectorMask<Integer>> ternaryOperations(int opc_) {
1050 switch (opc_) {
1051 default: return null;
1052 }
1053 }
1054
1055 /**
1056 * Combines the lane values of this vector
1057 * with the values of two broadcast scalars.
1058 *
1059 * This is a lane-wise ternary operation which applies
1060 * the selected operation to each lane.
1061 * The return value will be equal to this expression:
1062 * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
1063 *
1064 * @param op the operation used to combine lane values
1065 * @param e1 the first input scalar
1066 * @param e2 the second input scalar
1067 * @return the result of applying the operation lane-wise
1068 * to the input vector and the scalars
1069 * @throws UnsupportedOperationException if this vector does
1070 * not support the requested operation
1071 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1072 * @see #lanewise(VectorOperators.Ternary,int,int,VectorMask)
1089 * The return value will be equal to this expression:
1090 * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
1091 *
1092 * @param op the operation used to combine lane values
1093 * @param e1 the first input scalar
1094 * @param e2 the second input scalar
1095 * @param m the mask controlling lane selection
1096 * @return the result of applying the operation lane-wise
1097 * to the input vector and the scalars
1098 * @throws UnsupportedOperationException if this vector does
1099 * not support the requested operation
1100 * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1101 * @see #lanewise(VectorOperators.Ternary,int,int)
1102 */
1103 @ForceInline
1104 public final
1105 IntVector lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
1106 int e1,
1107 int e2,
1108 VectorMask<Integer> m) {
1109 return lanewise(op, broadcast(e1), broadcast(e2), m);
1110 }
1111
1112 /**
1113 * Combines the lane values of this vector
1114 * with the values of another vector and a broadcast scalar.
1115 *
1116 * This is a lane-wise ternary operation which applies
1117 * the selected operation to each lane.
1118 * The return value will be equal to this expression:
1119 * {@code this.lanewise(op, v1, this.broadcast(e2))}.
1120 *
1121 * @param op the operation used to combine lane values
1122 * @param v1 the other input vector
1123 * @param e2 the input scalar
1124 * @return the result of applying the operation lane-wise
1125 * to the input vectors and the scalar
1126 * @throws UnsupportedOperationException if this vector does
1127 * not support the requested operation
1128 * @see #lanewise(VectorOperators.Ternary,int,int)
1129 * @see #lanewise(VectorOperators.Ternary,Vector,int,VectorMask)
1147 * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
1148 *
1149 * @param op the operation used to combine lane values
1150 * @param v1 the other input vector
1151 * @param e2 the input scalar
1152 * @param m the mask controlling lane selection
1153 * @return the result of applying the operation lane-wise
1154 * to the input vectors and the scalar
1155 * @throws UnsupportedOperationException if this vector does
1156 * not support the requested operation
1157 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1158 * @see #lanewise(VectorOperators.Ternary,int,int,VectorMask)
1159 * @see #lanewise(VectorOperators.Ternary,Vector,int)
1160 */
1161 @ForceInline
1162 public final
1163 IntVector lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
1164 Vector<Integer> v1,
1165 int e2,
1166 VectorMask<Integer> m) {
1167 return lanewise(op, v1, broadcast(e2), m);
1168 }
1169
1170 /**
1171 * Combines the lane values of this vector
1172 * with the values of another vector and a broadcast scalar.
1173 *
1174 * This is a lane-wise ternary operation which applies
1175 * the selected operation to each lane.
1176 * The return value will be equal to this expression:
1177 * {@code this.lanewise(op, this.broadcast(e1), v2)}.
1178 *
1179 * @param op the operation used to combine lane values
1180 * @param e1 the input scalar
1181 * @param v2 the other input vector
1182 * @return the result of applying the operation lane-wise
1183 * to the input vectors and the scalar
1184 * @throws UnsupportedOperationException if this vector does
1185 * not support the requested operation
1186 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1187 * @see #lanewise(VectorOperators.Ternary,int,Vector,VectorMask)
1204 * The return value will be equal to this expression:
1205 * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
1206 *
1207 * @param op the operation used to combine lane values
1208 * @param e1 the input scalar
1209 * @param v2 the other input vector
1210 * @param m the mask controlling lane selection
1211 * @return the result of applying the operation lane-wise
1212 * to the input vectors and the scalar
1213 * @throws UnsupportedOperationException if this vector does
1214 * not support the requested operation
1215 * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1216 * @see #lanewise(VectorOperators.Ternary,int,Vector)
1217 */
1218 @ForceInline
1219 public final
1220 IntVector lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
1221 int e1,
1222 Vector<Integer> v2,
1223 VectorMask<Integer> m) {
1224 return lanewise(op, broadcast(e1), v2, m);
1225 }
1226
1227 // (Thus endeth the Great and Mighty Ternary Ogdoad.)
1228 // https://en.wikipedia.org/wiki/Ogdoad
1229
1230 /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1231 //
1232 // These include masked and non-masked versions.
1233 // This subclass adds broadcast (masked or not).
1234
1235 /**
1236 * {@inheritDoc} <!--workaround-->
1237 * @see #add(int)
1238 */
1239 @Override
1240 @ForceInline
1241 public final IntVector add(Vector<Integer> v) {
1242 return lanewise(ADD, v);
1243 }
1244
1876 @Override
1877 @ForceInline
1878 public final
1879 VectorMask<Integer> test(VectorOperators.Test op,
1880 VectorMask<Integer> m) {
1881 return test(op).and(m);
1882 }
1883
1884 /**
1885 * {@inheritDoc} <!--workaround-->
1886 */
1887 @Override
1888 public abstract
1889 VectorMask<Integer> compare(VectorOperators.Comparison op, Vector<Integer> v);
1890
1891 /*package-private*/
1892 @ForceInline
1893 final
1894 <M extends VectorMask<Integer>>
1895 M compareTemplate(Class<M> maskType, Comparison op, Vector<Integer> v) {
1896 IntVector that = (IntVector) v;
1897 that.check(this);
1898 int opc = opCode(op);
1899 return VectorSupport.compare(
1900 opc, getClass(), maskType, int.class, length(),
1901 this, that, null,
1902 (cond, v0, v1, m1) -> {
1903 AbstractMask<Integer> m
1904 = v0.bTest(cond, v1, (cond_, i, a, b)
1905 -> compareWithOp(cond, a, b));
1906 @SuppressWarnings("unchecked")
1907 M m2 = (M) m;
1908 return m2;
1909 });
1910 }
1911
1912 /*package-private*/
1913 @ForceInline
1914 final
1915 <M extends VectorMask<Integer>>
1916 M compareTemplate(Class<M> maskType, Comparison op, Vector<Integer> v, M m) {
1917 IntVector that = (IntVector) v;
1918 that.check(this);
1919 m.check(maskType, this);
1920 int opc = opCode(op);
1921 return VectorSupport.compare(
1922 opc, getClass(), maskType, int.class, length(),
1923 this, that, m,
1924 (cond, v0, v1, m1) -> {
1925 AbstractMask<Integer> cmpM
1926 = v0.bTest(cond, v1, (cond_, i, a, b)
1927 -> compareWithOp(cond, a, b));
1928 @SuppressWarnings("unchecked")
1929 M m2 = (M) cmpM.and(m1);
1930 return m2;
1931 });
1932 }
1933
1934 @ForceInline
1935 private static boolean compareWithOp(int cond, int a, int b) {
1936 return switch (cond) {
1937 case BT_eq -> a == b;
1938 case BT_ne -> a != b;
1939 case BT_lt -> a < b;
1940 case BT_le -> a <= b;
1941 case BT_gt -> a > b;
1942 case BT_ge -> a >= b;
1943 case BT_ult -> Integer.compareUnsigned(a, b) < 0;
1944 case BT_ule -> Integer.compareUnsigned(a, b) <= 0;
1945 case BT_ugt -> Integer.compareUnsigned(a, b) > 0;
1946 case BT_uge -> Integer.compareUnsigned(a, b) >= 0;
1947 default -> throw new AssertionError();
1948 };
1949 }
1950
1951 /**
1952 * Tests this vector by comparing it with an input scalar,
1953 * according to the given comparison operation.
1954 *
1955 * This is a lane-wise binary test operation which applies
1956 * the comparison operation to each lane.
1957 * <p>
1958 * The result is the same as
1959 * {@code compare(op, broadcast(species(), e))}.
1960 * That is, the scalar may be regarded as broadcast to
1961 * a vector of the same species, and then compared
1962 * against the original vector, using the selected
1963 * comparison operation.
1964 *
1965 * @param op the operation used to compare lane values
1966 * @param e the input scalar
1967 * @return the mask result of testing lane-wise if this vector
1968 * compares to the input, according to the selected
1969 * comparison operator
1970 * @see IntVector#compare(VectorOperators.Comparison,Vector)
1989 *
1990 * This is a masked lane-wise binary test operation which applies
1991 * to each pair of corresponding lane values.
1992 *
1993 * The returned result is equal to the expression
1994 * {@code compare(op,s).and(m)}.
1995 *
1996 * @param op the operation used to compare lane values
1997 * @param e the input scalar
1998 * @param m the mask controlling lane selection
1999 * @return the mask result of testing lane-wise if this vector
2000 * compares to the input, according to the selected
2001 * comparison operator,
2002 * and only in the lanes selected by the mask
2003 * @see IntVector#compare(VectorOperators.Comparison,Vector,VectorMask)
2004 */
2005 @ForceInline
2006 public final VectorMask<Integer> compare(VectorOperators.Comparison op,
2007 int e,
2008 VectorMask<Integer> m) {
2009 return compare(op, broadcast(e), m);
2010 }
2011
2012 /**
2013 * {@inheritDoc} <!--workaround-->
2014 */
2015 @Override
2016 public abstract
2017 VectorMask<Integer> compare(Comparison op, long e);
2018
2019 /*package-private*/
2020 @ForceInline
2021 final
2022 <M extends VectorMask<Integer>>
2023 M compareTemplate(Class<M> maskType, Comparison op, long e) {
2024 return compareTemplate(maskType, op, broadcast(e));
2025 }
2026
2027 /**
2028 * {@inheritDoc} <!--workaround-->
2029 */
2240 wrongPartForSlice(int part) {
2241 String msg = String.format("bad part number %d for slice operation",
2242 part);
2243 return new ArrayIndexOutOfBoundsException(msg);
2244 }
2245
2246 /**
2247 * {@inheritDoc} <!--workaround-->
2248 */
2249 @Override
2250 public abstract
2251 IntVector rearrange(VectorShuffle<Integer> m);
2252
2253 /*package-private*/
2254 @ForceInline
2255 final
2256 <S extends VectorShuffle<Integer>>
2257 IntVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
2258 shuffle.checkIndexes();
2259 return VectorSupport.rearrangeOp(
2260 getClass(), shuffletype, null, int.class, length(),
2261 this, shuffle, null,
2262 (v1, s_, m_) -> v1.uOp((i, a) -> {
2263 int ei = s_.laneSource(i);
2264 return v1.lane(ei);
2265 }));
2266 }
2267
2268 /**
2269 * {@inheritDoc} <!--workaround-->
2270 */
2271 @Override
2272 public abstract
2273 IntVector rearrange(VectorShuffle<Integer> s,
2274 VectorMask<Integer> m);
2275
2276 /*package-private*/
2277 @ForceInline
2278 final
2279 <S extends VectorShuffle<Integer>, M extends VectorMask<Integer>>
2280 IntVector rearrangeTemplate(Class<S> shuffletype,
2281 Class<M> masktype,
2282 S shuffle,
2283 M m) {
2284
2285 m.check(masktype, this);
2286 VectorMask<Integer> valid = shuffle.laneIsValid();
2287 if (m.andNot(valid).anyTrue()) {
2288 shuffle.checkIndexes();
2289 throw new AssertionError();
2290 }
2291 return VectorSupport.rearrangeOp(
2292 getClass(), shuffletype, masktype, int.class, length(),
2293 this, shuffle, m,
2294 (v1, s_, m_) -> v1.uOp((i, a) -> {
2295 int ei = s_.laneSource(i);
2296 return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
2297 }));
2298 }
2299
2300 /**
2301 * {@inheritDoc} <!--workaround-->
2302 */
2303 @Override
2304 public abstract
2305 IntVector rearrange(VectorShuffle<Integer> s,
2306 Vector<Integer> v);
2307
2308 /*package-private*/
2309 @ForceInline
2310 final
2311 <S extends VectorShuffle<Integer>>
2312 IntVector rearrangeTemplate(Class<S> shuffletype,
2313 S shuffle,
2314 IntVector v) {
2315 VectorMask<Integer> valid = shuffle.laneIsValid();
2316 @SuppressWarnings("unchecked")
2317 S ws = (S) shuffle.wrapIndexes();
2318 IntVector r0 =
2319 VectorSupport.rearrangeOp(
2320 getClass(), shuffletype, null, int.class, length(),
2321 this, ws, null,
2322 (v0, s_, m_) -> v0.uOp((i, a) -> {
2323 int ei = s_.laneSource(i);
2324 return v0.lane(ei);
2325 }));
2326 IntVector r1 =
2327 VectorSupport.rearrangeOp(
2328 getClass(), shuffletype, null, int.class, length(),
2329 v, ws, null,
2330 (v1, s_, m_) -> v1.uOp((i, a) -> {
2331 int ei = s_.laneSource(i);
2332 return v1.lane(ei);
2333 }));
2334 return r1.blend(r0, valid);
2335 }
2336
2337 @ForceInline
2338 private final
2339 VectorShuffle<Integer> toShuffle0(IntSpecies dsp) {
2340 int[] a = toArray();
2341 int[] sa = new int[a.length];
2342 for (int i = 0; i < a.length; i++) {
2343 sa[i] = (int) a[i];
2344 }
2345 return VectorShuffle.fromArray(dsp, sa, 0);
2346 }
2347
2348 /*package-private*/
2349 @ForceInline
2350 final
2573 * <li>
2574 * All other reduction operations are fully commutative and
2575 * associative. The implementation can choose any order of
2576 * processing, yet it will always produce the same result.
2577 * </ul>
2578 *
2579 * @param op the operation used to combine lane values
2580 * @param m the mask controlling lane selection
2581 * @return the reduced result accumulated from the selected lane values
2582 * @throws UnsupportedOperationException if this vector does
2583 * not support the requested operation
2584 * @see #reduceLanes(VectorOperators.Associative)
2585 */
2586 public abstract int reduceLanes(VectorOperators.Associative op,
2587 VectorMask<Integer> m);
2588
2589 /*package-private*/
2590 @ForceInline
2591 final
2592 int reduceLanesTemplate(VectorOperators.Associative op,
2593 Class<? extends VectorMask<Integer>> maskClass,
2594 VectorMask<Integer> m) {
2595 m.check(maskClass, this);
2596 if (op == FIRST_NONZERO) {
2597 IntVector v = reduceIdentityVector(op).blend(this, m);
2598 return v.reduceLanesTemplate(op);
2599 }
2600 int opc = opCode(op);
2601 return fromBits(VectorSupport.reductionCoerced(
2602 opc, getClass(), maskClass, int.class, length(),
2603 this, m,
2604 REDUCE_IMPL.find(op, opc, IntVector::reductionOperations)));
2605 }
2606
2607 /*package-private*/
2608 @ForceInline
2609 final
2610 int reduceLanesTemplate(VectorOperators.Associative op) {
2611 if (op == FIRST_NONZERO) {
2612 // FIXME: The JIT should handle this, and other scan ops alos.
2613 VectorMask<Integer> thisNZ
2614 = this.viewAsIntegralLanes().compare(NE, (int) 0);
2615 return this.lane(thisNZ.firstTrue());
2616 }
2617 int opc = opCode(op);
2618 return fromBits(VectorSupport.reductionCoerced(
2619 opc, getClass(), null, int.class, length(),
2620 this, null,
2621 REDUCE_IMPL.find(op, opc, IntVector::reductionOperations)));
2622 }
2623
2624 private static final
2625 ImplCache<Associative, ReductionOperation<IntVector, VectorMask<Integer>>>
2626 REDUCE_IMPL = new ImplCache<>(Associative.class, IntVector.class);
2627
2628 private static ReductionOperation<IntVector, VectorMask<Integer>> reductionOperations(int opc_) {
2629 switch (opc_) {
2630 case VECTOR_OP_ADD: return (v, m) ->
2631 toBits(v.rOp((int)0, m, (i, a, b) -> (int)(a + b)));
2632 case VECTOR_OP_MUL: return (v, m) ->
2633 toBits(v.rOp((int)1, m, (i, a, b) -> (int)(a * b)));
2634 case VECTOR_OP_MIN: return (v, m) ->
2635 toBits(v.rOp(MAX_OR_INF, m, (i, a, b) -> (int) Math.min(a, b)));
2636 case VECTOR_OP_MAX: return (v, m) ->
2637 toBits(v.rOp(MIN_OR_INF, m, (i, a, b) -> (int) Math.max(a, b)));
2638 case VECTOR_OP_AND: return (v, m) ->
2639 toBits(v.rOp((int)-1, m, (i, a, b) -> (int)(a & b)));
2640 case VECTOR_OP_OR: return (v, m) ->
2641 toBits(v.rOp((int)0, m, (i, a, b) -> (int)(a | b)));
2642 case VECTOR_OP_XOR: return (v, m) ->
2643 toBits(v.rOp((int)0, m, (i, a, b) -> (int)(a ^ b)));
2644 default: return null;
2645 }
2646 }
2647
2648 private
2649 @ForceInline
2650 IntVector reduceIdentityVector(VectorOperators.Associative op) {
2651 int opc = opCode(op);
2652 UnaryOperator<IntVector> fn
2653 = REDUCE_ID_IMPL.find(op, opc, (opc_) -> {
2654 switch (opc_) {
2655 case VECTOR_OP_ADD:
2656 case VECTOR_OP_OR:
2657 case VECTOR_OP_XOR:
2658 return v -> v.broadcast(0);
2659 case VECTOR_OP_MUL:
2660 return v -> v.broadcast(1);
2661 case VECTOR_OP_AND:
2662 return v -> v.broadcast(-1);
2663 case VECTOR_OP_MIN:
2664 return v -> v.broadcast(MAX_OR_INF);
2665 case VECTOR_OP_MAX:
2666 return v -> v.broadcast(MIN_OR_INF);
2845 * @param species species of desired vector
2846 * @param a the byte array
2847 * @param offset the offset into the array
2848 * @param bo the intended byte order
2849 * @param m the mask controlling lane selection
2850 * @return a vector loaded from a byte array
2851 * @throws IndexOutOfBoundsException
2852 * if {@code offset+N*ESIZE < 0}
2853 * or {@code offset+(N+1)*ESIZE > a.length}
2854 * for any lane {@code N} in the vector
2855 * where the mask is set
2856 */
2857 @ForceInline
2858 public static
2859 IntVector fromByteArray(VectorSpecies<Integer> species,
2860 byte[] a, int offset,
2861 ByteOrder bo,
2862 VectorMask<Integer> m) {
2863 IntSpecies vsp = (IntSpecies) species;
2864 if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
2865 return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
2866 }
2867
2868 // FIXME: optimize
2869 checkMaskFromIndexSize(offset, vsp, m, 4, a.length);
2870 ByteBuffer wb = wrapper(a, bo);
2871 return vsp.ldOp(wb, offset, (AbstractMask<Integer>)m,
2872 (wb_, o, i) -> wb_.getInt(o + i * 4));
2873 }
2874
2875 /**
2876 * Loads a vector from an array of type {@code int[]}
2877 * starting at an offset.
2878 * For each vector lane, where {@code N} is the vector lane index, the
2879 * array element at index {@code offset + N} is placed into the
2880 * resulting vector at lane index {@code N}.
2881 *
2882 * @param species species of desired vector
2883 * @param a the array
2884 * @param offset the offset into the array
2885 * @return the vector loaded from an array
2907 * {@code N}, otherwise the default element value is placed into the
2908 * resulting vector at lane index {@code N}.
2909 *
2910 * @param species species of desired vector
2911 * @param a the array
2912 * @param offset the offset into the array
2913 * @param m the mask controlling lane selection
2914 * @return the vector loaded from an array
2915 * @throws IndexOutOfBoundsException
2916 * if {@code offset+N < 0} or {@code offset+N >= a.length}
2917 * for any lane {@code N} in the vector
2918 * where the mask is set
2919 */
2920 @ForceInline
2921 public static
2922 IntVector fromArray(VectorSpecies<Integer> species,
2923 int[] a, int offset,
2924 VectorMask<Integer> m) {
2925 IntSpecies vsp = (IntSpecies) species;
2926 if (offset >= 0 && offset <= (a.length - species.length())) {
2927 return vsp.dummyVector().fromArray0(a, offset, m);
2928 }
2929
2930 // FIXME: optimize
2931 checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
2932 return vsp.vOp(m, i -> a[offset + i]);
2933 }
2934
2935 /**
2936 * Gathers a new vector composed of elements from an array of type
2937 * {@code int[]},
2938 * using indexes obtained by adding a fixed {@code offset} to a
2939 * series of secondary offsets from an <em>index map</em>.
2940 * The index map is a contiguous sequence of {@code VLENGTH}
2941 * elements in a second array of {@code int}s, starting at a given
2942 * {@code mapOffset}.
2943 * <p>
2944 * For each vector lane, where {@code N} is the vector lane index,
2945 * the lane is loaded from the array
2946 * element {@code a[f(N)]}, where {@code f(N)} is the
2947 * index mapping expression
2965 */
2966 @ForceInline
2967 public static
2968 IntVector fromArray(VectorSpecies<Integer> species,
2969 int[] a, int offset,
2970 int[] indexMap, int mapOffset) {
2971 IntSpecies vsp = (IntSpecies) species;
2972 IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
2973 Objects.requireNonNull(a);
2974 Objects.requireNonNull(indexMap);
2975 Class<? extends IntVector> vectorType = vsp.vectorType();
2976
2977 // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
2978 IntVector vix = IntVector
2979 .fromArray(isp, indexMap, mapOffset)
2980 .add(offset);
2981
2982 vix = VectorIntrinsics.checkIndex(vix, a.length);
2983
2984 return VectorSupport.loadWithMap(
2985 vectorType, null, int.class, vsp.laneCount(),
2986 isp.vectorType(),
2987 a, ARRAY_BASE, vix, null,
2988 a, offset, indexMap, mapOffset, vsp,
2989 (c, idx, iMap, idy, s, vm) ->
2990 s.vOp(n -> c[idx + iMap[idy+n]]));
2991 }
2992
2993 /**
2994 * Gathers a new vector composed of elements from an array of type
2995 * {@code int[]},
2996 * under the control of a mask, and
2997 * using indexes obtained by adding a fixed {@code offset} to a
2998 * series of secondary offsets from an <em>index map</em>.
2999 * The index map is a contiguous sequence of {@code VLENGTH}
3000 * elements in a second array of {@code int}s, starting at a given
3001 * {@code mapOffset}.
3002 * <p>
3003 * For each vector lane, where {@code N} is the vector lane index,
3004 * if the lane is set in the mask,
3005 * the lane is loaded from the array
3006 * element {@code a[f(N)]}, where {@code f(N)} is the
3007 * index mapping expression
3008 * {@code offset + indexMap[mapOffset + N]]}.
3009 * Unset lanes in the resulting vector are set to zero.
3010 *
3011 * @param species species of desired vector
3019 * @return the vector loaded from the indexed elements of the array
3020 * @throws IndexOutOfBoundsException
3021 * if {@code mapOffset+N < 0}
3022 * or if {@code mapOffset+N >= indexMap.length},
3023 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
3024 * is an invalid index into {@code a},
3025 * for any lane {@code N} in the vector
3026 * where the mask is set
3027 * @see IntVector#toIntArray()
3028 */
3029 @ForceInline
3030 public static
3031 IntVector fromArray(VectorSpecies<Integer> species,
3032 int[] a, int offset,
3033 int[] indexMap, int mapOffset,
3034 VectorMask<Integer> m) {
3035 if (m.allTrue()) {
3036 return fromArray(species, a, offset, indexMap, mapOffset);
3037 }
3038 else {
3039 IntSpecies vsp = (IntSpecies) species;
3040 return vsp.dummyVector().fromArray0(a, offset, indexMap, mapOffset, m);
3041 }
3042 }
3043
3044
3045
3046 /**
3047 * Loads a vector from a {@linkplain ByteBuffer byte buffer}
3048 * starting at an offset into the byte buffer.
3049 * Bytes are composed into primitive lane elements according
3050 * to the specified byte order.
3051 * The vector is arranged into lanes according to
3052 * <a href="Vector.html#lane-order">memory ordering</a>.
3053 * <p>
3054 * This method behaves as if it returns the result of calling
3055 * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
3056 * fromByteBuffer()} as follows:
3057 * <pre>{@code
3058 * var m = species.maskAll(true);
3059 * return fromByteBuffer(species, bb, offset, bo, m);
3060 * }</pre>
3114 * @param species species of desired vector
3115 * @param bb the byte buffer
3116 * @param offset the offset into the byte buffer
3117 * @param bo the intended byte order
3118 * @param m the mask controlling lane selection
3119 * @return a vector loaded from a byte buffer
3120 * @throws IndexOutOfBoundsException
3121 * if {@code offset+N*4 < 0}
3122 * or {@code offset+N*4 >= bb.limit()}
3123 * for any lane {@code N} in the vector
3124 * where the mask is set
3125 */
3126 @ForceInline
3127 public static
3128 IntVector fromByteBuffer(VectorSpecies<Integer> species,
3129 ByteBuffer bb, int offset,
3130 ByteOrder bo,
3131 VectorMask<Integer> m) {
3132 IntSpecies vsp = (IntSpecies) species;
3133 if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
3134 return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
3135 }
3136
3137 // FIXME: optimize
3138 checkMaskFromIndexSize(offset, vsp, m, 4, bb.limit());
3139 ByteBuffer wb = wrapper(bb, bo);
3140 return vsp.ldOp(wb, offset, (AbstractMask<Integer>)m,
3141 (wb_, o, i) -> wb_.getInt(o + i * 4));
3142 }
3143
3144 // Memory store operations
3145
3146 /**
3147 * Stores this vector into an array of type {@code int[]}
3148 * starting at an offset.
3149 * <p>
3150 * For each vector lane, where {@code N} is the vector lane index,
3151 * the lane element at index {@code N} is stored into the array
3152 * element {@code a[offset+N]}.
3153 *
3154 * @param a the array, of type {@code int[]}
3186 * Lanes where the mask is unset are not stored and do not need
3187 * to correspond to legitimate elements of {@code a}.
3188 * That is, unset lanes may correspond to array indexes less than
3189 * zero or beyond the end of the array.
3190 *
3191 * @param a the array, of type {@code int[]}
3192 * @param offset the offset into the array
3193 * @param m the mask controlling lane storage
3194 * @throws IndexOutOfBoundsException
3195 * if {@code offset+N < 0} or {@code offset+N >= a.length}
3196 * for any lane {@code N} in the vector
3197 * where the mask is set
3198 */
3199 @ForceInline
3200 public final
3201 void intoArray(int[] a, int offset,
3202 VectorMask<Integer> m) {
3203 if (m.allTrue()) {
3204 intoArray(a, offset);
3205 } else {
3206 IntSpecies vsp = vspecies();
3207 checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3208 intoArray0(a, offset, m);
3209 }
3210 }
3211
3212 /**
3213 * Scatters this vector into an array of type {@code int[]}
3214 * using indexes obtained by adding a fixed {@code offset} to a
3215 * series of secondary offsets from an <em>index map</em>.
3216 * The index map is a contiguous sequence of {@code VLENGTH}
3217 * elements in a second array of {@code int}s, starting at a given
3218 * {@code mapOffset}.
3219 * <p>
3220 * For each vector lane, where {@code N} is the vector lane index,
3221 * the lane element at index {@code N} is stored into the array
3222 * element {@code a[f(N)]}, where {@code f(N)} is the
3223 * index mapping expression
3224 * {@code offset + indexMap[mapOffset + N]]}.
3225 *
3226 * @param a the array
3227 * @param offset an offset to combine with the index map offsets
3228 * @param indexMap the index map
3232 * or if {@code mapOffset+N >= indexMap.length},
3233 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
3234 * is an invalid index into {@code a},
3235 * for any lane {@code N} in the vector
3236 * @see IntVector#toIntArray()
3237 */
3238 @ForceInline
3239 public final
3240 void intoArray(int[] a, int offset,
3241 int[] indexMap, int mapOffset) {
3242 IntSpecies vsp = vspecies();
3243 IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3244 // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
3245 IntVector vix = IntVector
3246 .fromArray(isp, indexMap, mapOffset)
3247 .add(offset);
3248
3249 vix = VectorIntrinsics.checkIndex(vix, a.length);
3250
3251 VectorSupport.storeWithMap(
3252 vsp.vectorType(), null, vsp.elementType(), vsp.laneCount(),
3253 isp.vectorType(),
3254 a, arrayAddress(a, 0), vix,
3255 this, null,
3256 a, offset, indexMap, mapOffset,
3257 (arr, off, v, map, mo, vm)
3258 -> v.stOp(arr, off,
3259 (arr_, off_, i, e) -> {
3260 int j = map[mo + i];
3261 arr[off + j] = e;
3262 }));
3263 }
3264
3265 /**
3266 * Scatters this vector into an array of type {@code int[]},
3267 * under the control of a mask, and
3268 * using indexes obtained by adding a fixed {@code offset} to a
3269 * series of secondary offsets from an <em>index map</em>.
3270 * The index map is a contiguous sequence of {@code VLENGTH}
3271 * elements in a second array of {@code int}s, starting at a given
3272 * {@code mapOffset}.
3273 * <p>
3274 * For each vector lane, where {@code N} is the vector lane index,
3275 * if the mask lane at index {@code N} is set then
3276 * the lane element at index {@code N} is stored into the array
3277 * element {@code a[f(N)]}, where {@code f(N)} is the
3284 * @param mapOffset the offset into the index map
3285 * @param m the mask
3286 * @throws IndexOutOfBoundsException
3287 * if {@code mapOffset+N < 0}
3288 * or if {@code mapOffset+N >= indexMap.length},
3289 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
3290 * is an invalid index into {@code a},
3291 * for any lane {@code N} in the vector
3292 * where the mask is set
3293 * @see IntVector#toIntArray()
3294 */
3295 @ForceInline
3296 public final
3297 void intoArray(int[] a, int offset,
3298 int[] indexMap, int mapOffset,
3299 VectorMask<Integer> m) {
3300 if (m.allTrue()) {
3301 intoArray(a, offset, indexMap, mapOffset);
3302 }
3303 else {
3304 intoArray0(a, offset, indexMap, mapOffset, m);
3305 }
3306 }
3307
3308
3309
3310 /**
3311 * {@inheritDoc} <!--workaround-->
3312 */
3313 @Override
3314 @ForceInline
3315 public final
3316 void intoByteArray(byte[] a, int offset,
3317 ByteOrder bo) {
3318 offset = checkFromIndexSize(offset, byteSize(), a.length);
3319 maybeSwap(bo).intoByteArray0(a, offset);
3320 }
3321
3322 /**
3323 * {@inheritDoc} <!--workaround-->
3324 */
3325 @Override
3326 @ForceInline
3327 public final
3328 void intoByteArray(byte[] a, int offset,
3329 ByteOrder bo,
3330 VectorMask<Integer> m) {
3331 if (m.allTrue()) {
3332 intoByteArray(a, offset, bo);
3333 } else {
3334 IntSpecies vsp = vspecies();
3335 checkMaskFromIndexSize(offset, vsp, m, 4, a.length);
3336 maybeSwap(bo).intoByteArray0(a, offset, m);
3337 }
3338 }
3339
3340 /**
3341 * {@inheritDoc} <!--workaround-->
3342 */
3343 @Override
3344 @ForceInline
3345 public final
3346 void intoByteBuffer(ByteBuffer bb, int offset,
3347 ByteOrder bo) {
3348 if (ScopedMemoryAccess.isReadOnly(bb)) {
3349 throw new ReadOnlyBufferException();
3350 }
3351 offset = checkFromIndexSize(offset, byteSize(), bb.limit());
3352 maybeSwap(bo).intoByteBuffer0(bb, offset);
3353 }
3354
3355 /**
3356 * {@inheritDoc} <!--workaround-->
3357 */
3358 @Override
3359 @ForceInline
3360 public final
3361 void intoByteBuffer(ByteBuffer bb, int offset,
3362 ByteOrder bo,
3363 VectorMask<Integer> m) {
3364 if (m.allTrue()) {
3365 intoByteBuffer(bb, offset, bo);
3366 } else {
3367 if (bb.isReadOnly()) {
3368 throw new ReadOnlyBufferException();
3369 }
3370 IntSpecies vsp = vspecies();
3371 checkMaskFromIndexSize(offset, vsp, m, 4, bb.limit());
3372 maybeSwap(bo).intoByteBuffer0(bb, offset, m);
3373 }
3374 }
3375
3376 // ================================================
3377
3378 // Low-level memory operations.
3379 //
3380 // Note that all of these operations *must* inline into a context
3381 // where the exact species of the involved vector is a
3382 // compile-time constant. Otherwise, the intrinsic generation
3383 // will fail and performance will suffer.
3384 //
3385 // In many cases this is achieved by re-deriving a version of the
3386 // method in each concrete subclass (per species). The re-derived
3387 // method simply calls one of these generic methods, with exact
3388 // parameters for the controlling metadata, which is either a
3389 // typed vector or constant species instance.
3390
3391 // Unchecked loading operations in native byte order.
3392 // Caller is responsible for applying index checks, masking, and
3393 // byte swapping.
3394
3395 /*package-private*/
3396 abstract
3397 IntVector fromArray0(int[] a, int offset);
3398 @ForceInline
3399 final
3400 IntVector fromArray0Template(int[] a, int offset) {
3401 IntSpecies vsp = vspecies();
3402 return VectorSupport.load(
3403 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3404 a, arrayAddress(a, offset),
3405 a, offset, vsp,
3406 (arr, off, s) -> s.ldOp(arr, off,
3407 (arr_, off_, i) -> arr_[off_ + i]));
3408 }
3409
3410 /*package-private*/
3411 abstract
3412 IntVector fromArray0(int[] a, int offset, VectorMask<Integer> m);
3413 @ForceInline
3414 final
3415 <M extends VectorMask<Integer>>
3416 IntVector fromArray0Template(Class<M> maskClass, int[] a, int offset, M m) {
3417 m.check(species());
3418 IntSpecies vsp = vspecies();
3419 return VectorSupport.loadMasked(
3420 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3421 a, arrayAddress(a, offset), m,
3422 a, offset, vsp,
3423 (arr, off, s, vm) -> s.ldOp(arr, off, vm,
3424 (arr_, off_, i) -> arr_[off_ + i]));
3425 }
3426
3427 /*package-private*/
3428 abstract
3429 IntVector fromArray0(int[] a, int offset,
3430 int[] indexMap, int mapOffset,
3431 VectorMask<Integer> m);
3432 @ForceInline
3433 final
3434 <M extends VectorMask<Integer>>
3435 IntVector fromArray0Template(Class<M> maskClass, int[] a, int offset,
3436 int[] indexMap, int mapOffset, M m) {
3437 IntSpecies vsp = vspecies();
3438 IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3439 Objects.requireNonNull(a);
3440 Objects.requireNonNull(indexMap);
3441 m.check(vsp);
3442 Class<? extends IntVector> vectorType = vsp.vectorType();
3443
3444 // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3445 IntVector vix = IntVector
3446 .fromArray(isp, indexMap, mapOffset)
3447 .add(offset);
3448
3449 // FIXME: Check index under mask controlling.
3450 vix = VectorIntrinsics.checkIndex(vix, a.length);
3451
3452 return VectorSupport.loadWithMap(
3453 vectorType, maskClass, int.class, vsp.laneCount(),
3454 isp.vectorType(),
3455 a, ARRAY_BASE, vix, m,
3456 a, offset, indexMap, mapOffset, vsp,
3457 (c, idx, iMap, idy, s, vm) ->
3458 s.vOp(vm, n -> c[idx + iMap[idy+n]]));
3459 }
3460
3461
3462
3463 @Override
3464 abstract
3465 IntVector fromByteArray0(byte[] a, int offset);
3466 @ForceInline
3467 final
3468 IntVector fromByteArray0Template(byte[] a, int offset) {
3469 IntSpecies vsp = vspecies();
3470 return VectorSupport.load(
3471 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3472 a, byteArrayAddress(a, offset),
3473 a, offset, vsp,
3474 (arr, off, s) -> {
3475 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3476 return s.ldOp(wb, off,
3477 (wb_, o, i) -> wb_.getInt(o + i * 4));
3478 });
3479 }
3480
3481 abstract
3482 IntVector fromByteArray0(byte[] a, int offset, VectorMask<Integer> m);
3483 @ForceInline
3484 final
3485 <M extends VectorMask<Integer>>
3486 IntVector fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
3487 IntSpecies vsp = vspecies();
3488 m.check(vsp);
3489 return VectorSupport.loadMasked(
3490 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3491 a, byteArrayAddress(a, offset), m,
3492 a, offset, vsp,
3493 (arr, off, s, vm) -> {
3494 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3495 return s.ldOp(wb, off, vm,
3496 (wb_, o, i) -> wb_.getInt(o + i * 4));
3497 });
3498 }
3499
3500 abstract
3501 IntVector fromByteBuffer0(ByteBuffer bb, int offset);
3502 @ForceInline
3503 final
3504 IntVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
3505 IntSpecies vsp = vspecies();
3506 return ScopedMemoryAccess.loadFromByteBuffer(
3507 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3508 bb, offset, vsp,
3509 (buf, off, s) -> {
3510 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3511 return s.ldOp(wb, off,
3512 (wb_, o, i) -> wb_.getInt(o + i * 4));
3513 });
3514 }
3515
3516 abstract
3517 IntVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m);
3518 @ForceInline
3519 final
3520 <M extends VectorMask<Integer>>
3521 IntVector fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
3522 IntSpecies vsp = vspecies();
3523 m.check(vsp);
3524 return ScopedMemoryAccess.loadFromByteBufferMasked(
3525 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3526 bb, offset, m, vsp,
3527 (buf, off, s, vm) -> {
3528 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3529 return s.ldOp(wb, off, vm,
3530 (wb_, o, i) -> wb_.getInt(o + i * 4));
3531 });
3532 }
3533
3534 // Unchecked storing operations in native byte order.
3535 // Caller is responsible for applying index checks, masking, and
3536 // byte swapping.
3537
3538 abstract
3539 void intoArray0(int[] a, int offset);
3540 @ForceInline
3541 final
3542 void intoArray0Template(int[] a, int offset) {
3543 IntSpecies vsp = vspecies();
3544 VectorSupport.store(
3545 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3546 a, arrayAddress(a, offset),
3547 this, a, offset,
3548 (arr, off, v)
3549 -> v.stOp(arr, off,
3550 (arr_, off_, i, e) -> arr_[off_+i] = e));
3551 }
3552
3553 abstract
3554 void intoArray0(int[] a, int offset, VectorMask<Integer> m);
3555 @ForceInline
3556 final
3557 <M extends VectorMask<Integer>>
3558 void intoArray0Template(Class<M> maskClass, int[] a, int offset, M m) {
3559 m.check(species());
3560 IntSpecies vsp = vspecies();
3561 VectorSupport.storeMasked(
3562 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3563 a, arrayAddress(a, offset),
3564 this, m, a, offset,
3565 (arr, off, v, vm)
3566 -> v.stOp(arr, off, vm,
3567 (arr_, off_, i, e) -> arr_[off_ + i] = e));
3568 }
3569
3570 abstract
3571 void intoArray0(int[] a, int offset,
3572 int[] indexMap, int mapOffset,
3573 VectorMask<Integer> m);
3574 @ForceInline
3575 final
3576 <M extends VectorMask<Integer>>
3577 void intoArray0Template(Class<M> maskClass, int[] a, int offset,
3578 int[] indexMap, int mapOffset, M m) {
3579 m.check(species());
3580 IntSpecies vsp = vspecies();
3581 IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3582 // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
3583 IntVector vix = IntVector
3584 .fromArray(isp, indexMap, mapOffset)
3585 .add(offset);
3586
3587 // FIXME: Check index under mask controlling.
3588 vix = VectorIntrinsics.checkIndex(vix, a.length);
3589
3590 VectorSupport.storeWithMap(
3591 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3592 isp.vectorType(),
3593 a, arrayAddress(a, 0), vix,
3594 this, m,
3595 a, offset, indexMap, mapOffset,
3596 (arr, off, v, map, mo, vm)
3597 -> v.stOp(arr, off, vm,
3598 (arr_, off_, i, e) -> {
3599 int j = map[mo + i];
3600 arr[off + j] = e;
3601 }));
3602 }
3603
3604
3605 abstract
3606 void intoByteArray0(byte[] a, int offset);
3607 @ForceInline
3608 final
3609 void intoByteArray0Template(byte[] a, int offset) {
3610 IntSpecies vsp = vspecies();
3611 VectorSupport.store(
3612 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3613 a, byteArrayAddress(a, offset),
3614 this, a, offset,
3615 (arr, off, v) -> {
3616 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3617 v.stOp(wb, off,
3618 (tb_, o, i, e) -> tb_.putInt(o + i * 4, e));
3619 });
3620 }
3621
3622 abstract
3623 void intoByteArray0(byte[] a, int offset, VectorMask<Integer> m);
3624 @ForceInline
3625 final
3626 <M extends VectorMask<Integer>>
3627 void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
3628 IntSpecies vsp = vspecies();
3629 m.check(vsp);
3630 VectorSupport.storeMasked(
3631 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3632 a, byteArrayAddress(a, offset),
3633 this, m, a, offset,
3634 (arr, off, v, vm) -> {
3635 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3636 v.stOp(wb, off, vm,
3637 (tb_, o, i, e) -> tb_.putInt(o + i * 4, e));
3638 });
3639 }
3640
3641 @ForceInline
3642 final
3643 void intoByteBuffer0(ByteBuffer bb, int offset) {
3644 IntSpecies vsp = vspecies();
3645 ScopedMemoryAccess.storeIntoByteBuffer(
3646 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3647 this, bb, offset,
3648 (buf, off, v) -> {
3649 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3650 v.stOp(wb, off,
3651 (wb_, o, i, e) -> wb_.putInt(o + i * 4, e));
3652 });
3653 }
3654
3655 abstract
3656 void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Integer> m);
3657 @ForceInline
3658 final
3659 <M extends VectorMask<Integer>>
3660 void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
3661 IntSpecies vsp = vspecies();
3662 m.check(vsp);
3663 ScopedMemoryAccess.storeIntoByteBufferMasked(
3664 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3665 this, m, bb, offset,
3666 (buf, off, v, vm) -> {
3667 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3668 v.stOp(wb, off, vm,
3669 (wb_, o, i, e) -> wb_.putInt(o + i * 4, e));
3670 });
3671 }
3672
3673
3674 // End of low-level memory operations.
3675
3676 private static
3677 void checkMaskFromIndexSize(int offset,
3678 IntSpecies vsp,
3679 VectorMask<Integer> m,
3680 int scale,
3681 int limit) {
3682 ((AbstractMask<Integer>)m)
3683 .checkIndexByLane(offset, limit, vsp.iota(), scale);
3684 }
3685
3686 @ForceInline
3687 private void conditionalStoreNYI(int offset,
3688 IntSpecies vsp,
3689 VectorMask<Integer> m,
3690 int scale,
3691 int limit) {
3692 if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
3693 String msg =
3971 int[] res = new int[laneCount()];
3972 boolean[] mbits = ((AbstractMask<Integer>)m).getBits();
3973 for (int i = 0; i < res.length; i++) {
3974 if (mbits[i]) {
3975 res[i] = f.apply(i);
3976 }
3977 }
3978 return dummyVector().vectorFactory(res);
3979 }
3980
3981 /*package-private*/
3982 @ForceInline
3983 <M> IntVector ldOp(M memory, int offset,
3984 FLdOp<M> f) {
3985 return dummyVector().ldOp(memory, offset, f);
3986 }
3987
3988 /*package-private*/
3989 @ForceInline
3990 <M> IntVector ldOp(M memory, int offset,
3991 VectorMask<Integer> m,
3992 FLdOp<M> f) {
3993 return dummyVector().ldOp(memory, offset, m, f);
3994 }
3995
3996 /*package-private*/
3997 @ForceInline
3998 <M> void stOp(M memory, int offset, FStOp<M> f) {
3999 dummyVector().stOp(memory, offset, f);
4000 }
4001
4002 /*package-private*/
4003 @ForceInline
4004 <M> void stOp(M memory, int offset,
4005 AbstractMask<Integer> m,
4006 FStOp<M> f) {
4007 dummyVector().stOp(memory, offset, m, f);
4008 }
4009
4010 // N.B. Make sure these constant vectors and
4011 // masks load up correctly into registers.
|