12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25 package jdk.incubator.vector;
26
27 import java.nio.ByteBuffer;
28 import java.nio.ByteOrder;
29 import java.nio.ReadOnlyBufferException;
30 import java.util.Arrays;
31 import java.util.Objects;
32 import java.util.function.BinaryOperator;
33 import java.util.function.Function;
34 import java.util.function.UnaryOperator;
35
36 import jdk.internal.misc.ScopedMemoryAccess;
37 import jdk.internal.misc.Unsafe;
38 import jdk.internal.vm.annotation.ForceInline;
39 import jdk.internal.vm.vector.VectorSupport;
40
41 import static jdk.internal.vm.vector.VectorSupport.*;
42 import static jdk.incubator.vector.VectorIntrinsics.*;
43
44 import static jdk.incubator.vector.VectorOperators.*;
45
46 // -- This file was mechanically generated: Do not edit! -- //
47
48 /**
49 * A specialized {@link Vector} representing an ordered immutable sequence of
50 * {@code long} values.
51 */
52 @SuppressWarnings("cast") // warning: redundant cast
156 LongVector uOp(FUnOp f);
157 @ForceInline
158 final
159 LongVector uOpTemplate(FUnOp f) {
160 long[] vec = vec();
161 long[] res = new long[length()];
162 for (int i = 0; i < res.length; i++) {
163 res[i] = f.apply(i, vec[i]);
164 }
165 return vectorFactory(res);
166 }
167
168 /*package-private*/
169 abstract
170 LongVector uOp(VectorMask<Long> m,
171 FUnOp f);
172 @ForceInline
173 final
174 LongVector uOpTemplate(VectorMask<Long> m,
175 FUnOp f) {
176 long[] vec = vec();
177 long[] res = new long[length()];
178 boolean[] mbits = ((AbstractMask<Long>)m).getBits();
179 for (int i = 0; i < res.length; i++) {
180 res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
181 }
182 return vectorFactory(res);
183 }
184
185 // Binary operator
186
187 /*package-private*/
188 interface FBinOp {
189 long apply(int i, long a, long b);
190 }
191
192 /*package-private*/
193 abstract
194 LongVector bOp(Vector<Long> o,
195 FBinOp f);
199 FBinOp f) {
200 long[] res = new long[length()];
201 long[] vec1 = this.vec();
202 long[] vec2 = ((LongVector)o).vec();
203 for (int i = 0; i < res.length; i++) {
204 res[i] = f.apply(i, vec1[i], vec2[i]);
205 }
206 return vectorFactory(res);
207 }
208
209 /*package-private*/
210 abstract
211 LongVector bOp(Vector<Long> o,
212 VectorMask<Long> m,
213 FBinOp f);
214 @ForceInline
215 final
216 LongVector bOpTemplate(Vector<Long> o,
217 VectorMask<Long> m,
218 FBinOp f) {
219 long[] res = new long[length()];
220 long[] vec1 = this.vec();
221 long[] vec2 = ((LongVector)o).vec();
222 boolean[] mbits = ((AbstractMask<Long>)m).getBits();
223 for (int i = 0; i < res.length; i++) {
224 res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
225 }
226 return vectorFactory(res);
227 }
228
229 // Ternary operator
230
231 /*package-private*/
232 interface FTriOp {
233 long apply(int i, long a, long b, long c);
234 }
235
236 /*package-private*/
237 abstract
238 LongVector tOp(Vector<Long> o1,
248 long[] vec2 = ((LongVector)o1).vec();
249 long[] vec3 = ((LongVector)o2).vec();
250 for (int i = 0; i < res.length; i++) {
251 res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
252 }
253 return vectorFactory(res);
254 }
255
256 /*package-private*/
257 abstract
258 LongVector tOp(Vector<Long> o1,
259 Vector<Long> o2,
260 VectorMask<Long> m,
261 FTriOp f);
262 @ForceInline
263 final
264 LongVector tOpTemplate(Vector<Long> o1,
265 Vector<Long> o2,
266 VectorMask<Long> m,
267 FTriOp f) {
268 long[] res = new long[length()];
269 long[] vec1 = this.vec();
270 long[] vec2 = ((LongVector)o1).vec();
271 long[] vec3 = ((LongVector)o2).vec();
272 boolean[] mbits = ((AbstractMask<Long>)m).getBits();
273 for (int i = 0; i < res.length; i++) {
274 res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
275 }
276 return vectorFactory(res);
277 }
278
279 // Reduction operator
280
281 /*package-private*/
282 abstract
283 long rOp(long v, FBinOp f);
284 @ForceInline
285 final
286 long rOpTemplate(long v, FBinOp f) {
287 long[] vec = vec();
288 for (int i = 0; i < vec.length; i++) {
289 v = f.apply(i, v, vec[i]);
290 }
291 return v;
292 }
293
294 // Memory reference
295
296 /*package-private*/
297 interface FLdOp<M> {
298 long apply(M memory, int offset, int i);
299 }
300
301 /*package-private*/
302 @ForceInline
303 final
490 return vsp.broadcast(e);
491 }
492
493
494 // Unary lanewise support
495
496 /**
497 * {@inheritDoc} <!--workaround-->
498 */
499 public abstract
500 LongVector lanewise(VectorOperators.Unary op);
501
502 @ForceInline
503 final
504 LongVector lanewiseTemplate(VectorOperators.Unary op) {
505 if (opKind(op, VO_SPECIAL)) {
506 if (op == ZOMO) {
507 return blend(broadcast(-1), compare(NE, 0));
508 }
509 if (op == NOT) {
510 return broadcast(-1).lanewiseTemplate(XOR, this);
511 } else if (op == NEG) {
512 // FIXME: Support this in the JIT.
513 return broadcast(0).lanewiseTemplate(SUB, this);
514 }
515 }
516 int opc = opCode(op);
517 return VectorSupport.unaryOp(
518 opc, getClass(), long.class, length(),
519 this,
520 UN_IMPL.find(op, opc, (opc_) -> {
521 switch (opc_) {
522 case VECTOR_OP_NEG: return v0 ->
523 v0.uOp((i, a) -> (long) -a);
524 case VECTOR_OP_ABS: return v0 ->
525 v0.uOp((i, a) -> (long) Math.abs(a));
526 default: return null;
527 }}));
528 }
529 private static final
530 ImplCache<Unary,UnaryOperator<LongVector>> UN_IMPL
531 = new ImplCache<>(Unary.class, LongVector.class);
532
533 /**
534 * {@inheritDoc} <!--workaround-->
535 */
536 @ForceInline
537 public final
538 LongVector lanewise(VectorOperators.Unary op,
539 VectorMask<Long> m) {
540 return blend(lanewise(op), m);
541 }
542
543 // Binary lanewise support
544
545 /**
546 * {@inheritDoc} <!--workaround-->
547 * @see #lanewise(VectorOperators.Binary,long)
548 * @see #lanewise(VectorOperators.Binary,long,VectorMask)
549 */
550 @Override
551 public abstract
552 LongVector lanewise(VectorOperators.Binary op,
553 Vector<Long> v);
554 @ForceInline
555 final
556 LongVector lanewiseTemplate(VectorOperators.Binary op,
557 Vector<Long> v) {
558 LongVector that = (LongVector) v;
559 that.check(this);
560 if (opKind(op, VO_SPECIAL | VO_SHIFT)) {
561 if (op == FIRST_NONZERO) {
562 // FIXME: Support this in the JIT.
563 VectorMask<Long> thisNZ
564 = this.viewAsIntegralLanes().compare(NE, (long) 0);
565 that = that.blend((long) 0, thisNZ.cast(vspecies()));
566 op = OR_UNCHECKED;
567 }
568 if (opKind(op, VO_SHIFT)) {
569 // As per shift specification for Java, mask the shift count.
570 // This allows the JIT to ignore some ISA details.
571 that = that.lanewise(AND, SHIFT_MASK);
572 }
573 if (op == AND_NOT) {
574 // FIXME: Support this in the JIT.
575 that = that.lanewise(NOT);
576 op = AND;
577 } else if (op == DIV) {
578 VectorMask<Long> eqz = that.eq((long)0);
579 if (eqz.anyTrue()) {
580 throw that.divZeroException();
581 }
582 }
583 }
584 int opc = opCode(op);
585 return VectorSupport.binaryOp(
586 opc, getClass(), long.class, length(),
587 this, that,
588 BIN_IMPL.find(op, opc, (opc_) -> {
589 switch (opc_) {
590 case VECTOR_OP_ADD: return (v0, v1) ->
591 v0.bOp(v1, (i, a, b) -> (long)(a + b));
592 case VECTOR_OP_SUB: return (v0, v1) ->
593 v0.bOp(v1, (i, a, b) -> (long)(a - b));
594 case VECTOR_OP_MUL: return (v0, v1) ->
595 v0.bOp(v1, (i, a, b) -> (long)(a * b));
596 case VECTOR_OP_DIV: return (v0, v1) ->
597 v0.bOp(v1, (i, a, b) -> (long)(a / b));
598 case VECTOR_OP_MAX: return (v0, v1) ->
599 v0.bOp(v1, (i, a, b) -> (long)Math.max(a, b));
600 case VECTOR_OP_MIN: return (v0, v1) ->
601 v0.bOp(v1, (i, a, b) -> (long)Math.min(a, b));
602 case VECTOR_OP_AND: return (v0, v1) ->
603 v0.bOp(v1, (i, a, b) -> (long)(a & b));
604 case VECTOR_OP_OR: return (v0, v1) ->
605 v0.bOp(v1, (i, a, b) -> (long)(a | b));
606 case VECTOR_OP_XOR: return (v0, v1) ->
607 v0.bOp(v1, (i, a, b) -> (long)(a ^ b));
608 case VECTOR_OP_LSHIFT: return (v0, v1) ->
609 v0.bOp(v1, (i, a, n) -> (long)(a << n));
610 case VECTOR_OP_RSHIFT: return (v0, v1) ->
611 v0.bOp(v1, (i, a, n) -> (long)(a >> n));
612 case VECTOR_OP_URSHIFT: return (v0, v1) ->
613 v0.bOp(v1, (i, a, n) -> (long)((a & LSHR_SETUP_MASK) >>> n));
614 case VECTOR_OP_LROTATE: return (v0, v1) ->
615 v0.bOp(v1, (i, a, n) -> rotateLeft(a, (int)n));
616 case VECTOR_OP_RROTATE: return (v0, v1) ->
617 v0.bOp(v1, (i, a, n) -> rotateRight(a, (int)n));
618 default: return null;
619 }}));
620 }
621 private static final
622 ImplCache<Binary,BinaryOperator<LongVector>> BIN_IMPL
623 = new ImplCache<>(Binary.class, LongVector.class);
624
625 /**
626 * {@inheritDoc} <!--workaround-->
627 * @see #lanewise(VectorOperators.Binary,long,VectorMask)
628 */
629 @ForceInline
630 public final
631 LongVector lanewise(VectorOperators.Binary op,
632 Vector<Long> v,
633 VectorMask<Long> m) {
634 LongVector that = (LongVector) v;
635 if (op == DIV) {
636 VectorMask<Long> eqz = that.eq((long)0);
637 if (eqz.and(m).anyTrue()) {
638 throw that.divZeroException();
639 }
640 // suppress div/0 exceptions in unset lanes
641 that = that.lanewise(NOT, eqz);
642 return blend(lanewise(DIV, that), m);
643 }
644 return blend(lanewise(op, v), m);
645 }
646 // FIXME: Maybe all of the public final methods in this file (the
647 // simple ones that just call lanewise) should be pushed down to
648 // the X-VectorBits template. They can't optimize properly at
649 // this level, and must rely on inlining. Does it work?
650 // (If it works, of course keep the code here.)
651
652 /**
653 * Combines the lane values of this vector
654 * with the value of a broadcast scalar.
655 *
656 * This is a lane-wise binary operation which applies
657 * the selected operation to each lane.
658 * The return value will be equal to this expression:
659 * {@code this.lanewise(op, this.broadcast(e))}.
660 *
661 * @param op the operation used to process lane values
662 * @param e the input scalar
663 * @return the result of applying the operation lane-wise
664 * to the two input vectors
665 * @throws UnsupportedOperationException if this vector does
688 * This is a masked lane-wise binary operation which applies
689 * the selected operation to each lane.
690 * The return value will be equal to this expression:
691 * {@code this.lanewise(op, this.broadcast(e), m)}.
692 *
693 * @param op the operation used to process lane values
694 * @param e the input scalar
695 * @param m the mask controlling lane selection
696 * @return the result of applying the operation lane-wise
697 * to the input vector and the scalar
698 * @throws UnsupportedOperationException if this vector does
699 * not support the requested operation
700 * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
701 * @see #lanewise(VectorOperators.Binary,long)
702 */
703 @ForceInline
704 public final
705 LongVector lanewise(VectorOperators.Binary op,
706 long e,
707 VectorMask<Long> m) {
708 return blend(lanewise(op, e), m);
709 }
710
711
712 /*package-private*/
713 abstract LongVector
714 lanewiseShift(VectorOperators.Binary op, int e);
715
716 /*package-private*/
717 @ForceInline
718 final LongVector
719 lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
720 // Special handling for these. FIXME: Refactor?
721 assert(opKind(op, VO_SHIFT));
722 // As per shift specification for Java, mask the shift count.
723 e &= SHIFT_MASK;
724 int opc = opCode(op);
725 return VectorSupport.broadcastInt(
726 opc, getClass(), long.class, length(),
727 this, e,
728 BIN_INT_IMPL.find(op, opc, (opc_) -> {
729 switch (opc_) {
730 case VECTOR_OP_LSHIFT: return (v, n) ->
731 v.uOp((i, a) -> (long)(a << n));
732 case VECTOR_OP_RSHIFT: return (v, n) ->
733 v.uOp((i, a) -> (long)(a >> n));
734 case VECTOR_OP_URSHIFT: return (v, n) ->
735 v.uOp((i, a) -> (long)((a & LSHR_SETUP_MASK) >>> n));
736 case VECTOR_OP_LROTATE: return (v, n) ->
737 v.uOp((i, a) -> rotateLeft(a, (int)n));
738 case VECTOR_OP_RROTATE: return (v, n) ->
739 v.uOp((i, a) -> rotateRight(a, (int)n));
740 default: return null;
741 }}));
742 }
743 private static final
744 ImplCache<Binary,VectorBroadcastIntOp<LongVector>> BIN_INT_IMPL
745 = new ImplCache<>(Binary.class, LongVector.class);
746
747 // As per shift specification for Java, mask the shift count.
748 // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
749 // The latter two maskings go beyond the JLS, but seem reasonable
750 // since our lane types are first-class types, not just dressed
751 // up ints.
752 private static final int SHIFT_MASK = (Long.SIZE - 1);
753 private static final long LSHR_SETUP_MASK = -1;
754
755 // Ternary lanewise support
756
757 // Ternary operators come in eight variations:
758 // lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
759 // lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
760
761 // It is annoying to support all of these variations of masking
762 // and broadcast, but it would be more surprising not to continue
763 // the obvious pattern started by unary and binary.
764
765 /**
766 * {@inheritDoc} <!--workaround-->
778 Vector<Long> v2);
779 @ForceInline
780 final
781 LongVector lanewiseTemplate(VectorOperators.Ternary op,
782 Vector<Long> v1,
783 Vector<Long> v2) {
784 LongVector that = (LongVector) v1;
785 LongVector tother = (LongVector) v2;
786 // It's a word: https://www.dictionary.com/browse/tother
787 // See also Chapter 11 of Dickens, Our Mutual Friend:
788 // "Totherest Governor," replied Mr Riderhood...
789 that.check(this);
790 tother.check(this);
791 if (op == BITWISE_BLEND) {
792 // FIXME: Support this in the JIT.
793 that = this.lanewise(XOR, that).lanewise(AND, tother);
794 return this.lanewise(XOR, that);
795 }
796 int opc = opCode(op);
797 return VectorSupport.ternaryOp(
798 opc, getClass(), long.class, length(),
799 this, that, tother,
800 TERN_IMPL.find(op, opc, (opc_) -> {
801 switch (opc_) {
802 default: return null;
803 }}));
804 }
805 private static final
806 ImplCache<Ternary,TernaryOperation<LongVector>> TERN_IMPL
807 = new ImplCache<>(Ternary.class, LongVector.class);
808
809 /**
810 * {@inheritDoc} <!--workaround-->
811 * @see #lanewise(VectorOperators.Ternary,long,long,VectorMask)
812 * @see #lanewise(VectorOperators.Ternary,Vector,long,VectorMask)
813 * @see #lanewise(VectorOperators.Ternary,long,Vector,VectorMask)
814 */
815 @ForceInline
816 public final
817 LongVector lanewise(VectorOperators.Ternary op,
818 Vector<Long> v1,
819 Vector<Long> v2,
820 VectorMask<Long> m) {
821 return blend(lanewise(op, v1, v2), m);
822 }
823
824 /**
825 * Combines the lane values of this vector
826 * with the values of two broadcast scalars.
827 *
828 * This is a lane-wise ternary operation which applies
829 * the selected operation to each lane.
830 * The return value will be equal to this expression:
831 * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
832 *
833 * @param op the operation used to combine lane values
834 * @param e1 the first input scalar
835 * @param e2 the second input scalar
836 * @return the result of applying the operation lane-wise
837 * to the input vector and the scalars
838 * @throws UnsupportedOperationException if this vector does
839 * not support the requested operation
840 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
841 * @see #lanewise(VectorOperators.Ternary,long,long,VectorMask)
858 * The return value will be equal to this expression:
859 * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
860 *
861 * @param op the operation used to combine lane values
862 * @param e1 the first input scalar
863 * @param e2 the second input scalar
864 * @param m the mask controlling lane selection
865 * @return the result of applying the operation lane-wise
866 * to the input vector and the scalars
867 * @throws UnsupportedOperationException if this vector does
868 * not support the requested operation
869 * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
870 * @see #lanewise(VectorOperators.Ternary,long,long)
871 */
872 @ForceInline
873 public final
874 LongVector lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
875 long e1,
876 long e2,
877 VectorMask<Long> m) {
878 return blend(lanewise(op, e1, e2), m);
879 }
880
881 /**
882 * Combines the lane values of this vector
883 * with the values of another vector and a broadcast scalar.
884 *
885 * This is a lane-wise ternary operation which applies
886 * the selected operation to each lane.
887 * The return value will be equal to this expression:
888 * {@code this.lanewise(op, v1, this.broadcast(e2))}.
889 *
890 * @param op the operation used to combine lane values
891 * @param v1 the other input vector
892 * @param e2 the input scalar
893 * @return the result of applying the operation lane-wise
894 * to the input vectors and the scalar
895 * @throws UnsupportedOperationException if this vector does
896 * not support the requested operation
897 * @see #lanewise(VectorOperators.Ternary,long,long)
898 * @see #lanewise(VectorOperators.Ternary,Vector,long,VectorMask)
916 * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
917 *
918 * @param op the operation used to combine lane values
919 * @param v1 the other input vector
920 * @param e2 the input scalar
921 * @param m the mask controlling lane selection
922 * @return the result of applying the operation lane-wise
923 * to the input vectors and the scalar
924 * @throws UnsupportedOperationException if this vector does
925 * not support the requested operation
926 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
927 * @see #lanewise(VectorOperators.Ternary,long,long,VectorMask)
928 * @see #lanewise(VectorOperators.Ternary,Vector,long)
929 */
930 @ForceInline
931 public final
932 LongVector lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
933 Vector<Long> v1,
934 long e2,
935 VectorMask<Long> m) {
936 return blend(lanewise(op, v1, e2), m);
937 }
938
939 /**
940 * Combines the lane values of this vector
941 * with the values of another vector and a broadcast scalar.
942 *
943 * This is a lane-wise ternary operation which applies
944 * the selected operation to each lane.
945 * The return value will be equal to this expression:
946 * {@code this.lanewise(op, this.broadcast(e1), v2)}.
947 *
948 * @param op the operation used to combine lane values
949 * @param e1 the input scalar
950 * @param v2 the other input vector
951 * @return the result of applying the operation lane-wise
952 * to the input vectors and the scalar
953 * @throws UnsupportedOperationException if this vector does
954 * not support the requested operation
955 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
956 * @see #lanewise(VectorOperators.Ternary,long,Vector,VectorMask)
973 * The return value will be equal to this expression:
974 * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
975 *
976 * @param op the operation used to combine lane values
977 * @param e1 the input scalar
978 * @param v2 the other input vector
979 * @param m the mask controlling lane selection
980 * @return the result of applying the operation lane-wise
981 * to the input vectors and the scalar
982 * @throws UnsupportedOperationException if this vector does
983 * not support the requested operation
984 * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
985 * @see #lanewise(VectorOperators.Ternary,long,Vector)
986 */
987 @ForceInline
988 public final
989 LongVector lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
990 long e1,
991 Vector<Long> v2,
992 VectorMask<Long> m) {
993 return blend(lanewise(op, e1, v2), m);
994 }
995
996 // (Thus endeth the Great and Mighty Ternary Ogdoad.)
997 // https://en.wikipedia.org/wiki/Ogdoad
998
999 /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1000 //
1001 // These include masked and non-masked versions.
1002 // This subclass adds broadcast (masked or not).
1003
1004 /**
1005 * {@inheritDoc} <!--workaround-->
1006 * @see #add(long)
1007 */
1008 @Override
1009 @ForceInline
1010 public final LongVector add(Vector<Long> v) {
1011 return lanewise(ADD, v);
1012 }
1013
1645 @Override
1646 @ForceInline
1647 public final
1648 VectorMask<Long> test(VectorOperators.Test op,
1649 VectorMask<Long> m) {
1650 return test(op).and(m);
1651 }
1652
1653 /**
1654 * {@inheritDoc} <!--workaround-->
1655 */
1656 @Override
1657 public abstract
1658 VectorMask<Long> compare(VectorOperators.Comparison op, Vector<Long> v);
1659
1660 /*package-private*/
1661 @ForceInline
1662 final
1663 <M extends VectorMask<Long>>
1664 M compareTemplate(Class<M> maskType, Comparison op, Vector<Long> v) {
1665 Objects.requireNonNull(v);
1666 LongSpecies vsp = vspecies();
1667 LongVector that = (LongVector) v;
1668 that.check(this);
1669 int opc = opCode(op);
1670 return VectorSupport.compare(
1671 opc, getClass(), maskType, long.class, length(),
1672 this, that,
1673 (cond, v0, v1) -> {
1674 AbstractMask<Long> m
1675 = v0.bTest(cond, v1, (cond_, i, a, b)
1676 -> compareWithOp(cond, a, b));
1677 @SuppressWarnings("unchecked")
1678 M m2 = (M) m;
1679 return m2;
1680 });
1681 }
1682
1683 @ForceInline
1684 private static boolean compareWithOp(int cond, long a, long b) {
1685 return switch (cond) {
1686 case BT_eq -> a == b;
1687 case BT_ne -> a != b;
1688 case BT_lt -> a < b;
1689 case BT_le -> a <= b;
1690 case BT_gt -> a > b;
1691 case BT_ge -> a >= b;
1692 case BT_ult -> Long.compareUnsigned(a, b) < 0;
1693 case BT_ule -> Long.compareUnsigned(a, b) <= 0;
1694 case BT_ugt -> Long.compareUnsigned(a, b) > 0;
1695 case BT_uge -> Long.compareUnsigned(a, b) >= 0;
1696 default -> throw new AssertionError();
1697 };
1698 }
1699
1700 /**
1701 * {@inheritDoc} <!--workaround-->
1702 */
1703 @Override
1704 @ForceInline
1705 public final
1706 VectorMask<Long> compare(VectorOperators.Comparison op,
1707 Vector<Long> v,
1708 VectorMask<Long> m) {
1709 return compare(op, v).and(m);
1710 }
1711
1712 /**
1713 * Tests this vector by comparing it with an input scalar,
1714 * according to the given comparison operation.
1715 *
1716 * This is a lane-wise binary test operation which applies
1717 * the comparison operation to each lane.
1718 * <p>
1719 * The result is the same as
1720 * {@code compare(op, broadcast(species(), e))}.
1721 * That is, the scalar may be regarded as broadcast to
1722 * a vector of the same species, and then compared
1723 * against the original vector, using the selected
1724 * comparison operation.
1725 *
1726 * @param op the operation used to compare lane values
1727 * @param e the input scalar
1728 * @return the mask result of testing lane-wise if this vector
1729 * compares to the input, according to the selected
1730 * comparison operator
1731 * @see LongVector#compare(VectorOperators.Comparison,Vector)
1750 *
1751 * This is a masked lane-wise binary test operation which applies
1752 * to each pair of corresponding lane values.
1753 *
1754 * The returned result is equal to the expression
1755 * {@code compare(op,s).and(m)}.
1756 *
1757 * @param op the operation used to compare lane values
1758 * @param e the input scalar
1759 * @param m the mask controlling lane selection
1760 * @return the mask result of testing lane-wise if this vector
1761 * compares to the input, according to the selected
1762 * comparison operator,
1763 * and only in the lanes selected by the mask
1764 * @see LongVector#compare(VectorOperators.Comparison,Vector,VectorMask)
1765 */
1766 @ForceInline
1767 public final VectorMask<Long> compare(VectorOperators.Comparison op,
1768 long e,
1769 VectorMask<Long> m) {
1770 return compare(op, e).and(m);
1771 }
1772
1773
1774 /**
1775 * {@inheritDoc} <!--workaround-->
1776 */
1777 @Override public abstract
1778 LongVector blend(Vector<Long> v, VectorMask<Long> m);
1779
1780 /*package-private*/
1781 @ForceInline
1782 final
1783 <M extends VectorMask<Long>>
1784 LongVector
1785 blendTemplate(Class<M> maskType, LongVector v, M m) {
1786 v.check(this);
1787 return VectorSupport.blend(
1788 getClass(), maskType, long.class, length(),
1789 this, v, m,
1790 (v0, v1, m_) -> v0.bOp(v1, m_, (i, a, b) -> b));
1954 wrongPartForSlice(int part) {
1955 String msg = String.format("bad part number %d for slice operation",
1956 part);
1957 return new ArrayIndexOutOfBoundsException(msg);
1958 }
1959
1960 /**
1961 * {@inheritDoc} <!--workaround-->
1962 */
1963 @Override
1964 public abstract
1965 LongVector rearrange(VectorShuffle<Long> m);
1966
1967 /*package-private*/
1968 @ForceInline
1969 final
1970 <S extends VectorShuffle<Long>>
1971 LongVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
1972 shuffle.checkIndexes();
1973 return VectorSupport.rearrangeOp(
1974 getClass(), shuffletype, long.class, length(),
1975 this, shuffle,
1976 (v1, s_) -> v1.uOp((i, a) -> {
1977 int ei = s_.laneSource(i);
1978 return v1.lane(ei);
1979 }));
1980 }
1981
1982 /**
1983 * {@inheritDoc} <!--workaround-->
1984 */
1985 @Override
1986 public abstract
1987 LongVector rearrange(VectorShuffle<Long> s,
1988 VectorMask<Long> m);
1989
1990 /*package-private*/
1991 @ForceInline
1992 final
1993 <S extends VectorShuffle<Long>>
1994 LongVector rearrangeTemplate(Class<S> shuffletype,
1995 S shuffle,
1996 VectorMask<Long> m) {
1997 LongVector unmasked =
1998 VectorSupport.rearrangeOp(
1999 getClass(), shuffletype, long.class, length(),
2000 this, shuffle,
2001 (v1, s_) -> v1.uOp((i, a) -> {
2002 int ei = s_.laneSource(i);
2003 return ei < 0 ? 0 : v1.lane(ei);
2004 }));
2005 VectorMask<Long> valid = shuffle.laneIsValid();
2006 if (m.andNot(valid).anyTrue()) {
2007 shuffle.checkIndexes();
2008 throw new AssertionError();
2009 }
2010 return broadcast((long)0).blend(unmasked, m);
2011 }
2012
2013 /**
2014 * {@inheritDoc} <!--workaround-->
2015 */
2016 @Override
2017 public abstract
2018 LongVector rearrange(VectorShuffle<Long> s,
2019 Vector<Long> v);
2020
2021 /*package-private*/
2022 @ForceInline
2023 final
2024 <S extends VectorShuffle<Long>>
2025 LongVector rearrangeTemplate(Class<S> shuffletype,
2026 S shuffle,
2027 LongVector v) {
2028 VectorMask<Long> valid = shuffle.laneIsValid();
2029 @SuppressWarnings("unchecked")
2030 S ws = (S) shuffle.wrapIndexes();
2031 LongVector r0 =
2032 VectorSupport.rearrangeOp(
2033 getClass(), shuffletype, long.class, length(),
2034 this, ws,
2035 (v0, s_) -> v0.uOp((i, a) -> {
2036 int ei = s_.laneSource(i);
2037 return v0.lane(ei);
2038 }));
2039 LongVector r1 =
2040 VectorSupport.rearrangeOp(
2041 getClass(), shuffletype, long.class, length(),
2042 v, ws,
2043 (v1, s_) -> v1.uOp((i, a) -> {
2044 int ei = s_.laneSource(i);
2045 return v1.lane(ei);
2046 }));
2047 return r1.blend(r0, valid);
2048 }
2049
2050 @ForceInline
2051 private final
2052 VectorShuffle<Long> toShuffle0(LongSpecies dsp) {
2053 long[] a = toArray();
2054 int[] sa = new int[a.length];
2055 for (int i = 0; i < a.length; i++) {
2056 sa[i] = (int) a[i];
2057 }
2058 return VectorShuffle.fromArray(dsp, sa, 0);
2059 }
2060
2061 /*package-private*/
2062 @ForceInline
2063 final
2286 * <li>
2287 * All other reduction operations are fully commutative and
2288 * associative. The implementation can choose any order of
2289 * processing, yet it will always produce the same result.
2290 * </ul>
2291 *
2292 * @param op the operation used to combine lane values
2293 * @param m the mask controlling lane selection
2294 * @return the reduced result accumulated from the selected lane values
2295 * @throws UnsupportedOperationException if this vector does
2296 * not support the requested operation
2297 * @see #reduceLanes(VectorOperators.Associative)
2298 */
2299 public abstract long reduceLanes(VectorOperators.Associative op,
2300 VectorMask<Long> m);
2301
2302 /*package-private*/
2303 @ForceInline
2304 final
2305 long reduceLanesTemplate(VectorOperators.Associative op,
2306 VectorMask<Long> m) {
2307 LongVector v = reduceIdentityVector(op).blend(this, m);
2308 return v.reduceLanesTemplate(op);
2309 }
2310
2311 /*package-private*/
2312 @ForceInline
2313 final
2314 long reduceLanesTemplate(VectorOperators.Associative op) {
2315 if (op == FIRST_NONZERO) {
2316 // FIXME: The JIT should handle this, and other scan ops alos.
2317 VectorMask<Long> thisNZ
2318 = this.viewAsIntegralLanes().compare(NE, (long) 0);
2319 return this.lane(thisNZ.firstTrue());
2320 }
2321 int opc = opCode(op);
2322 return fromBits(VectorSupport.reductionCoerced(
2323 opc, getClass(), long.class, length(),
2324 this,
2325 REDUCE_IMPL.find(op, opc, (opc_) -> {
2326 switch (opc_) {
2327 case VECTOR_OP_ADD: return v ->
2328 toBits(v.rOp((long)0, (i, a, b) -> (long)(a + b)));
2329 case VECTOR_OP_MUL: return v ->
2330 toBits(v.rOp((long)1, (i, a, b) -> (long)(a * b)));
2331 case VECTOR_OP_MIN: return v ->
2332 toBits(v.rOp(MAX_OR_INF, (i, a, b) -> (long) Math.min(a, b)));
2333 case VECTOR_OP_MAX: return v ->
2334 toBits(v.rOp(MIN_OR_INF, (i, a, b) -> (long) Math.max(a, b)));
2335 case VECTOR_OP_AND: return v ->
2336 toBits(v.rOp((long)-1, (i, a, b) -> (long)(a & b)));
2337 case VECTOR_OP_OR: return v ->
2338 toBits(v.rOp((long)0, (i, a, b) -> (long)(a | b)));
2339 case VECTOR_OP_XOR: return v ->
2340 toBits(v.rOp((long)0, (i, a, b) -> (long)(a ^ b)));
2341 default: return null;
2342 }})));
2343 }
2344 private static final
2345 ImplCache<Associative,Function<LongVector,Long>> REDUCE_IMPL
2346 = new ImplCache<>(Associative.class, LongVector.class);
2347
2348 private
2349 @ForceInline
2350 LongVector reduceIdentityVector(VectorOperators.Associative op) {
2351 int opc = opCode(op);
2352 UnaryOperator<LongVector> fn
2353 = REDUCE_ID_IMPL.find(op, opc, (opc_) -> {
2354 switch (opc_) {
2355 case VECTOR_OP_ADD:
2356 case VECTOR_OP_OR:
2357 case VECTOR_OP_XOR:
2358 return v -> v.broadcast(0);
2359 case VECTOR_OP_MUL:
2360 return v -> v.broadcast(1);
2361 case VECTOR_OP_AND:
2362 return v -> v.broadcast(-1);
2363 case VECTOR_OP_MIN:
2364 return v -> v.broadcast(MAX_OR_INF);
2365 case VECTOR_OP_MAX:
2366 return v -> v.broadcast(MIN_OR_INF);
2540 * @param species species of desired vector
2541 * @param a the byte array
2542 * @param offset the offset into the array
2543 * @param bo the intended byte order
2544 * @param m the mask controlling lane selection
2545 * @return a vector loaded from a byte array
2546 * @throws IndexOutOfBoundsException
2547 * if {@code offset+N*ESIZE < 0}
2548 * or {@code offset+(N+1)*ESIZE > a.length}
2549 * for any lane {@code N} in the vector
2550 * where the mask is set
2551 */
2552 @ForceInline
2553 public static
2554 LongVector fromByteArray(VectorSpecies<Long> species,
2555 byte[] a, int offset,
2556 ByteOrder bo,
2557 VectorMask<Long> m) {
2558 LongSpecies vsp = (LongSpecies) species;
2559 if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
2560 LongVector zero = vsp.zero();
2561 LongVector v = zero.fromByteArray0(a, offset);
2562 return zero.blend(v.maybeSwap(bo), m);
2563 }
2564
2565 // FIXME: optimize
2566 checkMaskFromIndexSize(offset, vsp, m, 8, a.length);
2567 ByteBuffer wb = wrapper(a, bo);
2568 return vsp.ldOp(wb, offset, (AbstractMask<Long>)m,
2569 (wb_, o, i) -> wb_.getLong(o + i * 8));
2570 }
2571
2572 /**
2573 * Loads a vector from an array of type {@code long[]}
2574 * starting at an offset.
2575 * For each vector lane, where {@code N} is the vector lane index, the
2576 * array element at index {@code offset + N} is placed into the
2577 * resulting vector at lane index {@code N}.
2578 *
2579 * @param species species of desired vector
2580 * @param a the array
2581 * @param offset the offset into the array
2582 * @return the vector loaded from an array
2604 * {@code N}, otherwise the default element value is placed into the
2605 * resulting vector at lane index {@code N}.
2606 *
2607 * @param species species of desired vector
2608 * @param a the array
2609 * @param offset the offset into the array
2610 * @param m the mask controlling lane selection
2611 * @return the vector loaded from an array
2612 * @throws IndexOutOfBoundsException
2613 * if {@code offset+N < 0} or {@code offset+N >= a.length}
2614 * for any lane {@code N} in the vector
2615 * where the mask is set
2616 */
2617 @ForceInline
2618 public static
2619 LongVector fromArray(VectorSpecies<Long> species,
2620 long[] a, int offset,
2621 VectorMask<Long> m) {
2622 LongSpecies vsp = (LongSpecies) species;
2623 if (offset >= 0 && offset <= (a.length - species.length())) {
2624 LongVector zero = vsp.zero();
2625 return zero.blend(zero.fromArray0(a, offset), m);
2626 }
2627
2628 // FIXME: optimize
2629 checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
2630 return vsp.vOp(m, i -> a[offset + i]);
2631 }
2632
2633 /**
2634 * Gathers a new vector composed of elements from an array of type
2635 * {@code long[]},
2636 * using indexes obtained by adding a fixed {@code offset} to a
2637 * series of secondary offsets from an <em>index map</em>.
2638 * The index map is a contiguous sequence of {@code VLENGTH}
2639 * elements in a second array of {@code int}s, starting at a given
2640 * {@code mapOffset}.
2641 * <p>
2642 * For each vector lane, where {@code N} is the vector lane index,
2643 * the lane is loaded from the array
2644 * element {@code a[f(N)]}, where {@code f(N)} is the
2645 * index mapping expression
2681 if (isp.laneCount() != vsp.laneCount()) {
2682 // For LongMaxVector, if vector length is non-power-of-two or
2683 // 2048 bits, indexShape of Long species is S_MAX_BIT.
2684 // Assume that vector length is 2048, then the lane count of Long
2685 // vector is 32. When converting Long species to int species,
2686 // indexShape is still S_MAX_BIT, but the lane count of int vector
2687 // is 64. So when loading index vector (IntVector), only lower half
2688 // of index data is needed.
2689 vix = IntVector
2690 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
2691 .add(offset);
2692 } else {
2693 vix = IntVector
2694 .fromArray(isp, indexMap, mapOffset)
2695 .add(offset);
2696 }
2697
2698 vix = VectorIntrinsics.checkIndex(vix, a.length);
2699
2700 return VectorSupport.loadWithMap(
2701 vectorType, long.class, vsp.laneCount(),
2702 IntVector.species(vsp.indexShape()).vectorType(),
2703 a, ARRAY_BASE, vix,
2704 a, offset, indexMap, mapOffset, vsp,
2705 (long[] c, int idx, int[] iMap, int idy, LongSpecies s) ->
2706 s.vOp(n -> c[idx + iMap[idy+n]]));
2707 }
2708
2709 /**
2710 * Gathers a new vector composed of elements from an array of type
2711 * {@code long[]},
2712 * under the control of a mask, and
2713 * using indexes obtained by adding a fixed {@code offset} to a
2714 * series of secondary offsets from an <em>index map</em>.
2715 * The index map is a contiguous sequence of {@code VLENGTH}
2716 * elements in a second array of {@code int}s, starting at a given
2717 * {@code mapOffset}.
2718 * <p>
2719 * For each vector lane, where {@code N} is the vector lane index,
2720 * if the lane is set in the mask,
2721 * the lane is loaded from the array
2722 * element {@code a[f(N)]}, where {@code f(N)} is the
2723 * index mapping expression
2724 * {@code offset + indexMap[mapOffset + N]]}.
2725 * Unset lanes in the resulting vector are set to zero.
2726 *
2727 * @param species species of desired vector
2735 * @return the vector loaded from the indexed elements of the array
2736 * @throws IndexOutOfBoundsException
2737 * if {@code mapOffset+N < 0}
2738 * or if {@code mapOffset+N >= indexMap.length},
2739 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
2740 * is an invalid index into {@code a},
2741 * for any lane {@code N} in the vector
2742 * where the mask is set
2743 * @see LongVector#toIntArray()
2744 */
2745 @ForceInline
2746 public static
2747 LongVector fromArray(VectorSpecies<Long> species,
2748 long[] a, int offset,
2749 int[] indexMap, int mapOffset,
2750 VectorMask<Long> m) {
2751 if (m.allTrue()) {
2752 return fromArray(species, a, offset, indexMap, mapOffset);
2753 }
2754 else {
2755 // FIXME: Cannot vectorize yet, if there's a mask.
2756 LongSpecies vsp = (LongSpecies) species;
2757 return vsp.vOp(m, n -> a[offset + indexMap[mapOffset + n]]);
2758 }
2759 }
2760
2761
2762
2763 /**
2764 * Loads a vector from a {@linkplain ByteBuffer byte buffer}
2765 * starting at an offset into the byte buffer.
2766 * Bytes are composed into primitive lane elements according
2767 * to the specified byte order.
2768 * The vector is arranged into lanes according to
2769 * <a href="Vector.html#lane-order">memory ordering</a>.
2770 * <p>
2771 * This method behaves as if it returns the result of calling
2772 * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
2773 * fromByteBuffer()} as follows:
2774 * <pre>{@code
2775 * var m = species.maskAll(true);
2776 * return fromByteBuffer(species, bb, offset, bo, m);
2777 * }</pre>
2831 * @param species species of desired vector
2832 * @param bb the byte buffer
2833 * @param offset the offset into the byte buffer
2834 * @param bo the intended byte order
2835 * @param m the mask controlling lane selection
2836 * @return a vector loaded from a byte buffer
2837 * @throws IndexOutOfBoundsException
2838 * if {@code offset+N*8 < 0}
2839 * or {@code offset+N*8 >= bb.limit()}
2840 * for any lane {@code N} in the vector
2841 * where the mask is set
2842 */
2843 @ForceInline
2844 public static
2845 LongVector fromByteBuffer(VectorSpecies<Long> species,
2846 ByteBuffer bb, int offset,
2847 ByteOrder bo,
2848 VectorMask<Long> m) {
2849 LongSpecies vsp = (LongSpecies) species;
2850 if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
2851 LongVector zero = vsp.zero();
2852 LongVector v = zero.fromByteBuffer0(bb, offset);
2853 return zero.blend(v.maybeSwap(bo), m);
2854 }
2855
2856 // FIXME: optimize
2857 checkMaskFromIndexSize(offset, vsp, m, 8, bb.limit());
2858 ByteBuffer wb = wrapper(bb, bo);
2859 return vsp.ldOp(wb, offset, (AbstractMask<Long>)m,
2860 (wb_, o, i) -> wb_.getLong(o + i * 8));
2861 }
2862
2863 // Memory store operations
2864
2865 /**
2866 * Stores this vector into an array of type {@code long[]}
2867 * starting at an offset.
2868 * <p>
2869 * For each vector lane, where {@code N} is the vector lane index,
2870 * the lane element at index {@code N} is stored into the array
2871 * element {@code a[offset+N]}.
2872 *
2873 * @param a the array, of type {@code long[]}
2905 * Lanes where the mask is unset are not stored and do not need
2906 * to correspond to legitimate elements of {@code a}.
2907 * That is, unset lanes may correspond to array indexes less than
2908 * zero or beyond the end of the array.
2909 *
2910 * @param a the array, of type {@code long[]}
2911 * @param offset the offset into the array
2912 * @param m the mask controlling lane storage
2913 * @throws IndexOutOfBoundsException
2914 * if {@code offset+N < 0} or {@code offset+N >= a.length}
2915 * for any lane {@code N} in the vector
2916 * where the mask is set
2917 */
2918 @ForceInline
2919 public final
2920 void intoArray(long[] a, int offset,
2921 VectorMask<Long> m) {
2922 if (m.allTrue()) {
2923 intoArray(a, offset);
2924 } else {
2925 // FIXME: optimize
2926 LongSpecies vsp = vspecies();
2927 checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
2928 stOp(a, offset, m, (arr, off, i, v) -> arr[off+i] = v);
2929 }
2930 }
2931
2932 /**
2933 * Scatters this vector into an array of type {@code long[]}
2934 * using indexes obtained by adding a fixed {@code offset} to a
2935 * series of secondary offsets from an <em>index map</em>.
2936 * The index map is a contiguous sequence of {@code VLENGTH}
2937 * elements in a second array of {@code int}s, starting at a given
2938 * {@code mapOffset}.
2939 * <p>
2940 * For each vector lane, where {@code N} is the vector lane index,
2941 * the lane element at index {@code N} is stored into the array
2942 * element {@code a[f(N)]}, where {@code f(N)} is the
2943 * index mapping expression
2944 * {@code offset + indexMap[mapOffset + N]]}.
2945 *
2946 * @param a the array
2947 * @param offset an offset to combine with the index map offsets
2948 * @param indexMap the index map
2971 if (isp.laneCount() != vsp.laneCount()) {
2972 // For LongMaxVector, if vector length is 2048 bits, indexShape
2973 // of Long species is S_MAX_BIT. and the lane count of Long
2974 // vector is 32. When converting Long species to int species,
2975 // indexShape is still S_MAX_BIT, but the lane count of int vector
2976 // is 64. So when loading index vector (IntVector), only lower half
2977 // of index data is needed.
2978 vix = IntVector
2979 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
2980 .add(offset);
2981 } else {
2982 vix = IntVector
2983 .fromArray(isp, indexMap, mapOffset)
2984 .add(offset);
2985 }
2986
2987
2988 vix = VectorIntrinsics.checkIndex(vix, a.length);
2989
2990 VectorSupport.storeWithMap(
2991 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
2992 isp.vectorType(),
2993 a, arrayAddress(a, 0), vix,
2994 this,
2995 a, offset, indexMap, mapOffset,
2996 (arr, off, v, map, mo)
2997 -> v.stOp(arr, off,
2998 (arr_, off_, i, e) -> {
2999 int j = map[mo + i];
3000 arr[off + j] = e;
3001 }));
3002 }
3003
3004 /**
3005 * Scatters this vector into an array of type {@code long[]},
3006 * under the control of a mask, and
3007 * using indexes obtained by adding a fixed {@code offset} to a
3008 * series of secondary offsets from an <em>index map</em>.
3009 * The index map is a contiguous sequence of {@code VLENGTH}
3010 * elements in a second array of {@code int}s, starting at a given
3011 * {@code mapOffset}.
3012 * <p>
3013 * For each vector lane, where {@code N} is the vector lane index,
3014 * if the mask lane at index {@code N} is set then
3015 * the lane element at index {@code N} is stored into the array
3016 * element {@code a[f(N)]}, where {@code f(N)} is the
3023 * @param mapOffset the offset into the index map
3024 * @param m the mask
3025 * @throws IndexOutOfBoundsException
3026 * if {@code mapOffset+N < 0}
3027 * or if {@code mapOffset+N >= indexMap.length},
3028 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
3029 * is an invalid index into {@code a},
3030 * for any lane {@code N} in the vector
3031 * where the mask is set
3032 * @see LongVector#toIntArray()
3033 */
3034 @ForceInline
3035 public final
3036 void intoArray(long[] a, int offset,
3037 int[] indexMap, int mapOffset,
3038 VectorMask<Long> m) {
3039 if (m.allTrue()) {
3040 intoArray(a, offset, indexMap, mapOffset);
3041 }
3042 else {
3043 // FIXME: Cannot vectorize yet, if there's a mask.
3044 stOp(a, offset, m,
3045 (arr, off, i, e) -> {
3046 int j = indexMap[mapOffset + i];
3047 arr[off + j] = e;
3048 });
3049 }
3050 }
3051
3052
3053
3054 /**
3055 * {@inheritDoc} <!--workaround-->
3056 */
3057 @Override
3058 @ForceInline
3059 public final
3060 void intoByteArray(byte[] a, int offset,
3061 ByteOrder bo) {
3062 offset = checkFromIndexSize(offset, byteSize(), a.length);
3063 maybeSwap(bo).intoByteArray0(a, offset);
3064 }
3065
3066 /**
3067 * {@inheritDoc} <!--workaround-->
3068 */
3069 @Override
3070 @ForceInline
3071 public final
3072 void intoByteArray(byte[] a, int offset,
3073 ByteOrder bo,
3074 VectorMask<Long> m) {
3075 if (m.allTrue()) {
3076 intoByteArray(a, offset, bo);
3077 } else {
3078 // FIXME: optimize
3079 LongSpecies vsp = vspecies();
3080 checkMaskFromIndexSize(offset, vsp, m, 8, a.length);
3081 ByteBuffer wb = wrapper(a, bo);
3082 this.stOp(wb, offset, m,
3083 (wb_, o, i, e) -> wb_.putLong(o + i * 8, e));
3084 }
3085 }
3086
3087 /**
3088 * {@inheritDoc} <!--workaround-->
3089 */
3090 @Override
3091 @ForceInline
3092 public final
3093 void intoByteBuffer(ByteBuffer bb, int offset,
3094 ByteOrder bo) {
3095 if (bb.isReadOnly()) {
3096 throw new ReadOnlyBufferException();
3097 }
3098 offset = checkFromIndexSize(offset, byteSize(), bb.limit());
3099 maybeSwap(bo).intoByteBuffer0(bb, offset);
3100 }
3101
3102 /**
3103 * {@inheritDoc} <!--workaround-->
3104 */
3105 @Override
3106 @ForceInline
3107 public final
3108 void intoByteBuffer(ByteBuffer bb, int offset,
3109 ByteOrder bo,
3110 VectorMask<Long> m) {
3111 if (m.allTrue()) {
3112 intoByteBuffer(bb, offset, bo);
3113 } else {
3114 // FIXME: optimize
3115 if (bb.isReadOnly()) {
3116 throw new ReadOnlyBufferException();
3117 }
3118 LongSpecies vsp = vspecies();
3119 checkMaskFromIndexSize(offset, vsp, m, 8, bb.limit());
3120 ByteBuffer wb = wrapper(bb, bo);
3121 this.stOp(wb, offset, m,
3122 (wb_, o, i, e) -> wb_.putLong(o + i * 8, e));
3123 }
3124 }
3125
3126 // ================================================
3127
3128 // Low-level memory operations.
3129 //
3130 // Note that all of these operations *must* inline into a context
3131 // where the exact species of the involved vector is a
3132 // compile-time constant. Otherwise, the intrinsic generation
3133 // will fail and performance will suffer.
3134 //
3135 // In many cases this is achieved by re-deriving a version of the
3136 // method in each concrete subclass (per species). The re-derived
3137 // method simply calls one of these generic methods, with exact
3138 // parameters for the controlling metadata, which is either a
3139 // typed vector or constant species instance.
3140
3141 // Unchecked loading operations in native byte order.
3142 // Caller is responsible for applying index checks, masking, and
3143 // byte swapping.
3144
3145 /*package-private*/
3146 abstract
3147 LongVector fromArray0(long[] a, int offset);
3148 @ForceInline
3149 final
3150 LongVector fromArray0Template(long[] a, int offset) {
3151 LongSpecies vsp = vspecies();
3152 return VectorSupport.load(
3153 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3154 a, arrayAddress(a, offset),
3155 a, offset, vsp,
3156 (arr, off, s) -> s.ldOp(arr, off,
3157 (arr_, off_, i) -> arr_[off_ + i]));
3158 }
3159
3160
3161
3162 @Override
3163 abstract
3164 LongVector fromByteArray0(byte[] a, int offset);
3165 @ForceInline
3166 final
3167 LongVector fromByteArray0Template(byte[] a, int offset) {
3168 LongSpecies vsp = vspecies();
3169 return VectorSupport.load(
3170 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3171 a, byteArrayAddress(a, offset),
3172 a, offset, vsp,
3173 (arr, off, s) -> {
3174 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3175 return s.ldOp(wb, off,
3176 (wb_, o, i) -> wb_.getLong(o + i * 8));
3177 });
3178 }
3179
3180 abstract
3181 LongVector fromByteBuffer0(ByteBuffer bb, int offset);
3182 @ForceInline
3183 final
3184 LongVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
3185 LongSpecies vsp = vspecies();
3186 return ScopedMemoryAccess.loadFromByteBuffer(
3187 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3188 bb, offset, vsp,
3189 (buf, off, s) -> {
3190 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3191 return s.ldOp(wb, off,
3192 (wb_, o, i) -> wb_.getLong(o + i * 8));
3193 });
3194 }
3195
3196 // Unchecked storing operations in native byte order.
3197 // Caller is responsible for applying index checks, masking, and
3198 // byte swapping.
3199
3200 abstract
3201 void intoArray0(long[] a, int offset);
3202 @ForceInline
3203 final
3204 void intoArray0Template(long[] a, int offset) {
3205 LongSpecies vsp = vspecies();
3206 VectorSupport.store(
3207 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3208 a, arrayAddress(a, offset),
3209 this, a, offset,
3210 (arr, off, v)
3211 -> v.stOp(arr, off,
3212 (arr_, off_, i, e) -> arr_[off_+i] = e));
3213 }
3214
3215 abstract
3216 void intoByteArray0(byte[] a, int offset);
3217 @ForceInline
3218 final
3219 void intoByteArray0Template(byte[] a, int offset) {
3220 LongSpecies vsp = vspecies();
3221 VectorSupport.store(
3222 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3223 a, byteArrayAddress(a, offset),
3224 this, a, offset,
3225 (arr, off, v) -> {
3226 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3227 v.stOp(wb, off,
3228 (tb_, o, i, e) -> tb_.putLong(o + i * 8, e));
3229 });
3230 }
3231
3232 @ForceInline
3233 final
3234 void intoByteBuffer0(ByteBuffer bb, int offset) {
3235 LongSpecies vsp = vspecies();
3236 ScopedMemoryAccess.storeIntoByteBuffer(
3237 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3238 this, bb, offset,
3239 (buf, off, v) -> {
3240 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3241 v.stOp(wb, off,
3242 (wb_, o, i, e) -> wb_.putLong(o + i * 8, e));
3243 });
3244 }
3245
3246 // End of low-level memory operations.
3247
3248 private static
3249 void checkMaskFromIndexSize(int offset,
3250 LongSpecies vsp,
3251 VectorMask<Long> m,
3252 int scale,
3253 int limit) {
3254 ((AbstractMask<Long>)m)
3255 .checkIndexByLane(offset, limit, vsp.iota(), scale);
3256 }
3257
3258 @ForceInline
3259 private void conditionalStoreNYI(int offset,
3260 LongSpecies vsp,
3261 VectorMask<Long> m,
3262 int scale,
3263 int limit) {
3264 if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
3265 String msg =
3534 long[] res = new long[laneCount()];
3535 boolean[] mbits = ((AbstractMask<Long>)m).getBits();
3536 for (int i = 0; i < res.length; i++) {
3537 if (mbits[i]) {
3538 res[i] = f.apply(i);
3539 }
3540 }
3541 return dummyVector().vectorFactory(res);
3542 }
3543
3544 /*package-private*/
3545 @ForceInline
3546 <M> LongVector ldOp(M memory, int offset,
3547 FLdOp<M> f) {
3548 return dummyVector().ldOp(memory, offset, f);
3549 }
3550
3551 /*package-private*/
3552 @ForceInline
3553 <M> LongVector ldOp(M memory, int offset,
3554 AbstractMask<Long> m,
3555 FLdOp<M> f) {
3556 return dummyVector().ldOp(memory, offset, m, f);
3557 }
3558
3559 /*package-private*/
3560 @ForceInline
3561 <M> void stOp(M memory, int offset, FStOp<M> f) {
3562 dummyVector().stOp(memory, offset, f);
3563 }
3564
3565 /*package-private*/
3566 @ForceInline
3567 <M> void stOp(M memory, int offset,
3568 AbstractMask<Long> m,
3569 FStOp<M> f) {
3570 dummyVector().stOp(memory, offset, m, f);
3571 }
3572
3573 // N.B. Make sure these constant vectors and
3574 // masks load up correctly into registers.
|
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25 package jdk.incubator.vector;
26
27 import java.nio.ByteBuffer;
28 import java.nio.ByteOrder;
29 import java.nio.ReadOnlyBufferException;
30 import java.util.Arrays;
31 import java.util.Objects;
32 import java.util.function.Function;
33 import java.util.function.UnaryOperator;
34
35 import jdk.internal.misc.ScopedMemoryAccess;
36 import jdk.internal.misc.Unsafe;
37 import jdk.internal.vm.annotation.ForceInline;
38 import jdk.internal.vm.vector.VectorSupport;
39
40 import static jdk.internal.vm.vector.VectorSupport.*;
41 import static jdk.incubator.vector.VectorIntrinsics.*;
42
43 import static jdk.incubator.vector.VectorOperators.*;
44
45 // -- This file was mechanically generated: Do not edit! -- //
46
47 /**
48 * A specialized {@link Vector} representing an ordered immutable sequence of
49 * {@code long} values.
50 */
51 @SuppressWarnings("cast") // warning: redundant cast
155 LongVector uOp(FUnOp f);
156 @ForceInline
157 final
158 LongVector uOpTemplate(FUnOp f) {
159 long[] vec = vec();
160 long[] res = new long[length()];
161 for (int i = 0; i < res.length; i++) {
162 res[i] = f.apply(i, vec[i]);
163 }
164 return vectorFactory(res);
165 }
166
167 /*package-private*/
168 abstract
169 LongVector uOp(VectorMask<Long> m,
170 FUnOp f);
171 @ForceInline
172 final
173 LongVector uOpTemplate(VectorMask<Long> m,
174 FUnOp f) {
175 if (m == null) {
176 return uOpTemplate(f);
177 }
178 long[] vec = vec();
179 long[] res = new long[length()];
180 boolean[] mbits = ((AbstractMask<Long>)m).getBits();
181 for (int i = 0; i < res.length; i++) {
182 res[i] = mbits[i] ? f.apply(i, vec[i]) : vec[i];
183 }
184 return vectorFactory(res);
185 }
186
187 // Binary operator
188
189 /*package-private*/
190 interface FBinOp {
191 long apply(int i, long a, long b);
192 }
193
194 /*package-private*/
195 abstract
196 LongVector bOp(Vector<Long> o,
197 FBinOp f);
201 FBinOp f) {
202 long[] res = new long[length()];
203 long[] vec1 = this.vec();
204 long[] vec2 = ((LongVector)o).vec();
205 for (int i = 0; i < res.length; i++) {
206 res[i] = f.apply(i, vec1[i], vec2[i]);
207 }
208 return vectorFactory(res);
209 }
210
211 /*package-private*/
212 abstract
213 LongVector bOp(Vector<Long> o,
214 VectorMask<Long> m,
215 FBinOp f);
216 @ForceInline
217 final
218 LongVector bOpTemplate(Vector<Long> o,
219 VectorMask<Long> m,
220 FBinOp f) {
221 if (m == null) {
222 return bOpTemplate(o, f);
223 }
224 long[] res = new long[length()];
225 long[] vec1 = this.vec();
226 long[] vec2 = ((LongVector)o).vec();
227 boolean[] mbits = ((AbstractMask<Long>)m).getBits();
228 for (int i = 0; i < res.length; i++) {
229 res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i]) : vec1[i];
230 }
231 return vectorFactory(res);
232 }
233
234 // Ternary operator
235
236 /*package-private*/
237 interface FTriOp {
238 long apply(int i, long a, long b, long c);
239 }
240
241 /*package-private*/
242 abstract
243 LongVector tOp(Vector<Long> o1,
253 long[] vec2 = ((LongVector)o1).vec();
254 long[] vec3 = ((LongVector)o2).vec();
255 for (int i = 0; i < res.length; i++) {
256 res[i] = f.apply(i, vec1[i], vec2[i], vec3[i]);
257 }
258 return vectorFactory(res);
259 }
260
261 /*package-private*/
262 abstract
263 LongVector tOp(Vector<Long> o1,
264 Vector<Long> o2,
265 VectorMask<Long> m,
266 FTriOp f);
267 @ForceInline
268 final
269 LongVector tOpTemplate(Vector<Long> o1,
270 Vector<Long> o2,
271 VectorMask<Long> m,
272 FTriOp f) {
273 if (m == null) {
274 return tOpTemplate(o1, o2, f);
275 }
276 long[] res = new long[length()];
277 long[] vec1 = this.vec();
278 long[] vec2 = ((LongVector)o1).vec();
279 long[] vec3 = ((LongVector)o2).vec();
280 boolean[] mbits = ((AbstractMask<Long>)m).getBits();
281 for (int i = 0; i < res.length; i++) {
282 res[i] = mbits[i] ? f.apply(i, vec1[i], vec2[i], vec3[i]) : vec1[i];
283 }
284 return vectorFactory(res);
285 }
286
287 // Reduction operator
288
289 /*package-private*/
290 abstract
291 long rOp(long v, VectorMask<Long> m, FBinOp f);
292
293 @ForceInline
294 final
295 long rOpTemplate(long v, VectorMask<Long> m, FBinOp f) {
296 if (m == null) {
297 return rOpTemplate(v, f);
298 }
299 long[] vec = vec();
300 boolean[] mbits = ((AbstractMask<Long>)m).getBits();
301 for (int i = 0; i < vec.length; i++) {
302 v = mbits[i] ? f.apply(i, v, vec[i]) : v;
303 }
304 return v;
305 }
306
307 @ForceInline
308 final
309 long rOpTemplate(long v, FBinOp f) {
310 long[] vec = vec();
311 for (int i = 0; i < vec.length; i++) {
312 v = f.apply(i, v, vec[i]);
313 }
314 return v;
315 }
316
317 // Memory reference
318
319 /*package-private*/
320 interface FLdOp<M> {
321 long apply(M memory, int offset, int i);
322 }
323
324 /*package-private*/
325 @ForceInline
326 final
513 return vsp.broadcast(e);
514 }
515
516
517 // Unary lanewise support
518
519 /**
520 * {@inheritDoc} <!--workaround-->
521 */
522 public abstract
523 LongVector lanewise(VectorOperators.Unary op);
524
525 @ForceInline
526 final
527 LongVector lanewiseTemplate(VectorOperators.Unary op) {
528 if (opKind(op, VO_SPECIAL)) {
529 if (op == ZOMO) {
530 return blend(broadcast(-1), compare(NE, 0));
531 }
532 if (op == NOT) {
533 return broadcast(-1).lanewise(XOR, this);
534 } else if (op == NEG) {
535 // FIXME: Support this in the JIT.
536 return broadcast(0).lanewise(SUB, this);
537 }
538 }
539 int opc = opCode(op);
540 return VectorSupport.unaryOp(
541 opc, getClass(), null, long.class, length(),
542 this, null,
543 UN_IMPL.find(op, opc, LongVector::unaryOperations));
544 }
545
546 /**
547 * {@inheritDoc} <!--workaround-->
548 */
549 @Override
550 public abstract
551 LongVector lanewise(VectorOperators.Unary op,
552 VectorMask<Long> m);
553 @ForceInline
554 final
555 LongVector lanewiseTemplate(VectorOperators.Unary op,
556 Class<? extends VectorMask<Long>> maskClass,
557 VectorMask<Long> m) {
558 m.check(maskClass, this);
559 if (opKind(op, VO_SPECIAL)) {
560 if (op == ZOMO) {
561 return blend(broadcast(-1), compare(NE, 0, m));
562 }
563 if (op == NOT) {
564 return lanewise(XOR, broadcast(-1), m);
565 } else if (op == NEG) {
566 return lanewise(NOT, m).lanewise(ADD, broadcast(1), m);
567 }
568 }
569 int opc = opCode(op);
570 return VectorSupport.unaryOp(
571 opc, getClass(), maskClass, long.class, length(),
572 this, m,
573 UN_IMPL.find(op, opc, LongVector::unaryOperations));
574 }
575
576 private static final
577 ImplCache<Unary, UnaryOperation<LongVector, VectorMask<Long>>>
578 UN_IMPL = new ImplCache<>(Unary.class, LongVector.class);
579
580 private static UnaryOperation<LongVector, VectorMask<Long>> unaryOperations(int opc_) {
581 switch (opc_) {
582 case VECTOR_OP_NEG: return (v0, m) ->
583 v0.uOp(m, (i, a) -> (long) -a);
584 case VECTOR_OP_ABS: return (v0, m) ->
585 v0.uOp(m, (i, a) -> (long) Math.abs(a));
586 default: return null;
587 }
588 }
589
590 // Binary lanewise support
591
592 /**
593 * {@inheritDoc} <!--workaround-->
594 * @see #lanewise(VectorOperators.Binary,long)
595 * @see #lanewise(VectorOperators.Binary,long,VectorMask)
596 */
597 @Override
598 public abstract
599 LongVector lanewise(VectorOperators.Binary op,
600 Vector<Long> v);
601 @ForceInline
602 final
603 LongVector lanewiseTemplate(VectorOperators.Binary op,
604 Vector<Long> v) {
605 LongVector that = (LongVector) v;
606 that.check(this);
607
608 if (opKind(op, VO_SPECIAL | VO_SHIFT)) {
609 if (op == FIRST_NONZERO) {
610 // FIXME: Support this in the JIT.
611 VectorMask<Long> thisNZ
612 = this.viewAsIntegralLanes().compare(NE, (long) 0);
613 that = that.blend((long) 0, thisNZ.cast(vspecies()));
614 op = OR_UNCHECKED;
615 }
616 if (opKind(op, VO_SHIFT)) {
617 // As per shift specification for Java, mask the shift count.
618 // This allows the JIT to ignore some ISA details.
619 that = that.lanewise(AND, SHIFT_MASK);
620 }
621 if (op == AND_NOT) {
622 // FIXME: Support this in the JIT.
623 that = that.lanewise(NOT);
624 op = AND;
625 } else if (op == DIV) {
626 VectorMask<Long> eqz = that.eq((long) 0);
627 if (eqz.anyTrue()) {
628 throw that.divZeroException();
629 }
630 }
631 }
632
633 int opc = opCode(op);
634 return VectorSupport.binaryOp(
635 opc, getClass(), null, long.class, length(),
636 this, that, null,
637 BIN_IMPL.find(op, opc, LongVector::binaryOperations));
638 }
639
640 /**
641 * {@inheritDoc} <!--workaround-->
642 * @see #lanewise(VectorOperators.Binary,long,VectorMask)
643 */
644 @Override
645 public abstract
646 LongVector lanewise(VectorOperators.Binary op,
647 Vector<Long> v,
648 VectorMask<Long> m);
649 @ForceInline
650 final
651 LongVector lanewiseTemplate(VectorOperators.Binary op,
652 Class<? extends VectorMask<Long>> maskClass,
653 Vector<Long> v, VectorMask<Long> m) {
654 LongVector that = (LongVector) v;
655 that.check(this);
656 m.check(maskClass, this);
657
658 if (opKind(op, VO_SPECIAL | VO_SHIFT)) {
659 if (op == FIRST_NONZERO) {
660 // FIXME: Support this in the JIT.
661 VectorMask<Long> thisNZ
662 = this.viewAsIntegralLanes().compare(NE, (long) 0);
663 that = that.blend((long) 0, thisNZ.cast(vspecies()));
664 op = OR_UNCHECKED;
665 }
666 if (opKind(op, VO_SHIFT)) {
667 // As per shift specification for Java, mask the shift count.
668 // This allows the JIT to ignore some ISA details.
669 that = that.lanewise(AND, SHIFT_MASK);
670 }
671 if (op == AND_NOT) {
672 // FIXME: Support this in the JIT.
673 that = that.lanewise(NOT);
674 op = AND;
675 } else if (op == DIV) {
676 VectorMask<Long> eqz = that.eq((long)0);
677 if (eqz.and(m).anyTrue()) {
678 throw that.divZeroException();
679 }
680 // suppress div/0 exceptions in unset lanes
681 that = that.lanewise(NOT, eqz);
682 }
683 }
684
685 int opc = opCode(op);
686 return VectorSupport.binaryOp(
687 opc, getClass(), maskClass, long.class, length(),
688 this, that, m,
689 BIN_IMPL.find(op, opc, LongVector::binaryOperations));
690 }
691
692 private static final
693 ImplCache<Binary, BinaryOperation<LongVector, VectorMask<Long>>>
694 BIN_IMPL = new ImplCache<>(Binary.class, LongVector.class);
695
696 private static BinaryOperation<LongVector, VectorMask<Long>> binaryOperations(int opc_) {
697 switch (opc_) {
698 case VECTOR_OP_ADD: return (v0, v1, vm) ->
699 v0.bOp(v1, vm, (i, a, b) -> (long)(a + b));
700 case VECTOR_OP_SUB: return (v0, v1, vm) ->
701 v0.bOp(v1, vm, (i, a, b) -> (long)(a - b));
702 case VECTOR_OP_MUL: return (v0, v1, vm) ->
703 v0.bOp(v1, vm, (i, a, b) -> (long)(a * b));
704 case VECTOR_OP_DIV: return (v0, v1, vm) ->
705 v0.bOp(v1, vm, (i, a, b) -> (long)(a / b));
706 case VECTOR_OP_MAX: return (v0, v1, vm) ->
707 v0.bOp(v1, vm, (i, a, b) -> (long)Math.max(a, b));
708 case VECTOR_OP_MIN: return (v0, v1, vm) ->
709 v0.bOp(v1, vm, (i, a, b) -> (long)Math.min(a, b));
710 case VECTOR_OP_AND: return (v0, v1, vm) ->
711 v0.bOp(v1, vm, (i, a, b) -> (long)(a & b));
712 case VECTOR_OP_OR: return (v0, v1, vm) ->
713 v0.bOp(v1, vm, (i, a, b) -> (long)(a | b));
714 case VECTOR_OP_XOR: return (v0, v1, vm) ->
715 v0.bOp(v1, vm, (i, a, b) -> (long)(a ^ b));
716 case VECTOR_OP_LSHIFT: return (v0, v1, vm) ->
717 v0.bOp(v1, vm, (i, a, n) -> (long)(a << n));
718 case VECTOR_OP_RSHIFT: return (v0, v1, vm) ->
719 v0.bOp(v1, vm, (i, a, n) -> (long)(a >> n));
720 case VECTOR_OP_URSHIFT: return (v0, v1, vm) ->
721 v0.bOp(v1, vm, (i, a, n) -> (long)((a & LSHR_SETUP_MASK) >>> n));
722 case VECTOR_OP_LROTATE: return (v0, v1, vm) ->
723 v0.bOp(v1, vm, (i, a, n) -> rotateLeft(a, (int)n));
724 case VECTOR_OP_RROTATE: return (v0, v1, vm) ->
725 v0.bOp(v1, vm, (i, a, n) -> rotateRight(a, (int)n));
726 default: return null;
727 }
728 }
729
730 // FIXME: Maybe all of the public final methods in this file (the
731 // simple ones that just call lanewise) should be pushed down to
732 // the X-VectorBits template. They can't optimize properly at
733 // this level, and must rely on inlining. Does it work?
734 // (If it works, of course keep the code here.)
735
736 /**
737 * Combines the lane values of this vector
738 * with the value of a broadcast scalar.
739 *
740 * This is a lane-wise binary operation which applies
741 * the selected operation to each lane.
742 * The return value will be equal to this expression:
743 * {@code this.lanewise(op, this.broadcast(e))}.
744 *
745 * @param op the operation used to process lane values
746 * @param e the input scalar
747 * @return the result of applying the operation lane-wise
748 * to the two input vectors
749 * @throws UnsupportedOperationException if this vector does
772 * This is a masked lane-wise binary operation which applies
773 * the selected operation to each lane.
774 * The return value will be equal to this expression:
775 * {@code this.lanewise(op, this.broadcast(e), m)}.
776 *
777 * @param op the operation used to process lane values
778 * @param e the input scalar
779 * @param m the mask controlling lane selection
780 * @return the result of applying the operation lane-wise
781 * to the input vector and the scalar
782 * @throws UnsupportedOperationException if this vector does
783 * not support the requested operation
784 * @see #lanewise(VectorOperators.Binary,Vector,VectorMask)
785 * @see #lanewise(VectorOperators.Binary,long)
786 */
787 @ForceInline
788 public final
789 LongVector lanewise(VectorOperators.Binary op,
790 long e,
791 VectorMask<Long> m) {
792 if (opKind(op, VO_SHIFT) && (long)(int)e == e) {
793 return lanewiseShift(op, (int) e, m);
794 }
795 if (op == AND_NOT) {
796 op = AND; e = (long) ~e;
797 }
798 return lanewise(op, broadcast(e), m);
799 }
800
801
802 /*package-private*/
803 abstract LongVector
804 lanewiseShift(VectorOperators.Binary op, int e);
805
806 /*package-private*/
807 @ForceInline
808 final LongVector
809 lanewiseShiftTemplate(VectorOperators.Binary op, int e) {
810 // Special handling for these. FIXME: Refactor?
811 assert(opKind(op, VO_SHIFT));
812 // As per shift specification for Java, mask the shift count.
813 e &= SHIFT_MASK;
814 int opc = opCode(op);
815 return VectorSupport.broadcastInt(
816 opc, getClass(), null, long.class, length(),
817 this, e, null,
818 BIN_INT_IMPL.find(op, opc, LongVector::broadcastIntOperations));
819 }
820
821 /*package-private*/
822 abstract LongVector
823 lanewiseShift(VectorOperators.Binary op, int e, VectorMask<Long> m);
824
825 /*package-private*/
826 @ForceInline
827 final LongVector
828 lanewiseShiftTemplate(VectorOperators.Binary op,
829 Class<? extends VectorMask<Long>> maskClass,
830 int e, VectorMask<Long> m) {
831 m.check(maskClass, this);
832 assert(opKind(op, VO_SHIFT));
833 // As per shift specification for Java, mask the shift count.
834 e &= SHIFT_MASK;
835 int opc = opCode(op);
836 return VectorSupport.broadcastInt(
837 opc, getClass(), maskClass, long.class, length(),
838 this, e, m,
839 BIN_INT_IMPL.find(op, opc, LongVector::broadcastIntOperations));
840 }
841
842 private static final
843 ImplCache<Binary,VectorBroadcastIntOp<LongVector, VectorMask<Long>>> BIN_INT_IMPL
844 = new ImplCache<>(Binary.class, LongVector.class);
845
846 private static VectorBroadcastIntOp<LongVector, VectorMask<Long>> broadcastIntOperations(int opc_) {
847 switch (opc_) {
848 case VECTOR_OP_LSHIFT: return (v, n, m) ->
849 v.uOp(m, (i, a) -> (long)(a << n));
850 case VECTOR_OP_RSHIFT: return (v, n, m) ->
851 v.uOp(m, (i, a) -> (long)(a >> n));
852 case VECTOR_OP_URSHIFT: return (v, n, m) ->
853 v.uOp(m, (i, a) -> (long)((a & LSHR_SETUP_MASK) >>> n));
854 case VECTOR_OP_LROTATE: return (v, n, m) ->
855 v.uOp(m, (i, a) -> rotateLeft(a, (int)n));
856 case VECTOR_OP_RROTATE: return (v, n, m) ->
857 v.uOp(m, (i, a) -> rotateRight(a, (int)n));
858 default: return null;
859 }
860 }
861
862 // As per shift specification for Java, mask the shift count.
863 // We mask 0X3F (long), 0X1F (int), 0x0F (short), 0x7 (byte).
864 // The latter two maskings go beyond the JLS, but seem reasonable
865 // since our lane types are first-class types, not just dressed
866 // up ints.
867 private static final int SHIFT_MASK = (Long.SIZE - 1);
868 private static final long LSHR_SETUP_MASK = -1;
869
870 // Ternary lanewise support
871
872 // Ternary operators come in eight variations:
873 // lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2])
874 // lanewise(op, [broadcast(e1)|v1], [broadcast(e2)|v2], mask)
875
876 // It is annoying to support all of these variations of masking
877 // and broadcast, but it would be more surprising not to continue
878 // the obvious pattern started by unary and binary.
879
880 /**
881 * {@inheritDoc} <!--workaround-->
893 Vector<Long> v2);
894 @ForceInline
895 final
896 LongVector lanewiseTemplate(VectorOperators.Ternary op,
897 Vector<Long> v1,
898 Vector<Long> v2) {
899 LongVector that = (LongVector) v1;
900 LongVector tother = (LongVector) v2;
901 // It's a word: https://www.dictionary.com/browse/tother
902 // See also Chapter 11 of Dickens, Our Mutual Friend:
903 // "Totherest Governor," replied Mr Riderhood...
904 that.check(this);
905 tother.check(this);
906 if (op == BITWISE_BLEND) {
907 // FIXME: Support this in the JIT.
908 that = this.lanewise(XOR, that).lanewise(AND, tother);
909 return this.lanewise(XOR, that);
910 }
911 int opc = opCode(op);
912 return VectorSupport.ternaryOp(
913 opc, getClass(), null, long.class, length(),
914 this, that, tother, null,
915 TERN_IMPL.find(op, opc, LongVector::ternaryOperations));
916 }
917
918 /**
919 * {@inheritDoc} <!--workaround-->
920 * @see #lanewise(VectorOperators.Ternary,long,long,VectorMask)
921 * @see #lanewise(VectorOperators.Ternary,Vector,long,VectorMask)
922 * @see #lanewise(VectorOperators.Ternary,long,Vector,VectorMask)
923 */
924 @Override
925 public abstract
926 LongVector lanewise(VectorOperators.Ternary op,
927 Vector<Long> v1,
928 Vector<Long> v2,
929 VectorMask<Long> m);
930 @ForceInline
931 final
932 LongVector lanewiseTemplate(VectorOperators.Ternary op,
933 Class<? extends VectorMask<Long>> maskClass,
934 Vector<Long> v1,
935 Vector<Long> v2,
936 VectorMask<Long> m) {
937 LongVector that = (LongVector) v1;
938 LongVector tother = (LongVector) v2;
939 // It's a word: https://www.dictionary.com/browse/tother
940 // See also Chapter 11 of Dickens, Our Mutual Friend:
941 // "Totherest Governor," replied Mr Riderhood...
942 that.check(this);
943 tother.check(this);
944 m.check(maskClass, this);
945
946 if (op == BITWISE_BLEND) {
947 // FIXME: Support this in the JIT.
948 that = this.lanewise(XOR, that).lanewise(AND, tother);
949 return this.lanewise(XOR, that, m);
950 }
951 int opc = opCode(op);
952 return VectorSupport.ternaryOp(
953 opc, getClass(), maskClass, long.class, length(),
954 this, that, tother, m,
955 TERN_IMPL.find(op, opc, LongVector::ternaryOperations));
956 }
957
958 private static final
959 ImplCache<Ternary, TernaryOperation<LongVector, VectorMask<Long>>>
960 TERN_IMPL = new ImplCache<>(Ternary.class, LongVector.class);
961
962 private static TernaryOperation<LongVector, VectorMask<Long>> ternaryOperations(int opc_) {
963 switch (opc_) {
964 default: return null;
965 }
966 }
967
968 /**
969 * Combines the lane values of this vector
970 * with the values of two broadcast scalars.
971 *
972 * This is a lane-wise ternary operation which applies
973 * the selected operation to each lane.
974 * The return value will be equal to this expression:
975 * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2))}.
976 *
977 * @param op the operation used to combine lane values
978 * @param e1 the first input scalar
979 * @param e2 the second input scalar
980 * @return the result of applying the operation lane-wise
981 * to the input vector and the scalars
982 * @throws UnsupportedOperationException if this vector does
983 * not support the requested operation
984 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
985 * @see #lanewise(VectorOperators.Ternary,long,long,VectorMask)
1002 * The return value will be equal to this expression:
1003 * {@code this.lanewise(op, this.broadcast(e1), this.broadcast(e2), m)}.
1004 *
1005 * @param op the operation used to combine lane values
1006 * @param e1 the first input scalar
1007 * @param e2 the second input scalar
1008 * @param m the mask controlling lane selection
1009 * @return the result of applying the operation lane-wise
1010 * to the input vector and the scalars
1011 * @throws UnsupportedOperationException if this vector does
1012 * not support the requested operation
1013 * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1014 * @see #lanewise(VectorOperators.Ternary,long,long)
1015 */
1016 @ForceInline
1017 public final
1018 LongVector lanewise(VectorOperators.Ternary op, //(op,e1,e2,m)
1019 long e1,
1020 long e2,
1021 VectorMask<Long> m) {
1022 return lanewise(op, broadcast(e1), broadcast(e2), m);
1023 }
1024
1025 /**
1026 * Combines the lane values of this vector
1027 * with the values of another vector and a broadcast scalar.
1028 *
1029 * This is a lane-wise ternary operation which applies
1030 * the selected operation to each lane.
1031 * The return value will be equal to this expression:
1032 * {@code this.lanewise(op, v1, this.broadcast(e2))}.
1033 *
1034 * @param op the operation used to combine lane values
1035 * @param v1 the other input vector
1036 * @param e2 the input scalar
1037 * @return the result of applying the operation lane-wise
1038 * to the input vectors and the scalar
1039 * @throws UnsupportedOperationException if this vector does
1040 * not support the requested operation
1041 * @see #lanewise(VectorOperators.Ternary,long,long)
1042 * @see #lanewise(VectorOperators.Ternary,Vector,long,VectorMask)
1060 * {@code this.lanewise(op, v1, this.broadcast(e2), m)}.
1061 *
1062 * @param op the operation used to combine lane values
1063 * @param v1 the other input vector
1064 * @param e2 the input scalar
1065 * @param m the mask controlling lane selection
1066 * @return the result of applying the operation lane-wise
1067 * to the input vectors and the scalar
1068 * @throws UnsupportedOperationException if this vector does
1069 * not support the requested operation
1070 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1071 * @see #lanewise(VectorOperators.Ternary,long,long,VectorMask)
1072 * @see #lanewise(VectorOperators.Ternary,Vector,long)
1073 */
1074 @ForceInline
1075 public final
1076 LongVector lanewise(VectorOperators.Ternary op, //(op,v1,e2,m)
1077 Vector<Long> v1,
1078 long e2,
1079 VectorMask<Long> m) {
1080 return lanewise(op, v1, broadcast(e2), m);
1081 }
1082
1083 /**
1084 * Combines the lane values of this vector
1085 * with the values of another vector and a broadcast scalar.
1086 *
1087 * This is a lane-wise ternary operation which applies
1088 * the selected operation to each lane.
1089 * The return value will be equal to this expression:
1090 * {@code this.lanewise(op, this.broadcast(e1), v2)}.
1091 *
1092 * @param op the operation used to combine lane values
1093 * @param e1 the input scalar
1094 * @param v2 the other input vector
1095 * @return the result of applying the operation lane-wise
1096 * to the input vectors and the scalar
1097 * @throws UnsupportedOperationException if this vector does
1098 * not support the requested operation
1099 * @see #lanewise(VectorOperators.Ternary,Vector,Vector)
1100 * @see #lanewise(VectorOperators.Ternary,long,Vector,VectorMask)
1117 * The return value will be equal to this expression:
1118 * {@code this.lanewise(op, this.broadcast(e1), v2, m)}.
1119 *
1120 * @param op the operation used to combine lane values
1121 * @param e1 the input scalar
1122 * @param v2 the other input vector
1123 * @param m the mask controlling lane selection
1124 * @return the result of applying the operation lane-wise
1125 * to the input vectors and the scalar
1126 * @throws UnsupportedOperationException if this vector does
1127 * not support the requested operation
1128 * @see #lanewise(VectorOperators.Ternary,Vector,Vector,VectorMask)
1129 * @see #lanewise(VectorOperators.Ternary,long,Vector)
1130 */
1131 @ForceInline
1132 public final
1133 LongVector lanewise(VectorOperators.Ternary op, //(op,e1,v2,m)
1134 long e1,
1135 Vector<Long> v2,
1136 VectorMask<Long> m) {
1137 return lanewise(op, broadcast(e1), v2, m);
1138 }
1139
1140 // (Thus endeth the Great and Mighty Ternary Ogdoad.)
1141 // https://en.wikipedia.org/wiki/Ogdoad
1142
1143 /// FULL-SERVICE BINARY METHODS: ADD, SUB, MUL, DIV
1144 //
1145 // These include masked and non-masked versions.
1146 // This subclass adds broadcast (masked or not).
1147
1148 /**
1149 * {@inheritDoc} <!--workaround-->
1150 * @see #add(long)
1151 */
1152 @Override
1153 @ForceInline
1154 public final LongVector add(Vector<Long> v) {
1155 return lanewise(ADD, v);
1156 }
1157
1789 @Override
1790 @ForceInline
1791 public final
1792 VectorMask<Long> test(VectorOperators.Test op,
1793 VectorMask<Long> m) {
1794 return test(op).and(m);
1795 }
1796
1797 /**
1798 * {@inheritDoc} <!--workaround-->
1799 */
1800 @Override
1801 public abstract
1802 VectorMask<Long> compare(VectorOperators.Comparison op, Vector<Long> v);
1803
1804 /*package-private*/
1805 @ForceInline
1806 final
1807 <M extends VectorMask<Long>>
1808 M compareTemplate(Class<M> maskType, Comparison op, Vector<Long> v) {
1809 LongVector that = (LongVector) v;
1810 that.check(this);
1811 int opc = opCode(op);
1812 return VectorSupport.compare(
1813 opc, getClass(), maskType, long.class, length(),
1814 this, that, null,
1815 (cond, v0, v1, m1) -> {
1816 AbstractMask<Long> m
1817 = v0.bTest(cond, v1, (cond_, i, a, b)
1818 -> compareWithOp(cond, a, b));
1819 @SuppressWarnings("unchecked")
1820 M m2 = (M) m;
1821 return m2;
1822 });
1823 }
1824
1825 /*package-private*/
1826 @ForceInline
1827 final
1828 <M extends VectorMask<Long>>
1829 M compareTemplate(Class<M> maskType, Comparison op, Vector<Long> v, M m) {
1830 LongVector that = (LongVector) v;
1831 that.check(this);
1832 m.check(maskType, this);
1833 int opc = opCode(op);
1834 return VectorSupport.compare(
1835 opc, getClass(), maskType, long.class, length(),
1836 this, that, m,
1837 (cond, v0, v1, m1) -> {
1838 AbstractMask<Long> cmpM
1839 = v0.bTest(cond, v1, (cond_, i, a, b)
1840 -> compareWithOp(cond, a, b));
1841 @SuppressWarnings("unchecked")
1842 M m2 = (M) cmpM.and(m1);
1843 return m2;
1844 });
1845 }
1846
1847 @ForceInline
1848 private static boolean compareWithOp(int cond, long a, long b) {
1849 return switch (cond) {
1850 case BT_eq -> a == b;
1851 case BT_ne -> a != b;
1852 case BT_lt -> a < b;
1853 case BT_le -> a <= b;
1854 case BT_gt -> a > b;
1855 case BT_ge -> a >= b;
1856 case BT_ult -> Long.compareUnsigned(a, b) < 0;
1857 case BT_ule -> Long.compareUnsigned(a, b) <= 0;
1858 case BT_ugt -> Long.compareUnsigned(a, b) > 0;
1859 case BT_uge -> Long.compareUnsigned(a, b) >= 0;
1860 default -> throw new AssertionError();
1861 };
1862 }
1863
1864 /**
1865 * Tests this vector by comparing it with an input scalar,
1866 * according to the given comparison operation.
1867 *
1868 * This is a lane-wise binary test operation which applies
1869 * the comparison operation to each lane.
1870 * <p>
1871 * The result is the same as
1872 * {@code compare(op, broadcast(species(), e))}.
1873 * That is, the scalar may be regarded as broadcast to
1874 * a vector of the same species, and then compared
1875 * against the original vector, using the selected
1876 * comparison operation.
1877 *
1878 * @param op the operation used to compare lane values
1879 * @param e the input scalar
1880 * @return the mask result of testing lane-wise if this vector
1881 * compares to the input, according to the selected
1882 * comparison operator
1883 * @see LongVector#compare(VectorOperators.Comparison,Vector)
1902 *
1903 * This is a masked lane-wise binary test operation which applies
1904 * to each pair of corresponding lane values.
1905 *
1906 * The returned result is equal to the expression
1907 * {@code compare(op,s).and(m)}.
1908 *
1909 * @param op the operation used to compare lane values
1910 * @param e the input scalar
1911 * @param m the mask controlling lane selection
1912 * @return the mask result of testing lane-wise if this vector
1913 * compares to the input, according to the selected
1914 * comparison operator,
1915 * and only in the lanes selected by the mask
1916 * @see LongVector#compare(VectorOperators.Comparison,Vector,VectorMask)
1917 */
1918 @ForceInline
1919 public final VectorMask<Long> compare(VectorOperators.Comparison op,
1920 long e,
1921 VectorMask<Long> m) {
1922 return compare(op, broadcast(e), m);
1923 }
1924
1925
1926 /**
1927 * {@inheritDoc} <!--workaround-->
1928 */
1929 @Override public abstract
1930 LongVector blend(Vector<Long> v, VectorMask<Long> m);
1931
1932 /*package-private*/
1933 @ForceInline
1934 final
1935 <M extends VectorMask<Long>>
1936 LongVector
1937 blendTemplate(Class<M> maskType, LongVector v, M m) {
1938 v.check(this);
1939 return VectorSupport.blend(
1940 getClass(), maskType, long.class, length(),
1941 this, v, m,
1942 (v0, v1, m_) -> v0.bOp(v1, m_, (i, a, b) -> b));
2106 wrongPartForSlice(int part) {
2107 String msg = String.format("bad part number %d for slice operation",
2108 part);
2109 return new ArrayIndexOutOfBoundsException(msg);
2110 }
2111
2112 /**
2113 * {@inheritDoc} <!--workaround-->
2114 */
2115 @Override
2116 public abstract
2117 LongVector rearrange(VectorShuffle<Long> m);
2118
2119 /*package-private*/
2120 @ForceInline
2121 final
2122 <S extends VectorShuffle<Long>>
2123 LongVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
2124 shuffle.checkIndexes();
2125 return VectorSupport.rearrangeOp(
2126 getClass(), shuffletype, null, long.class, length(),
2127 this, shuffle, null,
2128 (v1, s_, m_) -> v1.uOp((i, a) -> {
2129 int ei = s_.laneSource(i);
2130 return v1.lane(ei);
2131 }));
2132 }
2133
2134 /**
2135 * {@inheritDoc} <!--workaround-->
2136 */
2137 @Override
2138 public abstract
2139 LongVector rearrange(VectorShuffle<Long> s,
2140 VectorMask<Long> m);
2141
2142 /*package-private*/
2143 @ForceInline
2144 final
2145 <S extends VectorShuffle<Long>, M extends VectorMask<Long>>
2146 LongVector rearrangeTemplate(Class<S> shuffletype,
2147 Class<M> masktype,
2148 S shuffle,
2149 M m) {
2150
2151 m.check(masktype, this);
2152 VectorMask<Long> valid = shuffle.laneIsValid();
2153 if (m.andNot(valid).anyTrue()) {
2154 shuffle.checkIndexes();
2155 throw new AssertionError();
2156 }
2157 return VectorSupport.rearrangeOp(
2158 getClass(), shuffletype, masktype, long.class, length(),
2159 this, shuffle, m,
2160 (v1, s_, m_) -> v1.uOp((i, a) -> {
2161 int ei = s_.laneSource(i);
2162 return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
2163 }));
2164 }
2165
2166 /**
2167 * {@inheritDoc} <!--workaround-->
2168 */
2169 @Override
2170 public abstract
2171 LongVector rearrange(VectorShuffle<Long> s,
2172 Vector<Long> v);
2173
2174 /*package-private*/
2175 @ForceInline
2176 final
2177 <S extends VectorShuffle<Long>>
2178 LongVector rearrangeTemplate(Class<S> shuffletype,
2179 S shuffle,
2180 LongVector v) {
2181 VectorMask<Long> valid = shuffle.laneIsValid();
2182 @SuppressWarnings("unchecked")
2183 S ws = (S) shuffle.wrapIndexes();
2184 LongVector r0 =
2185 VectorSupport.rearrangeOp(
2186 getClass(), shuffletype, null, long.class, length(),
2187 this, ws, null,
2188 (v0, s_, m_) -> v0.uOp((i, a) -> {
2189 int ei = s_.laneSource(i);
2190 return v0.lane(ei);
2191 }));
2192 LongVector r1 =
2193 VectorSupport.rearrangeOp(
2194 getClass(), shuffletype, null, long.class, length(),
2195 v, ws, null,
2196 (v1, s_, m_) -> v1.uOp((i, a) -> {
2197 int ei = s_.laneSource(i);
2198 return v1.lane(ei);
2199 }));
2200 return r1.blend(r0, valid);
2201 }
2202
2203 @ForceInline
2204 private final
2205 VectorShuffle<Long> toShuffle0(LongSpecies dsp) {
2206 long[] a = toArray();
2207 int[] sa = new int[a.length];
2208 for (int i = 0; i < a.length; i++) {
2209 sa[i] = (int) a[i];
2210 }
2211 return VectorShuffle.fromArray(dsp, sa, 0);
2212 }
2213
2214 /*package-private*/
2215 @ForceInline
2216 final
2439 * <li>
2440 * All other reduction operations are fully commutative and
2441 * associative. The implementation can choose any order of
2442 * processing, yet it will always produce the same result.
2443 * </ul>
2444 *
2445 * @param op the operation used to combine lane values
2446 * @param m the mask controlling lane selection
2447 * @return the reduced result accumulated from the selected lane values
2448 * @throws UnsupportedOperationException if this vector does
2449 * not support the requested operation
2450 * @see #reduceLanes(VectorOperators.Associative)
2451 */
2452 public abstract long reduceLanes(VectorOperators.Associative op,
2453 VectorMask<Long> m);
2454
2455 /*package-private*/
2456 @ForceInline
2457 final
2458 long reduceLanesTemplate(VectorOperators.Associative op,
2459 Class<? extends VectorMask<Long>> maskClass,
2460 VectorMask<Long> m) {
2461 m.check(maskClass, this);
2462 if (op == FIRST_NONZERO) {
2463 LongVector v = reduceIdentityVector(op).blend(this, m);
2464 return v.reduceLanesTemplate(op);
2465 }
2466 int opc = opCode(op);
2467 return fromBits(VectorSupport.reductionCoerced(
2468 opc, getClass(), maskClass, long.class, length(),
2469 this, m,
2470 REDUCE_IMPL.find(op, opc, LongVector::reductionOperations)));
2471 }
2472
2473 /*package-private*/
2474 @ForceInline
2475 final
2476 long reduceLanesTemplate(VectorOperators.Associative op) {
2477 if (op == FIRST_NONZERO) {
2478 // FIXME: The JIT should handle this, and other scan ops alos.
2479 VectorMask<Long> thisNZ
2480 = this.viewAsIntegralLanes().compare(NE, (long) 0);
2481 return this.lane(thisNZ.firstTrue());
2482 }
2483 int opc = opCode(op);
2484 return fromBits(VectorSupport.reductionCoerced(
2485 opc, getClass(), null, long.class, length(),
2486 this, null,
2487 REDUCE_IMPL.find(op, opc, LongVector::reductionOperations)));
2488 }
2489
2490 private static final
2491 ImplCache<Associative, ReductionOperation<LongVector, VectorMask<Long>>>
2492 REDUCE_IMPL = new ImplCache<>(Associative.class, LongVector.class);
2493
2494 private static ReductionOperation<LongVector, VectorMask<Long>> reductionOperations(int opc_) {
2495 switch (opc_) {
2496 case VECTOR_OP_ADD: return (v, m) ->
2497 toBits(v.rOp((long)0, m, (i, a, b) -> (long)(a + b)));
2498 case VECTOR_OP_MUL: return (v, m) ->
2499 toBits(v.rOp((long)1, m, (i, a, b) -> (long)(a * b)));
2500 case VECTOR_OP_MIN: return (v, m) ->
2501 toBits(v.rOp(MAX_OR_INF, m, (i, a, b) -> (long) Math.min(a, b)));
2502 case VECTOR_OP_MAX: return (v, m) ->
2503 toBits(v.rOp(MIN_OR_INF, m, (i, a, b) -> (long) Math.max(a, b)));
2504 case VECTOR_OP_AND: return (v, m) ->
2505 toBits(v.rOp((long)-1, m, (i, a, b) -> (long)(a & b)));
2506 case VECTOR_OP_OR: return (v, m) ->
2507 toBits(v.rOp((long)0, m, (i, a, b) -> (long)(a | b)));
2508 case VECTOR_OP_XOR: return (v, m) ->
2509 toBits(v.rOp((long)0, m, (i, a, b) -> (long)(a ^ b)));
2510 default: return null;
2511 }
2512 }
2513
2514 private
2515 @ForceInline
2516 LongVector reduceIdentityVector(VectorOperators.Associative op) {
2517 int opc = opCode(op);
2518 UnaryOperator<LongVector> fn
2519 = REDUCE_ID_IMPL.find(op, opc, (opc_) -> {
2520 switch (opc_) {
2521 case VECTOR_OP_ADD:
2522 case VECTOR_OP_OR:
2523 case VECTOR_OP_XOR:
2524 return v -> v.broadcast(0);
2525 case VECTOR_OP_MUL:
2526 return v -> v.broadcast(1);
2527 case VECTOR_OP_AND:
2528 return v -> v.broadcast(-1);
2529 case VECTOR_OP_MIN:
2530 return v -> v.broadcast(MAX_OR_INF);
2531 case VECTOR_OP_MAX:
2532 return v -> v.broadcast(MIN_OR_INF);
2706 * @param species species of desired vector
2707 * @param a the byte array
2708 * @param offset the offset into the array
2709 * @param bo the intended byte order
2710 * @param m the mask controlling lane selection
2711 * @return a vector loaded from a byte array
2712 * @throws IndexOutOfBoundsException
2713 * if {@code offset+N*ESIZE < 0}
2714 * or {@code offset+(N+1)*ESIZE > a.length}
2715 * for any lane {@code N} in the vector
2716 * where the mask is set
2717 */
2718 @ForceInline
2719 public static
2720 LongVector fromByteArray(VectorSpecies<Long> species,
2721 byte[] a, int offset,
2722 ByteOrder bo,
2723 VectorMask<Long> m) {
2724 LongSpecies vsp = (LongSpecies) species;
2725 if (offset >= 0 && offset <= (a.length - species.vectorByteSize())) {
2726 return vsp.dummyVector().fromByteArray0(a, offset, m).maybeSwap(bo);
2727 }
2728
2729 // FIXME: optimize
2730 checkMaskFromIndexSize(offset, vsp, m, 8, a.length);
2731 ByteBuffer wb = wrapper(a, bo);
2732 return vsp.ldOp(wb, offset, (AbstractMask<Long>)m,
2733 (wb_, o, i) -> wb_.getLong(o + i * 8));
2734 }
2735
2736 /**
2737 * Loads a vector from an array of type {@code long[]}
2738 * starting at an offset.
2739 * For each vector lane, where {@code N} is the vector lane index, the
2740 * array element at index {@code offset + N} is placed into the
2741 * resulting vector at lane index {@code N}.
2742 *
2743 * @param species species of desired vector
2744 * @param a the array
2745 * @param offset the offset into the array
2746 * @return the vector loaded from an array
2768 * {@code N}, otherwise the default element value is placed into the
2769 * resulting vector at lane index {@code N}.
2770 *
2771 * @param species species of desired vector
2772 * @param a the array
2773 * @param offset the offset into the array
2774 * @param m the mask controlling lane selection
2775 * @return the vector loaded from an array
2776 * @throws IndexOutOfBoundsException
2777 * if {@code offset+N < 0} or {@code offset+N >= a.length}
2778 * for any lane {@code N} in the vector
2779 * where the mask is set
2780 */
2781 @ForceInline
2782 public static
2783 LongVector fromArray(VectorSpecies<Long> species,
2784 long[] a, int offset,
2785 VectorMask<Long> m) {
2786 LongSpecies vsp = (LongSpecies) species;
2787 if (offset >= 0 && offset <= (a.length - species.length())) {
2788 return vsp.dummyVector().fromArray0(a, offset, m);
2789 }
2790
2791 // FIXME: optimize
2792 checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
2793 return vsp.vOp(m, i -> a[offset + i]);
2794 }
2795
2796 /**
2797 * Gathers a new vector composed of elements from an array of type
2798 * {@code long[]},
2799 * using indexes obtained by adding a fixed {@code offset} to a
2800 * series of secondary offsets from an <em>index map</em>.
2801 * The index map is a contiguous sequence of {@code VLENGTH}
2802 * elements in a second array of {@code int}s, starting at a given
2803 * {@code mapOffset}.
2804 * <p>
2805 * For each vector lane, where {@code N} is the vector lane index,
2806 * the lane is loaded from the array
2807 * element {@code a[f(N)]}, where {@code f(N)} is the
2808 * index mapping expression
2844 if (isp.laneCount() != vsp.laneCount()) {
2845 // For LongMaxVector, if vector length is non-power-of-two or
2846 // 2048 bits, indexShape of Long species is S_MAX_BIT.
2847 // Assume that vector length is 2048, then the lane count of Long
2848 // vector is 32. When converting Long species to int species,
2849 // indexShape is still S_MAX_BIT, but the lane count of int vector
2850 // is 64. So when loading index vector (IntVector), only lower half
2851 // of index data is needed.
2852 vix = IntVector
2853 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
2854 .add(offset);
2855 } else {
2856 vix = IntVector
2857 .fromArray(isp, indexMap, mapOffset)
2858 .add(offset);
2859 }
2860
2861 vix = VectorIntrinsics.checkIndex(vix, a.length);
2862
2863 return VectorSupport.loadWithMap(
2864 vectorType, null, long.class, vsp.laneCount(),
2865 isp.vectorType(),
2866 a, ARRAY_BASE, vix, null,
2867 a, offset, indexMap, mapOffset, vsp,
2868 (c, idx, iMap, idy, s, vm) ->
2869 s.vOp(n -> c[idx + iMap[idy+n]]));
2870 }
2871
2872 /**
2873 * Gathers a new vector composed of elements from an array of type
2874 * {@code long[]},
2875 * under the control of a mask, and
2876 * using indexes obtained by adding a fixed {@code offset} to a
2877 * series of secondary offsets from an <em>index map</em>.
2878 * The index map is a contiguous sequence of {@code VLENGTH}
2879 * elements in a second array of {@code int}s, starting at a given
2880 * {@code mapOffset}.
2881 * <p>
2882 * For each vector lane, where {@code N} is the vector lane index,
2883 * if the lane is set in the mask,
2884 * the lane is loaded from the array
2885 * element {@code a[f(N)]}, where {@code f(N)} is the
2886 * index mapping expression
2887 * {@code offset + indexMap[mapOffset + N]]}.
2888 * Unset lanes in the resulting vector are set to zero.
2889 *
2890 * @param species species of desired vector
2898 * @return the vector loaded from the indexed elements of the array
2899 * @throws IndexOutOfBoundsException
2900 * if {@code mapOffset+N < 0}
2901 * or if {@code mapOffset+N >= indexMap.length},
2902 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
2903 * is an invalid index into {@code a},
2904 * for any lane {@code N} in the vector
2905 * where the mask is set
2906 * @see LongVector#toIntArray()
2907 */
2908 @ForceInline
2909 public static
2910 LongVector fromArray(VectorSpecies<Long> species,
2911 long[] a, int offset,
2912 int[] indexMap, int mapOffset,
2913 VectorMask<Long> m) {
2914 if (m.allTrue()) {
2915 return fromArray(species, a, offset, indexMap, mapOffset);
2916 }
2917 else {
2918 LongSpecies vsp = (LongSpecies) species;
2919 return vsp.dummyVector().fromArray0(a, offset, indexMap, mapOffset, m);
2920 }
2921 }
2922
2923
2924
2925 /**
2926 * Loads a vector from a {@linkplain ByteBuffer byte buffer}
2927 * starting at an offset into the byte buffer.
2928 * Bytes are composed into primitive lane elements according
2929 * to the specified byte order.
2930 * The vector is arranged into lanes according to
2931 * <a href="Vector.html#lane-order">memory ordering</a>.
2932 * <p>
2933 * This method behaves as if it returns the result of calling
2934 * {@link #fromByteBuffer(VectorSpecies,ByteBuffer,int,ByteOrder,VectorMask)
2935 * fromByteBuffer()} as follows:
2936 * <pre>{@code
2937 * var m = species.maskAll(true);
2938 * return fromByteBuffer(species, bb, offset, bo, m);
2939 * }</pre>
2993 * @param species species of desired vector
2994 * @param bb the byte buffer
2995 * @param offset the offset into the byte buffer
2996 * @param bo the intended byte order
2997 * @param m the mask controlling lane selection
2998 * @return a vector loaded from a byte buffer
2999 * @throws IndexOutOfBoundsException
3000 * if {@code offset+N*8 < 0}
3001 * or {@code offset+N*8 >= bb.limit()}
3002 * for any lane {@code N} in the vector
3003 * where the mask is set
3004 */
3005 @ForceInline
3006 public static
3007 LongVector fromByteBuffer(VectorSpecies<Long> species,
3008 ByteBuffer bb, int offset,
3009 ByteOrder bo,
3010 VectorMask<Long> m) {
3011 LongSpecies vsp = (LongSpecies) species;
3012 if (offset >= 0 && offset <= (bb.limit() - species.vectorByteSize())) {
3013 return vsp.dummyVector().fromByteBuffer0(bb, offset, m).maybeSwap(bo);
3014 }
3015
3016 // FIXME: optimize
3017 checkMaskFromIndexSize(offset, vsp, m, 8, bb.limit());
3018 ByteBuffer wb = wrapper(bb, bo);
3019 return vsp.ldOp(wb, offset, (AbstractMask<Long>)m,
3020 (wb_, o, i) -> wb_.getLong(o + i * 8));
3021 }
3022
3023 // Memory store operations
3024
3025 /**
3026 * Stores this vector into an array of type {@code long[]}
3027 * starting at an offset.
3028 * <p>
3029 * For each vector lane, where {@code N} is the vector lane index,
3030 * the lane element at index {@code N} is stored into the array
3031 * element {@code a[offset+N]}.
3032 *
3033 * @param a the array, of type {@code long[]}
3065 * Lanes where the mask is unset are not stored and do not need
3066 * to correspond to legitimate elements of {@code a}.
3067 * That is, unset lanes may correspond to array indexes less than
3068 * zero or beyond the end of the array.
3069 *
3070 * @param a the array, of type {@code long[]}
3071 * @param offset the offset into the array
3072 * @param m the mask controlling lane storage
3073 * @throws IndexOutOfBoundsException
3074 * if {@code offset+N < 0} or {@code offset+N >= a.length}
3075 * for any lane {@code N} in the vector
3076 * where the mask is set
3077 */
3078 @ForceInline
3079 public final
3080 void intoArray(long[] a, int offset,
3081 VectorMask<Long> m) {
3082 if (m.allTrue()) {
3083 intoArray(a, offset);
3084 } else {
3085 LongSpecies vsp = vspecies();
3086 checkMaskFromIndexSize(offset, vsp, m, 1, a.length);
3087 intoArray0(a, offset, m);
3088 }
3089 }
3090
3091 /**
3092 * Scatters this vector into an array of type {@code long[]}
3093 * using indexes obtained by adding a fixed {@code offset} to a
3094 * series of secondary offsets from an <em>index map</em>.
3095 * The index map is a contiguous sequence of {@code VLENGTH}
3096 * elements in a second array of {@code int}s, starting at a given
3097 * {@code mapOffset}.
3098 * <p>
3099 * For each vector lane, where {@code N} is the vector lane index,
3100 * the lane element at index {@code N} is stored into the array
3101 * element {@code a[f(N)]}, where {@code f(N)} is the
3102 * index mapping expression
3103 * {@code offset + indexMap[mapOffset + N]]}.
3104 *
3105 * @param a the array
3106 * @param offset an offset to combine with the index map offsets
3107 * @param indexMap the index map
3130 if (isp.laneCount() != vsp.laneCount()) {
3131 // For LongMaxVector, if vector length is 2048 bits, indexShape
3132 // of Long species is S_MAX_BIT. and the lane count of Long
3133 // vector is 32. When converting Long species to int species,
3134 // indexShape is still S_MAX_BIT, but the lane count of int vector
3135 // is 64. So when loading index vector (IntVector), only lower half
3136 // of index data is needed.
3137 vix = IntVector
3138 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3139 .add(offset);
3140 } else {
3141 vix = IntVector
3142 .fromArray(isp, indexMap, mapOffset)
3143 .add(offset);
3144 }
3145
3146
3147 vix = VectorIntrinsics.checkIndex(vix, a.length);
3148
3149 VectorSupport.storeWithMap(
3150 vsp.vectorType(), null, vsp.elementType(), vsp.laneCount(),
3151 isp.vectorType(),
3152 a, arrayAddress(a, 0), vix,
3153 this, null,
3154 a, offset, indexMap, mapOffset,
3155 (arr, off, v, map, mo, vm)
3156 -> v.stOp(arr, off,
3157 (arr_, off_, i, e) -> {
3158 int j = map[mo + i];
3159 arr[off + j] = e;
3160 }));
3161 }
3162
3163 /**
3164 * Scatters this vector into an array of type {@code long[]},
3165 * under the control of a mask, and
3166 * using indexes obtained by adding a fixed {@code offset} to a
3167 * series of secondary offsets from an <em>index map</em>.
3168 * The index map is a contiguous sequence of {@code VLENGTH}
3169 * elements in a second array of {@code int}s, starting at a given
3170 * {@code mapOffset}.
3171 * <p>
3172 * For each vector lane, where {@code N} is the vector lane index,
3173 * if the mask lane at index {@code N} is set then
3174 * the lane element at index {@code N} is stored into the array
3175 * element {@code a[f(N)]}, where {@code f(N)} is the
3182 * @param mapOffset the offset into the index map
3183 * @param m the mask
3184 * @throws IndexOutOfBoundsException
3185 * if {@code mapOffset+N < 0}
3186 * or if {@code mapOffset+N >= indexMap.length},
3187 * or if {@code f(N)=offset+indexMap[mapOffset+N]}
3188 * is an invalid index into {@code a},
3189 * for any lane {@code N} in the vector
3190 * where the mask is set
3191 * @see LongVector#toIntArray()
3192 */
3193 @ForceInline
3194 public final
3195 void intoArray(long[] a, int offset,
3196 int[] indexMap, int mapOffset,
3197 VectorMask<Long> m) {
3198 if (m.allTrue()) {
3199 intoArray(a, offset, indexMap, mapOffset);
3200 }
3201 else {
3202 intoArray0(a, offset, indexMap, mapOffset, m);
3203 }
3204 }
3205
3206
3207
3208 /**
3209 * {@inheritDoc} <!--workaround-->
3210 */
3211 @Override
3212 @ForceInline
3213 public final
3214 void intoByteArray(byte[] a, int offset,
3215 ByteOrder bo) {
3216 offset = checkFromIndexSize(offset, byteSize(), a.length);
3217 maybeSwap(bo).intoByteArray0(a, offset);
3218 }
3219
3220 /**
3221 * {@inheritDoc} <!--workaround-->
3222 */
3223 @Override
3224 @ForceInline
3225 public final
3226 void intoByteArray(byte[] a, int offset,
3227 ByteOrder bo,
3228 VectorMask<Long> m) {
3229 if (m.allTrue()) {
3230 intoByteArray(a, offset, bo);
3231 } else {
3232 LongSpecies vsp = vspecies();
3233 checkMaskFromIndexSize(offset, vsp, m, 8, a.length);
3234 maybeSwap(bo).intoByteArray0(a, offset, m);
3235 }
3236 }
3237
3238 /**
3239 * {@inheritDoc} <!--workaround-->
3240 */
3241 @Override
3242 @ForceInline
3243 public final
3244 void intoByteBuffer(ByteBuffer bb, int offset,
3245 ByteOrder bo) {
3246 if (ScopedMemoryAccess.isReadOnly(bb)) {
3247 throw new ReadOnlyBufferException();
3248 }
3249 offset = checkFromIndexSize(offset, byteSize(), bb.limit());
3250 maybeSwap(bo).intoByteBuffer0(bb, offset);
3251 }
3252
3253 /**
3254 * {@inheritDoc} <!--workaround-->
3255 */
3256 @Override
3257 @ForceInline
3258 public final
3259 void intoByteBuffer(ByteBuffer bb, int offset,
3260 ByteOrder bo,
3261 VectorMask<Long> m) {
3262 if (m.allTrue()) {
3263 intoByteBuffer(bb, offset, bo);
3264 } else {
3265 if (bb.isReadOnly()) {
3266 throw new ReadOnlyBufferException();
3267 }
3268 LongSpecies vsp = vspecies();
3269 checkMaskFromIndexSize(offset, vsp, m, 8, bb.limit());
3270 maybeSwap(bo).intoByteBuffer0(bb, offset, m);
3271 }
3272 }
3273
3274 // ================================================
3275
3276 // Low-level memory operations.
3277 //
3278 // Note that all of these operations *must* inline into a context
3279 // where the exact species of the involved vector is a
3280 // compile-time constant. Otherwise, the intrinsic generation
3281 // will fail and performance will suffer.
3282 //
3283 // In many cases this is achieved by re-deriving a version of the
3284 // method in each concrete subclass (per species). The re-derived
3285 // method simply calls one of these generic methods, with exact
3286 // parameters for the controlling metadata, which is either a
3287 // typed vector or constant species instance.
3288
3289 // Unchecked loading operations in native byte order.
3290 // Caller is responsible for applying index checks, masking, and
3291 // byte swapping.
3292
3293 /*package-private*/
3294 abstract
3295 LongVector fromArray0(long[] a, int offset);
3296 @ForceInline
3297 final
3298 LongVector fromArray0Template(long[] a, int offset) {
3299 LongSpecies vsp = vspecies();
3300 return VectorSupport.load(
3301 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3302 a, arrayAddress(a, offset),
3303 a, offset, vsp,
3304 (arr, off, s) -> s.ldOp(arr, off,
3305 (arr_, off_, i) -> arr_[off_ + i]));
3306 }
3307
3308 /*package-private*/
3309 abstract
3310 LongVector fromArray0(long[] a, int offset, VectorMask<Long> m);
3311 @ForceInline
3312 final
3313 <M extends VectorMask<Long>>
3314 LongVector fromArray0Template(Class<M> maskClass, long[] a, int offset, M m) {
3315 m.check(species());
3316 LongSpecies vsp = vspecies();
3317 return VectorSupport.loadMasked(
3318 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3319 a, arrayAddress(a, offset), m,
3320 a, offset, vsp,
3321 (arr, off, s, vm) -> s.ldOp(arr, off, vm,
3322 (arr_, off_, i) -> arr_[off_ + i]));
3323 }
3324
3325 /*package-private*/
3326 abstract
3327 LongVector fromArray0(long[] a, int offset,
3328 int[] indexMap, int mapOffset,
3329 VectorMask<Long> m);
3330 @ForceInline
3331 final
3332 <M extends VectorMask<Long>>
3333 LongVector fromArray0Template(Class<M> maskClass, long[] a, int offset,
3334 int[] indexMap, int mapOffset, M m) {
3335 LongSpecies vsp = vspecies();
3336 IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3337 Objects.requireNonNull(a);
3338 Objects.requireNonNull(indexMap);
3339 m.check(vsp);
3340 Class<? extends LongVector> vectorType = vsp.vectorType();
3341
3342 if (vsp.laneCount() == 1) {
3343 return LongVector.fromArray(vsp, a, offset + indexMap[mapOffset], m);
3344 }
3345
3346 // Index vector: vix[0:n] = k -> offset + indexMap[mapOffset + k]
3347 IntVector vix;
3348 if (isp.laneCount() != vsp.laneCount()) {
3349 // For LongMaxVector, if vector length is non-power-of-two or
3350 // 2048 bits, indexShape of Long species is S_MAX_BIT.
3351 // Assume that vector length is 2048, then the lane count of Long
3352 // vector is 32. When converting Long species to int species,
3353 // indexShape is still S_MAX_BIT, but the lane count of int vector
3354 // is 64. So when loading index vector (IntVector), only lower half
3355 // of index data is needed.
3356 vix = IntVector
3357 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3358 .add(offset);
3359 } else {
3360 vix = IntVector
3361 .fromArray(isp, indexMap, mapOffset)
3362 .add(offset);
3363 }
3364
3365 // FIXME: Check index under mask controlling.
3366 vix = VectorIntrinsics.checkIndex(vix, a.length);
3367
3368 return VectorSupport.loadWithMap(
3369 vectorType, maskClass, long.class, vsp.laneCount(),
3370 isp.vectorType(),
3371 a, ARRAY_BASE, vix, m,
3372 a, offset, indexMap, mapOffset, vsp,
3373 (c, idx, iMap, idy, s, vm) ->
3374 s.vOp(vm, n -> c[idx + iMap[idy+n]]));
3375 }
3376
3377
3378
3379 @Override
3380 abstract
3381 LongVector fromByteArray0(byte[] a, int offset);
3382 @ForceInline
3383 final
3384 LongVector fromByteArray0Template(byte[] a, int offset) {
3385 LongSpecies vsp = vspecies();
3386 return VectorSupport.load(
3387 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3388 a, byteArrayAddress(a, offset),
3389 a, offset, vsp,
3390 (arr, off, s) -> {
3391 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3392 return s.ldOp(wb, off,
3393 (wb_, o, i) -> wb_.getLong(o + i * 8));
3394 });
3395 }
3396
3397 abstract
3398 LongVector fromByteArray0(byte[] a, int offset, VectorMask<Long> m);
3399 @ForceInline
3400 final
3401 <M extends VectorMask<Long>>
3402 LongVector fromByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
3403 LongSpecies vsp = vspecies();
3404 m.check(vsp);
3405 return VectorSupport.loadMasked(
3406 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3407 a, byteArrayAddress(a, offset), m,
3408 a, offset, vsp,
3409 (arr, off, s, vm) -> {
3410 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3411 return s.ldOp(wb, off, vm,
3412 (wb_, o, i) -> wb_.getLong(o + i * 8));
3413 });
3414 }
3415
3416 abstract
3417 LongVector fromByteBuffer0(ByteBuffer bb, int offset);
3418 @ForceInline
3419 final
3420 LongVector fromByteBuffer0Template(ByteBuffer bb, int offset) {
3421 LongSpecies vsp = vspecies();
3422 return ScopedMemoryAccess.loadFromByteBuffer(
3423 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3424 bb, offset, vsp,
3425 (buf, off, s) -> {
3426 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3427 return s.ldOp(wb, off,
3428 (wb_, o, i) -> wb_.getLong(o + i * 8));
3429 });
3430 }
3431
3432 abstract
3433 LongVector fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m);
3434 @ForceInline
3435 final
3436 <M extends VectorMask<Long>>
3437 LongVector fromByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
3438 LongSpecies vsp = vspecies();
3439 m.check(vsp);
3440 return ScopedMemoryAccess.loadFromByteBufferMasked(
3441 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3442 bb, offset, m, vsp,
3443 (buf, off, s, vm) -> {
3444 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3445 return s.ldOp(wb, off, vm,
3446 (wb_, o, i) -> wb_.getLong(o + i * 8));
3447 });
3448 }
3449
3450 // Unchecked storing operations in native byte order.
3451 // Caller is responsible for applying index checks, masking, and
3452 // byte swapping.
3453
3454 abstract
3455 void intoArray0(long[] a, int offset);
3456 @ForceInline
3457 final
3458 void intoArray0Template(long[] a, int offset) {
3459 LongSpecies vsp = vspecies();
3460 VectorSupport.store(
3461 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3462 a, arrayAddress(a, offset),
3463 this, a, offset,
3464 (arr, off, v)
3465 -> v.stOp(arr, off,
3466 (arr_, off_, i, e) -> arr_[off_+i] = e));
3467 }
3468
3469 abstract
3470 void intoArray0(long[] a, int offset, VectorMask<Long> m);
3471 @ForceInline
3472 final
3473 <M extends VectorMask<Long>>
3474 void intoArray0Template(Class<M> maskClass, long[] a, int offset, M m) {
3475 m.check(species());
3476 LongSpecies vsp = vspecies();
3477 VectorSupport.storeMasked(
3478 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3479 a, arrayAddress(a, offset),
3480 this, m, a, offset,
3481 (arr, off, v, vm)
3482 -> v.stOp(arr, off, vm,
3483 (arr_, off_, i, e) -> arr_[off_ + i] = e));
3484 }
3485
3486 abstract
3487 void intoArray0(long[] a, int offset,
3488 int[] indexMap, int mapOffset,
3489 VectorMask<Long> m);
3490 @ForceInline
3491 final
3492 <M extends VectorMask<Long>>
3493 void intoArray0Template(Class<M> maskClass, long[] a, int offset,
3494 int[] indexMap, int mapOffset, M m) {
3495 m.check(species());
3496 LongSpecies vsp = vspecies();
3497 IntVector.IntSpecies isp = IntVector.species(vsp.indexShape());
3498 if (vsp.laneCount() == 1) {
3499 intoArray(a, offset + indexMap[mapOffset], m);
3500 return;
3501 }
3502
3503 // Index vector: vix[0:n] = i -> offset + indexMap[mo + i]
3504 IntVector vix;
3505 if (isp.laneCount() != vsp.laneCount()) {
3506 // For LongMaxVector, if vector length is 2048 bits, indexShape
3507 // of Long species is S_MAX_BIT. and the lane count of Long
3508 // vector is 32. When converting Long species to int species,
3509 // indexShape is still S_MAX_BIT, but the lane count of int vector
3510 // is 64. So when loading index vector (IntVector), only lower half
3511 // of index data is needed.
3512 vix = IntVector
3513 .fromArray(isp, indexMap, mapOffset, IntMaxVector.IntMaxMask.LOWER_HALF_TRUE_MASK)
3514 .add(offset);
3515 } else {
3516 vix = IntVector
3517 .fromArray(isp, indexMap, mapOffset)
3518 .add(offset);
3519 }
3520
3521
3522 // FIXME: Check index under mask controlling.
3523 vix = VectorIntrinsics.checkIndex(vix, a.length);
3524
3525 VectorSupport.storeWithMap(
3526 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3527 isp.vectorType(),
3528 a, arrayAddress(a, 0), vix,
3529 this, m,
3530 a, offset, indexMap, mapOffset,
3531 (arr, off, v, map, mo, vm)
3532 -> v.stOp(arr, off, vm,
3533 (arr_, off_, i, e) -> {
3534 int j = map[mo + i];
3535 arr[off + j] = e;
3536 }));
3537 }
3538
3539
3540 abstract
3541 void intoByteArray0(byte[] a, int offset);
3542 @ForceInline
3543 final
3544 void intoByteArray0Template(byte[] a, int offset) {
3545 LongSpecies vsp = vspecies();
3546 VectorSupport.store(
3547 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3548 a, byteArrayAddress(a, offset),
3549 this, a, offset,
3550 (arr, off, v) -> {
3551 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3552 v.stOp(wb, off,
3553 (tb_, o, i, e) -> tb_.putLong(o + i * 8, e));
3554 });
3555 }
3556
3557 abstract
3558 void intoByteArray0(byte[] a, int offset, VectorMask<Long> m);
3559 @ForceInline
3560 final
3561 <M extends VectorMask<Long>>
3562 void intoByteArray0Template(Class<M> maskClass, byte[] a, int offset, M m) {
3563 LongSpecies vsp = vspecies();
3564 m.check(vsp);
3565 VectorSupport.storeMasked(
3566 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3567 a, byteArrayAddress(a, offset),
3568 this, m, a, offset,
3569 (arr, off, v, vm) -> {
3570 ByteBuffer wb = wrapper(arr, NATIVE_ENDIAN);
3571 v.stOp(wb, off, vm,
3572 (tb_, o, i, e) -> tb_.putLong(o + i * 8, e));
3573 });
3574 }
3575
3576 @ForceInline
3577 final
3578 void intoByteBuffer0(ByteBuffer bb, int offset) {
3579 LongSpecies vsp = vspecies();
3580 ScopedMemoryAccess.storeIntoByteBuffer(
3581 vsp.vectorType(), vsp.elementType(), vsp.laneCount(),
3582 this, bb, offset,
3583 (buf, off, v) -> {
3584 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3585 v.stOp(wb, off,
3586 (wb_, o, i, e) -> wb_.putLong(o + i * 8, e));
3587 });
3588 }
3589
3590 abstract
3591 void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<Long> m);
3592 @ForceInline
3593 final
3594 <M extends VectorMask<Long>>
3595 void intoByteBuffer0Template(Class<M> maskClass, ByteBuffer bb, int offset, M m) {
3596 LongSpecies vsp = vspecies();
3597 m.check(vsp);
3598 ScopedMemoryAccess.storeIntoByteBufferMasked(
3599 vsp.vectorType(), maskClass, vsp.elementType(), vsp.laneCount(),
3600 this, m, bb, offset,
3601 (buf, off, v, vm) -> {
3602 ByteBuffer wb = wrapper(buf, NATIVE_ENDIAN);
3603 v.stOp(wb, off, vm,
3604 (wb_, o, i, e) -> wb_.putLong(o + i * 8, e));
3605 });
3606 }
3607
3608
3609 // End of low-level memory operations.
3610
3611 private static
3612 void checkMaskFromIndexSize(int offset,
3613 LongSpecies vsp,
3614 VectorMask<Long> m,
3615 int scale,
3616 int limit) {
3617 ((AbstractMask<Long>)m)
3618 .checkIndexByLane(offset, limit, vsp.iota(), scale);
3619 }
3620
3621 @ForceInline
3622 private void conditionalStoreNYI(int offset,
3623 LongSpecies vsp,
3624 VectorMask<Long> m,
3625 int scale,
3626 int limit) {
3627 if (offset < 0 || offset + vsp.laneCount() * scale > limit) {
3628 String msg =
3897 long[] res = new long[laneCount()];
3898 boolean[] mbits = ((AbstractMask<Long>)m).getBits();
3899 for (int i = 0; i < res.length; i++) {
3900 if (mbits[i]) {
3901 res[i] = f.apply(i);
3902 }
3903 }
3904 return dummyVector().vectorFactory(res);
3905 }
3906
3907 /*package-private*/
3908 @ForceInline
3909 <M> LongVector ldOp(M memory, int offset,
3910 FLdOp<M> f) {
3911 return dummyVector().ldOp(memory, offset, f);
3912 }
3913
3914 /*package-private*/
3915 @ForceInline
3916 <M> LongVector ldOp(M memory, int offset,
3917 VectorMask<Long> m,
3918 FLdOp<M> f) {
3919 return dummyVector().ldOp(memory, offset, m, f);
3920 }
3921
3922 /*package-private*/
3923 @ForceInline
3924 <M> void stOp(M memory, int offset, FStOp<M> f) {
3925 dummyVector().stOp(memory, offset, f);
3926 }
3927
3928 /*package-private*/
3929 @ForceInline
3930 <M> void stOp(M memory, int offset,
3931 AbstractMask<Long> m,
3932 FStOp<M> f) {
3933 dummyVector().stOp(memory, offset, m, f);
3934 }
3935
3936 // N.B. Make sure these constant vectors and
3937 // masks load up correctly into registers.
|