1 /*
  2  * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.  Oracle designates this
  8  * particular file as subject to the "Classpath" exception as provided
  9  * by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  */
 25 package jdk.incubator.vector;
 26 
 27 import jdk.incubator.foreign.MemorySegment;
 28 import jdk.internal.vm.annotation.ForceInline;
 29 import jdk.internal.vm.vector.VectorSupport;
 30 

 31 import java.nio.ByteOrder;
 32 import java.util.function.IntUnaryOperator;
 33 
 34 import static jdk.incubator.vector.VectorOperators.*;
 35 
 36 @SuppressWarnings("cast")
 37 abstract class AbstractVector<E> extends Vector<E> {
 38     /**
 39      * The order of vector bytes when stored in natural,
 40      * array elements of the same lane type.
 41      * This is the also the behavior of the
 42      * VectorSupport load/store instructions.
 43      * If these instructions gain the capability to do
 44      * byte swapping on the fly, add a bit to those
 45      * instructions, but let this polarity be the
 46      * "neutral" or "default" setting of the bit.
 47      */
 48     /*package-private*/
 49     static final ByteOrder NATIVE_ENDIAN = ByteOrder.nativeOrder();
 50 
 51     /**
 52      * The order of vector bytes as stored in the register
 53      * file.  This becomes visible with the asRaw[Type]Vector
 54      * operations, which convert between the internal byte-wise
 55      * representation and the typed lane-wise representation.
 56      * It is very possible for a platform to have big-endian
 57      * memory layout and little-endian register layout,
 58      * so this is a different setting from NATIVE_ENDIAN.
 59      * In fact, both Intel and ARM use LE conventions here.
 60      * Future work may be needed for resolutely BE platforms.
 61      */
 62     /*package-private*/
 63     static final ByteOrder REGISTER_ENDIAN = ByteOrder.LITTLE_ENDIAN;
 64 
 65     /*package-private*/
 66     AbstractVector(Object bits) {
 67         super(bits);
 68     }
 69 
 70     // Extractors
 71 
 72     /*package-private*/
 73     abstract AbstractSpecies<E> vspecies();
 74 
 75     @Override
 76     @ForceInline
 77     public final VectorSpecies<E> species() {
 78         return vspecies();
 79     }
 80 
 81     // Something to make types match up better:
 82 
 83     @Override
 84     @ForceInline
 85     public final
 86     <F> Vector<F> check(VectorSpecies<F> species) {
 87         return check0(species);
 88     }
 89 
 90     @ForceInline
 91     @SuppressWarnings("unchecked")
 92     /*package-private*/ final
 93     <F> AbstractVector<F> check0(VectorSpecies<F> species) {
 94         if (!sameSpecies(species)) {
 95             throw AbstractSpecies.checkFailed(this, species);
 96         }
 97         return (AbstractVector<F>) this;
 98     }
 99 
100     /**
101      * {@inheritDoc} <!--workaround-->
102      */
103     @Override
104     @ForceInline
105     public final
106     <F> Vector<F> check(Class<F> elementType) {
107         return check0(elementType);
108     }
109 
110     @ForceInline
111     @SuppressWarnings("unchecked")
112     /*package-private*/ final
113     <F> AbstractVector<F> check0(Class<F> elementType) {
114         if (this.elementType() != elementType) {
115             throw AbstractSpecies.checkFailed(this, elementType);
116         }
117         return (AbstractVector<F>) this;
118     }
119 
120     @ForceInline
121     @SuppressWarnings("unchecked")
122     /*package-private*/ final
123     <F> AbstractVector<F> check(Vector<F> other) {
124         if (!sameSpecies(other)) {
125             throw AbstractSpecies.checkFailed(this, other);
126         }
127         return (AbstractVector<F>) this;
128     }
129 
130     @ForceInline
131     private boolean sameSpecies(Vector<?> other) {
132         // It's simpler and faster to do a class check.
133         boolean same = (this.getClass() == other.getClass());
134         // Make sure it works, too!
135         assert(same == (this.species() == other.species())) : same;
136         return same;
137     }
138 
139     @ForceInline
140     private boolean sameSpecies(VectorSpecies<?> species) {
141         // It's simpler and faster to do a class check,
142         // even if you have to load a dummy vector.
143         AbstractVector<?> other = ((AbstractSpecies<?>)species).dummyVector();
144         boolean same = (this.getClass() == other.getClass());
145         // Make sure it works, too!
146         assert(same == (this.species() == species)) : same;
147         return same;
148     }
149 
150     /**
151      * {@inheritDoc} <!--workaround-->
152      */
153     @Override
154     @ForceInline
155     public final VectorMask<E> maskAll(boolean bit) {
156         return species().maskAll(bit);
157     }
158 
159     // Make myself into a vector of the same shape
160     // and same information content but different lane type
161     /*package-private*/
162     abstract AbstractVector<?> asVectorRaw(LaneType laneType);
163 
164     // Make myself into a byte vector of the same shape
165     /*package-private*/
166     abstract ByteVector asByteVectorRaw();
167 
168     /*package-private*/
169     @ForceInline
170     final AbstractVector<?> asVectorRawTemplate(LaneType laneType) {
171         // NOTE:  This assumes that convert0('X')
172         // respects REGISTER_ENDIAN order.
173         return convert0('X', vspecies().withLanes(laneType));
174     }
175 
176     /*package-private*/
177     @ForceInline
178     ByteVector asByteVectorRawTemplate() {
179         return (ByteVector) asVectorRawTemplate(LaneType.BYTE);
180     }
181 
182 
183     abstract AbstractMask<E> maskFromArray(boolean[] bits);
184 
185     abstract AbstractShuffle<E> iotaShuffle();
186 
187     abstract AbstractShuffle<E> iotaShuffle(int start, int step, boolean wrap);
188 
189     /*do not alias this byte array*/
190     abstract AbstractShuffle<E> shuffleFromBytes(byte[] reorder);
191 
192     abstract AbstractShuffle<E> shuffleFromArray(int[] indexes, int i);
193 
194     abstract AbstractShuffle<E> shuffleFromOp(IntUnaryOperator fn);
195 
196     /*package-private*/
197     abstract AbstractVector<E> fromMemorySegment0(MemorySegment ms, long offset);
198 
199     /*package-private*/
200     abstract AbstractVector<E> maybeSwap(ByteOrder bo);
201 
202     /*package-private*/
203     @ForceInline
204     VectorShuffle<Byte> swapBytesShuffle() {
205         return vspecies().swapBytesShuffle();
206     }
207 
208     /**
209      * {@inheritDoc} <!--workaround-->
210      */
211     @Override
212     @ForceInline
213     public ShortVector reinterpretAsShorts() {
214         return (ShortVector) asVectorRaw(LaneType.SHORT);
215     }
216 
217     /**
218      * {@inheritDoc} <!--workaround-->
219      */
220     @Override
221     @ForceInline
222     public IntVector reinterpretAsInts() {
223         return (IntVector) asVectorRaw(LaneType.INT);
224     }
225 
226     /**
227      * {@inheritDoc} <!--workaround-->
228      */
229     @Override
230     @ForceInline
231     public LongVector reinterpretAsLongs() {
232         return (LongVector) asVectorRaw(LaneType.LONG);
233     }
234 
235     /**
236      * {@inheritDoc} <!--workaround-->
237      */
238     @Override
239     @ForceInline
240     public FloatVector reinterpretAsFloats() {
241         return (FloatVector) asVectorRaw(LaneType.FLOAT);
242     }
243 
244     /**
245      * {@inheritDoc} <!--workaround-->
246      */
247     @Override
248     @ForceInline
249     public DoubleVector reinterpretAsDoubles() {
250         return (DoubleVector) asVectorRaw(LaneType.DOUBLE);
251     }
252 
253     /**
254      * {@inheritDoc} <!--workaround-->
255      */
256     @Override
257     @ForceInline
258     public final <F>
259     Vector<F> convert(Conversion<E,F> conv, int part) {
260         // Shape invariance is simple to implement.
261         // It's part of the API because shape invariance
262         // is the default mode of operation, and shape
263         // shifting operations must advertise themselves.
264         ConversionImpl<E,F> c = (ConversionImpl<E,F>) conv;
265         @SuppressWarnings("unchecked")
266         VectorSpecies<F> rsp = (VectorSpecies<F>)
267             vspecies().withLanes(c.range());
268         return convertShape(conv, rsp, part);
269     }
270 
271     /**
272      * {@inheritDoc} <!--workaround-->
273      */
274     @Override
275     @ForceInline
276     public final <F>
277     Vector<F> castShape(VectorSpecies<F> toSpecies, int part) {
278         // This is an odd mix of shape conversion plus
279         // lanewise conversions.  It seems to be useful
280         // sometimes as a shorthand, though maybe we
281         // can drop it.
282         AbstractSpecies<E> vsp = vspecies();
283         AbstractSpecies<F> rsp = (AbstractSpecies<F>) toSpecies;
284         @SuppressWarnings("unchecked")
285         ConversionImpl<E,F> c = (ConversionImpl<E,F>)
286             ConversionImpl.ofCast(vsp.laneType, rsp.laneType);
287         return convertShape(c, rsp, part);
288     }
289 
290     /**
291      * {@inheritDoc} <!--workaround-->
292      */
293     @Override
294     @ForceInline
295     public abstract <F>
296     Vector<F> convertShape(Conversion<E,F> conv, VectorSpecies<F> rsp, int part);
297 
298     /**
299      * This is the template for Vector::reinterpretShape, to be
300      * specialized by each distinct vector class.
301      */
302     /*package-private*/
303     @ForceInline
304     final <F>
305     AbstractVector<F> reinterpretShapeTemplate(VectorSpecies<F> toSpecies, int part) {
306         AbstractSpecies<F> rsp = (AbstractSpecies<F>) toSpecies;
307         AbstractSpecies<E> vsp = vspecies();
308         if (part == 0) {
309             // Works the same for in-place, expand, or contract.
310             return convert0('X', rsp);
311         } else {
312             int origin = shapeChangeOrigin(vsp, rsp, false, part);
313             //System.out.println("*** origin = "+origin+", part = "+part+", reinterpret");
314             if (part > 0) {  // Expansion: slice first then cast.
315                 return slice(origin).convert0('X', rsp);
316             } else {  // Contraction: cast first then unslice.
317                 return rsp.zero().slice(rsp.laneCount() - origin,
318                                         convert0('X', rsp));
319             }
320         }
321     }
322 
323     @Override
324     public abstract AbstractVector<E> slice(int origin, Vector<E> v1);
325 
326     @Override
327     public abstract AbstractVector<E> slice(int origin);
328 
329     /**
330      * This is the template for Vector::convertShape, to be
331      * specialized by each distinct vector class.
332      */
333     /*package-private*/
334     @ForceInline
335     final <F>
336     AbstractVector<F> convertShapeTemplate(Conversion<E,F> conv, VectorSpecies<F> toSpecies, int part) {
337         ConversionImpl<E,F> c = (ConversionImpl<E,F>) conv;
338         AbstractSpecies<F> rsp = (AbstractSpecies<F>) toSpecies;
339         AbstractSpecies<E> vsp = vspecies();
340         char kind = c.kind();
341         switch (kind) {
342         case 'C': // Regular cast conversion, known to the JIT.
343             break;
344         case 'I':  // Identity conversion => reinterpret.
345             assert(c.sizeChangeLog2() == 0);
346             kind = 'X';
347             break;
348         case 'Z':  // Lane-wise expansion with zero padding.
349             assert(c.sizeChangeLog2() > 0);
350             assert(c.range().elementKind == 'I');
351             break;
352         case 'R':  // Lane-wise reinterpret conversion.
353             if (c.sizeChangeLog2() != 0) {
354                 kind = 'Z';  // some goofy stuff here
355                 break;
356             }
357             kind = 'X';  // No size change => reinterpret whole vector
358             break;
359         default:
360             throw new AssertionError(c);
361         }
362         vsp.check(c.domain());  // apply dynamic check to conv
363         rsp.check(c.range());   // apply dynamic check to conv
364         if (part == 0) {
365             // Works the same for in-place, expand, or contract.
366             return convert0(kind, rsp);
367         } else {
368             int origin = shapeChangeOrigin(vsp, rsp, true, part);
369             //System.out.println("*** origin = "+origin+", part = "+part+", lanewise");
370             if (part > 0) {  // Expansion: slice first then cast.
371                 return slice(origin).convert0(kind, rsp);
372             } else {  // Contraction: cast first then unslice.
373                 return rsp.zero().slice(rsp.laneCount() - origin,
374                                         convert0(kind, rsp));
375             }
376         }
377     }
378 
379     /**
380      * Check a part number and return it multiplied by the appropriate
381      * block factor to yield the origin of the operand block, as a
382      * lane number.  For expansions the origin is reckoned in the
383      * domain vector, since the domain vector has too much information
384      * and must be sliced.  For contractions the origin is reckoned in
385      * the range vector, since the range vector has too many lanes and
386      * the result must be unsliced at the same position as the inverse
387      * expansion.  If the conversion is lanewise, then lane sizes may
388      * be changing as well.  This affects the logical size of the
389      * result, and so the domain size is multiplied or divided by the
390      * lane size change.
391      */
392     /*package-private*/
393     @ForceInline
394     static
395     int shapeChangeOrigin(AbstractSpecies<?> dsp,
396                           AbstractSpecies<?> rsp,
397                           boolean lanewise,
398                           int part) {
399         int domSizeLog2 = dsp.vectorShape.vectorBitSizeLog2;
400         int phySizeLog2 = rsp.vectorShape.vectorBitSizeLog2;
401         int laneChangeLog2 = 0;
402         if (lanewise) {
403             laneChangeLog2 = (rsp.laneType.elementSizeLog2 -
404                               dsp.laneType.elementSizeLog2);
405         }
406         int resSizeLog2 = domSizeLog2 + laneChangeLog2;
407         // resSizeLog2 = 0 => 1-lane vector shrinking to 1-byte lane-size
408         // resSizeLog2 < 0 => small vector shrinking by more than a lane-size
409         assert(resSizeLog2 >= 0);
410         // Expansion ratio: expansionLog2 = resSizeLog2 - phySizeLog2;
411         if (!partInRange(resSizeLog2, phySizeLog2, part)) {
412             // fall through...
413         } else if (resSizeLog2 > phySizeLog2) {
414             // Expansion by M means we must slice a block from the domain.
415             // What is that block size?  It is 1/M of the domain.
416             // Let's compute the log2 of that block size, as 's'.
417             //s = (dsp.laneCountLog2() - expansionLog2);
418             //s = ((domSizeLog2 - dsp.laneType.elementSizeLog2) - expansionLog2);
419             //s = (domSizeLog2 - expansionLog2 - dsp.laneType.elementSizeLog2);
420             int s = phySizeLog2 - laneChangeLog2 - dsp.laneType.elementSizeLog2;
421             // Scale the part number by the input block size, in input lanes.
422             if ((s & 31) == s)  // sanity check
423                 return part << s;
424         } else {
425             // Contraction by M means we must drop a block into the range.
426             // What is that block size?  It is 1/M of the range.
427             // Let's compute the log2 of that block size, as 's'.
428             //s = (rsp.laneCountLog2() + expansionLog2);
429             //s = ((phySizeLog2 - rsp.laneType.elementSizeLog2) + expansionLog2);
430             //s = (phySizeLog2 + expansionLog2 - rsp.laneType.elementSizeLog2);
431             int s = resSizeLog2 - rsp.laneType.elementSizeLog2;
432             // Scale the part number by the output block size, in output lanes.
433             if ((s & 31) == s)  // sanity check
434                 return -part << s;
435         }
436         throw wrongPart(dsp, rsp, lanewise, part);
437     }
438 
439     @ForceInline
440     private static boolean partInRange(int resSizeLog2, int phySizeLog2, int part) {
441         // Let's try a branch-free version of this.
442         int diff = (resSizeLog2 - phySizeLog2);
443         int sign = (diff >> -1);
444         //d = Math.abs(diff);
445         //d = (sign == 0 ? diff : sign == -1 ? 1 + ~diff);
446         int d = (diff ^ sign) - sign;
447         assert(d == Math.abs(diff) && d <= 16);  // let's not go crazy here
448         //p = part * sign;
449         int p = (part ^ sign) - sign;
450         // z = sign == 0  ? 0<=part<(1<<d),  == (part & (-1 << d)) == 0
451         // z = sign == -1 ? 0<=-part<(1<<d), == (-part & (-1 << d)) == 0
452         boolean z = (p & (-1 << d)) == 0;
453         assert(z == partInRangeSlow(resSizeLog2, phySizeLog2, part)) : z;
454         return z;
455     }
456 
457     private static boolean partInRangeSlow(int resSizeLog2, int phySizeLog2, int part) {
458         if (resSizeLog2 > phySizeLog2) {  // expansion
459             int limit = 1 << (resSizeLog2 - phySizeLog2);
460             return part >= 0 && part < limit;
461         } else if (resSizeLog2 < phySizeLog2) {  // contraction
462             int limit = 1 << (phySizeLog2 - resSizeLog2);
463             return part > -limit && part <= 0;
464         } else {
465             return (part == 0);
466         }
467     }
468 
469     private static
470     ArrayIndexOutOfBoundsException
471     wrongPart(AbstractSpecies<?> dsp,
472               AbstractSpecies<?> rsp,
473               boolean lanewise,
474               int part) {
475         String laneChange = "";
476         String converting = "converting";
477         int dsize = dsp.elementSize(), rsize = rsp.elementSize();
478         if (!lanewise) {
479             converting = "reinterpreting";
480         } else if (dsize < rsize) {
481             laneChange = String.format(" (lanes are expanding by %d)",
482                                        rsize / dsize);
483         } else if (dsize > rsize) {
484             laneChange = String.format(" (lanes are contracting by %d)",
485                                        dsize / rsize);
486         }
487         String msg = String.format("bad part number %d %s %s -> %s%s",
488                                    part, converting, dsp, rsp, laneChange);
489         return new ArrayIndexOutOfBoundsException(msg);
490     }
491 
492     /*package-private*/
493     ArithmeticException divZeroException() {
494         throw new ArithmeticException("zero vector lane in dividend "+this);
495     }
496 
497     /**
498      * Helper function for all sorts of byte-wise reinterpretation casts.
499      * This function kicks in after intrinsic failure.
500      */
501     /*package-private*/
502     @ForceInline
503     final <F>
504     AbstractVector<F> defaultReinterpret(AbstractSpecies<F> rsp) {
505         int blen = Math.max(this.bitSize(), rsp.vectorBitSize()) / Byte.SIZE;
506         ByteOrder bo = ByteOrder.nativeOrder();
507         MemorySegment ms = MemorySegment.ofArray(new byte[blen]);
508         this.intoMemorySegment(ms, 0, bo);
509         VectorMask<F> m = rsp.maskAll(true);
510         // enum-switches don't optimize properly JDK-8161245
511         switch (rsp.laneType.switchKey) {
512         case LaneType.SK_BYTE:
513             return ByteVector.fromMemorySegment(rsp.check(byte.class), ms, 0, bo, m.check(byte.class)).check0(rsp);
514         case LaneType.SK_SHORT:
515             return ShortVector.fromMemorySegment(rsp.check(short.class), ms, 0, bo, m.check(short.class)).check0(rsp);
516         case LaneType.SK_INT:
517             return IntVector.fromMemorySegment(rsp.check(int.class), ms, 0, bo, m.check(int.class)).check0(rsp);
518         case LaneType.SK_LONG:
519             return LongVector.fromMemorySegment(rsp.check(long.class), ms, 0, bo, m.check(long.class)).check0(rsp);
520         case LaneType.SK_FLOAT:
521             return FloatVector.fromMemorySegment(rsp.check(float.class), ms, 0, bo, m.check(float.class)).check0(rsp);
522         case LaneType.SK_DOUBLE:
523             return DoubleVector.fromMemorySegment(rsp.check(double.class), ms, 0, bo, m.check(double.class)).check0(rsp);
524         default:
525             throw new AssertionError(rsp.toString());
526         }
527     }
528 
529     /**
530      * Helper function for all sorts of lane-wise conversions.
531      * This function kicks in after intrinsic failure.
532      */
533     /*package-private*/
534     @ForceInline
535     final <F>
536     AbstractVector<F> defaultCast(AbstractSpecies<F> dsp) {
537         int rlength = dsp.laneCount;
538         if (vspecies().laneType.elementKind == 'F') {
539             // Buffer input values in a double array.
540             double[] lanes = toDoubleArray();
541             int limit = Math.min(lanes.length, rlength);
542             // enum-switches don't optimize properly JDK-8161245
543             switch (dsp.laneType.switchKey) {
544             case LaneType.SK_BYTE: {
545                 byte[] a = new byte[rlength];
546                 for (int i = 0; i < limit; i++) {
547                     a[i] = (byte) lanes[i];
548                 }
549                 return ByteVector.fromArray(dsp.check(byte.class), a, 0).check0(dsp);
550             }
551             case LaneType.SK_SHORT: {
552                 short[] a = new short[rlength];
553                 for (int i = 0; i < limit; i++) {
554                     a[i] = (short) lanes[i];
555                 }
556                 return ShortVector.fromArray(dsp.check(short.class), a, 0).check0(dsp);
557             }
558             case LaneType.SK_INT: {
559                 int[] a = new int[rlength];
560                 for (int i = 0; i < limit; i++) {
561                     a[i] = (int) lanes[i];
562                 }
563                 return IntVector.fromArray(dsp.check(int.class), a, 0).check0(dsp);
564             }
565             case LaneType.SK_LONG: {
566                 long[] a = new long[rlength];
567                 for (int i = 0; i < limit; i++) {
568                     a[i] = (long) lanes[i];
569                 }
570                 return LongVector.fromArray(dsp.check(long.class), a, 0).check0(dsp);
571             }
572             case LaneType.SK_FLOAT: {
573                 float[] a = new float[rlength];
574                 for (int i = 0; i < limit; i++) {
575                     a[i] = (float) lanes[i];
576                 }
577                 return FloatVector.fromArray(dsp.check(float.class), a, 0).check0(dsp);
578             }
579             case LaneType.SK_DOUBLE: {
580                 double[] a = new double[rlength];
581                 for (int i = 0; i < limit; i++) {
582                     a[i] = (double) lanes[i];
583                 }
584                 return DoubleVector.fromArray(dsp.check(double.class), a, 0).check0(dsp);
585             }
586             default: break;
587             }
588         } else {
589             // Buffer input values in a long array.
590             long[] lanes = toLongArray();
591             int limit = Math.min(lanes.length, rlength);
592             // enum-switches don't optimize properly JDK-8161245
593             switch (dsp.laneType.switchKey) {
594             case LaneType.SK_BYTE: {
595                 byte[] a = new byte[rlength];
596                 for (int i = 0; i < limit; i++) {
597                     a[i] = (byte) lanes[i];
598                 }
599                 return ByteVector.fromArray(dsp.check(byte.class), a, 0).check0(dsp);
600             }
601             case LaneType.SK_SHORT: {
602                 short[] a = new short[rlength];
603                 for (int i = 0; i < limit; i++) {
604                     a[i] = (short) lanes[i];
605                 }
606                 return ShortVector.fromArray(dsp.check(short.class), a, 0).check0(dsp);
607             }
608             case LaneType.SK_INT: {
609                 int[] a = new int[rlength];
610                 for (int i = 0; i < limit; i++) {
611                     a[i] = (int) lanes[i];
612                 }
613                 return IntVector.fromArray(dsp.check(int.class), a, 0).check0(dsp);
614             }
615             case LaneType.SK_LONG: {
616                 long[] a = new long[rlength];
617                 for (int i = 0; i < limit; i++) {
618                     a[i] = (long) lanes[i];
619                 }
620                 return LongVector.fromArray(dsp.check(long.class), a, 0).check0(dsp);
621             }
622             case LaneType.SK_FLOAT: {
623                 float[] a = new float[rlength];
624                 for (int i = 0; i < limit; i++) {
625                     a[i] = (float) lanes[i];
626                 }
627                 return FloatVector.fromArray(dsp.check(float.class), a, 0).check0(dsp);
628             }
629             case LaneType.SK_DOUBLE: {
630                 double[] a = new double[rlength];
631                 for (int i = 0; i < limit; i++) {
632                     a[i] = (double) lanes[i];
633                 }
634                 return DoubleVector.fromArray(dsp.check(double.class), a, 0).check0(dsp);
635             }
636             default: break;
637             }
638         }
639         throw new AssertionError();
640     }
641 
642     /**
643      * Helper function for all sorts of lane-wise unsigned conversions.
644      * This function kicks in after intrinsic failure.
645      */
646     /*package-private*/
647     @ForceInline
648     final <F>
649     AbstractVector<F> defaultUCast(AbstractSpecies<F> dsp) {
650         AbstractSpecies<?> vsp = this.vspecies();
651         if (vsp.elementSize() >= dsp.elementSize()) {
652             // clip in place
653             return this.convert0('C', dsp);
654         } else {
655             // extend in place, but remove unwanted sign extension
656             long mask = -1L >>> -vsp.elementSize();
657             return (AbstractVector<F>) this.convert0('C', dsp).lanewise(AND, dsp.broadcast(mask));
658         }
659     }
660 
661     // Constant-folded access to conversion intrinsics:
662 
663     /**
664      * Dispatch on conversion kind and target species.
665      * The code of this is arranged to fold up if the
666      * vector class is constant and the target species
667      * is also constant.  This is often the case.
668      * Residual non-folded code may also perform acceptably
669      * in some cases due to type profiling, especially
670      * of rvtype.  If only one shape is being used,
671      * the profiling of rvtype should help speculatively
672      * fold the code even when the target species is
673      * not a constant.
674      */
675     /*package-private*/
676     @ForceInline
677     final <F>
678     AbstractVector<F> convert0(char kind, AbstractSpecies<F> rsp) {
679         // Derive some JIT-time constants:
680         Class<?> vtype;
681         Class<?> etype;   // fill in after switch (constant)
682         int vlength;      // fill in after switch (mark type profile?)
683         Class<?> rvtype;  // fill in after switch (mark type profile)
684         Class<?> rtype;
685         int rlength;
686         switch (kind) {
687         case 'Z':  // lane-wise size change, maybe with sign clip
688             AbstractSpecies<?> rspi = rsp.asIntegral();
689             AbstractSpecies<?> vsp = this.vspecies();
690             AbstractSpecies<?> vspi = vsp.asIntegral();
691             AbstractVector<?> biti = vspi == vsp ? this : this.convert0('X', vspi);
692             rtype = rspi.elementType();
693             rlength = rspi.laneCount();
694             etype = vspi.elementType();
695             vlength = vspi.laneCount();
696             rvtype = rspi.dummyVector().getClass();
697             vtype = vspi.dummyVector().getClass();
698             int opc = vspi.elementSize() < rspi.elementSize() ? VectorSupport.VECTOR_OP_UCAST : VectorSupport.VECTOR_OP_CAST;
699             AbstractVector<?> bitv = VectorSupport.convert(opc,
700                     vtype, etype, vlength,
701                     rvtype, rtype, rlength,
702                     biti, rspi,
703                     AbstractVector::defaultUCast);
704             return (rspi == rsp ? bitv.check0(rsp) : bitv.convert0('X', rsp));
705         case 'C':  // lane-wise cast (but not identity)
706             rtype = rsp.elementType();
707             rlength = rsp.laneCount();
708             etype = this.elementType(); // (profile)
709             vlength = this.length();  // (profile)
710             rvtype = rsp.dummyVector().getClass();  // (profile)
711             vtype = this.getClass();
712             return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
713                     vtype, etype, vlength,
714                     rvtype, rtype, rlength,
715                     this, rsp,
716                     AbstractVector::defaultCast);
717         case 'X':  // reinterpret cast, not lane-wise if lane sizes differ
718             rtype = rsp.elementType();
719             rlength = rsp.laneCount();
720             etype = this.elementType(); // (profile)
721             vlength = this.length();  // (profile)
722             rvtype = rsp.dummyVector().getClass();  // (profile)
723             vtype = this.getClass();
724             return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET,
725                     vtype, etype, vlength,
726                     rvtype, rtype, rlength,
727                     this, rsp,
728                     AbstractVector::defaultReinterpret);
729         }
730         throw new AssertionError();
731     }
732 









733     static {
734         // Recode uses of VectorSupport.reinterpret if this assertion fails:
735         assert(REGISTER_ENDIAN == ByteOrder.LITTLE_ENDIAN);
736     }
737 }
--- EOF ---