1 /* 2 * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 package jdk.incubator.vector; 26 27 import java.util.Arrays; 28 import java.util.Objects; 29 import java.util.function.IntUnaryOperator; 30 31 import jdk.incubator.foreign.MemorySegment; 32 import jdk.internal.vm.annotation.ForceInline; 33 import jdk.internal.vm.vector.VectorSupport; 34 35 import static jdk.internal.vm.vector.VectorSupport.*; 36 37 import static jdk.incubator.vector.VectorOperators.*; 38 39 #warn This file is preprocessed before being compiled 40 41 @SuppressWarnings("cast") // warning: redundant cast 42 final class $vectortype$ extends $abstractvectortype$ { 43 static final $Type$Species VSPECIES = 44 ($Type$Species) $Type$Vector.SPECIES_$BITS$; 45 46 static final VectorShape VSHAPE = 47 VSPECIES.vectorShape(); 48 49 static final Class<$vectortype$> VCLASS = $vectortype$.class; 50 51 static final int VSIZE = VSPECIES.vectorBitSize(); 52 53 static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM 54 55 static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM 56 57 $vectortype$($type$[] v) { 58 super(v); 59 } 60 61 // For compatibility as $vectortype$::new, 62 // stored into species.vectorFactory. 63 $vectortype$(Object v) { 64 this(($type$[]) v); 65 } 66 67 static final $vectortype$ ZERO = new $vectortype$(new $type$[VLENGTH]); 68 static final $vectortype$ IOTA = new $vectortype$(VSPECIES.iotaArray()); 69 70 static { 71 // Warm up a few species caches. 72 // If we do this too much we will 73 // get NPEs from bootstrap circularity. 74 VSPECIES.dummyVector(); 75 VSPECIES.withLanes(LaneType.BYTE); 76 } 77 78 // Specialized extractors 79 80 @ForceInline 81 final @Override 82 public $Type$Species vspecies() { 83 // ISSUE: This should probably be a @Stable 84 // field inside AbstractVector, rather than 85 // a megamorphic method. 86 return VSPECIES; 87 } 88 89 @ForceInline 90 @Override 91 public final Class<$Boxtype$> elementType() { return $type$.class; } 92 93 @ForceInline 94 @Override 95 public final int elementSize() { return $Boxtype$.SIZE; } 96 97 @ForceInline 98 @Override 99 public final VectorShape shape() { return VSHAPE; } 100 101 @ForceInline 102 @Override 103 public final int length() { return VLENGTH; } 104 105 @ForceInline 106 @Override 107 public final int bitSize() { return VSIZE; } 108 109 @ForceInline 110 @Override 111 public final int byteSize() { return VSIZE / Byte.SIZE; } 112 113 /*package-private*/ 114 @ForceInline 115 final @Override 116 $type$[] vec() { 117 return ($type$[])getPayload(); 118 } 119 120 // Virtualized constructors 121 122 @Override 123 @ForceInline 124 public final $vectortype$ broadcast($type$ e) { 125 return ($vectortype$) super.broadcastTemplate(e); // specialize 126 } 127 128 #if[!long] 129 @Override 130 @ForceInline 131 public final $vectortype$ broadcast(long e) { 132 return ($vectortype$) super.broadcastTemplate(e); // specialize 133 } 134 #end[!long] 135 136 @Override 137 @ForceInline 138 $masktype$ maskFromArray(boolean[] bits) { 139 return new $masktype$(bits); 140 } 141 142 @Override 143 @ForceInline 144 $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; } 145 146 @ForceInline 147 $shuffletype$ iotaShuffle(int start, int step, boolean wrap) { 148 if (wrap) { 149 return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1, 150 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); 151 } else { 152 return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0, 153 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); 154 } 155 } 156 157 @Override 158 @ForceInline 159 $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); } 160 161 @Override 162 @ForceInline 163 $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); } 164 165 @Override 166 @ForceInline 167 $shuffletype$ shuffleFromOp(IntUnaryOperator fn) { return new $shuffletype$(fn); } 168 169 // Make a vector of the same species but the given elements: 170 @ForceInline 171 final @Override 172 $vectortype$ vectorFactory($type$[] vec) { 173 return new $vectortype$(vec); 174 } 175 176 @ForceInline 177 final @Override 178 Byte$bits$Vector asByteVectorRaw() { 179 return (Byte$bits$Vector) super.asByteVectorRawTemplate(); // specialize 180 } 181 182 @ForceInline 183 final @Override 184 AbstractVector<?> asVectorRaw(LaneType laneType) { 185 return super.asVectorRawTemplate(laneType); // specialize 186 } 187 188 // Unary operator 189 190 @ForceInline 191 final @Override 192 $vectortype$ uOp(FUnOp f) { 193 return ($vectortype$) super.uOpTemplate(f); // specialize 194 } 195 196 @ForceInline 197 final @Override 198 $vectortype$ uOp(VectorMask<$Boxtype$> m, FUnOp f) { 199 return ($vectortype$) 200 super.uOpTemplate(($masktype$)m, f); // specialize 201 } 202 203 // Binary operator 204 205 @ForceInline 206 final @Override 207 $vectortype$ bOp(Vector<$Boxtype$> v, FBinOp f) { 208 return ($vectortype$) super.bOpTemplate(($vectortype$)v, f); // specialize 209 } 210 211 @ForceInline 212 final @Override 213 $vectortype$ bOp(Vector<$Boxtype$> v, 214 VectorMask<$Boxtype$> m, FBinOp f) { 215 return ($vectortype$) 216 super.bOpTemplate(($vectortype$)v, ($masktype$)m, 217 f); // specialize 218 } 219 220 // Ternary operator 221 222 @ForceInline 223 final @Override 224 $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, FTriOp f) { 225 return ($vectortype$) 226 super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2, 227 f); // specialize 228 } 229 230 @ForceInline 231 final @Override 232 $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, 233 VectorMask<$Boxtype$> m, FTriOp f) { 234 return ($vectortype$) 235 super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2, 236 ($masktype$)m, f); // specialize 237 } 238 239 @ForceInline 240 final @Override 241 $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f) { 242 return super.rOpTemplate(v, m, f); // specialize 243 } 244 245 @Override 246 @ForceInline 247 public final <F> 248 Vector<F> convertShape(VectorOperators.Conversion<$Boxtype$,F> conv, 249 VectorSpecies<F> rsp, int part) { 250 return super.convertShapeTemplate(conv, rsp, part); // specialize 251 } 252 253 @Override 254 @ForceInline 255 public final <F> 256 Vector<F> reinterpretShape(VectorSpecies<F> toSpecies, int part) { 257 return super.reinterpretShapeTemplate(toSpecies, part); // specialize 258 } 259 260 // Specialized algebraic operations: 261 262 // The following definition forces a specialized version of this 263 // crucial method into the v-table of this class. A call to add() 264 // will inline to a call to lanewise(ADD,), at which point the JIT 265 // intrinsic will have the opcode of ADD, plus all the metadata 266 // for this particular class, enabling it to generate precise 267 // code. 268 // 269 // There is probably no benefit to the JIT to specialize the 270 // masked or broadcast versions of the lanewise method. 271 272 @Override 273 @ForceInline 274 public $vectortype$ lanewise(Unary op) { 275 return ($vectortype$) super.lanewiseTemplate(op); // specialize 276 } 277 278 @Override 279 @ForceInline 280 public $vectortype$ lanewise(Unary op, VectorMask<$Boxtype$> m) { 281 return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, ($masktype$) m); // specialize 282 } 283 284 @Override 285 @ForceInline 286 public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v) { 287 return ($vectortype$) super.lanewiseTemplate(op, v); // specialize 288 } 289 290 @Override 291 @ForceInline 292 public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) { 293 return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v, ($masktype$) m); // specialize 294 } 295 296 #if[!FP] 297 /*package-private*/ 298 @Override 299 @ForceInline $vectortype$ 300 lanewiseShift(VectorOperators.Binary op, int e) { 301 return ($vectortype$) super.lanewiseShiftTemplate(op, e); // specialize 302 } 303 304 /*package-private*/ 305 @Override 306 @ForceInline $vectortype$ 307 lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m) { 308 return ($vectortype$) super.lanewiseShiftTemplate(op, $masktype$.class, e, ($masktype$) m); // specialize 309 } 310 #end[!FP] 311 312 /*package-private*/ 313 @Override 314 @ForceInline 315 public final 316 $vectortype$ 317 lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2) { 318 return ($vectortype$) super.lanewiseTemplate(op, v1, v2); // specialize 319 } 320 321 @Override 322 @ForceInline 323 public final 324 $vectortype$ 325 lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, VectorMask<$Boxtype$> m) { 326 return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v1, v2, ($masktype$) m); // specialize 327 } 328 329 @Override 330 @ForceInline 331 public final 332 $vectortype$ addIndex(int scale) { 333 return ($vectortype$) super.addIndexTemplate(scale); // specialize 334 } 335 336 // Type specific horizontal reductions 337 338 @Override 339 @ForceInline 340 public final $type$ reduceLanes(VectorOperators.Associative op) { 341 return super.reduceLanesTemplate(op); // specialized 342 } 343 344 @Override 345 @ForceInline 346 public final $type$ reduceLanes(VectorOperators.Associative op, 347 VectorMask<$Boxtype$> m) { 348 return super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized 349 } 350 351 @Override 352 @ForceInline 353 public final long reduceLanesToLong(VectorOperators.Associative op) { 354 return (long) super.reduceLanesTemplate(op); // specialized 355 } 356 357 @Override 358 @ForceInline 359 public final long reduceLanesToLong(VectorOperators.Associative op, 360 VectorMask<$Boxtype$> m) { 361 return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized 362 } 363 364 @ForceInline 365 public VectorShuffle<$Boxtype$> toShuffle() { 366 return super.toShuffleTemplate($shuffletype$.class); // specialize 367 } 368 369 // Specialized unary testing 370 371 @Override 372 @ForceInline 373 public final $masktype$ test(Test op) { 374 return super.testTemplate($masktype$.class, op); // specialize 375 } 376 377 @Override 378 @ForceInline 379 public final $masktype$ test(Test op, VectorMask<$Boxtype$> m) { 380 return super.testTemplate($masktype$.class, op, ($masktype$) m); // specialize 381 } 382 383 // Specialized comparisons 384 385 @Override 386 @ForceInline 387 public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v) { 388 return super.compareTemplate($masktype$.class, op, v); // specialize 389 } 390 391 @Override 392 @ForceInline 393 public final $masktype$ compare(Comparison op, $type$ s) { 394 return super.compareTemplate($masktype$.class, op, s); // specialize 395 } 396 397 #if[!long] 398 @Override 399 @ForceInline 400 public final $masktype$ compare(Comparison op, long s) { 401 return super.compareTemplate($masktype$.class, op, s); // specialize 402 } 403 #end[!long] 404 405 @Override 406 @ForceInline 407 public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) { 408 return super.compareTemplate($masktype$.class, op, v, ($masktype$) m); 409 } 410 411 412 @Override 413 @ForceInline 414 public $vectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) { 415 return ($vectortype$) 416 super.blendTemplate($masktype$.class, 417 ($vectortype$) v, 418 ($masktype$) m); // specialize 419 } 420 421 @Override 422 @ForceInline 423 public $vectortype$ slice(int origin, Vector<$Boxtype$> v) { 424 return ($vectortype$) super.sliceTemplate(origin, v); // specialize 425 } 426 427 @Override 428 @ForceInline 429 public $vectortype$ slice(int origin) { 430 return ($vectortype$) super.sliceTemplate(origin); // specialize 431 } 432 433 @Override 434 @ForceInline 435 public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part) { 436 return ($vectortype$) super.unsliceTemplate(origin, w, part); // specialize 437 } 438 439 @Override 440 @ForceInline 441 public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m) { 442 return ($vectortype$) 443 super.unsliceTemplate($masktype$.class, 444 origin, w, part, 445 ($masktype$) m); // specialize 446 } 447 448 @Override 449 @ForceInline 450 public $vectortype$ unslice(int origin) { 451 return ($vectortype$) super.unsliceTemplate(origin); // specialize 452 } 453 454 @Override 455 @ForceInline 456 public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s) { 457 return ($vectortype$) 458 super.rearrangeTemplate($shuffletype$.class, 459 ($shuffletype$) s); // specialize 460 } 461 462 @Override 463 @ForceInline 464 public $vectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle, 465 VectorMask<$Boxtype$> m) { 466 return ($vectortype$) 467 super.rearrangeTemplate($shuffletype$.class, 468 $masktype$.class, 469 ($shuffletype$) shuffle, 470 ($masktype$) m); // specialize 471 } 472 473 @Override 474 @ForceInline 475 public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s, 476 Vector<$Boxtype$> v) { 477 return ($vectortype$) 478 super.rearrangeTemplate($shuffletype$.class, 479 ($shuffletype$) s, 480 ($vectortype$) v); // specialize 481 } 482 483 @Override 484 @ForceInline 485 public $vectortype$ compress(VectorMask<$Boxtype$> m) { 486 return ($vectortype$) 487 super.compressTemplate($masktype$.class, 488 ($masktype$) m); // specialize 489 } 490 491 @Override 492 @ForceInline 493 public $vectortype$ expand(VectorMask<$Boxtype$> m) { 494 return ($vectortype$) 495 super.expandTemplate($masktype$.class, 496 ($masktype$) m); // specialize 497 } 498 499 @Override 500 @ForceInline 501 public $vectortype$ selectFrom(Vector<$Boxtype$> v) { 502 return ($vectortype$) 503 super.selectFromTemplate(($vectortype$) v); // specialize 504 } 505 506 @Override 507 @ForceInline 508 public $vectortype$ selectFrom(Vector<$Boxtype$> v, 509 VectorMask<$Boxtype$> m) { 510 return ($vectortype$) 511 super.selectFromTemplate(($vectortype$) v, 512 ($masktype$) m); // specialize 513 } 514 515 516 #if[FP] 517 @ForceInline 518 @Override 519 public $type$ lane(int i) { 520 #if[!Max] 521 $bitstype$ bits; 522 switch(i) { 523 case 0: bits = laneHelper(0); break; 524 #if[!1L] 525 case 1: bits = laneHelper(1); break; 526 #if[!2L] 527 case 2: bits = laneHelper(2); break; 528 case 3: bits = laneHelper(3); break; 529 #if[!4L] 530 case 4: bits = laneHelper(4); break; 531 case 5: bits = laneHelper(5); break; 532 case 6: bits = laneHelper(6); break; 533 case 7: bits = laneHelper(7); break; 534 #if[!8L] 535 case 8: bits = laneHelper(8); break; 536 case 9: bits = laneHelper(9); break; 537 case 10: bits = laneHelper(10); break; 538 case 11: bits = laneHelper(11); break; 539 case 12: bits = laneHelper(12); break; 540 case 13: bits = laneHelper(13); break; 541 case 14: bits = laneHelper(14); break; 542 case 15: bits = laneHelper(15); break; 543 #end[!8L] 544 #end[!4L] 545 #end[!2L] 546 #end[!1L] 547 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH); 548 } 549 #else[!Max] 550 if (i < 0 || i >= VLENGTH) { 551 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH); 552 } 553 $bitstype$ bits = laneHelper(i); 554 #end[!Max] 555 return $Type$.$bitstype$BitsTo$Fptype$(bits); 556 } 557 558 public $bitstype$ laneHelper(int i) { 559 return ($bitstype$) VectorSupport.extract( 560 VCLASS, ETYPE, VLENGTH, 561 this, i, 562 (vec, ix) -> { 563 $type$[] vecarr = vec.vec(); 564 return (long)$Type$.$type$To$Bitstype$Bits(vecarr[ix]); 565 }); 566 } 567 568 @ForceInline 569 @Override 570 public $vectortype$ withLane(int i, $type$ e) { 571 #if[!Max] 572 switch(i) { 573 case 0: return withLaneHelper(0, e); 574 #if[!1L] 575 case 1: return withLaneHelper(1, e); 576 #if[!2L] 577 case 2: return withLaneHelper(2, e); 578 case 3: return withLaneHelper(3, e); 579 #if[!4L] 580 case 4: return withLaneHelper(4, e); 581 case 5: return withLaneHelper(5, e); 582 case 6: return withLaneHelper(6, e); 583 case 7: return withLaneHelper(7, e); 584 #if[!8L] 585 case 8: return withLaneHelper(8, e); 586 case 9: return withLaneHelper(9, e); 587 case 10: return withLaneHelper(10, e); 588 case 11: return withLaneHelper(11, e); 589 case 12: return withLaneHelper(12, e); 590 case 13: return withLaneHelper(13, e); 591 case 14: return withLaneHelper(14, e); 592 case 15: return withLaneHelper(15, e); 593 #end[!8L] 594 #end[!4L] 595 #end[!2L] 596 #end[!1L] 597 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH); 598 } 599 #else[!Max] 600 if (i < 0 || i >= VLENGTH) { 601 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH); 602 } 603 return withLaneHelper(i, e); 604 #end[!Max] 605 } 606 607 public $vectortype$ withLaneHelper(int i, $type$ e) { 608 return VectorSupport.insert( 609 VCLASS, ETYPE, VLENGTH, 610 this, i, (long)$Type$.$type$To$Bitstype$Bits(e), 611 (v, ix, bits) -> { 612 $type$[] res = v.vec().clone(); 613 res[ix] = $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits); 614 return v.vectorFactory(res); 615 }); 616 } 617 #else[FP] 618 @ForceInline 619 @Override 620 public $type$ lane(int i) { 621 #if[!Max] 622 switch(i) { 623 case 0: return laneHelper(0); 624 #if[!1L] 625 case 1: return laneHelper(1); 626 #if[!2L] 627 case 2: return laneHelper(2); 628 case 3: return laneHelper(3); 629 #if[!4L] 630 case 4: return laneHelper(4); 631 case 5: return laneHelper(5); 632 case 6: return laneHelper(6); 633 case 7: return laneHelper(7); 634 #if[!8L] 635 case 8: return laneHelper(8); 636 case 9: return laneHelper(9); 637 case 10: return laneHelper(10); 638 case 11: return laneHelper(11); 639 case 12: return laneHelper(12); 640 case 13: return laneHelper(13); 641 case 14: return laneHelper(14); 642 case 15: return laneHelper(15); 643 #if[!16L] 644 case 16: return laneHelper(16); 645 case 17: return laneHelper(17); 646 case 18: return laneHelper(18); 647 case 19: return laneHelper(19); 648 case 20: return laneHelper(20); 649 case 21: return laneHelper(21); 650 case 22: return laneHelper(22); 651 case 23: return laneHelper(23); 652 case 24: return laneHelper(24); 653 case 25: return laneHelper(25); 654 case 26: return laneHelper(26); 655 case 27: return laneHelper(27); 656 case 28: return laneHelper(28); 657 case 29: return laneHelper(29); 658 case 30: return laneHelper(30); 659 case 31: return laneHelper(31); 660 #if[!32L] 661 case 32: return laneHelper(32); 662 case 33: return laneHelper(33); 663 case 34: return laneHelper(34); 664 case 35: return laneHelper(35); 665 case 36: return laneHelper(36); 666 case 37: return laneHelper(37); 667 case 38: return laneHelper(38); 668 case 39: return laneHelper(39); 669 case 40: return laneHelper(40); 670 case 41: return laneHelper(41); 671 case 42: return laneHelper(42); 672 case 43: return laneHelper(43); 673 case 44: return laneHelper(44); 674 case 45: return laneHelper(45); 675 case 46: return laneHelper(46); 676 case 47: return laneHelper(47); 677 case 48: return laneHelper(48); 678 case 49: return laneHelper(49); 679 case 50: return laneHelper(50); 680 case 51: return laneHelper(51); 681 case 52: return laneHelper(52); 682 case 53: return laneHelper(53); 683 case 54: return laneHelper(54); 684 case 55: return laneHelper(55); 685 case 56: return laneHelper(56); 686 case 57: return laneHelper(57); 687 case 58: return laneHelper(58); 688 case 59: return laneHelper(59); 689 case 60: return laneHelper(60); 690 case 61: return laneHelper(61); 691 case 62: return laneHelper(62); 692 case 63: return laneHelper(63); 693 #end[!32L] 694 #end[!16L] 695 #end[!8L] 696 #end[!4L] 697 #end[!2L] 698 #end[!1L] 699 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH); 700 } 701 #else[!Max] 702 if (i < 0 || i >= VLENGTH) { 703 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH); 704 } 705 return laneHelper(i); 706 #end[!Max] 707 } 708 709 public $type$ laneHelper(int i) { 710 return ($type$) VectorSupport.extract( 711 VCLASS, ETYPE, VLENGTH, 712 this, i, 713 (vec, ix) -> { 714 $type$[] vecarr = vec.vec(); 715 return (long)vecarr[ix]; 716 }); 717 } 718 719 @ForceInline 720 @Override 721 public $vectortype$ withLane(int i, $type$ e) { 722 #if[!Max] 723 switch (i) { 724 case 0: return withLaneHelper(0, e); 725 #if[!1L] 726 case 1: return withLaneHelper(1, e); 727 #if[!2L] 728 case 2: return withLaneHelper(2, e); 729 case 3: return withLaneHelper(3, e); 730 #if[!4L] 731 case 4: return withLaneHelper(4, e); 732 case 5: return withLaneHelper(5, e); 733 case 6: return withLaneHelper(6, e); 734 case 7: return withLaneHelper(7, e); 735 #if[!8L] 736 case 8: return withLaneHelper(8, e); 737 case 9: return withLaneHelper(9, e); 738 case 10: return withLaneHelper(10, e); 739 case 11: return withLaneHelper(11, e); 740 case 12: return withLaneHelper(12, e); 741 case 13: return withLaneHelper(13, e); 742 case 14: return withLaneHelper(14, e); 743 case 15: return withLaneHelper(15, e); 744 #if[!16L] 745 case 16: return withLaneHelper(16, e); 746 case 17: return withLaneHelper(17, e); 747 case 18: return withLaneHelper(18, e); 748 case 19: return withLaneHelper(19, e); 749 case 20: return withLaneHelper(20, e); 750 case 21: return withLaneHelper(21, e); 751 case 22: return withLaneHelper(22, e); 752 case 23: return withLaneHelper(23, e); 753 case 24: return withLaneHelper(24, e); 754 case 25: return withLaneHelper(25, e); 755 case 26: return withLaneHelper(26, e); 756 case 27: return withLaneHelper(27, e); 757 case 28: return withLaneHelper(28, e); 758 case 29: return withLaneHelper(29, e); 759 case 30: return withLaneHelper(30, e); 760 case 31: return withLaneHelper(31, e); 761 #if[!32L] 762 case 32: return withLaneHelper(32, e); 763 case 33: return withLaneHelper(33, e); 764 case 34: return withLaneHelper(34, e); 765 case 35: return withLaneHelper(35, e); 766 case 36: return withLaneHelper(36, e); 767 case 37: return withLaneHelper(37, e); 768 case 38: return withLaneHelper(38, e); 769 case 39: return withLaneHelper(39, e); 770 case 40: return withLaneHelper(40, e); 771 case 41: return withLaneHelper(41, e); 772 case 42: return withLaneHelper(42, e); 773 case 43: return withLaneHelper(43, e); 774 case 44: return withLaneHelper(44, e); 775 case 45: return withLaneHelper(45, e); 776 case 46: return withLaneHelper(46, e); 777 case 47: return withLaneHelper(47, e); 778 case 48: return withLaneHelper(48, e); 779 case 49: return withLaneHelper(49, e); 780 case 50: return withLaneHelper(50, e); 781 case 51: return withLaneHelper(51, e); 782 case 52: return withLaneHelper(52, e); 783 case 53: return withLaneHelper(53, e); 784 case 54: return withLaneHelper(54, e); 785 case 55: return withLaneHelper(55, e); 786 case 56: return withLaneHelper(56, e); 787 case 57: return withLaneHelper(57, e); 788 case 58: return withLaneHelper(58, e); 789 case 59: return withLaneHelper(59, e); 790 case 60: return withLaneHelper(60, e); 791 case 61: return withLaneHelper(61, e); 792 case 62: return withLaneHelper(62, e); 793 case 63: return withLaneHelper(63, e); 794 #end[!32L] 795 #end[!16L] 796 #end[!8L] 797 #end[!4L] 798 #end[!2L] 799 #end[!1L] 800 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH); 801 } 802 #else[!Max] 803 if (i < 0 || i >= VLENGTH) { 804 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH); 805 } 806 return withLaneHelper(i, e); 807 #end[!Max] 808 } 809 810 public $vectortype$ withLaneHelper(int i, $type$ e) { 811 return VectorSupport.insert( 812 VCLASS, ETYPE, VLENGTH, 813 this, i, (long)e, 814 (v, ix, bits) -> { 815 $type$[] res = v.vec().clone(); 816 res[ix] = ($type$)bits; 817 return v.vectorFactory(res); 818 }); 819 } 820 #end[FP] 821 822 // Mask 823 824 static final class $masktype$ extends AbstractMask<$Boxtype$> { 825 static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM 826 static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM 827 828 $masktype$(boolean[] bits) { 829 this(bits, 0); 830 } 831 832 $masktype$(boolean[] bits, int offset) { 833 super(prepare(bits, offset)); 834 } 835 836 $masktype$(boolean val) { 837 super(prepare(val)); 838 } 839 840 private static boolean[] prepare(boolean[] bits, int offset) { 841 boolean[] newBits = new boolean[VSPECIES.laneCount()]; 842 for (int i = 0; i < newBits.length; i++) { 843 newBits[i] = bits[offset + i]; 844 } 845 return newBits; 846 } 847 848 private static boolean[] prepare(boolean val) { 849 boolean[] bits = new boolean[VSPECIES.laneCount()]; 850 Arrays.fill(bits, val); 851 return bits; 852 } 853 854 @ForceInline 855 final @Override 856 public $Type$Species vspecies() { 857 // ISSUE: This should probably be a @Stable 858 // field inside AbstractMask, rather than 859 // a megamorphic method. 860 return VSPECIES; 861 } 862 863 @ForceInline 864 boolean[] getBits() { 865 return (boolean[])getPayload(); 866 } 867 868 @Override 869 $masktype$ uOp(MUnOp f) { 870 boolean[] res = new boolean[vspecies().laneCount()]; 871 boolean[] bits = getBits(); 872 for (int i = 0; i < res.length; i++) { 873 res[i] = f.apply(i, bits[i]); 874 } 875 return new $masktype$(res); 876 } 877 878 @Override 879 $masktype$ bOp(VectorMask<$Boxtype$> m, MBinOp f) { 880 boolean[] res = new boolean[vspecies().laneCount()]; 881 boolean[] bits = getBits(); 882 boolean[] mbits = (($masktype$)m).getBits(); 883 for (int i = 0; i < res.length; i++) { 884 res[i] = f.apply(i, bits[i], mbits[i]); 885 } 886 return new $masktype$(res); 887 } 888 889 @ForceInline 890 @Override 891 public final 892 $vectortype$ toVector() { 893 return ($vectortype$) super.toVectorTemplate(); // specialize 894 } 895 896 /** 897 * Helper function for lane-wise mask conversions. 898 * This function kicks in after intrinsic failure. 899 */ 900 @ForceInline 901 private final <E> 902 VectorMask<E> defaultMaskCast(AbstractSpecies<E> dsp) { 903 if (length() != dsp.laneCount()) 904 throw new IllegalArgumentException("VectorMask length and species length differ"); 905 boolean[] maskArray = toArray(); 906 return dsp.maskFactory(maskArray).check(dsp); 907 } 908 909 @Override 910 @ForceInline 911 public <E> VectorMask<E> cast(VectorSpecies<E> dsp) { 912 AbstractSpecies<E> species = (AbstractSpecies<E>) dsp; 913 if (length() != species.laneCount()) 914 throw new IllegalArgumentException("VectorMask length and species length differ"); 915 916 return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, 917 this.getClass(), ETYPE, VLENGTH, 918 species.maskType(), species.elementType(), VLENGTH, 919 this, species, 920 (m, s) -> s.maskFactory(m.toArray()).check(s)); 921 } 922 923 @Override 924 @ForceInline 925 public $masktype$ eq(VectorMask<$Boxtype$> mask) { 926 Objects.requireNonNull(mask); 927 $masktype$ m = ($masktype$)mask; 928 return xor(m.not()); 929 } 930 931 // Unary operations 932 933 @Override 934 @ForceInline 935 public $masktype$ not() { 936 return xor(maskAll(true)); 937 } 938 939 @Override 940 @ForceInline 941 public $masktype$ compress() { 942 return ($masktype$)VectorSupport.comExpOp(VectorSupport.VECTOR_OP_MASK_COMPRESS, 943 $vectortype$.class, $masktype$.class, ETYPE, VLENGTH, null, this, 944 (v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount())); 945 } 946 947 948 // Binary operations 949 950 @Override 951 @ForceInline 952 public $masktype$ and(VectorMask<$Boxtype$> mask) { 953 Objects.requireNonNull(mask); 954 $masktype$ m = ($masktype$)mask; 955 return VectorSupport.binaryOp(VECTOR_OP_AND, $masktype$.class, null, $bitstype$.class, VLENGTH, 956 this, m, null, 957 (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a & b)); 958 } 959 960 @Override 961 @ForceInline 962 public $masktype$ or(VectorMask<$Boxtype$> mask) { 963 Objects.requireNonNull(mask); 964 $masktype$ m = ($masktype$)mask; 965 return VectorSupport.binaryOp(VECTOR_OP_OR, $masktype$.class, null, $bitstype$.class, VLENGTH, 966 this, m, null, 967 (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a | b)); 968 } 969 970 @ForceInline 971 /* package-private */ 972 $masktype$ xor(VectorMask<$Boxtype$> mask) { 973 Objects.requireNonNull(mask); 974 $masktype$ m = ($masktype$)mask; 975 return VectorSupport.binaryOp(VECTOR_OP_XOR, $masktype$.class, null, $bitstype$.class, VLENGTH, 976 this, m, null, 977 (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a ^ b)); 978 } 979 980 // Mask Query operations 981 982 @Override 983 @ForceInline 984 public int trueCount() { 985 return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, $masktype$.class, $bitstype$.class, VLENGTH, this, 986 (m) -> trueCountHelper(m.getBits())); 987 } 988 989 @Override 990 @ForceInline 991 public int firstTrue() { 992 return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this, 993 (m) -> firstTrueHelper(m.getBits())); 994 } 995 996 @Override 997 @ForceInline 998 public int lastTrue() { 999 return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this, 1000 (m) -> lastTrueHelper(m.getBits())); 1001 } 1002 1003 @Override 1004 @ForceInline 1005 public long toLong() { 1006 if (length() > Long.SIZE) { 1007 throw new UnsupportedOperationException("too many lanes for one long"); 1008 } 1009 return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TOLONG, $masktype$.class, $bitstype$.class, VLENGTH, this, 1010 (m) -> toLongHelper(m.getBits())); 1011 } 1012 1013 // Reductions 1014 1015 @Override 1016 @ForceInline 1017 public boolean anyTrue() { 1018 return VectorSupport.test(BT_ne, $masktype$.class, $bitstype$.class, VLENGTH, 1019 this, vspecies().maskAll(true), 1020 (m, __) -> anyTrueHelper((($masktype$)m).getBits())); 1021 } 1022 1023 @Override 1024 @ForceInline 1025 public boolean allTrue() { 1026 return VectorSupport.test(BT_overflow, $masktype$.class, $bitstype$.class, VLENGTH, 1027 this, vspecies().maskAll(true), 1028 (m, __) -> allTrueHelper((($masktype$)m).getBits())); 1029 } 1030 1031 @ForceInline 1032 /*package-private*/ 1033 static $masktype$ maskAll(boolean bit) { 1034 return VectorSupport.fromBitsCoerced($masktype$.class, $bitstype$.class, VLENGTH, 1035 (bit ? -1 : 0), MODE_BROADCAST, null, 1036 (v, __) -> (v != 0 ? TRUE_MASK : FALSE_MASK)); 1037 } 1038 private static final $masktype$ TRUE_MASK = new $masktype$(true); 1039 private static final $masktype$ FALSE_MASK = new $masktype$(false); 1040 1041 #if[intAndMax] 1042 1043 static boolean[] maskLowerHalf() { 1044 boolean[] a = new boolean[VLENGTH]; 1045 int len = a.length >> 1; 1046 for (int i = 0; i < len; i++) { 1047 a[i] = true; 1048 } 1049 return a; 1050 } 1051 1052 #end[intAndMax] 1053 #if[intAndMax] 1054 static final IntMaxMask LOWER_HALF_TRUE_MASK = new IntMaxMask(maskLowerHalf()); 1055 #end[intAndMax] 1056 } 1057 1058 // Shuffle 1059 1060 static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> { 1061 static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM 1062 static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM 1063 1064 $shuffletype$(byte[] reorder) { 1065 super(VLENGTH, reorder); 1066 } 1067 1068 public $shuffletype$(int[] reorder) { 1069 super(VLENGTH, reorder); 1070 } 1071 1072 public $shuffletype$(int[] reorder, int i) { 1073 super(VLENGTH, reorder, i); 1074 } 1075 1076 public $shuffletype$(IntUnaryOperator fn) { 1077 super(VLENGTH, fn); 1078 } 1079 1080 @Override 1081 public $Type$Species vspecies() { 1082 return VSPECIES; 1083 } 1084 1085 static { 1086 // There must be enough bits in the shuffle lanes to encode 1087 // VLENGTH valid indexes and VLENGTH exceptional ones. 1088 assert(VLENGTH < Byte.MAX_VALUE); 1089 assert(Byte.MIN_VALUE <= -VLENGTH); 1090 } 1091 static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY); 1092 1093 @Override 1094 @ForceInline 1095 public $vectortype$ toVector() { 1096 return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH, 1097 (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate()))); 1098 } 1099 1100 @Override 1101 @ForceInline 1102 public <F> VectorShuffle<F> cast(VectorSpecies<F> s) { 1103 AbstractSpecies<F> species = (AbstractSpecies<F>) s; 1104 if (length() != species.laneCount()) 1105 throw new IllegalArgumentException("VectorShuffle length and species length differ"); 1106 int[] shuffleArray = toArray(); 1107 return s.shuffleFromArray(shuffleArray, 0).check(s); 1108 } 1109 1110 @ForceInline 1111 @Override 1112 public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) { 1113 $shuffletype$ s = ($shuffletype$) shuffle; 1114 byte[] reorder1 = reorder(); 1115 byte[] reorder2 = s.reorder(); 1116 byte[] r = new byte[reorder1.length]; 1117 for (int i = 0; i < reorder1.length; i++) { 1118 int ssi = reorder2[i]; 1119 r[i] = reorder1[ssi]; // throws on exceptional index 1120 } 1121 return new $shuffletype$(r); 1122 } 1123 } 1124 1125 // ================================================ 1126 1127 // Specialized low-level memory operations. 1128 1129 @ForceInline 1130 @Override 1131 final 1132 $abstractvectortype$ fromArray0($type$[] a, int offset) { 1133 return super.fromArray0Template(a, offset); // specialize 1134 } 1135 1136 @ForceInline 1137 @Override 1138 final 1139 $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) { 1140 return super.fromArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize 1141 } 1142 1143 #if[!byteOrShort] 1144 @ForceInline 1145 @Override 1146 final 1147 $abstractvectortype$ fromArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) { 1148 return super.fromArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m); 1149 } 1150 #end[!byteOrShort] 1151 1152 #if[short] 1153 @ForceInline 1154 @Override 1155 final 1156 $abstractvectortype$ fromCharArray0(char[] a, int offset) { 1157 return super.fromCharArray0Template(a, offset); // specialize 1158 } 1159 1160 @ForceInline 1161 @Override 1162 final 1163 $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) { 1164 return super.fromCharArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize 1165 } 1166 #end[short] 1167 1168 #if[byte] 1169 @ForceInline 1170 @Override 1171 final 1172 $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset) { 1173 return super.fromBooleanArray0Template(a, offset); // specialize 1174 } 1175 1176 @ForceInline 1177 @Override 1178 final 1179 $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) { 1180 return super.fromBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize 1181 } 1182 #end[byte] 1183 1184 @ForceInline 1185 @Override 1186 final 1187 $abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset) { 1188 return super.fromMemorySegment0Template(ms, offset); // specialize 1189 } 1190 1191 @ForceInline 1192 @Override 1193 final 1194 $abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m) { 1195 return super.fromMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m); // specialize 1196 } 1197 1198 @ForceInline 1199 @Override 1200 final 1201 void intoArray0($type$[] a, int offset) { 1202 super.intoArray0Template(a, offset); // specialize 1203 } 1204 1205 @ForceInline 1206 @Override 1207 final 1208 void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) { 1209 super.intoArray0Template($masktype$.class, a, offset, ($masktype$) m); 1210 } 1211 1212 #if[!byteOrShort] 1213 @ForceInline 1214 @Override 1215 final 1216 void intoArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) { 1217 super.intoArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m); 1218 } 1219 #end[!byteOrShort] 1220 1221 #if[byte] 1222 @ForceInline 1223 @Override 1224 final 1225 void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) { 1226 super.intoBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m); 1227 } 1228 #end[byte] 1229 1230 @ForceInline 1231 @Override 1232 final 1233 void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m) { 1234 super.intoMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m); 1235 } 1236 1237 #if[short] 1238 @ForceInline 1239 @Override 1240 final 1241 void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) { 1242 super.intoCharArray0Template($masktype$.class, a, offset, ($masktype$) m); 1243 } 1244 #end[short] 1245 1246 // End of specialized low-level memory operations. 1247 1248 // ================================================ 1249 1250 }