1 /* 2 * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 package jdk.incubator.vector; 26 27 import java.util.Objects; 28 29 import jdk.internal.vm.annotation.ForceInline; 30 31 import jdk.internal.misc.Unsafe; 32 33 import jdk.internal.vm.vector.VectorSupport; 34 35 import static jdk.incubator.vector.VectorOperators.*; 36 37 abstract class AbstractMask<E> extends VectorMask<E> { 38 AbstractMask(boolean[] bits) { 39 super(bits); 40 } 41 42 /*package-private*/ 43 abstract boolean[] getBits(); 44 45 // Unary operator 46 47 interface MUnOp { 48 boolean apply(int i, boolean a); 49 } 50 51 abstract AbstractMask<E> uOp(MUnOp f); 52 53 // Binary operator 54 55 interface MBinOp { 56 boolean apply(int i, boolean a, boolean b); 57 } 58 59 abstract AbstractMask<E> bOp(VectorMask<E> o, MBinOp f); 60 61 /*package-private*/ 62 abstract AbstractSpecies<E> vspecies(); 63 64 @Override 65 @ForceInline 66 public final VectorSpecies<E> vectorSpecies() { 67 return vspecies(); 68 } 69 70 @Override 71 @ForceInline 72 public boolean laneIsSet(int i) { 73 int length = length(); 74 Objects.checkIndex(i, length); 75 if (length <= Long.SIZE) { 76 return ((toLong() >>> i) & 1L) == 1; 77 } else { 78 return getBits()[i]; 79 } 80 } 81 82 @Override 83 public void intoArray(boolean[] bits, int i) { 84 AbstractSpecies<E> vsp = (AbstractSpecies<E>) vectorSpecies(); 85 int laneCount = vsp.laneCount(); 86 i = VectorIntrinsics.checkFromIndexSize(i, laneCount, bits.length); 87 VectorSupport.store( 88 vsp.maskType(), vsp.elementType(), laneCount, 89 bits, (long) i + Unsafe.ARRAY_BOOLEAN_BASE_OFFSET, 90 this, bits, i, 91 (c, idx, s) -> System.arraycopy(s.getBits(), 0, c, (int) idx, s.length())); 92 93 } 94 95 @Override 96 public boolean[] toArray() { 97 return getBits().clone(); 98 } 99 100 @Override 101 @ForceInline 102 @SuppressWarnings("unchecked") 103 public 104 <F> VectorMask<F> check(Class<F> elementType) { 105 if (vectorSpecies().elementType() != elementType) { 106 throw AbstractSpecies.checkFailed(this, elementType); 107 } 108 return (VectorMask<F>) this; 109 } 110 111 @Override 112 @ForceInline 113 @SuppressWarnings("unchecked") 114 public 115 <F> VectorMask<F> check(VectorSpecies<F> species) { 116 if (species != vectorSpecies()) { 117 throw AbstractSpecies.checkFailed(this, species); 118 } 119 return (VectorMask<F>) this; 120 } 121 122 @Override 123 @ForceInline 124 @SuppressWarnings("unchecked") 125 <F> VectorMask<F> check(Class<? extends VectorMask<F>> maskClass, Vector<F> vector) { 126 if (!sameSpecies(maskClass, vector)) { 127 throw AbstractSpecies.checkFailed(this, vector); 128 } 129 return (VectorMask<F>) this; 130 } 131 132 @ForceInline 133 private <F> boolean sameSpecies(Class<? extends VectorMask<F>> maskClass, Vector<F> vector) { 134 boolean same = getClass() == maskClass; 135 assert (same == (vectorSpecies() == vector.species())) : same; 136 return same; 137 } 138 139 @Override 140 public VectorMask<E> andNot(VectorMask<E> m) { 141 return and(m.not()); 142 } 143 144 /*package-private*/ 145 static boolean anyTrueHelper(boolean[] bits) { 146 // FIXME: Maybe use toLong() != 0 here. 147 for (boolean i : bits) { 148 if (i) return true; 149 } 150 return false; 151 } 152 153 /*package-private*/ 154 static boolean allTrueHelper(boolean[] bits) { 155 // FIXME: Maybe use not().toLong() == 0 here. 156 for (boolean i : bits) { 157 if (!i) return false; 158 } 159 return true; 160 } 161 162 /*package-private*/ 163 static int trueCountHelper(boolean[] bits) { 164 int c = 0; 165 for (boolean i : bits) { 166 if (i) c++; 167 } 168 return c; 169 } 170 171 /*package-private*/ 172 static int firstTrueHelper(boolean[] bits) { 173 for (int i = 0; i < bits.length; i++) { 174 if (bits[i]) return i; 175 } 176 return bits.length; 177 } 178 179 /*package-private*/ 180 static int lastTrueHelper(boolean[] bits) { 181 for (int i = bits.length-1; i >= 0; i--) { 182 if (bits[i]) return i; 183 } 184 return -1; 185 } 186 187 /*package-private*/ 188 static long toLongHelper(boolean[] bits) { 189 long res = 0; 190 long set = 1; 191 for (int i = 0; i < bits.length; i++) { 192 res = bits[i] ? res | set : res; 193 set = set << 1; 194 } 195 return res; 196 } 197 198 @Override 199 @ForceInline 200 public VectorMask<E> indexInRange(int offset, int limit) { 201 int vlength = length(); 202 Vector<E> iota = vectorSpecies().zero().addIndex(1); 203 VectorMask<E> badMask = checkIndex0(offset, limit, iota, vlength); 204 return this.andNot(badMask); 205 } 206 207 @Override 208 @ForceInline 209 public VectorMask<E> indexInRange(long offset, long limit) { 210 int vlength = length(); 211 Vector<E> iota = vectorSpecies().zero().addIndex(1); 212 VectorMask<E> badMask = checkIndex0(offset, limit, iota, vlength); 213 return this.andNot(badMask); 214 } 215 216 /*package-private*/ 217 @ForceInline 218 AbstractVector<E> 219 toVectorTemplate() { 220 AbstractSpecies<E> vsp = vspecies(); 221 Vector<E> zero = vsp.broadcast(0); 222 Vector<E> mone = vsp.broadcast(-1); 223 // -1 will result in the most significant bit being set in 224 // addition to some or all other lane bits. 225 // For integral types, *all* lane bits will be set. 226 // The bits for -1.0 are like {0b10111*0000*}. 227 // FIXME: Use a conversion intrinsic for this operation. 228 // https://bugs.openjdk.java.net/browse/JDK-8225740 229 return (AbstractVector<E>) zero.blend(mone, this); 230 } 231 232 /** 233 * Test if a masked memory access at a given offset into an array 234 * of the given length will stay within the array. 235 * The per-lane offsets are iota*esize. 236 */ 237 /*package-private*/ 238 @ForceInline 239 void checkIndexByLane(int offset, int length, 240 Vector<E> iota, 241 int esize) { 242 if (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK == 0) { 243 return; 244 } 245 // Although the specification is simple, the implementation is 246 // tricky, because the value iota*esize might possibly 247 // overflow. So we calculate our test values as scalars, 248 // clipping to the range [-1..VLENGTH], and test them against 249 // the unscaled iota vector, whose values are in [0..VLENGTH-1]. 250 int vlength = length(); 251 VectorMask<E> badMask; 252 if (esize == 1) { 253 badMask = checkIndex0(offset, length, iota, vlength); 254 } else if (offset >= 0) { 255 // Masked access to multi-byte lanes in byte array. 256 // It could be aligned anywhere. 257 int elemCount = Math.min(vlength, (length - offset) / esize); 258 badMask = checkIndex0(0, elemCount, iota, vlength); 259 } else { 260 int clipOffset = Math.max(offset, -(vlength * esize)); 261 badMask = checkIndex0(clipOffset, length, 262 iota.lanewise(VectorOperators.MUL, esize), 263 vlength * esize); 264 } 265 badMask = badMask.and(this); 266 if (badMask.anyTrue()) { 267 int badLane = badMask.firstTrue(); 268 throw ((AbstractMask<E>)badMask) 269 .checkIndexFailed(offset, badLane, length, esize); 270 } 271 } 272 273 private 274 @ForceInline 275 VectorMask<E> checkIndex0(int offset, int length, 276 Vector<E> iota, int vlength) { 277 // An active lane is bad if its number is greater than 278 // length-offset, since when added to offset it will step off 279 // of the end of the array. To avoid overflow when 280 // converting, clip the comparison value to [0..vlength] 281 // inclusive. 282 int indexLimit = Math.max(0, Math.min(length - offset, vlength)); 283 VectorMask<E> badMask = 284 iota.compare(GE, iota.broadcast(indexLimit)); 285 if (offset < 0) { 286 // An active lane is bad if its number is less than 287 // -offset, because when added to offset it will then 288 // address an array element at a negative index. To avoid 289 // overflow when converting, clip the comparison value at 290 // vlength. This specific expression works correctly even 291 // when offset is Integer.MIN_VALUE. 292 int firstGoodIndex = -Math.max(offset, -vlength); 293 VectorMask<E> badMask2 = 294 iota.compare(LT, iota.broadcast(firstGoodIndex)); 295 if (indexLimit >= vlength) { 296 badMask = badMask2; // 1st badMask is all true 297 } else { 298 badMask = badMask.or(badMask2); 299 } 300 } 301 return badMask; 302 } 303 304 /** 305 * Test if a masked memory access at a given offset into an array 306 * of the given length will stay within the array. 307 * The per-lane offsets are iota*esize. 308 */ 309 /*package-private*/ 310 @ForceInline 311 void checkIndexByLane(long offset, long length, 312 Vector<E> iota, 313 int esize) { 314 if (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK == 0) { 315 return; 316 } 317 // Although the specification is simple, the implementation is 318 // tricky, because the value iota*esize might possibly 319 // overflow. So we calculate our test values as scalars, 320 // clipping to the range [-1..VLENGTH], and test them against 321 // the unscaled iota vector, whose values are in [0..VLENGTH-1]. 322 int vlength = length(); 323 VectorMask<E> badMask; 324 if (esize == 1) { 325 badMask = checkIndex0(offset, length, iota, vlength); 326 } else if (offset >= 0) { 327 // Masked access to multi-byte lanes in byte array. 328 // It could be aligned anywhere. 329 // 0 <= elemCount <= vlength 330 int elemCount = (int) Math.min(vlength, (length - offset) / esize); 331 badMask = checkIndex0(0, elemCount, iota, vlength); 332 } else { 333 // -vlength * esize <= clipOffset <= 0 334 int clipOffset = (int) Math.max(offset, -(vlength * esize)); 335 badMask = checkIndex0(clipOffset, length, 336 iota.lanewise(VectorOperators.MUL, esize), 337 vlength * esize); 338 } 339 badMask = badMask.and(this); 340 if (badMask.anyTrue()) { 341 int badLane = badMask.firstTrue(); 342 throw ((AbstractMask<E>)badMask) 343 .checkIndexFailed(offset, badLane, length, esize); 344 } 345 } 346 347 private 348 @ForceInline 349 VectorMask<E> checkIndex0(long offset, long length, 350 Vector<E> iota, int vlength) { 351 // An active lane is bad if its number is greater than 352 // length-offset, since when added to offset it will step off 353 // of the end of the array. To avoid overflow when 354 // converting, clip the comparison value to [0..vlength] 355 // inclusive. 356 // 0 <= indexLimit <= vlength 357 int indexLimit = (int) Math.max(0, Math.min(length - offset, vlength)); 358 VectorMask<E> badMask = 359 iota.compare(GE, iota.broadcast(indexLimit)); 360 if (offset < 0) { 361 // An active lane is bad if its number is less than 362 // -offset, because when added to offset it will then 363 // address an array element at a negative index. To avoid 364 // overflow when converting, clip the comparison value at 365 // vlength. This specific expression works correctly even 366 // when offset is Integer.MIN_VALUE. 367 // 0 <= firstGoodIndex <= vlength 368 int firstGoodIndex = (int) -Math.max(offset, -vlength); 369 VectorMask<E> badMask2 = 370 iota.compare(LT, iota.broadcast(firstGoodIndex)); 371 if (indexLimit >= vlength) { 372 badMask = badMask2; // 1st badMask is all true 373 } else { 374 badMask = badMask.or(badMask2); 375 } 376 } 377 return badMask; 378 } 379 380 private IndexOutOfBoundsException checkIndexFailed(long offset, int lane, 381 long length, int esize) { 382 String msg = String.format("Masked range check failed: "+ 383 "vector mask %s out of bounds at "+ 384 "index %d+%d for length %d", 385 this, offset, lane * esize, length); 386 if (esize != 1) { 387 msg += String.format(" (each lane spans %d elements)", esize); 388 } 389 throw new IndexOutOfBoundsException(msg); 390 } 391 392 }