154 int i;
155 for (i = 0; i < Math.max(offset, 0); i++) {
156 if (UNSAFE.getByte(null, baseOffHeap + i) != 0) {
157 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
158 }
159 }
160 for (; i < Math.min(size * 8 + offset, size * 8); i++) {
161 if (UNSAFE.getByte(null, baseOffHeap + i) != verifyByteArray[i - offset]) {
162 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i-offset]);
163 }
164 }
165 for (; i < byteArray.length; i++) {
166 if (UNSAFE.getByte(null, baseOffHeap + i) != 0) {
167 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
168 }
169 }
170 }
171
172 @Test
173 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
174 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
175 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
176 applyIfPlatform = {"64-bit", "true"})
177 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
178 // might get fixed with JDK-8325155.
179 public static void testByteLong1a(byte[] dest, long[] src) {
180 for (int i = 0; i < src.length; i++) {
181 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i, handleByteOrder(src[i]));
182 // With AlignVector, we need 8-byte alignment of vector loads/stores.
183 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
184 // B_adr = base + 16 + 8*i -> always B_adr = base + 12 + 8*i -> never
185 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
186 // -> vectorize -> no vectorization
187 }
188 }
189
190 @Test
191 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
192 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
193 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
194 applyIfPlatform = {"64-bit", "true"})
195 // 32-bit: address has ConvL2I for cast of long to address, not supported.
196 public static void testByteLong1b(byte[] dest, long[] src) {
197 for (int i = 0; i < src.length; i++) {
198 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i, handleByteOrder(src[i]));
199 // With AlignVector, we need 8-byte alignment of vector loads/stores.
200 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
201 // B_adr = base + 16 + 8*i -> always B_adr = base + 12 + 8*i -> never
202 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
203 // -> vectorize -> no vectorization
204 }
205 }
206
207 @Test
208 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
209 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"})
210 public static void testByteLong1c(byte[] dest, long[] src) {
211 long base = 64; // make sure it is big enough and 8 byte aligned (required for 32-bit)
212 for (int i = 0; i < src.length - 8; i++) {
213 UNSAFE.putLongUnaligned(dest, base + 8 * i, handleByteOrder(src[i]));
214 // With AlignVector, we need 8-byte alignment of vector loads/stores.
215 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
216 // B_adr = base + 64 + 8*i -> always B_adr = base + 64 + 8*i -> always
217 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
218 // -> vectorize -> vectorize
219 }
220 }
221
222 @Test
223 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
224 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
225 applyIfPlatform = {"64-bit", "true"})
226 // 32-bit: address has ConvL2I for cast of long to address, not supported.
227 public static void testByteLong1d(byte[] dest, long[] src) {
228 long base = 64; // make sure it is big enough and 8 byte aligned (required for 32-bit)
229 for (int i = 0; i < src.length - 8; i++) {
230 UNSAFE.putLongUnaligned(dest, base + 8L * i, handleByteOrder(src[i]));
231 // With AlignVector, we need 8-byte alignment of vector loads/stores.
232 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
233 // B_adr = base + 64 + 8*i -> always B_adr = base + 64 + 8*i -> always
234 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
235 // -> vectorize -> vectorize
236 }
237 }
238
239 @Run(test = {"testByteLong1a", "testByteLong1b", "testByteLong1c", "testByteLong1d"})
240 public static void testByteLong1_runner() {
241 runAndVerify(() -> testByteLong1a(byteArray, longArray), 0);
242 runAndVerify(() -> testByteLong1b(byteArray, longArray), 0);
243 testByteLong1c(byteArray, longArray);
244 testByteLong1d(byteArray, longArray);
245 }
246
247 @Test
248 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
249 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
250 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
251 applyIfPlatform = {"64-bit", "true"})
252 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
253 // might get fixed with JDK-8325155.
254 public static void testByteLong2a(byte[] dest, long[] src) {
255 for (int i = 1; i < src.length; i++) {
256 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i - 1), handleByteOrder(src[i]));
257 // With AlignVector, we need 8-byte alignment of vector loads/stores.
258 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
259 // B_adr = base + 16 + 8*(i-1) -> always B_adr = base + 12 + 8*(i-1) -> never
260 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
261 // -> vectorize -> no vectorization
262 }
263 }
264
265 @Test
266 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
267 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
268 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
269 applyIfPlatform = {"64-bit", "true"})
270 // 32-bit: address has ConvL2I for cast of long to address, not supported.
271 public static void testByteLong2b(byte[] dest, long[] src) {
272 for (int i = 1; i < src.length; i++) {
273 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i - 1), handleByteOrder(src[i]));
274 // With AlignVector, we need 8-byte alignment of vector loads/stores.
275 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
276 // B_adr = base + 16 + 8*(i-1) -> always B_adr = base + 12 + 8*(i-1) -> never
277 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
278 // -> vectorize -> no vectorization
279 }
280 }
281
282 @Run(test = {"testByteLong2a", "testByteLong2b"})
283 public static void testByteLong2_runner() {
284 runAndVerify(() -> testByteLong2a(byteArray, longArray), -8);
285 runAndVerify(() -> testByteLong2b(byteArray, longArray), -8);
286 }
287
288 @Test
289 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
290 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
291 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
292 applyIfPlatform = {"64-bit", "true"})
293 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
294 // might get fixed with JDK-8325155.
295 public static void testByteLong3a(byte[] dest, long[] src) {
296 for (int i = 0; i < src.length - 1; i++) {
297 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + 1), handleByteOrder(src[i]));
298 // With AlignVector, we need 8-byte alignment of vector loads/stores.
299 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
300 // B_adr = base + 16 + 8*(i+1) -> always B_adr = base + 12 + 8*(i+1) -> never
301 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
302 // -> vectorize -> no vectorization
303 }
304 }
305
306 @Test
307 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
308 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
309 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
310 applyIfPlatform = {"64-bit", "true"})
311 // 32-bit: address has ConvL2I for cast of long to address, not supported.
312 public static void testByteLong3b(byte[] dest, long[] src) {
313 for (int i = 0; i < src.length - 1; i++) {
314 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i + 1), handleByteOrder(src[i]));
315 // With AlignVector, we need 8-byte alignment of vector loads/stores.
316 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
317 // B_adr = base + 16 + 8*(i+1) -> always B_adr = base + 12 + 8*(i+1) -> never
318 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
319 // -> vectorize -> no vectorization
320 }
321 }
322
323 @Run(test = {"testByteLong3a", "testByteLong3b"})
324 public static void testByteLong3_runner() {
325 runAndVerify(() -> testByteLong3a(byteArray, longArray), 8);
326 runAndVerify(() -> testByteLong3b(byteArray, longArray), 8);
327 }
328
329 @Test
330 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
331 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
332 applyIfPlatform = {"64-bit", "true"},
333 applyIf = {"AlignVector", "false"})
334 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
335 // might get fixed with JDK-8325155.
336 // AlignVector cannot guarantee that invar is aligned.
337 public static void testByteLong4a(byte[] dest, long[] src, int start, int stop) {
338 for (int i = start; i < stop; i++) {
339 UNSAFE.putLongUnaligned(dest, 8 * i + baseOffset, handleByteOrder(src[i]));
345 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
346 applyIfPlatform = {"64-bit", "true"},
347 applyIf = {"AlignVector", "false"})
348 // 32-bit: address has ConvL2I for cast of long to address, not supported.
349 // AlignVector cannot guarantee that invar is aligned.
350 public static void testByteLong4b(byte[] dest, long[] src, int start, int stop) {
351 for (int i = start; i < stop; i++) {
352 UNSAFE.putLongUnaligned(dest, 8L * i + baseOffset, handleByteOrder(src[i]));
353 }
354 }
355
356 @Run(test = {"testByteLong4a", "testByteLong4b"})
357 public static void testByteLong4_runner() {
358 baseOffset = UNSAFE.ARRAY_BYTE_BASE_OFFSET;
359 runAndVerify(() -> testByteLong4a(byteArray, longArray, 0, size), 0);
360 runAndVerify(() -> testByteLong4b(byteArray, longArray, 0, size), 0);
361 }
362
363 @Test
364 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
365 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
366 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
367 applyIfPlatform = {"64-bit", "true"})
368 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
369 // might get fixed with JDK-8325155.
370 public static void testByteLong5a(byte[] dest, long[] src, int start, int stop) {
371 for (int i = start; i < stop; i++) {
372 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + baseOffset), handleByteOrder(src[i]));
373 // With AlignVector, we need 8-byte alignment of vector loads/stores.
374 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
375 // B_adr = base + 16 + 8*(i+x) -> always B_adr = base + 12 + 8*(i+x) -> never
376 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
377 // -> vectorize -> no vectorization
378 }
379 }
380
381 @Test
382 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
383 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
384 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
385 applyIfPlatform = {"64-bit", "true"})
386 // 32-bit: address has ConvL2I for cast of long to address, not supported.
387 public static void testByteLong5b(byte[] dest, long[] src, int start, int stop) {
388 for (int i = start; i < stop; i++) {
389 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i + baseOffset), handleByteOrder(src[i]));
390 // With AlignVector, we need 8-byte alignment of vector loads/stores.
391 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
392 // B_adr = base + 16 + 8*(i+x) -> always B_adr = base + 12 + 8*(i+x) -> never
393 // L_adr = base + 16 + 8*i -> always L_adr = base + 16 + 8*i -> always
394 // -> vectorize -> no vectorization
395 }
396 }
397
398 @Run(test = {"testByteLong5a", "testByteLong5b"})
399 public static void testByteLong5_runner() {
400 baseOffset = 1;
401 runAndVerify(() -> testByteLong5a(byteArray, longArray, 0, size-1), 8);
402 runAndVerify(() -> testByteLong5b(byteArray, longArray, 0, size-1), 8);
403 }
404
405 @Test
406 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
407 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
408 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
409 applyIfPlatform = {"64-bit", "true"})
410 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
411 // might get fixed with JDK-8325155.
412 public static void testByteByte1a(byte[] dest, byte[] src) {
413 for (int i = 0; i < src.length / 8; i++) {
414 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
415 // With AlignVector, we need 8-byte alignment of vector loads/stores.
416 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
417 // src_adr = base + 16 + 8*i -> always src_adr = base + 12 + 8*i -> never
418 // dst_adr = base + 16 + 8*i -> always dst_adr = base + 12 + 8*i -> never
419 // -> vectorize -> no vectorization
420 }
421 }
422
423 @Test
424 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
425 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
426 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
427 applyIfPlatform = {"64-bit", "true"})
428 // 32-bit: address has ConvL2I for cast of long to address, not supported.
429 public static void testByteByte1b(byte[] dest, byte[] src) {
430 for (int i = 0; i < src.length / 8; i++) {
431 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i));
432 // With AlignVector, we need 8-byte alignment of vector loads/stores.
433 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
434 // src_adr = base + 16 + 8*i -> always src_adr = base + 12 + 8*i -> never
435 // dst_adr = base + 16 + 8*i -> always dst_adr = base + 12 + 8*i -> never
436 // -> vectorize -> no vectorization
437 }
438 }
439
440 @Run(test = {"testByteByte1a", "testByteByte1b"})
441 public static void testByteByte1_runner() {
442 runAndVerify2(() -> testByteByte1a(byteArray, byteArray), 0);
443 runAndVerify2(() -> testByteByte1b(byteArray, byteArray), 0);
444 }
445
446 @Test
447 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
448 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
449 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
450 applyIfPlatform = {"64-bit", "true"})
451 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
452 // might get fixed with JDK-8325155.
453 public static void testByteByte2a(byte[] dest, byte[] src) {
454 for (int i = 1; i < src.length / 8; i++) {
455 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i - 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
456 // With AlignVector, we need 8-byte alignment of vector loads/stores.
457 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
458 // src_adr = base + 16 + 8*i -> always src_adr = base + 12 + 8*i -> never
459 // dst_adr = base + 16 + 8*(i-1) -> always dst_adr = base + 12 + 8*(i-1) -> never
460 // -> vectorize -> no vectorization
461 }
462 }
463
464 @Test
465 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
466 applyIfOr = {"UseCompactObjectHeaders", "false", "AlignVector", "false"},
467 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
468 applyIfPlatform = {"64-bit", "true"})
469 // 32-bit: address has ConvL2I for cast of long to address, not supported.
470 public static void testByteByte2b(byte[] dest, byte[] src) {
471 for (int i = 1; i < src.length / 8; i++) {
472 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i - 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i));
473 // With AlignVector, we need 8-byte alignment of vector loads/stores.
474 // UseCompactObjectHeaders=false UseCompactObjectHeaders=true
475 // src_adr = base + 16 + 8*i -> always src_adr = base + 12 + 8*i -> never
476 // dst_adr = base + 16 + 8*(i-1) -> always dst_adr = base + 12 + 8*(i-1) -> never
477 // -> vectorize -> no vectorization
478 }
479 }
480
481 @Run(test = {"testByteByte2a", "testByteByte2b"})
482 public static void testByteByte2_runner() {
483 runAndVerify2(() -> testByteByte2a(byteArray, byteArray), -8);
484 runAndVerify2(() -> testByteByte2b(byteArray, byteArray), -8);
485 }
486
487 @Test
488 @IR(failOn = { IRNode.LOAD_VECTOR_L, IRNode.STORE_VECTOR },
489 applyIf = {"UseAutoVectorizationSpeculativeAliasingChecks", "false"})
490 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1", ".*multiversion.*", ">=1"},
491 phase = CompilePhase.PRINT_IDEAL,
492 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
493 applyIfAnd = {"UseAutoVectorizationSpeculativeAliasingChecks", "true", "AlignVector", "false"},
494 applyIfPlatform = {"64-bit", "true"})
495 // We have unknown aliasing. At runtime "dest == src", so the AutoVectorization Predicate fails, and recompiles with Multiversioning.
496 public static void testByteByte3a(byte[] dest, byte[] src) {
497 for (int i = 0; i < src.length / 8 - 1; i++) {
|
154 int i;
155 for (i = 0; i < Math.max(offset, 0); i++) {
156 if (UNSAFE.getByte(null, baseOffHeap + i) != 0) {
157 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
158 }
159 }
160 for (; i < Math.min(size * 8 + offset, size * 8); i++) {
161 if (UNSAFE.getByte(null, baseOffHeap + i) != verifyByteArray[i - offset]) {
162 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i-offset]);
163 }
164 }
165 for (; i < byteArray.length; i++) {
166 if (UNSAFE.getByte(null, baseOffHeap + i) != 0) {
167 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
168 }
169 }
170 }
171
172 @Test
173 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
174 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
175 applyIfPlatform = {"64-bit", "true"})
176 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
177 // might get fixed with JDK-8325155.
178 public static void testByteLong1a(byte[] dest, long[] src) {
179 for (int i = 0; i < src.length; i++) {
180 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i, handleByteOrder(src[i]));
181 }
182 }
183
184 @Test
185 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
186 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
187 applyIfPlatform = {"64-bit", "true"})
188 // 32-bit: address has ConvL2I for cast of long to address, not supported.
189 public static void testByteLong1b(byte[] dest, long[] src) {
190 for (int i = 0; i < src.length; i++) {
191 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i, handleByteOrder(src[i]));
192 }
193 }
194
195 @Test
196 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
197 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"})
198 public static void testByteLong1c(byte[] dest, long[] src) {
199 long base = 64; // make sure it is big enough and 8 byte aligned (required for 32-bit)
200 for (int i = 0; i < src.length - 8; i++) {
201 UNSAFE.putLongUnaligned(dest, base + 8 * i, handleByteOrder(src[i]));
202 }
203 }
204
205 @Test
206 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
207 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
208 applyIfPlatform = {"64-bit", "true"})
209 // 32-bit: address has ConvL2I for cast of long to address, not supported.
210 public static void testByteLong1d(byte[] dest, long[] src) {
211 long base = 64; // make sure it is big enough and 8 byte aligned (required for 32-bit)
212 for (int i = 0; i < src.length - 8; i++) {
213 UNSAFE.putLongUnaligned(dest, base + 8L * i, handleByteOrder(src[i]));
214 }
215 }
216
217 @Run(test = {"testByteLong1a", "testByteLong1b", "testByteLong1c", "testByteLong1d"})
218 public static void testByteLong1_runner() {
219 runAndVerify(() -> testByteLong1a(byteArray, longArray), 0);
220 runAndVerify(() -> testByteLong1b(byteArray, longArray), 0);
221 testByteLong1c(byteArray, longArray);
222 testByteLong1d(byteArray, longArray);
223 }
224
225 @Test
226 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
227 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
228 applyIfPlatform = {"64-bit", "true"})
229 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
230 // might get fixed with JDK-8325155.
231 public static void testByteLong2a(byte[] dest, long[] src) {
232 for (int i = 1; i < src.length; i++) {
233 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i - 1), handleByteOrder(src[i]));
234 }
235 }
236
237 @Test
238 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
239 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
240 applyIfPlatform = {"64-bit", "true"})
241 // 32-bit: address has ConvL2I for cast of long to address, not supported.
242 public static void testByteLong2b(byte[] dest, long[] src) {
243 for (int i = 1; i < src.length; i++) {
244 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i - 1), handleByteOrder(src[i]));
245 }
246 }
247
248 @Run(test = {"testByteLong2a", "testByteLong2b"})
249 public static void testByteLong2_runner() {
250 runAndVerify(() -> testByteLong2a(byteArray, longArray), -8);
251 runAndVerify(() -> testByteLong2b(byteArray, longArray), -8);
252 }
253
254 @Test
255 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
256 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
257 applyIfPlatform = {"64-bit", "true"})
258 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
259 // might get fixed with JDK-8325155.
260 public static void testByteLong3a(byte[] dest, long[] src) {
261 for (int i = 0; i < src.length - 1; i++) {
262 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + 1), handleByteOrder(src[i]));
263 }
264 }
265
266 @Test
267 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
268 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
269 applyIfPlatform = {"64-bit", "true"})
270 // 32-bit: address has ConvL2I for cast of long to address, not supported.
271 public static void testByteLong3b(byte[] dest, long[] src) {
272 for (int i = 0; i < src.length - 1; i++) {
273 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i + 1), handleByteOrder(src[i]));
274 }
275 }
276
277 @Run(test = {"testByteLong3a", "testByteLong3b"})
278 public static void testByteLong3_runner() {
279 runAndVerify(() -> testByteLong3a(byteArray, longArray), 8);
280 runAndVerify(() -> testByteLong3b(byteArray, longArray), 8);
281 }
282
283 @Test
284 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
285 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
286 applyIfPlatform = {"64-bit", "true"},
287 applyIf = {"AlignVector", "false"})
288 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
289 // might get fixed with JDK-8325155.
290 // AlignVector cannot guarantee that invar is aligned.
291 public static void testByteLong4a(byte[] dest, long[] src, int start, int stop) {
292 for (int i = start; i < stop; i++) {
293 UNSAFE.putLongUnaligned(dest, 8 * i + baseOffset, handleByteOrder(src[i]));
299 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
300 applyIfPlatform = {"64-bit", "true"},
301 applyIf = {"AlignVector", "false"})
302 // 32-bit: address has ConvL2I for cast of long to address, not supported.
303 // AlignVector cannot guarantee that invar is aligned.
304 public static void testByteLong4b(byte[] dest, long[] src, int start, int stop) {
305 for (int i = start; i < stop; i++) {
306 UNSAFE.putLongUnaligned(dest, 8L * i + baseOffset, handleByteOrder(src[i]));
307 }
308 }
309
310 @Run(test = {"testByteLong4a", "testByteLong4b"})
311 public static void testByteLong4_runner() {
312 baseOffset = UNSAFE.ARRAY_BYTE_BASE_OFFSET;
313 runAndVerify(() -> testByteLong4a(byteArray, longArray, 0, size), 0);
314 runAndVerify(() -> testByteLong4b(byteArray, longArray, 0, size), 0);
315 }
316
317 @Test
318 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
319 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
320 applyIfPlatform = {"64-bit", "true"})
321 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
322 // might get fixed with JDK-8325155.
323 public static void testByteLong5a(byte[] dest, long[] src, int start, int stop) {
324 for (int i = start; i < stop; i++) {
325 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + baseOffset), handleByteOrder(src[i]));
326 }
327 }
328
329 @Test
330 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
331 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
332 applyIfPlatform = {"64-bit", "true"})
333 // 32-bit: address has ConvL2I for cast of long to address, not supported.
334 public static void testByteLong5b(byte[] dest, long[] src, int start, int stop) {
335 for (int i = start; i < stop; i++) {
336 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i + baseOffset), handleByteOrder(src[i]));
337 }
338 }
339
340 @Run(test = {"testByteLong5a", "testByteLong5b"})
341 public static void testByteLong5_runner() {
342 baseOffset = 1;
343 runAndVerify(() -> testByteLong5a(byteArray, longArray, 0, size-1), 8);
344 runAndVerify(() -> testByteLong5b(byteArray, longArray, 0, size-1), 8);
345 }
346
347 @Test
348 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
349 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
350 applyIfPlatform = {"64-bit", "true"})
351 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
352 // might get fixed with JDK-8325155.
353 public static void testByteByte1a(byte[] dest, byte[] src) {
354 for (int i = 0; i < src.length / 8; i++) {
355 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
356 }
357 }
358
359 @Test
360 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
361 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
362 applyIfPlatform = {"64-bit", "true"})
363 // 32-bit: address has ConvL2I for cast of long to address, not supported.
364 public static void testByteByte1b(byte[] dest, byte[] src) {
365 for (int i = 0; i < src.length / 8; i++) {
366 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i));
367 }
368 }
369
370 @Run(test = {"testByteByte1a", "testByteByte1b"})
371 public static void testByteByte1_runner() {
372 runAndVerify2(() -> testByteByte1a(byteArray, byteArray), 0);
373 runAndVerify2(() -> testByteByte1b(byteArray, byteArray), 0);
374 }
375
376 @Test
377 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
378 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
379 applyIfPlatform = {"64-bit", "true"})
380 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
381 // might get fixed with JDK-8325155.
382 public static void testByteByte2a(byte[] dest, byte[] src) {
383 for (int i = 1; i < src.length / 8; i++) {
384 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i - 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
385 }
386 }
387
388 @Test
389 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
390 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
391 applyIfPlatform = {"64-bit", "true"})
392 // 32-bit: address has ConvL2I for cast of long to address, not supported.
393 public static void testByteByte2b(byte[] dest, byte[] src) {
394 for (int i = 1; i < src.length / 8; i++) {
395 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i - 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i));
396 }
397 }
398
399 @Run(test = {"testByteByte2a", "testByteByte2b"})
400 public static void testByteByte2_runner() {
401 runAndVerify2(() -> testByteByte2a(byteArray, byteArray), -8);
402 runAndVerify2(() -> testByteByte2b(byteArray, byteArray), -8);
403 }
404
405 @Test
406 @IR(failOn = { IRNode.LOAD_VECTOR_L, IRNode.STORE_VECTOR },
407 applyIf = {"UseAutoVectorizationSpeculativeAliasingChecks", "false"})
408 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1", ".*multiversion.*", ">=1"},
409 phase = CompilePhase.PRINT_IDEAL,
410 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
411 applyIfAnd = {"UseAutoVectorizationSpeculativeAliasingChecks", "true", "AlignVector", "false"},
412 applyIfPlatform = {"64-bit", "true"})
413 // We have unknown aliasing. At runtime "dest == src", so the AutoVectorization Predicate fails, and recompiles with Multiversioning.
414 public static void testByteByte3a(byte[] dest, byte[] src) {
415 for (int i = 0; i < src.length / 8 - 1; i++) {
|