1 /*
2 * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
3 * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 */
24
25 package compiler.c2.irTests;
26
27 import compiler.lib.ir_framework.*;
28 import jdk.test.lib.Utils;
29 import jdk.test.whitebox.WhiteBox;
30 import jdk.internal.misc.Unsafe;
31 import java.util.Random;
32 import java.util.Arrays;
33 import java.nio.ByteOrder;
34 import java.util.List;
35
36 /*
37 * @test
38 * @bug 8300258
39 * @key randomness
40 * @summary C2: vectorization fails on simple ByteBuffer loop
41 * @modules java.base/jdk.internal.misc
42 * @library /test/lib /
43 * @build jdk.test.whitebox.WhiteBox
44 * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
45 * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI compiler.c2.irTests.TestVectorizationMismatchedAccess
46 */
47
48 public class TestVectorizationMismatchedAccess {
49 private static final Unsafe UNSAFE = Unsafe.getUnsafe();
50 private static final Random RANDOM = Utils.getRandomInstance();
51 private final static WhiteBox wb = WhiteBox.getWhiteBox();
52
53 public static void main(String[] args) {
54 TestFramework framework = new TestFramework();
55 framework.addFlags("--add-modules", "java.base", "--add-exports", "java.base/jdk.internal.misc=ALL-UNNAMED",
56 "-XX:+UnlockExperimentalVMOptions");
57
58 // Cross-product:
59 // +-AlignVector
60 // +-UseCompactObjectHeaders
61 // +-UseAutoVectorizationSpeculativeAliasingChecks
62 int idx = 0;
63 for (String av : List.of("-XX:-AlignVector", "-XX:+AlignVector")) {
64 for (String coh : List.of("-XX:-UseCompactObjectHeaders", "-XX:+UseCompactObjectHeaders")) {
65 for (String sac : List.of("-XX:-UseAutoVectorizationSpeculativeAliasingChecks", "-XX:+UseAutoVectorizationSpeculativeAliasingChecks")) {
66 framework.addScenarios(new Scenario(idx++, av, coh, sac));
67 }
68 }
69 }
70
71 framework.start();
72 }
73
74 static int size = 1024;
75 static byte[] byteArray = new byte[size * 8];
76 static long[] longArray = new long[size];
77 static byte[] verifyByteArray = new byte[size * 8];
78 static long[] verifyLongArray = new long[size];
79 static long baseOffset = 0;
80 static long baseOffHeap = UNSAFE.allocateMemory(size * 8);
81
82
83 static {
84 for (int i = 0; i < verifyByteArray.length; i++) {
85 verifyByteArray[i] = (byte)RANDOM.nextInt(Byte.MAX_VALUE);
86 }
87 for (int i = 0; i < verifyLongArray.length; i++) {
88 verifyLongArray[i] = 0;
89 for (int j = 0; j < 8; j++) {
90 verifyLongArray[i] = verifyLongArray[i] | (((long)verifyByteArray[8 * i + j]) << 8 * j);
91 }
92 }
93 }
94
95 // Method to adjust the value for the native byte order
96 static private long handleByteOrder(long value) {
97 if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) {
98 value = Long.reverseBytes(value);
99 }
100 return value;
101 }
102
103 static private void runAndVerify(Runnable test, int offset) {
104 System.arraycopy(verifyLongArray, 0, longArray, 0, longArray.length);
105 Arrays.fill(byteArray, (byte)0);
106 test.run();
107 int i;
108 for (i = 0; i < Math.max(offset, 0); i++) {
109 if (byteArray[i] != 0) {
110 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
111 }
112 }
113 for (; i < Math.min(byteArray.length + offset, byteArray.length); i++) {
114 if (byteArray[i] != verifyByteArray[i - offset]) {
115 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i-offset]);
116 }
117 }
118 for (; i < byteArray.length; i++) {
119 if (byteArray[i] != 0) {
120 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
121 }
122 }
123 }
124
125 static private void runAndVerify2(Runnable test, int offset) {
126 System.arraycopy(verifyByteArray, 0, byteArray, 0, byteArray.length);
127 test.run();
128 int i;
129 for (i = 0; i < Math.max(offset, 0); i++) {
130 if (byteArray[i] != verifyByteArray[i]) {
131 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i]);
132 }
133 }
134 for (; i < Math.min(byteArray.length + offset, byteArray.length); i++) {
135 int val = offset >=1 ? verifyByteArray[(i-offset) % 8] : verifyByteArray[i-offset];
136 if (byteArray[i] != val) {
137 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i-offset]);
138 }
139 }
140 for (; i < byteArray.length; i++) {
141 if (byteArray[i] != verifyByteArray[i]) {
142 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i]);
143 }
144 }
145 }
146
147
148 static private void runAndVerify3(Runnable test, int offset) {
149 System.arraycopy(verifyLongArray, 0, longArray, 0, longArray.length);
150 for (int i = 0; i < size * 8; i++) {
151 UNSAFE.putByte(null, baseOffHeap + i, (byte)0);
152 }
153 test.run();
154 int i;
155 for (i = 0; i < Math.max(offset, 0); i++) {
156 if (UNSAFE.getByte(null, baseOffHeap + i) != 0) {
157 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
158 }
159 }
160 for (; i < Math.min(size * 8 + offset, size * 8); i++) {
161 if (UNSAFE.getByte(null, baseOffHeap + i) != verifyByteArray[i - offset]) {
162 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i-offset]);
163 }
164 }
165 for (; i < byteArray.length; i++) {
166 if (UNSAFE.getByte(null, baseOffHeap + i) != 0) {
167 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
168 }
169 }
170 }
171
172 @Test
173 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
174 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
175 applyIfPlatform = {"64-bit", "true"})
176 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
177 // might get fixed with JDK-8325155.
178 public static void testByteLong1a(byte[] dest, long[] src) {
179 for (int i = 0; i < src.length; i++) {
180 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i, handleByteOrder(src[i]));
181 }
182 }
183
184 @Test
185 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
186 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
187 applyIfPlatform = {"64-bit", "true"})
188 // 32-bit: address has ConvL2I for cast of long to address, not supported.
189 public static void testByteLong1b(byte[] dest, long[] src) {
190 for (int i = 0; i < src.length; i++) {
191 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i, handleByteOrder(src[i]));
192 }
193 }
194
195 @Test
196 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
197 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"})
198 public static void testByteLong1c(byte[] dest, long[] src) {
199 long base = 64; // make sure it is big enough and 8 byte aligned (required for 32-bit)
200 for (int i = 0; i < src.length - 8; i++) {
201 UNSAFE.putLongUnaligned(dest, base + 8 * i, handleByteOrder(src[i]));
202 }
203 }
204
205 @Test
206 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
207 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
208 applyIfPlatform = {"64-bit", "true"})
209 // 32-bit: address has ConvL2I for cast of long to address, not supported.
210 public static void testByteLong1d(byte[] dest, long[] src) {
211 long base = 64; // make sure it is big enough and 8 byte aligned (required for 32-bit)
212 for (int i = 0; i < src.length - 8; i++) {
213 UNSAFE.putLongUnaligned(dest, base + 8L * i, handleByteOrder(src[i]));
214 }
215 }
216
217 @Run(test = {"testByteLong1a", "testByteLong1b", "testByteLong1c", "testByteLong1d"})
218 public static void testByteLong1_runner() {
219 runAndVerify(() -> testByteLong1a(byteArray, longArray), 0);
220 runAndVerify(() -> testByteLong1b(byteArray, longArray), 0);
221 testByteLong1c(byteArray, longArray);
222 testByteLong1d(byteArray, longArray);
223 }
224
225 @Test
226 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
227 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
228 applyIfPlatform = {"64-bit", "true"})
229 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
230 // might get fixed with JDK-8325155.
231 public static void testByteLong2a(byte[] dest, long[] src) {
232 for (int i = 1; i < src.length; i++) {
233 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i - 1), handleByteOrder(src[i]));
234 }
235 }
236
237 @Test
238 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
239 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
240 applyIfPlatform = {"64-bit", "true"})
241 // 32-bit: address has ConvL2I for cast of long to address, not supported.
242 public static void testByteLong2b(byte[] dest, long[] src) {
243 for (int i = 1; i < src.length; i++) {
244 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i - 1), handleByteOrder(src[i]));
245 }
246 }
247
248 @Run(test = {"testByteLong2a", "testByteLong2b"})
249 public static void testByteLong2_runner() {
250 runAndVerify(() -> testByteLong2a(byteArray, longArray), -8);
251 runAndVerify(() -> testByteLong2b(byteArray, longArray), -8);
252 }
253
254 @Test
255 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
256 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
257 applyIfPlatform = {"64-bit", "true"})
258 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
259 // might get fixed with JDK-8325155.
260 public static void testByteLong3a(byte[] dest, long[] src) {
261 for (int i = 0; i < src.length - 1; i++) {
262 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + 1), handleByteOrder(src[i]));
263 }
264 }
265
266 @Test
267 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
268 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
269 applyIfPlatform = {"64-bit", "true"})
270 // 32-bit: address has ConvL2I for cast of long to address, not supported.
271 public static void testByteLong3b(byte[] dest, long[] src) {
272 for (int i = 0; i < src.length - 1; i++) {
273 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i + 1), handleByteOrder(src[i]));
274 }
275 }
276
277 @Run(test = {"testByteLong3a", "testByteLong3b"})
278 public static void testByteLong3_runner() {
279 runAndVerify(() -> testByteLong3a(byteArray, longArray), 8);
280 runAndVerify(() -> testByteLong3b(byteArray, longArray), 8);
281 }
282
283 @Test
284 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
285 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
286 applyIfPlatform = {"64-bit", "true"},
287 applyIf = {"AlignVector", "false"})
288 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
289 // might get fixed with JDK-8325155.
290 // AlignVector cannot guarantee that invar is aligned.
291 public static void testByteLong4a(byte[] dest, long[] src, int start, int stop) {
292 for (int i = start; i < stop; i++) {
293 UNSAFE.putLongUnaligned(dest, 8 * i + baseOffset, handleByteOrder(src[i]));
294 }
295 }
296
297 @Test
298 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
299 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
300 applyIfPlatform = {"64-bit", "true"},
301 applyIf = {"AlignVector", "false"})
302 // 32-bit: address has ConvL2I for cast of long to address, not supported.
303 // AlignVector cannot guarantee that invar is aligned.
304 public static void testByteLong4b(byte[] dest, long[] src, int start, int stop) {
305 for (int i = start; i < stop; i++) {
306 UNSAFE.putLongUnaligned(dest, 8L * i + baseOffset, handleByteOrder(src[i]));
307 }
308 }
309
310 @Run(test = {"testByteLong4a", "testByteLong4b"})
311 public static void testByteLong4_runner() {
312 baseOffset = UNSAFE.ARRAY_BYTE_BASE_OFFSET;
313 runAndVerify(() -> testByteLong4a(byteArray, longArray, 0, size), 0);
314 runAndVerify(() -> testByteLong4b(byteArray, longArray, 0, size), 0);
315 }
316
317 @Test
318 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
319 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
320 applyIfPlatform = {"64-bit", "true"})
321 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
322 // might get fixed with JDK-8325155.
323 public static void testByteLong5a(byte[] dest, long[] src, int start, int stop) {
324 for (int i = start; i < stop; i++) {
325 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + baseOffset), handleByteOrder(src[i]));
326 }
327 }
328
329 @Test
330 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
331 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
332 applyIfPlatform = {"64-bit", "true"})
333 // 32-bit: address has ConvL2I for cast of long to address, not supported.
334 public static void testByteLong5b(byte[] dest, long[] src, int start, int stop) {
335 for (int i = start; i < stop; i++) {
336 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i + baseOffset), handleByteOrder(src[i]));
337 }
338 }
339
340 @Run(test = {"testByteLong5a", "testByteLong5b"})
341 public static void testByteLong5_runner() {
342 baseOffset = 1;
343 runAndVerify(() -> testByteLong5a(byteArray, longArray, 0, size-1), 8);
344 runAndVerify(() -> testByteLong5b(byteArray, longArray, 0, size-1), 8);
345 }
346
347 @Test
348 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
349 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
350 applyIfPlatform = {"64-bit", "true"})
351 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
352 // might get fixed with JDK-8325155.
353 public static void testByteByte1a(byte[] dest, byte[] src) {
354 for (int i = 0; i < src.length / 8; i++) {
355 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
356 }
357 }
358
359 @Test
360 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
361 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
362 applyIfPlatform = {"64-bit", "true"})
363 // 32-bit: address has ConvL2I for cast of long to address, not supported.
364 public static void testByteByte1b(byte[] dest, byte[] src) {
365 for (int i = 0; i < src.length / 8; i++) {
366 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i));
367 }
368 }
369
370 @Run(test = {"testByteByte1a", "testByteByte1b"})
371 public static void testByteByte1_runner() {
372 runAndVerify2(() -> testByteByte1a(byteArray, byteArray), 0);
373 runAndVerify2(() -> testByteByte1b(byteArray, byteArray), 0);
374 }
375
376 @Test
377 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
378 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
379 applyIfPlatform = {"64-bit", "true"})
380 // 32-bit: offsets are badly aligned (UNSAFE.ARRAY_BYTE_BASE_OFFSET is 4 byte aligned, but not 8 byte aligned).
381 // might get fixed with JDK-8325155.
382 public static void testByteByte2a(byte[] dest, byte[] src) {
383 for (int i = 1; i < src.length / 8; i++) {
384 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i - 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
385 }
386 }
387
388 @Test
389 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
390 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
391 applyIfPlatform = {"64-bit", "true"})
392 // 32-bit: address has ConvL2I for cast of long to address, not supported.
393 public static void testByteByte2b(byte[] dest, byte[] src) {
394 for (int i = 1; i < src.length / 8; i++) {
395 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i - 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i));
396 }
397 }
398
399 @Run(test = {"testByteByte2a", "testByteByte2b"})
400 public static void testByteByte2_runner() {
401 runAndVerify2(() -> testByteByte2a(byteArray, byteArray), -8);
402 runAndVerify2(() -> testByteByte2b(byteArray, byteArray), -8);
403 }
404
405 @Test
406 @IR(failOn = { IRNode.LOAD_VECTOR_L, IRNode.STORE_VECTOR },
407 applyIf = {"UseAutoVectorizationSpeculativeAliasingChecks", "false"})
408 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1", ".*multiversion.*", ">=1"},
409 phase = CompilePhase.PRINT_IDEAL,
410 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
411 applyIfAnd = {"UseAutoVectorizationSpeculativeAliasingChecks", "true", "AlignVector", "false"},
412 applyIfPlatform = {"64-bit", "true"})
413 // We have unknown aliasing. At runtime "dest == src", so the AutoVectorization Predicate fails, and recompiles with Multiversioning.
414 public static void testByteByte3a(byte[] dest, byte[] src) {
415 for (int i = 0; i < src.length / 8 - 1; i++) {
416 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
417 }
418 }
419
420 @Test
421 @IR(failOn = { IRNode.LOAD_VECTOR_L, IRNode.STORE_VECTOR },
422 applyIf = {"UseAutoVectorizationSpeculativeAliasingChecks", "false"})
423 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1", ".*multiversion.*", ">=1"},
424 phase = CompilePhase.PRINT_IDEAL,
425 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
426 applyIfAnd = {"UseAutoVectorizationSpeculativeAliasingChecks", "true", "AlignVector", "false"},
427 applyIfPlatform = {"64-bit", "true"})
428 // We have unknown aliasing. At runtime "dest == src", so the AutoVectorization Predicate fails, and recompiles with Multiversioning.
429 public static void testByteByte3b(byte[] dest, byte[] src) {
430 for (int i = 0; i < src.length / 8 - 1; i++) {
431 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i + 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i));
432 }
433 }
434
435 @Run(test = {"testByteByte3a", "testByteByte3b"})
436 public static void testByteByte3_runner() {
437 runAndVerify2(() -> testByteByte3a(byteArray, byteArray), 8);
438 runAndVerify2(() -> testByteByte3b(byteArray, byteArray), 8);
439 }
440
441 @Test
442 @IR(failOn = { IRNode.LOAD_VECTOR_L, IRNode.STORE_VECTOR },
443 applyIf = {"UseAutoVectorizationSpeculativeAliasingChecks", "false"})
444 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1", ".*multiversion.*", ">=1"},
445 phase = CompilePhase.PRINT_IDEAL,
446 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
447 applyIfAnd = {"UseAutoVectorizationSpeculativeAliasingChecks", "true", "AlignVector", "false"},
448 applyIfPlatform = {"64-bit", "true"})
449 // We have unknown aliasing. At runtime "dest == src", so the AutoVectorization Predicate fails, and recompiles with Multiversioning.
450 public static void testByteByte4a(byte[] dest, byte[] src, int start, int stop) {
451 for (int i = start; i < stop; i++) {
452 UNSAFE.putLongUnaligned(dest, 8 * i + baseOffset, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
453 }
454 }
455
456 @Test
457 @IR(failOn = { IRNode.LOAD_VECTOR_L, IRNode.STORE_VECTOR },
458 applyIf = {"UseAutoVectorizationSpeculativeAliasingChecks", "false"})
459 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1", ".*multiversion.*", ">=1"},
460 phase = CompilePhase.PRINT_IDEAL,
461 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
462 applyIfAnd = {"UseAutoVectorizationSpeculativeAliasingChecks", "true", "AlignVector", "false"},
463 applyIfPlatform = {"64-bit", "true"})
464 // We have unknown aliasing. At runtime "dest == src", so the AutoVectorization Predicate fails, and recompiles with Multiversioning.
465 public static void testByteByte4b(byte[] dest, byte[] src, int start, int stop) {
466 for (int i = start; i < stop; i++) {
467 UNSAFE.putLongUnaligned(dest, 8L * i + baseOffset, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i));
468 }
469 }
470
471 @Run(test = {"testByteByte4a", "testByteByte4b"})
472 public static void testByteByte4_runner() {
473 baseOffset = UNSAFE.ARRAY_BYTE_BASE_OFFSET;
474 runAndVerify2(() -> testByteByte4a(byteArray, byteArray, 0, size), 0);
475 runAndVerify2(() -> testByteByte4b(byteArray, byteArray, 0, size), 0);
476 }
477
478 @Test
479 @IR(failOn = { IRNode.LOAD_VECTOR_L, IRNode.STORE_VECTOR },
480 applyIf = {"UseAutoVectorizationSpeculativeAliasingChecks", "false"})
481 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1", ".*multiversion.*", ">=1"},
482 phase = CompilePhase.PRINT_IDEAL,
483 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
484 applyIfAnd = {"UseAutoVectorizationSpeculativeAliasingChecks", "true", "AlignVector", "false"},
485 applyIfPlatform = {"64-bit", "true"})
486 // We have unknown aliasing. At runtime "dest == src", so the AutoVectorization Predicate fails, and recompiles with Multiversioning.
487 public static void testByteByte5a(byte[] dest, byte[] src, int start, int stop) {
488 for (int i = start; i < stop; i++) {
489 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + baseOffset), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
490 }
491 }
492
493 @Test
494 @IR(failOn = { IRNode.LOAD_VECTOR_L, IRNode.STORE_VECTOR },
495 applyIf = {"UseAutoVectorizationSpeculativeAliasingChecks", "false"})
496 @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1", ".*multiversion.*", ">=1"},
497 phase = CompilePhase.PRINT_IDEAL,
498 applyIfCPUFeatureOr = {"sse2", "true", "asimd", "true", "rvv", "true"},
499 applyIfAnd = {"UseAutoVectorizationSpeculativeAliasingChecks", "true", "AlignVector", "false"},
500 applyIfPlatform = {"64-bit", "true"})
501 // We have unknown aliasing. At runtime "dest == src", so the AutoVectorization Predicate fails, and recompiles with Multiversioning.
502 public static void testByteByte5b(byte[] dest, byte[] src, int start, int stop) {
503 for (int i = start; i < stop; i++) {
504 UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * (i + baseOffset), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8L * i));
505 }
506 }
507
508 @Run(test = {"testByteByte5a", "testByteByte5b"})
509 public static void testByteByte5_runner() {
510 baseOffset = 1;
511 runAndVerify2(() -> testByteByte5a(byteArray, byteArray, 0, size-1), 8);
512 runAndVerify2(() -> testByteByte5b(byteArray, byteArray, 0, size-1), 8);
513 }
514
515 @Test
516 @IR(counts = { IRNode.LOAD_VECTOR_L, "=0", IRNode.STORE_VECTOR, "=0" }) // temporary
517 // @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" })
518 // FAILS: adr is CastX2P(dest + 8 * (i + int_con))
519 // See: JDK-8331576
520 public static void testOffHeapLong1a(long dest, long[] src) {
521 for (int i = 0; i < src.length; i++) {
522 UNSAFE.putLongUnaligned(null, dest + 8 * i, handleByteOrder(src[i]));
523 }
524 }
525
526 @Test
527 @IR(counts = { IRNode.LOAD_VECTOR_L, "=0", IRNode.STORE_VECTOR, "=0" }) // temporary
528 // @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" })
529 // FAILS: adr is CastX2P(dest + 8L * (i + int_con))
530 // See: JDK-8331576
531 public static void testOffHeapLong1b(long dest, long[] src) {
532 for (int i = 0; i < src.length; i++) {
533 UNSAFE.putLongUnaligned(null, dest + 8L * i, handleByteOrder(src[i]));
534 }
535 }
536
537 @Run(test = {"testOffHeapLong1a", "testOffHeapLong1b"})
538 public static void testOffHeapLong1_runner() {
539 runAndVerify3(() -> testOffHeapLong1a(baseOffHeap, longArray), 0);
540 runAndVerify3(() -> testOffHeapLong1b(baseOffHeap, longArray), 0);
541 }
542
543 @Test
544 @IR(counts = { IRNode.LOAD_VECTOR_L, "=0", IRNode.STORE_VECTOR, "=0" }) // temporary
545 // @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" })
546 // FAILS: adr is CastX2P
547 // See: JDK-8331576
548 public static void testOffHeapLong2a(long dest, long[] src) {
549 for (int i = 1; i < src.length; i++) {
550 UNSAFE.putLongUnaligned(null, dest + 8 * (i - 1), handleByteOrder(src[i]));
551 }
552 }
553
554 @Test
555 @IR(counts = { IRNode.LOAD_VECTOR_L, "=0", IRNode.STORE_VECTOR, "=0" }) // temporary
556 // @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" })
557 // FAILS: adr is CastX2P
558 // See: JDK-8331576
559 public static void testOffHeapLong2b(long dest, long[] src) {
560 for (int i = 1; i < src.length; i++) {
561 UNSAFE.putLongUnaligned(null, dest + 8L * (i - 1), handleByteOrder(src[i]));
562 }
563 }
564
565 @Run(test = {"testOffHeapLong2a", "testOffHeapLong2b"})
566 public static void testOffHeapLong2_runner() {
567 runAndVerify3(() -> testOffHeapLong2a(baseOffHeap, longArray), -8);
568 runAndVerify3(() -> testOffHeapLong2b(baseOffHeap, longArray), -8);
569 }
570
571 @Test
572 @IR(counts = { IRNode.LOAD_VECTOR_L, "=0", IRNode.STORE_VECTOR, "=0" }) // temporary
573 // @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" })
574 // FAILS: adr is CastX2P
575 // See: JDK-8331576
576 public static void testOffHeapLong3a(long dest, long[] src) {
577 for (int i = 0; i < src.length - 1; i++) {
578 UNSAFE.putLongUnaligned(null, dest + 8 * (i + 1), handleByteOrder(src[i]));
579 }
580 }
581
582 @Test
583 @IR(counts = { IRNode.LOAD_VECTOR_L, "=0", IRNode.STORE_VECTOR, "=0" }) // temporary
584 // @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" })
585 // FAILS: adr is CastX2P
586 // See: JDK-8331576
587 public static void testOffHeapLong3b(long dest, long[] src) {
588 for (int i = 0; i < src.length - 1; i++) {
589 UNSAFE.putLongUnaligned(null, dest + 8L * (i + 1), handleByteOrder(src[i]));
590 }
591 }
592
593 @Run(test = {"testOffHeapLong3a", "testOffHeapLong3b"})
594 public static void testOffHeapLong3_runner() {
595 runAndVerify3(() -> testOffHeapLong3a(baseOffHeap, longArray), 8);
596 runAndVerify3(() -> testOffHeapLong3b(baseOffHeap, longArray), 8);
597 }
598
599 @Test
600 @IR(counts = { IRNode.LOAD_VECTOR_L, "=0", IRNode.STORE_VECTOR, "=0" }) // temporary
601 // @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
602 // applyIf = {"AlignVector", "false"})
603 // FAILS: adr is CastX2P
604 // See: JDK-8331576
605 // AlignVector cannot guarantee that invar is aligned.
606 public static void testOffHeapLong4a(long dest, long[] src, int start, int stop) {
607 for (int i = start; i < stop; i++) {
608 UNSAFE.putLongUnaligned(null, dest + 8 * i + baseOffset, handleByteOrder(src[i]));
609 }
610 }
611
612 @Test
613 @IR(counts = { IRNode.LOAD_VECTOR_L, "=0", IRNode.STORE_VECTOR, "=0" }) // temporary
614 // @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
615 // applyIf = {"AlignVector", "false"})
616 // FAILS: adr is CastX2P
617 // See: JDK-8331576
618 // AlignVector cannot guarantee that invar is aligned.
619 public static void testOffHeapLong4b(long dest, long[] src, int start, int stop) {
620 for (int i = start; i < stop; i++) {
621 UNSAFE.putLongUnaligned(null, dest + 8L * i + baseOffset, handleByteOrder(src[i]));
622 }
623 }
624
625 @Run(test = {"testOffHeapLong4a", "testOffHeapLong4b"})
626 public static void testOffHeapLong4_runner() {
627 baseOffset = 8;
628 runAndVerify3(() -> testOffHeapLong4a(baseOffHeap, longArray, 0, size-1), 8);
629 runAndVerify3(() -> testOffHeapLong4b(baseOffHeap, longArray, 0, size-1), 8);
630 }
631 }