1 /*
  2  * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 package compiler.c2.irTests;
 25 
 26 import compiler.lib.ir_framework.*;
 27 import jdk.test.lib.Utils;
 28 import jdk.test.whitebox.WhiteBox;
 29 import jdk.internal.misc.Unsafe;
 30 import java.util.Random;
 31 import java.util.Arrays;
 32 import java.nio.ByteOrder;
 33 
 34 // Note Lilliput:
 35 // Tests test that vectorization is used to fill destination byte array if alignment allows. That works in Stock VM
 36 // since for both byte[] and long[] members start at the same offset. It does not work in Lilliput, nor would it work
 37 // in stock if we fix "8139457: Array bases are aligned at HeapWord granularity", since bytes start at offset 12, long
 38 // at offset 16.
 39 // For now I just enforce -CompactObjectHeaders.
 40 
 41 /*
 42  * @test
 43  * @bug 8300258
 44  * @key randomness
 45  * @requires (os.simpleArch == "x64") | (os.simpleArch == "aarch64")
 46  * @summary C2: vectorization fails on simple ByteBuffer loop
 47  * @modules java.base/jdk.internal.misc
 48  * @library /test/lib /
 49  * @build jdk.test.whitebox.WhiteBox
 50  * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
 51  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI compiler.c2.irTests.TestVectorizationMismatchedAccess
 52  */
 53 
 54 public class TestVectorizationMismatchedAccess {
 55     private static final Unsafe UNSAFE = Unsafe.getUnsafe();
 56     private static final Random RANDOM = Utils.getRandomInstance();
 57     private final static WhiteBox wb = WhiteBox.getWhiteBox();
 58 
 59     public static void main(String[] args) {
 60         Object alignVector = wb.getVMFlag("AlignVector");
 61         if (alignVector != null && !((Boolean)alignVector)) {
 62             if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) {
 63                 throw new RuntimeException("fix test that was written for a little endian platform");
 64             }
 65             TestFramework.runWithFlags("-XX:+UnlockExperimentalVMOptions", "-XX:-UseCompactObjectHeaders", "--add-modules", "java.base", "--add-exports", "java.base/jdk.internal.misc=ALL-UNNAMED");
 66         }
 67     }
 68 
 69     static int size = 1024;
 70     static byte[] byteArray = new byte[size * 8];
 71     static long[] longArray = new long[size];
 72     static byte[] verifyByteArray = new byte[size * 8];
 73     static long[] verifyLongArray = new long[size];
 74     static long baseOffset = 0;
 75     static long baseOffHeap = UNSAFE.allocateMemory(size * 8);
 76 
 77 
 78     static {
 79         for (int i = 0; i < verifyByteArray.length; i++) {
 80             verifyByteArray[i] = (byte)RANDOM.nextInt(Byte.MAX_VALUE);
 81         }
 82         for (int i = 0; i < verifyLongArray.length; i++) {
 83             verifyLongArray[i] = 0;
 84             for (int j = 0; j < 8; j++) {
 85                 verifyLongArray[i] = verifyLongArray[i] | (((long)verifyByteArray[8 * i + j]) << 8 * j);
 86             }
 87         }
 88     }
 89 
 90     static private void runAndVerify(Runnable test, int offset) {
 91         System.arraycopy(verifyLongArray, 0, longArray, 0, longArray.length);
 92         Arrays.fill(byteArray, (byte)0);
 93         test.run();
 94         int i;
 95         for (i = 0; i < Math.max(offset, 0); i++) {
 96             if (byteArray[i] != 0) {
 97                 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
 98             }
 99         }
100         for (; i < Math.min(byteArray.length + offset, byteArray.length); i++) {
101             if (byteArray[i] != verifyByteArray[i - offset]) {
102                 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i-offset]);
103             }
104         }
105         for (; i < byteArray.length; i++) {
106             if (byteArray[i] != 0) {
107                 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
108             }
109         }
110     }
111 
112     static private void runAndVerify2(Runnable test, int offset) {
113         System.arraycopy(verifyByteArray, 0, byteArray, 0, byteArray.length);
114         test.run();
115         int i;
116         for (i = 0; i < Math.max(offset, 0); i++) {
117             if (byteArray[i] != verifyByteArray[i]) {
118                 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i]);
119             }
120         }
121         for (; i < Math.min(byteArray.length + offset, byteArray.length); i++) {
122             int val = offset > 0 ? verifyByteArray[(i-offset) % 8] : verifyByteArray[i-offset];
123             if (byteArray[i] != val) {
124                 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i-offset]);
125             }
126         }
127         for (; i < byteArray.length; i++) {
128             if (byteArray[i] != verifyByteArray[i]) {
129                 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i]);
130             }
131         }
132     }
133 
134 
135     static private void runAndVerify3(Runnable test, int offset) {
136         System.arraycopy(verifyLongArray, 0, longArray, 0, longArray.length);
137         for (int i = 0; i < size * 8; i++) {
138             UNSAFE.putByte(null, baseOffHeap + i, (byte)0);
139         }
140         test.run();
141         int i;
142         for (i = 0; i < Math.max(offset, 0); i++) {
143             if (UNSAFE.getByte(null, baseOffHeap + i) != 0) {
144                 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
145             }
146         }
147         for (; i < Math.min(size * 8 + offset, size * 8); i++) {
148             if (UNSAFE.getByte(null, baseOffHeap + i) != verifyByteArray[i - offset]) {
149                 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != " + verifyByteArray[i-offset]);
150             }
151         }
152         for (; i < byteArray.length; i++) {
153             if (UNSAFE.getByte(null, baseOffHeap + i) != 0) {
154                 throw new RuntimeException("Incorrect result at " + i + " " + byteArray[i] + " != 0");
155             }
156         }
157     }
158 
159     @Test
160     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
161     public static void testByteLong1(byte[] dest, long[] src) {
162         for (int i = 0; i < src.length; i++) {
163             UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i, src[i]);
164         }
165     }
166 
167     @Run(test = "testByteLong1")
168     public static void testByteLong1_runner() {
169         runAndVerify(() -> testByteLong1(byteArray, longArray), 0);
170     }
171 
172     @Test
173     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
174     public static void testByteLong2(byte[] dest, long[] src) {
175         for (int i = 1; i < src.length; i++) {
176             UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i - 1), src[i]);
177         }
178     }
179 
180     @Run(test = "testByteLong2")
181     public static void testByteLong2_runner() {
182         runAndVerify(() -> testByteLong2(byteArray, longArray), -8);
183     }
184 
185     @Test
186     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
187     public static void testByteLong3(byte[] dest, long[] src) {
188         for (int i = 0; i < src.length - 1; i++) {
189             UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + 1), src[i]);
190         }
191     }
192 
193     @Run(test = "testByteLong3")
194     public static void testByteLong3_runner() {
195         runAndVerify(() -> testByteLong3(byteArray, longArray), 8);
196     }
197 
198     @Test
199     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
200     public static void testByteLong4(byte[] dest, long[] src, int start, int stop) {
201         for (int i = start; i < stop; i++) {
202             UNSAFE.putLongUnaligned(dest, 8 * i + baseOffset, src[i]);
203         }
204     }
205 
206     @Run(test = "testByteLong4")
207     public static void testByteLong4_runner() {
208         baseOffset = UNSAFE.ARRAY_BYTE_BASE_OFFSET;
209         runAndVerify(() -> testByteLong4(byteArray, longArray, 0, size), 0);
210     }
211 
212     @Test
213     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
214     public static void testByteLong5(byte[] dest, long[] src, int start, int stop) {
215         for (int i = start; i < stop; i++) {
216             UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + baseOffset), src[i]);
217         }
218     }
219 
220     @Run(test = "testByteLong5")
221     public static void testByteLong5_runner() {
222         baseOffset = 1;
223         runAndVerify(() -> testByteLong5(byteArray, longArray, 0, size-1), 8);
224     }
225 
226     @Test
227     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
228     public static void testByteByte1(byte[] dest, byte[] src) {
229         for (int i = 0; i < src.length / 8; i++) {
230             UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
231         }
232     }
233 
234     @Run(test = "testByteByte1")
235     public static void testByteByte1_runner() {
236         runAndVerify2(() -> testByteByte1(byteArray, byteArray), 0);
237     }
238 
239     // It would be legal to vectorize this one but it's not currently
240     @Test
241     //@IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
242     public static void testByteByte2(byte[] dest, byte[] src) {
243         for (int i = 1; i < src.length / 8; i++) {
244             UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i - 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
245         }
246     }
247 
248     @Run(test = "testByteByte2")
249     public static void testByteByte2_runner() {
250         runAndVerify2(() -> testByteByte2(byteArray, byteArray), -8);
251     }
252 
253     @Test
254     @IR(failOn = { IRNode.LOAD_VECTOR, IRNode.STORE_VECTOR })
255     public static void testByteByte3(byte[] dest, byte[] src) {
256         for (int i = 0; i < src.length / 8 - 1; i++) {
257             UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + 1), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
258         }
259     }
260 
261     @Run(test = "testByteByte3")
262     public static void testByteByte3_runner() {
263         runAndVerify2(() -> testByteByte3(byteArray, byteArray), 8);
264     }
265 
266     @Test
267     @IR(failOn = { IRNode.LOAD_VECTOR, IRNode.STORE_VECTOR })
268     public static void testByteByte4(byte[] dest, byte[] src, int start, int stop) {
269         for (int i = start; i < stop; i++) {
270             UNSAFE.putLongUnaligned(dest, 8 * i + baseOffset, UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
271         }
272     }
273 
274     @Run(test = "testByteByte4")
275     public static void testByteByte4_runner() {
276         baseOffset = UNSAFE.ARRAY_BYTE_BASE_OFFSET;
277         runAndVerify2(() -> testByteByte4(byteArray, byteArray, 0, size), 0);
278     }
279 
280     @Test
281     @IR(failOn = { IRNode.LOAD_VECTOR, IRNode.STORE_VECTOR })
282     public static void testByteByte5(byte[] dest, byte[] src, int start, int stop) {
283         for (int i = start; i < stop; i++) {
284             UNSAFE.putLongUnaligned(dest, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * (i + baseOffset), UNSAFE.getLongUnaligned(src, UNSAFE.ARRAY_BYTE_BASE_OFFSET + 8 * i));
285         }
286     }
287 
288     @Run(test = "testByteByte5")
289     public static void testByteByte5_runner() {
290         baseOffset = 1;
291         runAndVerify2(() -> testByteByte5(byteArray, byteArray, 0, size-1), 8);
292     }
293 
294     @Test
295     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
296     public static void testOffHeapLong1(long dest, long[] src) {
297         for (int i = 0; i < src.length; i++) {
298             UNSAFE.putLongUnaligned(null, dest + 8 * i, src[i]);
299         }
300     }
301 
302     @Run(test = "testOffHeapLong1")
303     public static void testOffHeapLong1_runner() {
304         runAndVerify3(() -> testOffHeapLong1(baseOffHeap, longArray), 0);
305     }
306 
307     @Test
308     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
309     public static void testOffHeapLong2(long dest, long[] src) {
310         for (int i = 1; i < src.length; i++) {
311             UNSAFE.putLongUnaligned(null, dest + 8 * (i - 1), src[i]);
312         }
313     }
314 
315     @Run(test = "testOffHeapLong2")
316     public static void testOffHeapLong2_runner() {
317         runAndVerify3(() -> testOffHeapLong2(baseOffHeap, longArray), -8);
318     }
319 
320     @Test
321     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
322     public static void testOffHeapLong3(long dest, long[] src) {
323         for (int i = 0; i < src.length - 1; i++) {
324             UNSAFE.putLongUnaligned(null, dest + 8 * (i + 1), src[i]);
325         }
326     }
327 
328     @Run(test = "testOffHeapLong3")
329     public static void testOffHeapLong3_runner() {
330         runAndVerify3(() -> testOffHeapLong3(baseOffHeap, longArray), 8);
331     }
332 
333     @Test
334     @IR(counts = { IRNode.LOAD_VECTOR, ">=1", IRNode.STORE_VECTOR, ">=1" })
335     public static void testOffHeapLong4(long dest, long[] src, int start, int stop) {
336         for (int i = start; i < stop; i++) {
337             UNSAFE.putLongUnaligned(null, dest + 8 * i + baseOffset, src[i]);
338         }
339     }
340 
341     @Run(test = "testOffHeapLong4")
342     public static void testOffHeapLong4_runner() {
343         baseOffset = 8;
344         runAndVerify3(() -> testOffHeapLong4(baseOffHeap, longArray, 0, size-1), 8);
345     }
346 }