1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 /**
25 * @test
26 * @bug 8387073
27 * @key randomness
28 * @summary Narrower stores preceding masked vector stores must not be eliminated.
29 * @modules jdk.incubator.vector
30 * @library /test/lib /
31 * @run driver ${test.main.class}
32 */
33
34 package compiler.igvn;
35
36 import java.util.Arrays;
37 import java.util.ArrayList;
38 import java.util.Collections;
39 import java.util.List;
40 import java.util.Random;
41 import java.util.Set;
42 import java.util.stream.*;
43
44 import jdk.incubator.vector.VectorShape;
45
46 import jdk.test.lib.Utils;
47 import compiler.lib.compile_framework.*;
48 import compiler.lib.template_framework.*;
49 import static compiler.lib.template_framework.Template.scope;
50 import static compiler.lib.template_framework.Template.let;
51 import compiler.lib.template_framework.library.*;
52
53 public class TestMaskedStoreIdealization {
54 private static final Random RANDOM = Utils.getRandomInstance();
55 private static final String PACKAGE = "compiler.igvn.generated";
56 private static final String CLASS_NAME = "TestMaskedStoreIdealizationGenerated";
57
58 public static void main(String[] args) {
59 final CompileFramework comp = new CompileFramework();
60 comp.addJavaSourceCode(PACKAGE + "." + CLASS_NAME, generate(comp));
61 comp.compile("--add-modules=jdk.incubator.vector");
62
63 List<String> vmArgs = new ArrayList<>(List.of(
64 "--add-modules=jdk.incubator.vector",
65 "--add-opens", "jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED"
66 ));
67 vmArgs.addAll(Arrays.asList(args)); // Forward args
68 // Temporarily disable stress flag due to unrelated test failures.
69 // TODO: Remove when JDK-8388490 is fixed.
70 vmArgs.addAll(List.of("-XX:+IgnoreUnrecognizedVMOptions", "-XX:-StressReflectiveCode"));
71 String[] vmArgsArray = vmArgs.toArray(new String[0]);
72
73 comp.invoke(PACKAGE + "." + CLASS_NAME, "main", new Object[] { vmArgsArray });
74 }
75
76 private static String generate(CompileFramework comp) {
77 final Set<String> imports = Set.of("java.util.Arrays",
78 "java.util.Random",
79 "jdk.incubator.vector.*",
80 "jdk.test.lib.Utils",
81 "compiler.lib.generators.*");
82
83 // The preferred vector shape is the largest possible vector size.
84 final int maxVecByteSize = VectorShape.preferredShape().vectorBitSize() / 8;
85
86 final List<TemplateToken> tests = new ArrayList<>();
87 // Add tests only for the vector shapes that
88 tests.addAll(CodeGenerationDataNameType.VECTOR_VECTOR_TYPES
89 .stream()
90 .filter(vec -> vec.byteSize() <= maxVecByteSize && vec.elementType instanceof PrimitiveType)
91 .map(vec -> new TestPerShape(vec).generate())
92 .collect(Collectors.toList()));
93 tests.add(PrimitiveType.generateLibraryRNG());
94
95 return TestFrameworkClass.render(PACKAGE, CLASS_NAME, imports, comp.getEscapedClassPathOfCompiledClasses(), tests);
96 }
97
98 enum Operation {
99 STORE_SCATTER,
100 STORE_MASK,
101 STORE_SCATTER_MASK,
102 STORE_VECTOR_AFTER_SCATTER,
103 RANDOM
104 }
105
106 record TestPerShape(VectorType.Vector vec) {
107 TemplateToken generate() {
108 final String testName = vec.elementType.boxedTypeName() + vec.length;
109
110 // Select the index where we set the mask to false. The index is biased to
111 // zero, as the original bug only triggered with the first element.
112 final int idx = RANDOM.nextBoolean() ? RANDOM.nextInt(0, vec.length) : 0;
113
114 var irVerification = Template.make("op", "arraySize", (Operation op, Integer arraySize) -> {
115 // No IR-verification for random test cases.
116 if (op == Operation.RANDOM) {
117 return scope("");
118 }
119
120 final PrimitiveType pty = (PrimitiveType) vec.elementType;
121 final String ptyIR = pty.abbrev().equals("S") ? "C" : pty.abbrev();
122
123 // Verify that the Scatter store for STORE_VECTOR_AFTER_SCATTER is not eliminated.
124 var opVerification = Template.make(() -> {
125 if (op != Operation.STORE_VECTOR_AFTER_SCATTER) {
126 return scope("");
127 }
128
129 if (vec.length <= 2) {
130 return scope(" // No Vector nodes are emitted for vectors of length 2 or shorter.\n");
131 }
132
133 if (vec.elementType.byteSize() < 4) {
134 return scope(" // StoreVectorScatter is not emitted for vectors of subword types.\n");
135 }
136
137 return scope(
138 """
139 @IR(counts = {IRNode.STORE_VECTOR_SCATTER, "=1"},
140 applyIfCPUFeatureOr = {"avx512", "true", "sve", "true"})
141 """
142 );
143 });
144
145 if (op == Operation.STORE_MASK && vec.elementType.name().equals("int") && vec.length == 4) {
146 return scope(
147 """
148 // No IR-verification for testInteger4Mask because it intermittently
149 // compiles to a different code shape. Probably due to a fully set mask.
150 """
151 );
152 }
153
154 return scope(
155 let("pty", vec.elementType.name()),
156 let("ptyIR", ptyIR),
157 let("idx", idx),
158 switch (op) {
159 case STORE_MASK, STORE_SCATTER_MASK ->
160 // For masked operations, depending on the generated mask and index map C2 does not manage to elide a branch from
161 // if (mask.allTrue()) {
162 // intoArray(a, offset);
163 // } else {
164 // intoArray(a, offset, mask);
165 // }
166 // leading to both stores of the diamond being live. This is highly profile dependent and cannot be predicted.
167 """
168 @IR(counts = {IRNode.START + "Store#{ptyIR}" + IRNode.MID + "(Memory: @aryptr:[a-z_]*:#{pty}\\\\[int:#{arraySize}\\\\]).*(:NotNull:exact:[a-z_:]*\\\\[\\\\d+\\\\]).*" + IRNode.END, ">=1",
169 IRNode.START + "Store#{ptyIR}" + IRNode.MID + "(Memory: @aryptr:[a-z_]*:#{pty}\\\\[int:#{arraySize}\\\\]).*(:NotNull:exact:[a-z_:].*\\\\[\\\\d+\\\\]).*" + IRNode.END, "<=2"},
170 applyIfCPUFeatureOr = {"avx512", "true", "sve", "true"},
171 phase = CompilePhase.BEFORE_MATCHING)
172 """;
173 case STORE_SCATTER ->
174 """
175 @IR(counts = {IRNode.START + "Store#{ptyIR}" + IRNode.MID + "(Memory: @aryptr:[a-z_]*:#{pty}\\\\[int:#{arraySize}\\\\]).*(:NotNull:exact:[a-z_:]*\\\\[\\\\d+\\\\]).*" + IRNode.END, "=1"},
176 applyIfCPUFeatureOr = {"avx512", "true", "sve", "true"},
177 phase = CompilePhase.BEFORE_MATCHING)
178 """;
179 case STORE_VECTOR_AFTER_SCATTER -> opVerification.asToken();
180 case RANDOM -> "";
181 }
182 );
183 });
184
185 var testBody = Template.make("testCaseRandom", "op", "arraySize", (Random testCaseRandom, Operation op, Integer arraySize) -> {
186 if (op == Operation.RANDOM) {
187 return scope(generateRandomTest(testCaseRandom).asToken());
188 }
189
190 var maskGeneration = Template.make(() -> scope(
191 let("idx", idx),
192 let("boxedTy", vec.elementType.boxedTypeName()),
193 let("species", vec.speciesName),
194 " VectorMask<#boxedTy> mask = VectorMask.fromLong(#species, ",
195 testCaseRandom.nextInt(0, 10) == 0 ? testCaseRandom.nextLong() : "-1 - (1 << #idx)",
196 ");\n"
197 ));
198
199 var indexMapGeneration = Template.make(() -> {
200 // For the scatter tests, the array is one larger than the number of lanes so we can
201 // map indices starting at idx to the next index, omitting idx.
202 int[] indexMap = IntStream.range(0, vec.length)
203 .map(i -> i >= idx ? i + 1 : i)
204 .toArray();
205 String indexMapStr = Arrays.toString(indexMap)
206 .replace('[', '{')
207 .replace(']', '}');
208 return scope(
209 let("idx", idx),
210 let("len", vec.length),
211 let("idxMap", indexMapStr),
212 """
213 final int[] indexMap = #{idxMap};
214 """
215 );
216 });
217
218 var generation = switch (op) {
219 case STORE_SCATTER, STORE_VECTOR_AFTER_SCATTER -> indexMapGeneration.asToken();
220 case STORE_MASK -> maskGeneration.asToken();
221 case STORE_SCATTER_MASK ->
222 Template.make(() -> scope(indexMapGeneration.asToken(), maskGeneration.asToken())).asToken();
223 case RANDOM -> throw new RuntimeException("unreachable");
224 };
225
226 var initStore = switch (op) {
227 case STORE_SCATTER, STORE_VECTOR_AFTER_SCATTER -> " v.intoArray(a, 0, indexMap, 0);\n";
228 case STORE_MASK -> " v.intoArray(a, 0, mask);\n";
229 case STORE_SCATTER_MASK -> " v.intoArray(a, 0, indexMap, 0, mask);\n";
230 case RANDOM -> throw new RuntimeException("unreachable");
231 };
232
233 var keepStore = switch (op) {
234 case STORE_MASK, STORE_SCATTER, STORE_SCATTER_MASK -> " a[#idx] = arrVal;\n";
235 case STORE_VECTOR_AFTER_SCATTER -> " v.intoArray(a, 0);\n";
236 case RANDOM -> throw new RuntimeException("unreachable");
237 };
238
239 var holeStore = switch (op) {
240 case STORE_SCATTER -> " v.intoArray(a, 0, indexMap, 0);\n";
241 case STORE_MASK -> " v.intoArray(a, 0, mask);\n";
242 case STORE_SCATTER_MASK -> " v.intoArray(a, 0, indexMap, 0, mask);\n";
243 case STORE_VECTOR_AFTER_SCATTER -> "";
244 case RANDOM -> throw new RuntimeException("unreachable");
245 };
246
247 return scope(
248 let("pty", vec.elementType.name()),
249 let("vecTy", vec.name()),
250 let("species", vec.speciesName),
251 let("idx", idx),
252 """
253 #pty[] a = new #pty[#arraySize];
254 """,
255 generation,
256 """
257
258 var v = #vecTy.broadcast(#species, broadcastVal);
259 """,
260 initStore,
261 keepStore,
262 holeStore,
263 """
264 return a;
265 """
266 );
267 });
268
269 var testCase = Template.make("op", (Operation op) -> {
270 // To get the same test body twice, use a new random instance for each generated test body with a seed fixed per test case.
271 final int testCaseSeed = RANDOM.nextInt();
272
273 String testCaseName = testName + switch (op) {
274 case STORE_SCATTER -> "Scatter";
275 case STORE_MASK -> "Mask";
276 case STORE_SCATTER_MASK -> "ScatterMask";
277 case STORE_VECTOR_AFTER_SCATTER -> "VectorAfterScatter";
278 case RANDOM -> "Random";
279 };
280
281 // The array size needs to be one larger than the number of lanes for scatter tests, so we can not write to one element.
282 final int arraySize = switch (op) {
283 case STORE_SCATTER, STORE_SCATTER_MASK, STORE_VECTOR_AFTER_SCATTER -> vec.length + 1;
284 case STORE_MASK, RANDOM -> vec.length;
285 };
286
287 return scope(
288 let("pty", vec.elementType.name()),
289 let("testCaseName", testCaseName),
290 let("boxedTy", vec.elementType.boxedTypeName()),
291 let("vecTy", vec.name()),
292 let("lanes", vec.length),
293 let("species", vec.speciesName),
294 let("idx", idx),
295 let("rngCall", vec.elementType.callLibraryRNG()),
296 let("broadcastVal", vec.elementType.con()),
297 let("arrVal", vec.elementType.con()),
298 """
299 @Run(test = "test#{testCaseName}")
300 @Warmup(10_000)
301 static void run#{testCaseName}(RunInfo info) {
302 final #pty broadcastVal = #broadcastVal;
303 final #pty arrVal = #arrVal;
304 final #pty[] compiledResult = test#{testCaseName}(broadcastVal, arrVal);
305
306 if (!info.isWarmUp()) {
307 final #pty[] interpreterResult = reference#{testCaseName}(broadcastVal, arrVal);
308 if (!Arrays.equals(interpreterResult, compiledResult)) {
309 throw new RuntimeException("wrong result for test${testCaseName}:\\n" +
310 " interpreter result: " + Arrays.toString(interpreterResult) + "\\n" +
311 " compiled result: " + Arrays.toString(compiledResult));
312 }
313 }
314 }
315
316 @Test
317 """,
318 irVerification.asToken(op, arraySize),
319 """
320 static #pty[] test#{testCaseName}(#pty broadcastVal, #pty arrVal) {
321 """,
322 testBody.asToken(new Random(testCaseSeed), op, arraySize),
323 """
324 }
325
326 @DontCompile
327 static #pty[] reference#{testCaseName}(#pty broadcastVal, #pty arrVal) {
328 """,
329 testBody.asToken(new Random(testCaseSeed), op, arraySize),
330 """
331 }
332
333 """
334 );
335 });
336
337 return Template.make(() -> scope(
338 Stream.of(Operation.class.getEnumConstants())
339 .map(op -> testCase.asToken(op))
340 .toList()
341 )).asToken();
342 }
343
344 Template.ZeroArgs generateRandomTest(Random testCaseRandom) {
345 final int arraySize = testCaseRandom.nextInt(vec.length + 1, 5 * vec.length);
346 var maskHook = new Hook("MaskHook");
347 var genRandomMask = Template.make("maskName", "vecTy", (String maskName, VectorType.Vector vecTy) -> scope(
348 let("boxedTy", vecTy.elementType.boxedTypeName()),
349 let("species", vecTy.speciesName),
350 let("maskVal", testCaseRandom.nextLong()),
351 """
352 VectorMask<#{boxedTy}> #maskName = VectorMask.fromLong(#species, #maskVal);
353 """
354 ));
355 var genRandomIdxMap = Template.make("mapName", "maxIdx", "len", (String mapName, Integer maxIdx, Integer len) -> {
356 ArrayList<Integer> possibleIndices = new ArrayList(IntStream.range(0, maxIdx).boxed().toList());
357 Collections.shuffle(possibleIndices, testCaseRandom);
358 return scope(
359 let("map", String.join(", ", possibleIndices.stream().limit(vec.length).map(i -> i.toString()).toList())),
360 """
361 int[] #mapName = { #map };
362 """
363 );
364 });
365 var initialStore = Template.make(() -> {
366 final int offset = testCaseRandom.nextInt(0, 4) == 0 ? testCaseRandom.nextInt(0, arraySize - vec.length) : 0;
367 return scope(
368 let("offset", offset),
369 switch (testCaseRandom.nextInt(0,4)) {
370 case 0 -> scope(
371 """
372 v.intoArray(a, #offset);
373 """
374 );
375 case 1 -> scope(
376 maskHook.insert(genRandomMask.asToken("initMask", vec)),
377 """
378 v.intoArray(a, #offset, initMask);
379 """
380 );
381 case 2 -> scope(
382 maskHook.insert(genRandomIdxMap.asToken("initMap", arraySize - offset, vec.length)),
383 """
384 v.intoArray(a, #offset, initMap, 0);
385 """
386 );
387 case 3 -> scope(
388 maskHook.insert(scope(
389 genRandomIdxMap.asToken("initMap", arraySize - offset, vec.length),
390 genRandomMask.asToken("initMask", vec)
391 )),
392 """
393 v.intoArray(a, #offset, initMap, 0, initMask);
394 """
395 );
396 default -> throw new RuntimeException("unreachable");
397 }
398 );
399 });
400
401 var storeThatShouldNotBeLost = Template.make(() -> {
402 final int selection = testCaseRandom.nextInt(0,6);
403 final VectorType.Vector narrowerVector = CodeGenerationDataNameType.VECTOR_VECTOR_TYPES
404 .stream()
405 .filter(v -> v.elementType == vec.elementType && v.length <= vec.length)
406 // Flip a coin on each reduction step to pick a random element.
407 .reduce(null, (l, r) -> l == null ? r : (testCaseRandom.nextBoolean() ? l : r));
408 final int offset = testCaseRandom.nextInt(0, 4) == 0 ? testCaseRandom.nextInt(0, arraySize - narrowerVector.length) : 0;
409 return scope(
410 let("offset", offset),
411 switch (selection) {
412 case 0, 1, 2, 3 -> scope(
413 let("vecTy", narrowerVector),
414 let("species", narrowerVector.speciesName),
415 """
416 var vKeep = #vecTy.broadcast(#species, arrVal);
417 """
418 );
419 default -> "";
420 },
421 switch (selection) {
422 case 0 -> scope(
423 """
424 vKeep.intoArray(a, #offset);
425 """
426 );
427 case 1 -> scope(
428 maskHook.insert(genRandomMask.asToken("keepMask", narrowerVector)),
429 """
430 vKeep.intoArray(a, #offset, keepMask);
431 """
432 );
433 case 2 -> scope(
434 maskHook.insert(genRandomIdxMap.asToken("keepMap", arraySize - offset, narrowerVector.length)),
435 """
436 vKeep.intoArray(a, #offset, keepMap, 0);
437 """
438 );
439 case 3 -> scope(
440 maskHook.insert(scope(
441 genRandomIdxMap.asToken("keepMap", arraySize - offset, narrowerVector.length),
442 genRandomMask.asToken("keepMask", narrowerVector)
443 )),
444 """
445 vKeep.intoArray(a, #offset, keepMap, 0, keepMask);
446 """
447 );
448 case 4 -> scope(
449 let("idx", testCaseRandom.nextInt(0, arraySize)),
450 """
451 a[#idx] = arrVal;
452 """
453 );
454 case 5 -> scope(
455 let("idx", testCaseRandom.nextInt(0, arraySize - vec.length)),
456 let("len", testCaseRandom.nextInt(1, vec.length + 1)),
457 """
458 Arrays.fill(a, #idx, #idx + #len, arrVal);
459 """
460 );
461 default -> throw new RuntimeException("unreachable");
462 }
463 );
464 });
465
466 var storeWithHole = Template.make(() -> {
467 final int offset = testCaseRandom.nextInt(0, 4) == 0 ? testCaseRandom.nextInt(0, arraySize - vec.length) : 0;
468 return scope(
469 let("offset", offset),
470 switch (testCaseRandom.nextInt(0,3)) {
471 case 0 -> scope(
472 maskHook.insert(genRandomMask.asToken("holeMask", vec)),
473 """
474 v.intoArray(a, #offset, holeMask);
475 """
476 );
477 case 1 -> scope(
478 maskHook.insert(genRandomIdxMap.asToken("holeMap", arraySize - offset, vec.length)),
479 """
480 v.intoArray(a, #offset, holeMap, 0);
481 """
482 );
483 case 2 -> scope(
484 maskHook.insert(scope(
485 genRandomIdxMap.asToken("holeMap", arraySize - offset, vec.length),
486 genRandomMask.asToken("holeMask", vec)
487 )),
488 """
489 v.intoArray(a, #offset, holeMap, 0, holeMask);
490 """
491 );
492 default -> throw new RuntimeException("unreachable");
493 }
494 );
495 });
496
497 return Template.make(() -> scope(
498 let("pty", vec.elementType.name()),
499 let("arraySize", arraySize),
500 """
501 #pty[] a = new #pty[#arraySize];
502
503 """,
504 maskHook.anchor(scope(
505 // The code for generating masks and index maps goes here.
506 let("vecTy", vec.name()),
507 let("species", vec.speciesName),
508 """
509 var v = #vecTy.broadcast(#species, broadcastVal);
510 """,
511 initialStore.asToken(),
512 storeThatShouldNotBeLost.asToken(),
513 storeWithHole.asToken()
514 )),
515 """
516 return a;
517 """
518 ));
519 }
520 }
521 }
--- EOF ---