125 if (op != Operation.STORE_VECTOR_AFTER_SCATTER) {
126 return scope("");
127 }
128
129 if (vec.length <= 2) {
130 return scope(" // No Vector nodes are emitted for vectors of length 2 or shorter.\n");
131 }
132
133 if (vec.elementType.byteSize() < 4) {
134 return scope(" // StoreVectorScatter is not emitted for vectors of subword types.\n");
135 }
136
137 return scope(
138 """
139 @IR(counts = {IRNode.STORE_VECTOR_SCATTER, "=1"},
140 applyIfCPUFeatureOr = {"avx512", "true", "sve", "true"})
141 """
142 );
143 });
144
145 return scope(
146 let("pty", vec.elementType.name()),
147 let("ptyIR", ptyIR),
148 let("idx", idx),
149 switch (op) {
150 case STORE_MASK, STORE_SCATTER_MASK ->
151 // For masked operations, depending on the generated mask and index map C2 does not manage to elide a branch from
152 // if (mask.allTrue()) {
153 // intoArray(a, offset);
154 // } else {
155 // intoArray(a, offset, mask);
156 // }
157 // leading to both stores of the diamond being live. This is highly profile dependent and cannot be predicted.
158 """
159 @IR(counts = {IRNode.START + "Store#{ptyIR}" + IRNode.MID + "(Memory: @aryptr:#{pty}\\\\[int:#{arraySize}\\\\]).*(:NotNull:exact\\\\[\\\\d+\\\\]).*" + IRNode.END, ">=1",
160 IRNode.START + "Store#{ptyIR}" + IRNode.MID + "(Memory: @aryptr:#{pty}\\\\[int:#{arraySize}\\\\]).*(:NotNull:exact\\\\[\\\\d+\\\\]).*" + IRNode.END, "<=2"},
161 applyIfCPUFeatureOr = {"avx512", "true", "sve", "true"},
162 phase = CompilePhase.BEFORE_MATCHING)
163 """;
164 case STORE_SCATTER ->
165 """
166 @IR(counts = {IRNode.START + "Store#{ptyIR}" + IRNode.MID + "(Memory: @aryptr:#{pty}\\\\[int:#{arraySize}\\\\]).*(:NotNull:exact\\\\[\\\\d+\\\\]).*" + IRNode.END, "=1"},
167 applyIfCPUFeatureOr = {"avx512", "true", "sve", "true"},
168 phase = CompilePhase.BEFORE_MATCHING)
169 """;
170 case STORE_VECTOR_AFTER_SCATTER -> opVerification.asToken();
171 case RANDOM -> "";
172 }
173 );
174 });
175
176 var testBody = Template.make("testCaseRandom", "op", "arraySize", (Random testCaseRandom, Operation op, Integer arraySize) -> {
177 if (op == Operation.RANDOM) {
178 return scope(generateRandomTest(testCaseRandom).asToken());
179 }
180
181 var maskGeneration = Template.make(() -> scope(
182 let("idx", idx),
183 let("boxedTy", vec.elementType.boxedTypeName()),
184 let("species", vec.speciesName),
185 " VectorMask<#boxedTy> mask = VectorMask.fromLong(#species, ",
186 testCaseRandom.nextInt(0, 10) == 0 ? testCaseRandom.nextLong() : "-1 - (1 << #idx)",
|
125 if (op != Operation.STORE_VECTOR_AFTER_SCATTER) {
126 return scope("");
127 }
128
129 if (vec.length <= 2) {
130 return scope(" // No Vector nodes are emitted for vectors of length 2 or shorter.\n");
131 }
132
133 if (vec.elementType.byteSize() < 4) {
134 return scope(" // StoreVectorScatter is not emitted for vectors of subword types.\n");
135 }
136
137 return scope(
138 """
139 @IR(counts = {IRNode.STORE_VECTOR_SCATTER, "=1"},
140 applyIfCPUFeatureOr = {"avx512", "true", "sve", "true"})
141 """
142 );
143 });
144
145 if (op == Operation.STORE_MASK && vec.elementType.name().equals("int") && vec.length == 4) {
146 return scope(
147 """
148 // No IR-verification for testInteger4Mask because it intermittently
149 // compiles to a different code shape. Probably due to a fully set mask.
150 """
151 );
152 }
153
154 return scope(
155 let("pty", vec.elementType.name()),
156 let("ptyIR", ptyIR),
157 let("idx", idx),
158 switch (op) {
159 case STORE_MASK, STORE_SCATTER_MASK ->
160 // For masked operations, depending on the generated mask and index map C2 does not manage to elide a branch from
161 // if (mask.allTrue()) {
162 // intoArray(a, offset);
163 // } else {
164 // intoArray(a, offset, mask);
165 // }
166 // leading to both stores of the diamond being live. This is highly profile dependent and cannot be predicted.
167 """
168 @IR(counts = {IRNode.START + "Store#{ptyIR}" + IRNode.MID + "(Memory: @aryptr:[a-z_]*:#{pty}\\\\[int:#{arraySize}\\\\]).*(:NotNull:exact:[a-z_:]*\\\\[\\\\d+\\\\]).*" + IRNode.END, ">=1",
169 IRNode.START + "Store#{ptyIR}" + IRNode.MID + "(Memory: @aryptr:[a-z_]*:#{pty}\\\\[int:#{arraySize}\\\\]).*(:NotNull:exact:[a-z_:].*\\\\[\\\\d+\\\\]).*" + IRNode.END, "<=2"},
170 applyIfCPUFeatureOr = {"avx512", "true", "sve", "true"},
171 phase = CompilePhase.BEFORE_MATCHING)
172 """;
173 case STORE_SCATTER ->
174 """
175 @IR(counts = {IRNode.START + "Store#{ptyIR}" + IRNode.MID + "(Memory: @aryptr:[a-z_]*:#{pty}\\\\[int:#{arraySize}\\\\]).*(:NotNull:exact:[a-z_:]*\\\\[\\\\d+\\\\]).*" + IRNode.END, "=1"},
176 applyIfCPUFeatureOr = {"avx512", "true", "sve", "true"},
177 phase = CompilePhase.BEFORE_MATCHING)
178 """;
179 case STORE_VECTOR_AFTER_SCATTER -> opVerification.asToken();
180 case RANDOM -> "";
181 }
182 );
183 });
184
185 var testBody = Template.make("testCaseRandom", "op", "arraySize", (Random testCaseRandom, Operation op, Integer arraySize) -> {
186 if (op == Operation.RANDOM) {
187 return scope(generateRandomTest(testCaseRandom).asToken());
188 }
189
190 var maskGeneration = Template.make(() -> scope(
191 let("idx", idx),
192 let("boxedTy", vec.elementType.boxedTypeName()),
193 let("species", vec.speciesName),
194 " VectorMask<#boxedTy> mask = VectorMask.fromLong(#species, ",
195 testCaseRandom.nextInt(0, 10) == 0 ? testCaseRandom.nextLong() : "-1 - (1 << #idx)",
|