1 /*
2 * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 package compiler.lib.ir_framework;
25
26 import compiler.lib.ir_framework.driver.FlagVMProcess;
27 import compiler.lib.ir_framework.driver.TestVMException;
28 import compiler.lib.ir_framework.driver.TestVMProcess;
29 import compiler.lib.ir_framework.driver.irmatching.IRMatcher;
30 import compiler.lib.ir_framework.driver.irmatching.IRViolationException;
31 import compiler.lib.ir_framework.driver.irmatching.Matchable;
32 import compiler.lib.ir_framework.driver.irmatching.parser.TestClassParser;
33 import compiler.lib.ir_framework.shared.*;
34 import compiler.lib.ir_framework.test.TestVM;
35 import jdk.test.lib.Platform;
36 import jdk.test.lib.Utils;
37 import jdk.test.lib.helpers.ClassFileInstaller;
38 import jdk.test.whitebox.WhiteBox;
39
40 import java.io.PrintWriter;
41 import java.io.StringWriter;
42 import java.lang.reflect.Method;
43 import java.net.MalformedURLException;
44 import java.net.URL;
45 import java.net.URLClassLoader;
46 import java.nio.file.Path;
47 import java.util.*;
48 import java.util.stream.Collectors;
49
50 /**
51 * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on
52 * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can
53 * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used
54 * testing patterns and compiler control flags.
55 * <p>
56 * The framework offers various annotations to control how your test code should be invoked and being checked. There are
57 * three kinds of tests depending on how much control is needed over the test invocation:
58 * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b>
59 * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test}
60 * annotation which represents the test code that is eventually executed by the test framework. More information about
61 * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}.
62 * <p>
63 * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which
64 * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}.
65 * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage
66 * and how to write different IR rules can be found at {@link IR}.
67 * <p>
68 * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>:
69 * <pre>
70 * {@literal @}library /test/lib
71 * {@literal @}run driver some.package.Test
72 * </pre>
73 * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the
74 * JTreg test if the test itself is not utilizing any Whitebox features directly.
75 * <p>
76 * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or
77 * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations
78 * (instead of specifying multiple JTreg {@code @run} entries).
79 * <p>
80 * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the
81 * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various
82 * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new
83 * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing.
84 * <p>
85 * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code
86 * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions.
87 * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are
88 * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}
89 * for IR matching).
90 * <p>
91 * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified
92 * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}.
93 * <p>
94 * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR
95 * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details).
96 * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing
97 * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}.
98 *
99 * @see Test
100 * @see Check
101 * @see Run
102 * @see IR
103 */
104 public class TestFramework {
105 /**
106 * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only
107 * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags
108 * by user code) are whitelisted.
109 *
110 * <p>
111 * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of
112 * the entries of this list as a substring (partial match).
113 */
114 public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>(
115 Arrays.asList(
116 // The following substrings are part of more than one VM flag
117 "RAM",
118 "Heap",
119 "Trace",
120 "Print",
121 "Verify",
122 "UseNewCode",
123 "Xmn",
124 "Xms",
125 "Xmx",
126 "Xss",
127 // The following substrings are only part of one VM flag (=exact match)
128 "CreateCoredumpOnCrash",
129 "IgnoreUnrecognizedVMOptions",
130 "UnlockDiagnosticVMOptions",
131 "UnlockExperimentalVMOptions",
132 "BackgroundCompilation",
133 "Xbatch",
134 "TieredCompilation",
135 "CompileThreshold",
136 "Xmixed",
137 "server",
138 "UseAVX",
139 "UseSSE",
140 "UseSVE",
141 "UseZbb",
142 "UseRVV",
143 "Xlog",
144 "LogCompilation"
145 )
146 );
147
148 public static final boolean VERBOSE = Boolean.getBoolean("Verbose");
149 public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty();
150 public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty();
151 private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout");
152 // Only used for internal testing and should not be used for normal user testing.
153
154 private static final String RERUN_HINT = """
155 #############################################################
156 - To only run the failed tests use -DTest, -DExclude,
157 and/or -DScenarios.
158 - To also get the standard output of the test VM run with
159 -DReportStdout=true or for even more fine-grained logging
160 use -DVerbose=true.
161 #############################################################
162 """ + System.lineSeparator();
163
164 private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true"));
165 private boolean shouldVerifyIR; // Should we perform IR matching?
166 private static boolean toggleBool;
167
168 private final Class<?> testClass;
169 private Set<Class<?>> helperClasses;
170 private List<Scenario> scenarios;
171 private Set<Integer> scenarioIndices;
172 private List<String> flags;
173 private int defaultWarmup = -1;
174
175 /*
176 * Public interface methods
177 */
178
179 /**
180 * Creates an instance acting as a builder to test the class from which this constructor was invoked from.
181 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
182 * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
183 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
184 */
185 public TestFramework() {
186 this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass());
187 }
188
189 /**
190 * Creates an instance acting as a builder to test {@code testClass}.
191 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
192 * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
193 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
194 *
195 * @param testClass the class to be tested by the framework.
196 * @see #TestFramework()
197 */
198 public TestFramework(Class<?> testClass) {
199 TestRun.check(testClass != null, "Test class cannot be null");
200 this.testClass = testClass;
201 if (VERBOSE) {
202 System.out.println("Test class: " + testClass);
203 }
204 }
205
206 /**
207 * Tests the class from which this method was invoked from.
208 */
209 public static void run() {
210 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
211 run(walker.getCallerClass());
212 }
213
214 /**
215 * Tests {@code testClass}.
216 *
217 * @param testClass the class to be tested by the framework.
218 * @see #run()
219 */
220 public static void run(Class<?> testClass) {
221 TestFramework framework = new TestFramework(testClass);
222 framework.start();
223 }
224
225 /**
226 * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}.
227 * <ul>
228 * <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p>
229 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and Javaoptions flags over
230 * the specified {@code flags} of this method.</li>
231 * <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li>
232 * <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one
233 * from which this method was called from, use {@link #addFlags(String...)}, use this method.</li>
234 * <li><p>If you want to run your JTreg test with multiple flag combinations, use
235 * {@link #addScenarios(Scenario...)}</li>
236 * </ul>
237 *
238 * @param flags VM flags to be used for the test VM.
239 */
240 public static void runWithFlags(String... flags) {
241 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
242 TestFramework framework = new TestFramework(walker.getCallerClass());
243 framework.addFlags(flags);
244 framework.start();
245 }
246
247 /**
248 * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p>
249 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags.
250 *
251 * <p>
252 * The testing can be started by invoking {@link #start()}
253 *
254 * @param flags VM options to be applied to the test VM.
255 * @return the same framework instance.
256 */
257 public TestFramework addFlags(String... flags) {
258 TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null");
259 if (this.flags == null) {
260 this.flags = new ArrayList<>();
261 }
262 this.flags.addAll(Arrays.asList(flags));
263 return this;
264 }
265
266 /**
267 * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile},
268 * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied
269 * while testing {@code testClass} (also see description of {@link TestFramework}).
270 *
271 * <p>
272 * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any
273 * compile command annotations, you do not need to include it with this method. If no helper class specifies any
274 * compile commands, you do not need to call this method at all.
275 *
276 * <p>
277 * The testing can be started by invoking {@link #start()}.
278 *
279 * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile},
280 * {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied
281 * while testing {@code testClass} (also see description of {@link TestFramework}).
282 * @return the same framework instance.
283 */
284 public TestFramework addHelperClasses(Class<?>... helperClasses) {
285 TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull),
286 "A Helper class cannot be null");
287 if (this.helperClasses == null) {
288 this.helperClasses = new HashSet<>();
289 }
290
291 this.helperClasses.addAll(Arrays.asList(helperClasses));
292 return this;
293 }
294
295 /**
296 * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the
297 * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)}
298 * and thus also override any VM or Javaoptions set by JTreg by default.<p>
299 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags.
300 *
301 * <p>
302 * The testing can be started by invoking {@link #start()}
303 *
304 * @param scenarios scenarios which specify specific flags for the test VM.
305 * @return the same framework instance.
306 */
307 public TestFramework addScenarios(Scenario... scenarios) {
308 TestFormat.checkAndReport(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull),
309 "A scenario cannot be null");
310 if (this.scenarios == null) {
311 this.scenarios = new ArrayList<>();
312 this.scenarioIndices = new HashSet<>();
313 }
314
315 for (Scenario scenario : scenarios) {
316 int scenarioIndex = scenario.getIndex();
317 TestFormat.checkNoThrow(scenarioIndices.add(scenarioIndex),
318 "Cannot define two scenarios with the same index " + scenarioIndex);
319 this.scenarios.add(scenario);
320 }
321 TestFormat.throwIfAnyFailures();
322 return this;
323 }
324
325 /**
326 * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)})
327 * set test class.
328 */
329 public void start() {
330 if (shouldInstallWhiteBox()) {
331 installWhiteBox();
332 }
333 checkIRRuleCompilePhasesFormat();
334 disableIRVerificationIfNotFeasible();
335
336 if (scenarios == null) {
337 try {
338 start(null);
339 } catch (TestVMException e) {
340 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT);
341 throw e;
342 } catch (IRViolationException e) {
343 System.out.println(e.getCompilations());
344 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT);
345 throw e;
346 }
347 } else {
348 startWithScenarios();
349 }
350 }
351
352 private void checkIRRuleCompilePhasesFormat() {
353 for (Method method : testClass.getDeclaredMethods()) {
354 for (IR irAnno : method.getAnnotationsByType(IR.class)) {
355 TestFormat.checkNoThrow(irAnno.phase().length > 0,
356 "@IR rule " + irAnno + " must specify a non-empty list of compile " +
357 "phases \"phase\" at " + method);
358 }
359 }
360 TestFormat.throwIfAnyFailures();
361 }
362
363 /**
364 * Try to load the Whitebox class from the user directory with a custom class loader. If the user has already built the
365 * Whitebox, we can load it. Otherwise, the framework needs to install it.
366 *
367 * @return true if the framework needs to install the Whitebox
368 */
369 private boolean shouldInstallWhiteBox() {
370 try {
371 URL url = Path.of(System.getProperty("user.dir")).toUri().toURL();
372 URLClassLoader userDirClassLoader =
373 URLClassLoader.newInstance(new URL[] {url}, TestFramework.class.getClassLoader().getParent());
374 Class.forName(WhiteBox.class.getName(), false, userDirClassLoader);
375 } catch (MalformedURLException e) {
376 throw new TestFrameworkException("corrupted user.dir property", e);
377 } catch (ClassNotFoundException e) {
378 // We need to manually install the WhiteBox if we cannot load the WhiteBox class from the user directory.
379 // This happens when the user test does not explicitly install the WhiteBox as part of the test.
380 return true;
381 }
382 return false;
383 }
384
385 /**
386 * Set a new default warm-up (overriding the framework default of 2000 at
387 * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit
388 * warm-up with {@link Warmup @Warmup}.
389 *
390 * @param defaultWarmup a new non-negative default warm-up.
391 * @return the same framework instance.
392 */
393 public TestFramework setDefaultWarmup(int defaultWarmup) {
394 TestFormat.checkAndReport(defaultWarmup >= 0, "Cannot specify a negative default warm-up");
395 this.defaultWarmup = defaultWarmup;
396 return this;
397 }
398
399 /**
400 * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios
401 * were run, use {@link Scenario#getTestVMOutput()}.
402 *
403 * @return the last test VM output.
404 */
405 public static String getLastTestVMOutput() {
406 return TestVMProcess.getLastTestVMOutput();
407 }
408
409 /*
410 * The following methods are only intended to be called from actual @Test methods and not from the main() method of
411 * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled).
412 */
413
414 /**
415 * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled,
416 * yet, upon returning from this method.
417 *
418 * @param m the method to be compiled.
419 * @param compLevel the (valid) compilation level at which the method should be compiled.
420 * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}.
421 */
422 public static void compile(Method m, CompLevel compLevel) {
423 TestVM.compile(m, compLevel);
424 }
425
426 /**
427 * Deoptimize {@code m}.
428 *
429 * @param m the method to be deoptimized.
430 */
431 public static void deoptimize(Method m) {
432 TestVM.deoptimize(m);
433 }
434
435 /**
436 * Returns a boolean indicating if {@code m} is compiled at any level.
437 *
438 * @param m the method to be checked.
439 * @return {@code true} if {@code m} is compiled at any level;
440 * {@code false} otherwise.
441 */
442 public static boolean isCompiled(Method m) {
443 return TestVM.isCompiled(m);
444 }
445
446 /**
447 * Returns a boolean indicating if {@code m} is compiled with C1.
448 *
449 * @param m the method to be checked.
450 * @return {@code true} if {@code m} is compiled with C1;
451 * {@code false} otherwise.
452 */
453 public static boolean isC1Compiled(Method m) {
454 return TestVM.isC1Compiled(m);
455 }
456
457 /**
458 * Returns a boolean indicating if {@code m} is compiled with C2.
459 *
460 * @param m the method to be checked.
461 * @return {@code true} if {@code m} is compiled with C2;
462 * {@code false} otherwise.
463 */
464 public static boolean isC2Compiled(Method m) {
465 return TestVM.isC2Compiled(m);
466 }
467
468 /**
469 * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}.
470 *
471 * @param m the method to be checked.
472 * @param compLevel the compilation level.
473 * @return {@code true} if {@code m} is compiled at {@code compLevel};
474 * {@code false} otherwise.
475 */
476 public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
477 return TestVM.isCompiledAtLevel(m, compLevel);
478 }
479
480 /**
481 * Checks if {@code m} is compiled at any level.
482 *
483 * @param m the method to be checked.
484 * @throws TestRunException if {@code m} is not compiled at any level.
485 */
486 public static void assertCompiled(Method m) {
487 TestVM.assertCompiled(m);
488 }
489
490 /**
491 * Checks if {@code m} is not compiled at any level.
492 *
493 * @param m the method to be checked.
494 * @throws TestRunException if {@code m} is compiled at any level.
495 */
496 public static void assertNotCompiled(Method m) {
497 TestVM.assertNotCompiled(m);
498 }
499
500 /**
501 * Verifies that {@code m} is compiled with C1.
502 *
503 * @param m the method to be verified.
504 * @throws TestRunException if {@code m} is not compiled with C1.
505 */
506 public static void assertCompiledByC1(Method m) {
507 TestVM.assertCompiledByC1(m);
508 }
509
510 /**
511 * Verifies that {@code m} is compiled with C2.
512 *
513 * @param m the method to be checked.
514 * @throws TestRunException if {@code m} is not compiled with C2.
515 */
516 public static void assertCompiledByC2(Method m) {
517 TestVM.assertCompiledByC2(m);
518 }
519
520 /**
521 * Verifies that {@code m} is compiled at the specified {@code compLevel}.
522 *
523 * @param m the method to be checked.
524 * @param compLevel the compilation level.
525 * @throws TestRunException if {@code m} is not compiled at {@code compLevel}.
526 */
527 public static void assertCompiledAtLevel(Method m, CompLevel compLevel) {
528 TestVM.assertCompiledAtLevel(m, compLevel);
529 }
530
531 /**
532 * Verifies that {@code m} was deoptimized after being C1 compiled.
533 *
534 * @param m the method to be checked.
535 * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled.
536 */
537 public static void assertDeoptimizedByC1(Method m) {
538 TestVM.assertDeoptimizedByC1(m);
539 }
540
541 /**
542 * Verifies that {@code m} was deoptimized after being C2 compiled.
543 *
544 * @param m the method to be checked.
545 * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled.
546 */
547 public static void assertDeoptimizedByC2(Method m) {
548 TestVM.assertDeoptimizedByC2(m);
549 }
550
551 /**552 * Checks if deopt of {@code m} is stable at the specified {@code compLevel}.553 *554 * @param m the method to be checked.555 * @param compLevel the compilation level.556 * @return {@code true} if deopt of {@code m} is stable at {@code compLevel};557 * {@code false} otherwise.558 */559 public static boolean isStableDeopt(Method m, CompLevel compLevel) {560 return TestVM.isStableDeopt(m, compLevel);561 }562
563 /**
564 * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}).
565 * The very first invocation returns {@code false}. Note that this method could be used by different tests and
566 * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times
567 * other tests have already invoked this method.
568 *
569 * @return an inverted boolean of the result of the last invocation of this method.
570 */
571 public static boolean toggleBoolean() {
572 toggleBool = !toggleBool;
573 return toggleBool;
574 }
575
576 /*
577 * End of public interface methods
578 */
579
580 /**
581 * Used to move Whitebox class to the right folder in the JTreg test
582 */
583 private void installWhiteBox() {
584 try {
585 ClassFileInstaller.main(WhiteBox.class.getName());
586 } catch (Exception e) {
587 throw new Error("failed to install whitebox classes", e);
588 }
589 }
590
591 /**
592 * Disable IR verification completely in certain cases.
593 */
594 private void disableIRVerificationIfNotFeasible() {
595 if (irVerificationPossible) {
596 irVerificationPossible = Platform.isDebugBuild() && !Platform.isInt() && !Platform.isComp();
597 if (!irVerificationPossible) {
598 System.out.println("IR verification disabled due to not running a debug build (required for PrintIdeal" +
599 "and PrintOptoAssembly), running with -Xint, or -Xcomp (use warm-up of 0 instead)");
600 return;
601 }
602
603 irVerificationPossible = hasIRAnnotations();
604 if (!irVerificationPossible) {
605 System.out.println("IR verification disabled due to test " + testClass + " not specifying any @IR annotations");
606 return;
607 }
608
609 // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified.
610 irVerificationPossible = onlyWhitelistedJTregVMAndJavaOptsFlags();
611 if (!irVerificationPossible) {
612 System.out.println("IR verification disabled due to using non-whitelisted JTreg VM or Javaoptions flag(s)."
613 + System.lineSeparator());
614 }
615 }
616 }
617
618 /**
619 * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all
620 * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios
621 * and thus is reported immediately on the first scenario execution.
622 */
623 private void startWithScenarios() {
624 Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex));
625 for (Scenario scenario : scenarios) {
626 try {
627 start(scenario);
628 } catch (TestFormatException e) {
629 // Test format violation is wrong for all the scenarios. Only report once.
630 throw e;
631 } catch (Exception e) {
632 exceptionMap.put(scenario, e);
633 }
634 }
635 if (!exceptionMap.isEmpty()) {
636 reportScenarioFailures(exceptionMap);
637 }
638 }
639
640 private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) {
641 String failedScenarios = "The following scenarios have failed: #"
642 + exceptionMap.keySet().stream()
643 .map(s -> String.valueOf(s.getIndex()))
644 .collect(Collectors.joining(", #"));
645 StringBuilder builder = new StringBuilder(failedScenarios);
646 builder.append(System.lineSeparator()).append(System.lineSeparator());
647 for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) {
648 Exception e = entry.getValue();
649 Scenario scenario = entry.getKey();
650 String errorMsg = "";
651 if (scenario != null) {
652 errorMsg = getScenarioTitleAndFlags(scenario);
653 }
654 if (e instanceof IRViolationException irException) {
655 // For IR violations, only show the actual violations and not the (uninteresting) stack trace.
656 if (scenario != null) {
657 System.out.println("Scenario #" + scenario.getIndex());
658 }
659 System.out.println(irException.getCompilations());
660 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo());
661 } else if (e instanceof TestVMException testVMException) {
662 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo());
663 } else {
664 // Print stack trace otherwise
665 StringWriter errors = new StringWriter();
666 e.printStackTrace(new PrintWriter(errors));
667 builder.append(errors);
668 }
669 builder.append(System.lineSeparator());
670 }
671 System.err.println(builder);
672 if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) {
673 // Provide a hint to the user how to get additional output/debugging information.
674 System.err.println(RERUN_HINT);
675 }
676 throw new TestRunException(failedScenarios + ". Please check stderr for more information.");
677 }
678
679 private static String getScenarioTitleAndFlags(Scenario scenario) {
680 StringBuilder builder = new StringBuilder();
681 String title = "Scenario #" + scenario.getIndex();
682 builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length()))
683 .append(System.lineSeparator());
684 builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]")
685 .append(System.lineSeparator());
686 return builder.toString();
687 }
688
689 /**
690 * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of
691 * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the
692 * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests.
693 */
694 private void start(Scenario scenario) {
695 if (scenario != null && !scenario.isEnabled()) {
696 System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " +
697 "-DScenarios and is therefore not executed.");
698 return;
699 }
700 shouldVerifyIR = irVerificationPossible;
701 try {
702 // Use TestFramework flags and scenario flags for new VMs.
703 List<String> additionalFlags = new ArrayList<>();
704 if (flags != null) {
705 additionalFlags.addAll(flags);
706 }
707 if (scenario != null) {
708 List<String> scenarioFlags = scenario.getFlags();
709 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]";
710 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":");
711 additionalFlags.addAll(scenarioFlags);
712 }
713 String frameworkAndScenarioFlags = additionalFlags.isEmpty() ?
714 "" : " - [" + String.join(", ", additionalFlags) + "]";
715
716 if (shouldVerifyIR) {
717 // Only need to use flag VM if an IR verification is possibly done.
718 System.out.println("Run Flag VM:");
719 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags);
720 shouldVerifyIR = flagVMProcess.shouldVerifyIR();
721 if (shouldVerifyIR) {
722 // Add more flags for the test VM which are required to do IR verification.
723 additionalFlags.addAll(flagVMProcess.getTestVMFlags());
724 } // else: Flag VM found a reason to not do IR verification.
725 } else {
726 System.out.println("Skip Flag VM due to not performing IR verification.");
727 }
728
729 System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":");
730 runTestVM(additionalFlags);
731 } finally {
732 if (scenario != null) {
733 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput());
734 }
735 System.out.println();
736 }
737 }
738
739 private boolean hasIRAnnotations() {
740 return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class).length > 0);
741 }
742
743 private boolean onlyWhitelistedJTregVMAndJavaOptsFlags() {
744 List<String> flags = Arrays.stream(Utils.getTestJavaOpts())
745 .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", ""))
746 .collect(Collectors.toList());
747 for (String flag : flags) {
748 // Property flags (prefix -D), -ea and -esa are whitelisted.
749 if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) {
750 // Found VM flag that is not whitelisted
751 return false;
752 }
753 }
754 return true;
755 }
756
757 private void runTestVM(List<String> additionalFlags) {
758 TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup);
759 if (shouldVerifyIR) {
760 try {
761 TestClassParser testClassParser = new TestClassParser(testClass);
762 Matchable testClassMatchable = testClassParser.parse(testVMProcess.getHotspotPidFileName(),
763 testVMProcess.getIrEncoding());
764 IRMatcher matcher = new IRMatcher(testClassMatchable);
765 matcher.match();
766 } catch (IRViolationException e) {
767 e.addCommandLine(testVMProcess.getCommandLine());
768 throw e;
769 }
770 } else {
771 System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " +
772 "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " +
773 "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " +
774 "that make the IR verification impossible (e.g. -XX:-UseCompile, " +
775 "-XX:TieredStopAtLevel=[1,2,3], etc.).");
776 }
777 }
778
779 public static void check(boolean test, String failureMessage) {
780 if (!test) {
781 throw new TestFrameworkException(failureMessage);
782 }
783 }
784 }
--- EOF ---