1 /*
2 * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 package compiler.lib.ir_framework;
25
26 import compiler.lib.ir_framework.driver.FlagVMProcess;
27 import compiler.lib.ir_framework.driver.TestVMException;
28 import compiler.lib.ir_framework.driver.TestVMProcess;
29 import compiler.lib.ir_framework.driver.irmatching.IRMatcher;
30 import compiler.lib.ir_framework.driver.irmatching.IRViolationException;
31 import compiler.lib.ir_framework.driver.irmatching.Matchable;
32 import compiler.lib.ir_framework.driver.irmatching.parser.TestClassParser;
33 import compiler.lib.ir_framework.shared.*;
34 import compiler.lib.ir_framework.test.TestVM;
35 import jdk.test.lib.Platform;
36 import jdk.test.lib.Utils;
37 import jdk.test.lib.helpers.ClassFileInstaller;
38 import jdk.test.whitebox.WhiteBox;
39
40 import java.io.PrintWriter;
41 import java.io.StringWriter;
42 import java.lang.reflect.Method;
43 import java.net.MalformedURLException;
44 import java.net.URL;
45 import java.net.URLClassLoader;
46 import java.nio.file.Path;
47 import java.util.*;
48 import java.util.stream.Collectors;
49
50 /**
51 * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on
52 * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can
53 * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used
54 * testing patterns and compiler control flags.
55 * <p>
56 * The framework offers various annotations to control how your test code should be invoked and being checked. There are
57 * three kinds of tests depending on how much control is needed over the test invocation:
58 * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b>
59 * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test}
60 * annotation which represents the test code that is eventually executed by the test framework. More information about
61 * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}.
62 * <p>
63 * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which
64 * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}.
65 * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage
66 * and how to write different IR rules can be found at {@link IR}.
67 * <p>
68 * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>:
69 * <pre>
70 * {@literal @}library /test/lib
71 * {@literal @}run driver some.package.Test
72 * </pre>
73 * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the
74 * JTreg test if the test itself is not utilizing any Whitebox features directly.
75 * <p>
76 * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or
77 * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations
78 * (instead of specifying multiple JTreg {@code @run} entries).
79 * <p>
80 * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the
81 * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various
82 * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new
83 * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing.
84 * <p>
85 * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code
86 * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions.
87 * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are
88 * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}
89 * for IR matching).
90 * <p>
91 * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified
92 * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}.
93 * <p>
94 * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR
95 * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details).
96 * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing
97 * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}.
98 *
99 * @see Test
100 * @see Check
101 * @see Run
102 * @see IR
103 */
104 public class TestFramework {
105 /**
106 * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only
107 * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags
108 * by user code) are whitelisted.
109 *
110 * <p>
111 * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of
112 * the entries of this list as a substring (partial match).
113 */
114 public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>(
115 Arrays.asList(
116 // The following substrings are part of more than one VM flag
117 "RAM",
118 "Heap",
119 "Trace",
120 "Print",
121 "Verify",
122 "UseNewCode",
123 "Xmn",
124 "Xms",
125 "Xmx",
126 "Xss",
127 // The following substrings are only part of one VM flag (=exact match)
128 "CreateCoredumpOnCrash",
129 "IgnoreUnrecognizedVMOptions",
130 "UnlockDiagnosticVMOptions",
131 "UnlockExperimentalVMOptions",
132 "BackgroundCompilation",
133 "Xbatch",
134 "TieredCompilation",
135 "CompileThreshold",
136 "Xmixed",
137 "server",
138 "UseAVX",
139 "UseSSE",
140 "UseSVE",
141 "UseZbb",
142 "UseRVV",
143 "Xlog",
144 "LogCompilation"
145 )
146 );
147
148 public static final boolean VERBOSE = Boolean.getBoolean("Verbose");
149 public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty();
150 public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty();
151 private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout");
152 // Only used for internal testing and should not be used for normal user testing.
153
154 private static final String RERUN_HINT = """
155 #############################################################
156 - To only run the failed tests use -DTest, -DExclude,
157 and/or -DScenarios.
158 - To also get the standard output of the test VM run with
159 -DReportStdout=true or for even more fine-grained logging
160 use -DVerbose=true.
161 #############################################################
162 """ + System.lineSeparator();
163
164 private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true"));
165 private boolean shouldVerifyIR; // Should we perform IR matching?
166 private static boolean toggleBool;
167
168 private final Class<?> testClass;
169 private Set<Class<?>> helperClasses;
170 private List<Scenario> scenarios;
171 private Set<Integer> scenarioIndices;
172 private List<String> flags;
173 private int defaultWarmup = -1;
174
175 /*
176 * Public interface methods
177 */
178
179 /**
180 * Creates an instance acting as a builder to test the class from which this constructor was invoked from.
181 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
182 * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
183 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
184 */
185 public TestFramework() {
186 this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass());
187 }
188
189 /**
190 * Creates an instance acting as a builder to test {@code testClass}.
191 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
192 * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
193 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
194 *
195 * @param testClass the class to be tested by the framework.
196 * @see #TestFramework()
197 */
198 public TestFramework(Class<?> testClass) {
199 TestRun.check(testClass != null, "Test class cannot be null");
200 this.testClass = testClass;
201 if (VERBOSE) {
202 System.out.println("Test class: " + testClass);
203 }
204 }
205
206 /**
207 * Tests the class from which this method was invoked from.
208 */
209 public static void run() {
210 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
211 run(walker.getCallerClass());
212 }
213
214 /**
215 * Tests {@code testClass}.
216 *
217 * @param testClass the class to be tested by the framework.
218 * @see #run()
219 */
220 public static void run(Class<?> testClass) {
221 TestFramework framework = new TestFramework(testClass);
222 framework.start();
223 }
224
225 /**
226 * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}.
227 * <ul>
228 * <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p>
229 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and Javaoptions flags over
230 * the specified {@code flags} of this method.</li>
231 * <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li>
232 * <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one
233 * from which this method was called from, use {@link #addFlags(String...)}, use this method.</li>
234 * <li><p>If you want to run your JTreg test with multiple flag combinations, use
235 * {@link #addScenarios(Scenario...)}</li>
236 * </ul>
237 *
238 * @param flags VM flags to be used for the test VM.
239 */
240 public static void runWithFlags(String... flags) {
241 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
242 TestFramework framework = new TestFramework(walker.getCallerClass());
243 framework.addFlags(flags);
244 framework.start();
245 }
246
247 /**
248 * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p>
249 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags.
250 *
251 * <p>
252 * The testing can be started by invoking {@link #start()}
253 *
254 * @param flags VM options to be applied to the test VM.
255 * @return the same framework instance.
256 */
257 public TestFramework addFlags(String... flags) {
258 TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null");
259 if (this.flags == null) {
260 this.flags = new ArrayList<>();
261 }
262 this.flags.addAll(Arrays.asList(flags));
263 return this;
264 }
265
266 /**
267 * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile},
268 * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied
269 * while testing {@code testClass} (also see description of {@link TestFramework}).
270 *
271 * <p>
272 * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any
273 * compile command annotations, you do not need to include it with this method. If no helper class specifies any
274 * compile commands, you do not need to call this method at all.
275 *
276 * <p>
277 * The testing can be started by invoking {@link #start()}.
278 *
279 * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile},
280 * {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied
281 * while testing {@code testClass} (also see description of {@link TestFramework}).
282 * @return the same framework instance.
283 */
284 public TestFramework addHelperClasses(Class<?>... helperClasses) {
285 TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull),
286 "A Helper class cannot be null");
287 if (this.helperClasses == null) {
288 this.helperClasses = new HashSet<>();
289 }
290
291 this.helperClasses.addAll(Arrays.asList(helperClasses));
292 return this;
293 }
294
295 /**
296 * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the
297 * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)}
298 * and thus also override any VM or Javaoptions set by JTreg by default.<p>
299 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags.
300 *
301 * <p>
302 * The testing can be started by invoking {@link #start()}
303 *
304 * @param scenarios scenarios which specify specific flags for the test VM.
305 * @return the same framework instance.
306 */
307 public TestFramework addScenarios(Scenario... scenarios) {
308 TestFormat.checkAndReport(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull),
309 "A scenario cannot be null");
310 if (this.scenarios == null) {
311 this.scenarios = new ArrayList<>();
312 this.scenarioIndices = new HashSet<>();
313 }
314
315 for (Scenario scenario : scenarios) {
316 int scenarioIndex = scenario.getIndex();
317 TestFormat.checkNoThrow(scenarioIndices.add(scenarioIndex),
318 "Cannot define two scenarios with the same index " + scenarioIndex);
319 this.scenarios.add(scenario);
320 }
321 TestFormat.throwIfAnyFailures();
322 return this;
323 }
324
325 /**
326 * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)})
327 * set test class.
328 */
329 public void start() {
330 if (shouldInstallWhiteBox()) {
331 installWhiteBox();
332 }
333 checkIRRuleCompilePhasesFormat();
334 disableIRVerificationIfNotFeasible();
335
336 if (scenarios == null) {
337 try {
338 start(null);
339 } catch (TestVMException e) {
340 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT);
341 throw e;
342 } catch (IRViolationException e) {
343 System.out.println(e.getCompilations());
344 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT);
345 throw e;
346 }
347 } else {
348 startWithScenarios();
349 }
350 }
351
352 private void checkIRRuleCompilePhasesFormat() {
353 for (Method method : testClass.getDeclaredMethods()) {
354 for (IR irAnno : method.getAnnotationsByType(IR.class)) {
355 TestFormat.checkNoThrow(irAnno.phase().length > 0,
356 "@IR rule " + irAnno + " must specify a non-empty list of compile " +
357 "phases \"phase\" at " + method);
358 }
359 }
360 TestFormat.throwIfAnyFailures();
361 }
362
363 /**
364 * Try to load the Whitebox class from the user directory with a custom class loader. If the user has already built the
365 * Whitebox, we can load it. Otherwise, the framework needs to install it.
366 *
367 * @return true if the framework needs to install the Whitebox
368 */
369 private boolean shouldInstallWhiteBox() {
370 try {
371 URL url = Path.of(System.getProperty("user.dir")).toUri().toURL();
372 URLClassLoader userDirClassLoader =
373 URLClassLoader.newInstance(new URL[] {url}, TestFramework.class.getClassLoader().getParent());
374 Class.forName(WhiteBox.class.getName(), false, userDirClassLoader);
375 } catch (MalformedURLException e) {
376 throw new TestFrameworkException("corrupted user.dir property", e);
377 } catch (ClassNotFoundException e) {
378 // We need to manually install the WhiteBox if we cannot load the WhiteBox class from the user directory.
379 // This happens when the user test does not explicitly install the WhiteBox as part of the test.
380 return true;
381 }
382 return false;
383 }
384
385 /**
386 * Set a new default warm-up (overriding the framework default of 2000 at
387 * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit
388 * warm-up with {@link Warmup @Warmup}.
389 *
390 * @param defaultWarmup a new non-negative default warm-up.
391 * @return the same framework instance.
392 */
393 public TestFramework setDefaultWarmup(int defaultWarmup) {
394 TestFormat.checkAndReport(defaultWarmup >= 0, "Cannot specify a negative default warm-up");
395 this.defaultWarmup = defaultWarmup;
396 return this;
397 }
398
399 /**
400 * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios
401 * were run, use {@link Scenario#getTestVMOutput()}.
402 *
403 * @return the last test VM output.
404 */
405 public static String getLastTestVMOutput() {
406 return TestVMProcess.getLastTestVMOutput();
407 }
408
409 /*
410 * The following methods are only intended to be called from actual @Test methods and not from the main() method of
411 * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled).
412 */
413
414 /**
415 * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled,
416 * yet, upon returning from this method.
417 *
418 * @param m the method to be compiled.
419 * @param compLevel the (valid) compilation level at which the method should be compiled.
420 * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}.
421 */
422 public static void compile(Method m, CompLevel compLevel) {
423 TestVM.compile(m, compLevel);
424 }
425
426 /**
427 * Deoptimize {@code m}.
428 *
429 * @param m the method to be deoptimized.
430 */
431 public static void deoptimize(Method m) {
432 TestVM.deoptimize(m);
433 }
434
435 /**
436 * Returns a boolean indicating if {@code m} is compiled at any level.
437 *
438 * @param m the method to be checked.
439 * @return {@code true} if {@code m} is compiled at any level;
440 * {@code false} otherwise.
441 */
442 public static boolean isCompiled(Method m) {
443 return TestVM.isCompiled(m);
444 }
445
446 /**
447 * Returns a boolean indicating if {@code m} is compiled with C1.
448 *
449 * @param m the method to be checked.
450 * @return {@code true} if {@code m} is compiled with C1;
451 * {@code false} otherwise.
452 */
453 public static boolean isC1Compiled(Method m) {
454 return TestVM.isC1Compiled(m);
455 }
456
457 /**
458 * Returns a boolean indicating if {@code m} is compiled with C2.
459 *
460 * @param m the method to be checked.
461 * @return {@code true} if {@code m} is compiled with C2;
462 * {@code false} otherwise.
463 */
464 public static boolean isC2Compiled(Method m) {
465 return TestVM.isC2Compiled(m);
466 }
467
468 /**
469 * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}.
470 *
471 * @param m the method to be checked.
472 * @param compLevel the compilation level.
473 * @return {@code true} if {@code m} is compiled at {@code compLevel};
474 * {@code false} otherwise.
475 */
476 public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
477 return TestVM.isCompiledAtLevel(m, compLevel);
478 }
479
480 /**
481 * Checks if {@code m} is compiled at any level.
482 *
483 * @param m the method to be checked.
484 * @throws TestRunException if {@code m} is not compiled at any level.
485 */
486 public static void assertCompiled(Method m) {
487 TestVM.assertCompiled(m);
488 }
489
490 /**
491 * Checks if {@code m} is not compiled at any level.
492 *
493 * @param m the method to be checked.
494 * @throws TestRunException if {@code m} is compiled at any level.
495 */
496 public static void assertNotCompiled(Method m) {
497 TestVM.assertNotCompiled(m);
498 }
499
500 /**
501 * Verifies that {@code m} is compiled with C1.
502 *
503 * @param m the method to be verified.
504 * @throws TestRunException if {@code m} is not compiled with C1.
505 */
506 public static void assertCompiledByC1(Method m) {
507 TestVM.assertCompiledByC1(m);
508 }
509
510 /**
511 * Verifies that {@code m} is compiled with C2.
512 *
513 * @param m the method to be checked.
514 * @throws TestRunException if {@code m} is not compiled with C2.
515 */
516 public static void assertCompiledByC2(Method m) {
517 TestVM.assertCompiledByC2(m);
518 }
519
520 /**
521 * Verifies that {@code m} is compiled at the specified {@code compLevel}.
522 *
523 * @param m the method to be checked.
524 * @param compLevel the compilation level.
525 * @throws TestRunException if {@code m} is not compiled at {@code compLevel}.
526 */
527 public static void assertCompiledAtLevel(Method m, CompLevel compLevel) {
528 TestVM.assertCompiledAtLevel(m, compLevel);
529 }
530
531 /**
532 * Verifies that {@code m} was deoptimized after being C1 compiled.
533 *
534 * @param m the method to be checked.
535 * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled.
536 */
537 public static void assertDeoptimizedByC1(Method m) {
538 TestVM.assertDeoptimizedByC1(m);
539 }
540
541 /**
542 * Verifies that {@code m} was deoptimized after being C2 compiled.
543 *
544 * @param m the method to be checked.
545 * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled.
546 */
547 public static void assertDeoptimizedByC2(Method m) {
548 TestVM.assertDeoptimizedByC2(m);
549 }
550
551 /**
552 * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}).
553 * The very first invocation returns {@code false}. Note that this method could be used by different tests and
554 * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times
555 * other tests have already invoked this method.
556 *
557 * @return an inverted boolean of the result of the last invocation of this method.
558 */
559 public static boolean toggleBoolean() {
560 toggleBool = !toggleBool;
561 return toggleBool;
562 }
563
564 /*
565 * End of public interface methods
566 */
567
568 /**
569 * Used to move Whitebox class to the right folder in the JTreg test
570 */
571 private void installWhiteBox() {
572 try {
573 ClassFileInstaller.main(WhiteBox.class.getName());
574 } catch (Exception e) {
575 throw new Error("failed to install whitebox classes", e);
576 }
577 }
578
579 /**
580 * Disable IR verification completely in certain cases.
581 */
582 private void disableIRVerificationIfNotFeasible() {
583 if (irVerificationPossible) {
584 irVerificationPossible = Platform.isDebugBuild() && !Platform.isInt() && !Platform.isComp();
585 if (!irVerificationPossible) {
586 System.out.println("IR verification disabled due to not running a debug build (required for PrintIdeal" +
587 "and PrintOptoAssembly), running with -Xint, or -Xcomp (use warm-up of 0 instead)");
588 return;
589 }
590
591 irVerificationPossible = hasIRAnnotations();
592 if (!irVerificationPossible) {
593 System.out.println("IR verification disabled due to test " + testClass + " not specifying any @IR annotations");
594 return;
595 }
596
597 // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified.
598 irVerificationPossible = onlyWhitelistedJTregVMAndJavaOptsFlags();
599 if (!irVerificationPossible) {
600 System.out.println("IR verification disabled due to using non-whitelisted JTreg VM or Javaoptions flag(s)."
601 + System.lineSeparator());
602 }
603 }
604 }
605
606 /**
607 * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all
608 * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios
609 * and thus is reported immediately on the first scenario execution.
610 */
611 private void startWithScenarios() {
612 Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex));
613 for (Scenario scenario : scenarios) {
614 try {
615 start(scenario);
616 } catch (TestFormatException e) {
617 // Test format violation is wrong for all the scenarios. Only report once.
618 throw e;
619 } catch (Exception e) {
620 exceptionMap.put(scenario, e);
621 }
622 }
623 if (!exceptionMap.isEmpty()) {
624 reportScenarioFailures(exceptionMap);
625 }
626 }
627
628 private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) {
629 String failedScenarios = "The following scenarios have failed: #"
630 + exceptionMap.keySet().stream()
631 .map(s -> String.valueOf(s.getIndex()))
632 .collect(Collectors.joining(", #"));
633 StringBuilder builder = new StringBuilder(failedScenarios);
634 builder.append(System.lineSeparator()).append(System.lineSeparator());
635 for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) {
636 Exception e = entry.getValue();
637 Scenario scenario = entry.getKey();
638 String errorMsg = "";
639 if (scenario != null) {
640 errorMsg = getScenarioTitleAndFlags(scenario);
641 }
642 if (e instanceof IRViolationException irException) {
643 // For IR violations, only show the actual violations and not the (uninteresting) stack trace.
644 if (scenario != null) {
645 System.out.println("Scenario #" + scenario.getIndex());
646 }
647 System.out.println(irException.getCompilations());
648 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo());
649 } else if (e instanceof TestVMException testVMException) {
650 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo());
651 } else {
652 // Print stack trace otherwise
653 StringWriter errors = new StringWriter();
654 e.printStackTrace(new PrintWriter(errors));
655 builder.append(errors);
656 }
657 builder.append(System.lineSeparator());
658 }
659 System.err.println(builder);
660 if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) {
661 // Provide a hint to the user how to get additional output/debugging information.
662 System.err.println(RERUN_HINT);
663 }
664 throw new TestRunException(failedScenarios + ". Please check stderr for more information.");
665 }
666
667 private static String getScenarioTitleAndFlags(Scenario scenario) {
668 StringBuilder builder = new StringBuilder();
669 String title = "Scenario #" + scenario.getIndex();
670 builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length()))
671 .append(System.lineSeparator());
672 builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]")
673 .append(System.lineSeparator());
674 return builder.toString();
675 }
676
677 /**
678 * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of
679 * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the
680 * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests.
681 */
682 private void start(Scenario scenario) {
683 if (scenario != null && !scenario.isEnabled()) {
684 System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " +
685 "-DScenarios and is therefore not executed.");
686 return;
687 }
688 shouldVerifyIR = irVerificationPossible;
689 try {
690 // Use TestFramework flags and scenario flags for new VMs.
691 List<String> additionalFlags = new ArrayList<>();
692 if (flags != null) {
693 additionalFlags.addAll(flags);
694 }
695 if (scenario != null) {
696 List<String> scenarioFlags = scenario.getFlags();
697 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]";
698 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":");
699 additionalFlags.addAll(scenarioFlags);
700 }
701 String frameworkAndScenarioFlags = additionalFlags.isEmpty() ?
702 "" : " - [" + String.join(", ", additionalFlags) + "]";
703
704 if (shouldVerifyIR) {
705 // Only need to use flag VM if an IR verification is possibly done.
706 System.out.println("Run Flag VM:");
707 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags);
708 shouldVerifyIR = flagVMProcess.shouldVerifyIR();
709 if (shouldVerifyIR) {
710 // Add more flags for the test VM which are required to do IR verification.
711 additionalFlags.addAll(flagVMProcess.getTestVMFlags());
712 } // else: Flag VM found a reason to not do IR verification.
713 } else {
714 System.out.println("Skip Flag VM due to not performing IR verification.");
715 }
716
717 System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":");
718 runTestVM(additionalFlags);
719 } finally {
720 if (scenario != null) {
721 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput());
722 }
723 System.out.println();
724 }
725 }
726
727 private boolean hasIRAnnotations() {
728 return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class).length > 0);
729 }
730
731 private boolean onlyWhitelistedJTregVMAndJavaOptsFlags() {
732 List<String> flags = Arrays.stream(Utils.getTestJavaOpts())
733 .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", ""))
734 .collect(Collectors.toList());
735 for (String flag : flags) {
736 // Property flags (prefix -D), -ea and -esa are whitelisted.
737 if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) {
738 // Found VM flag that is not whitelisted
739 return false;
740 }
741 }
742 return true;
743 }
744
745 private void runTestVM(List<String> additionalFlags) {
746 TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup);
747 if (shouldVerifyIR) {
748 try {
749 TestClassParser testClassParser = new TestClassParser(testClass);
750 Matchable testClassMatchable = testClassParser.parse(testVMProcess.getHotspotPidFileName(),
751 testVMProcess.getIrEncoding());
752 IRMatcher matcher = new IRMatcher(testClassMatchable);
753 matcher.match();
754 } catch (IRViolationException e) {
755 e.addCommandLine(testVMProcess.getCommandLine());
756 throw e;
757 }
758 } else {
759 System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " +
760 "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " +
761 "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " +
762 "that make the IR verification impossible (e.g. -XX:-UseCompile, " +
763 "-XX:TieredStopAtLevel=[1,2,3], etc.).");
764 }
765 }
766
767 public static void check(boolean test, String failureMessage) {
768 if (!test) {
769 throw new TestFrameworkException(failureMessage);
770 }
771 }
772 }