1 /*
2 * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 package compiler.lib.ir_framework;
25
26 import compiler.lib.ir_framework.driver.*;
27 import compiler.lib.ir_framework.driver.irmatching.IRMatcher;
28 import compiler.lib.ir_framework.driver.irmatching.IRViolationException;
29 import compiler.lib.ir_framework.shared.*;
30 import compiler.lib.ir_framework.test.*;
31 import jdk.test.lib.Platform;
32 import jdk.test.lib.Utils;
33 import jdk.test.lib.helpers.ClassFileInstaller;
34 import sun.hotspot.WhiteBox;
35
36 import java.io.PrintWriter;
37 import java.io.StringWriter;
38 import java.lang.reflect.Method;
39 import java.util.*;
40 import java.util.stream.Collectors;
41
42 /**
43 * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on
44 * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can
45 * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used
46 * testing patterns and compiler control flags.
47 * <p>
48 * The framework offers various annotations to control how your test code should be invoked and being checked. There are
49 * three kinds of tests depending on how much control is needed over the test invocation:
50 * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b>
51 * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test}
52 * annotation which represents the test code that is eventually executed by the test framework. More information about
53 * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}.
54 * <p>
55 * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which
56 * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}.
57 * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage
58 * and how to write different IR rules can be found at {@link IR}.
59 * <p>
60 * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>:
61 * <pre>
62 * {@literal @}library /test/lib
63 * {@literal @}run driver some.package.Test
64 * </pre>
65 * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the
66 * JTreg test if the test itself is not utilizing any Whitebox features directly.
67 * <p>
68 * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or
69 * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations
70 * (instead of specifying multiple JTreg {@code @run} entries).
71 * <p>
72 * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the
73 * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various
74 * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new
75 * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing.
76 * <p>
77 * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code
78 * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions.
79 * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are
80 * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}
81 * for IR matching).
82 * <p>
83 * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified
84 * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}.
85 * <p>
86 * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR
87 * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details).
88 * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing
89 * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}.
90 *
91 * @see Test
92 * @see Check
93 * @see Run
94 * @see IR
95 */
96 public class TestFramework {
97 /**
98 * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only
99 * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags
100 * by user code) are whitelisted.
101 *
102 * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of
103 * the entries of this list as a substring (partial match).
104 */
105 public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>(
106 Arrays.asList(
107 // The following substrings are part of more than one VM flag
108 "RAM",
109 "Heap",
110 "Trace",
111 "Print",
112 "Verify",
113 "TLAB",
114 "UseNewCode",
115 "Xmn",
116 "Xms",
117 "Xmx",
118 "Xss",
119 // The following substrings are only part of one VM flag (=exact match)
120 "CreateCoredumpOnCrash",
121 "IgnoreUnrecognizedVMOptions",
122 "UnlockDiagnosticVMOptions",
123 "UnlockExperimentalVMOptions",
124 "BackgroundCompilation",
125 "Xbatch",
126 "TieredCompilation",
127 "CompileThreshold",
128 "Xmixed",
129 "server",
130 "Xlog",
131 "LogCompilation"
132 )
133 );
134
135 public static final boolean VERBOSE = Boolean.getBoolean("Verbose");
136 public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty();
137 public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty();
138 private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout");
139 // Only used for internal testing and should not be used for normal user testing.
140 private static final boolean SKIP_WHITEBOX_INSTALL = Boolean.getBoolean("SkipWhiteBoxInstall");
141
142 private static final String RERUN_HINT = """
143 #############################################################
144 - To only run the failed tests use -DTest, -DExclude,
145 and/or -DScenarios.
146 - To also get the standard output of the test VM run with
147 -DReportStdout=true or for even more fine-grained logging
148 use -DVerbose=true.
149 #############################################################
150 """ + System.lineSeparator();
151
152 private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true"));
153 private boolean shouldVerifyIR; // Should we perform IR matching?
154 private static boolean toggleBool;
155
156 private final Class<?> testClass;
157 private Set<Class<?>> helperClasses;
158 private List<Scenario> scenarios;
159 private Set<Integer> scenarioIndices;
160 private List<String> flags;
161 private int defaultWarmup = -1;
162
163 /*
164 * Public interface methods
165 */
166
167 /**
168 * Creates an instance acting as a builder to test the class from which this constructor was invoked from.
169 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
170 * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
171 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
172 */
173 public TestFramework() {
174 this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass());
175 }
176
177 /**
178 * Creates an instance acting as a builder to test {@code testClass}.
179 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
180 * Use the associated add methods ({@link #addFlags(String...)}, @link #addScenarios(Scenario...)},
181 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
182 *
183 * @param testClass the class to be tested by the framework.
184 * @see #TestFramework()
185 */
186 public TestFramework(Class<?> testClass) {
187 TestRun.check(testClass != null, "Test class cannot be null");
188 this.testClass = testClass;
189 if (VERBOSE) {
190 System.out.println("Test class: " + testClass);
191 }
192 }
193
194 /**
195 * Tests the class from which this method was invoked from.
196 */
197 public static void run() {
198 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
199 run(walker.getCallerClass());
200 }
201
202 /**
203 * Tests {@code testClass}.
204 *
205 * @param testClass the class to be tested by the framework.
206 * @see #run()
207 */
208 public static void run(Class<?> testClass) {
209 TestFramework framework = new TestFramework(testClass);
210 framework.start();
211 }
212
213 /**
214 * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}.
215 * <ul>
216 * <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p>
217 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and Javaoptions flags over
218 * the specified {@code flags} of this method.</li>
219 * <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li>
220 * <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one
221 * from which this method was called from, use {@link #addFlags(String...)}, use this method.</li>
222 * <li><p>If you want to run your JTreg test with multiple flag combinations, use
223 * {@link #addScenarios(Scenario...)}</li>
224 * </ul>
225 *
226 * @param flags VM flags to be used for the test VM.
227 */
228 public static void runWithFlags(String... flags) {
229 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
230 TestFramework framework = new TestFramework(walker.getCallerClass());
231 framework.addFlags(flags);
232 framework.start();
233 }
234
235 /**
236 * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p>
237 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags.
238 *
239 * <p>
240 * The testing can be started by invoking {@link #start()}
241 *
242 * @param flags VM options to be applied to the test VM.
243 * @return the same framework instance.
244 */
245 public TestFramework addFlags(String... flags) {
246 TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null");
247 if (this.flags == null) {
248 this.flags = new ArrayList<>();
249 }
250 this.flags.addAll(Arrays.asList(flags));
251 return this;
252 }
253
254 /**
255 * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile},
256 * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied
257 * while testing {@code testClass} (also see description of {@link TestFramework}).
258 *
259 * <p>
260 * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any
261 * compile command annotations, you do not need to include it with this method. If no helper class specifies any
262 * compile commands, you do not need to call this method at all.
263 *
264 * <p>
265 * The testing can be started by invoking {@link #start()}.
266 *
267 * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile},
268 * {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied
269 * while testing {@code testClass} (also see description of {@link TestFramework}).
270 * @return the same framework instance.
271 */
272 public TestFramework addHelperClasses(Class<?>... helperClasses) {
273 TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull),
274 "A Helper class cannot be null");
275 if (this.helperClasses == null) {
276 this.helperClasses = new HashSet<>();
277 }
278
279 this.helperClasses.addAll(Arrays.asList(helperClasses));
280 return this;
281 }
282
283 /**
284 * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the
285 * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)}
286 * and thus also override any VM or Javaoptions set by JTreg by default.<p>
287 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags.
288 *
289 * <p>
290 * The testing can be started by invoking {@link #start()}
291 *
292 * @param scenarios scenarios which specify specific flags for the test VM.
293 * @return the same framework instance.
294 */
295 public TestFramework addScenarios(Scenario... scenarios) {
296 TestFormat.check(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull),
297 "A scenario cannot be null");
298 if (this.scenarios == null) {
299 this.scenarios = new ArrayList<>();
300 this.scenarioIndices = new HashSet<>();
301 }
302
303 for (Scenario scenario : scenarios) {
304 int scenarioIndex = scenario.getIndex();
305 TestFormat.check(scenarioIndices.add(scenarioIndex),
306 "Cannot define two scenarios with the same index " + scenarioIndex);
307 this.scenarios.add(scenario);
308 }
309 return this;
310 }
311
312 /**
313 * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)})
314 * set test class.
315 */
316 public void start() {
317 if (!SKIP_WHITEBOX_INSTALL) {
318 installWhiteBox();
319 }
320 disableIRVerificationIfNotFeasible();
321
322 if (scenarios == null) {
323 try {
324 start(null);
325 } catch (TestVMException e) {
326 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT);
327 throw e;
328 } catch (IRViolationException e) {
329 System.out.println("Compilation(s) of failed match(es):");
330 System.out.println(e.getCompilations());
331 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT);
332 throw e;
333 }
334 } else {
335 startWithScenarios();
336 }
337 }
338
339 /**
340 * Set a new default warm-up (overriding the framework default of 2000 at
341 * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit
342 * warm-up with {@link Warmup @Warmup}.
343 *
344 * @param defaultWarmup a new non-negative default warm-up.
345 * @return the same framework instance.
346 */
347 public TestFramework setDefaultWarmup(int defaultWarmup) {
348 TestFormat.check(defaultWarmup >= 0, "Cannot specify a negative default warm-up");
349 this.defaultWarmup = defaultWarmup;
350 return this;
351 }
352
353 /**
354 * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios
355 * were run, use {@link Scenario#getTestVMOutput()}.
356 *
357 * @return the last test VM output.
358 */
359 public static String getLastTestVMOutput() {
360 return TestVMProcess.getLastTestVMOutput();
361 }
362
363 /*
364 * The following methods are only intended to be called from actual @Test methods and not from the main() method of
365 * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled).
366 */
367
368 /**
369 * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled,
370 * yet, upon returning from this method.
371 *
372 * @param m the method to be compiled.
373 * @param compLevel the (valid) compilation level at which the method should be compiled.
374 * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}.
375 */
376 public static void compile(Method m, CompLevel compLevel) {
377 TestVM.compile(m, compLevel);
378 }
379
380 /**
381 * Deoptimize {@code m}.
382 *
383 * @param m the method to be deoptimized.
384 */
385 public static void deoptimize(Method m) {
386 TestVM.deoptimize(m);
387 }
388
389 /**
390 * Returns a boolean indicating if {@code m} is compiled at any level.
391 *
392 * @param m the method to be checked.
393 * @return {@code true} if {@code m} is compiled at any level;
394 * {@code false} otherwise.
395 */
396 public static boolean isCompiled(Method m) {
397 return TestVM.isCompiled(m);
398 }
399
400 /**
401 * Returns a boolean indicating if {@code m} is compiled with C1.
402 *
403 * @param m the method to be checked.
404 * @return {@code true} if {@code m} is compiled with C1;
405 * {@code false} otherwise.
406 */
407 public static boolean isC1Compiled(Method m) {
408 return TestVM.isC1Compiled(m);
409 }
410
411 /**
412 * Returns a boolean indicating if {@code m} is compiled with C2.
413 *
414 * @param m the method to be checked.
415 * @return {@code true} if {@code m} is compiled with C2;
416 * {@code false} otherwise.
417 */
418 public static boolean isC2Compiled(Method m) {
419 return TestVM.isC2Compiled(m);
420 }
421
422 /**
423 * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}.
424 *
425 * @param m the method to be checked.
426 * @param compLevel the compilation level.
427 * @return {@code true} if {@code m} is compiled at {@code compLevel};
428 * {@code false} otherwise.
429 */
430 public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
431 return TestVM.isCompiledAtLevel(m, compLevel);
432 }
433
434 /**
435 * Checks if {@code m} is compiled at any level.
436 *
437 * @param m the method to be checked.
438 * @throws TestRunException if {@code m} is not compiled at any level.
439 */
440 public static void assertCompiled(Method m) {
441 TestVM.assertCompiled(m);
442 }
443
444 /**
445 * Checks if {@code m} is not compiled at any level.
446 *
447 * @param m the method to be checked.
448 * @throws TestRunException if {@code m} is compiled at any level.
449 */
450 public static void assertNotCompiled(Method m) {
451 TestVM.assertNotCompiled(m);
452 }
453
454 /**
455 * Verifies that {@code m} is compiled with C1.
456 *
457 * @param m the method to be verified.
458 * @throws TestRunException if {@code m} is not compiled with C1.
459 */
460 public static void assertCompiledByC1(Method m) {
461 TestVM.assertCompiledByC1(m);
462 }
463
464 /**
465 * Verifies that {@code m} is compiled with C2.
466 *
467 * @param m the method to be checked.
468 * @throws TestRunException if {@code m} is not compiled with C2.
469 */
470 public static void assertCompiledByC2(Method m) {
471 TestVM.assertCompiledByC2(m);
472 }
473
474 /**
475 * Verifies that {@code m} is compiled at the specified {@code compLevel}.
476 *
477 * @param m the method to be checked.
478 * @param compLevel the compilation level.
479 * @throws TestRunException if {@code m} is not compiled at {@code compLevel}.
480 */
481 public static void assertCompiledAtLevel(Method m, CompLevel compLevel) {
482 TestVM.assertCompiledAtLevel(m, compLevel);
483 }
484
485 /**
486 * Verifies that {@code m} was deoptimized after being C1 compiled.
487 *
488 * @param m the method to be checked.
489 * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled.
490 */
491 public static void assertDeoptimizedByC1(Method m) {
492 TestVM.assertDeoptimizedByC1(m);
493 }
494
495 /**
496 * Verifies that {@code m} was deoptimized after being C2 compiled.
497 *
498 * @param m the method to be checked.
499 * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled.
500 */
501 public static void assertDeoptimizedByC2(Method m) {
502 TestVM.assertDeoptimizedByC2(m);
503 }
504
505 /**
506 * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}).
507 * The very first invocation returns {@code false}. Note that this method could be used by different tests and
508 * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times
509 * other tests have already invoked this method.
510 *
511 * @return an inverted boolean of the result of the last invocation of this method.
512 */
513 public static boolean toggleBoolean() {
514 toggleBool = !toggleBool;
515 return toggleBool;
516 }
517
518 /*
519 * End of public interface methods
520 */
521
522 /**
523 * Used to move Whitebox class to the right folder in the JTreg test
524 */
525 private void installWhiteBox() {
526 try {
527 ClassFileInstaller.main(WhiteBox.class.getName());
528 } catch (Exception e) {
529 throw new Error("failed to install whitebox classes", e);
530 }
531 }
532
533 /**
534 * Disable IR verification completely in certain cases.
535 */
536 private void disableIRVerificationIfNotFeasible() {
537 if (irVerificationPossible) {
538 irVerificationPossible = Platform.isDebugBuild() && !Platform.isInt() && !Platform.isComp();
539 if (!irVerificationPossible) {
540 System.out.println("IR verification disabled due to not running a debug build (required for PrintIdeal" +
541 "and PrintOptoAssembly), running with -Xint, or -Xcomp (use warm-up of 0 instead)");
542 return;
543 }
544
545 irVerificationPossible = hasIRAnnotations();
546 if (!irVerificationPossible) {
547 System.out.println("IR verification disabled due to test " + testClass + " not specifying any @IR annotations");
548 return;
549 }
550
551 // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified.
552 irVerificationPossible = onlyWhitelistedJTregVMAndJavaOptsFlags();
553 if (!irVerificationPossible) {
554 System.out.println("IR verification disabled due to using non-whitelisted JTreg VM or Javaoptions flag(s)."
555 + System.lineSeparator());
556 }
557 }
558 }
559
560 /**
561 * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all
562 * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios
563 * and thus is reported immediately on the first scenario execution.
564 */
565 private void startWithScenarios() {
566 Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex));
567 for (Scenario scenario : scenarios) {
568 try {
569 start(scenario);
570 } catch (TestFormatException e) {
571 // Test format violation is wrong for all the scenarios. Only report once.
572 throw e;
573 } catch (Exception e) {
574 exceptionMap.put(scenario, e);
575 }
576 }
577 if (!exceptionMap.isEmpty()) {
578 reportScenarioFailures(exceptionMap);
579 }
580 }
581
582 private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) {
583 String failedScenarios = "The following scenarios have failed: #"
584 + exceptionMap.keySet().stream()
585 .map(s -> String.valueOf(s.getIndex()))
586 .collect(Collectors.joining(", #"));
587 StringBuilder builder = new StringBuilder(failedScenarios);
588 builder.append(System.lineSeparator()).append(System.lineSeparator());
589 for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) {
590 Exception e = entry.getValue();
591 Scenario scenario = entry.getKey();
592 String errorMsg = "";
593 if (scenario != null) {
594 errorMsg = getScenarioTitleAndFlags(scenario);
595 }
596 if (e instanceof IRViolationException irException) {
597 // For IR violations, only show the actual violations and not the (uninteresting) stack trace.
598 System.out.println((scenario != null ? "Scenario #" + scenario.getIndex() + " - " : "")
599 + "Compilation(s) of failed matche(s):");
600 System.out.println(irException.getCompilations());
601 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo());
602 } else if (e instanceof TestVMException testVMException) {
603 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo());
604 } else {
605 // Print stack trace otherwise
606 StringWriter errors = new StringWriter();
607 e.printStackTrace(new PrintWriter(errors));
608 builder.append(errors);
609 }
610 builder.append(System.lineSeparator());
611 }
612 System.err.println(builder);
613 if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) {
614 // Provide a hint to the user how to get additional output/debugging information.
615 System.err.println(RERUN_HINT);
616 }
617 throw new TestRunException(failedScenarios + ". Please check stderr for more information.");
618 }
619
620 private static String getScenarioTitleAndFlags(Scenario scenario) {
621 StringBuilder builder = new StringBuilder();
622 String title = "Scenario #" + scenario.getIndex();
623 builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length()))
624 .append(System.lineSeparator());
625 builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]")
626 .append(System.lineSeparator());
627 return builder.toString();
628 }
629
630 /**
631 * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of
632 * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the
633 * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests.
634 */
635 private void start(Scenario scenario) {
636 if (scenario != null && !scenario.isEnabled()) {
637 System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " +
638 "-DScenarios and is therefore not executed.");
639 return;
640 }
641 shouldVerifyIR = irVerificationPossible;
642 try {
643 // Use TestFramework flags and scenario flags for new VMs.
644 List<String> additionalFlags = new ArrayList<>();
645 if (flags != null) {
646 additionalFlags.addAll(flags);
647 }
648 if (scenario != null) {
649 List<String> scenarioFlags = scenario.getFlags();
650 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]";
651 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":");
652 additionalFlags.addAll(scenarioFlags);
653 }
654 String frameworkAndScenarioFlags = additionalFlags.isEmpty() ?
655 "" : " - [" + String.join(", ", additionalFlags) + "]";
656
657 if (shouldVerifyIR) {
658 // Only need to use flag VM if an IR verification is possibly done.
659 System.out.println("Run Flag VM:");
660 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags);
661 shouldVerifyIR = flagVMProcess.shouldVerifyIR();
662 if (shouldVerifyIR) {
663 // Add more flags for the test VM which are required to do IR verification.
664 additionalFlags.addAll(flagVMProcess.getTestVMFlags());
665 } // else: Flag VM found a reason to not do IR verification.
666 } else {
667 System.out.println("Skip Flag VM due to not performing IR verification.");
668 }
669
670 System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":");
671 runTestVM(additionalFlags);
672 } finally {
673 if (scenario != null) {
674 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput());
675 }
676 System.out.println();
677 }
678 }
679
680 private boolean hasIRAnnotations() {
681 return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class) != null);
682 }
683
684 private boolean onlyWhitelistedJTregVMAndJavaOptsFlags() {
685 List<String> flags = Arrays.stream(Utils.getTestJavaOpts())
686 .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", ""))
687 .collect(Collectors.toList());
688 for (String flag : flags) {
689 // Property flags (prefix -D), -ea and -esa are whitelisted.
690 if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) {
691 // Found VM flag that is not whitelisted
692 return false;
693 }
694 }
695 return true;
696 }
697
698 private void runTestVM(List<String> additionalFlags) {
699 TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup);
700 if (shouldVerifyIR) {
701 try {
702 new IRMatcher(testVMProcess.getHotspotPidFileName(), testVMProcess.getIrEncoding(), testClass);
703 } catch (IRViolationException e) {
704 e.addCommandLine(testVMProcess.getCommandLine());
705 throw e;
706 }
707 } else {
708 System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " +
709 "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " +
710 "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " +
711 "that make the IR verification impossible (e.g. -XX:-UseCompile, " +
712 "-XX:TieredStopAtLevel=[1,2,3], etc.).");
713 }
714 }
715
716 public static void check(boolean test, String failureMessage) {
717 if (!test) {
718 throw new TestFrameworkException(failureMessage);
719 }
720 }
721 }