1 /*
  2  * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 package compiler.lib.ir_framework;
 25 
 26 import compiler.lib.ir_framework.driver.*;
 27 import compiler.lib.ir_framework.driver.irmatching.IRMatcher;
 28 import compiler.lib.ir_framework.driver.irmatching.IRViolationException;
 29 import compiler.lib.ir_framework.shared.*;
 30 import compiler.lib.ir_framework.test.*;
 31 import jdk.test.lib.Platform;
 32 import jdk.test.lib.Utils;
 33 import jdk.test.lib.helpers.ClassFileInstaller;
 34 import sun.hotspot.WhiteBox;
 35 
 36 import java.io.PrintWriter;
 37 import java.io.StringWriter;
 38 import java.lang.reflect.Method;
 39 import java.util.*;
 40 import java.util.stream.Collectors;
 41 
 42 /**
 43  * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on
 44  * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can
 45  * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used
 46  * testing patterns and compiler control flags.
 47  * <p>
 48  * The framework offers various annotations to control how your test code should be invoked and being checked. There are
 49  * three kinds of tests depending on how much control is needed over the test invocation:
 50  * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b>
 51  * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test}
 52  * annotation which represents the test code that is eventually executed by the test framework. More information about
 53  * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}.
 54  * <p>
 55  * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which
 56  * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}.
 57  * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage
 58  * and how to write different IR rules can be found at {@link IR}.
 59  * <p>
 60  * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>:
 61  * <pre>
 62  * {@literal @}library /test/lib
 63  * {@literal @}run driver some.package.Test
 64  * </pre>
 65  * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the
 66  * JTreg test if the test itself is not utilizing any Whitebox features directly.
 67  * <p>
 68  * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or
 69  * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations
 70  * (instead of specifying multiple JTreg {@code @run} entries).
 71  * <p>
 72  * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the
 73  * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various
 74  * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new
 75  * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing.
 76  * <p>
 77  * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code
 78  * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions.
 79  * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are
 80  * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}
 81  * for IR matching).
 82  * <p>
 83  * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified
 84  * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}.
 85  * <p>
 86  * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR
 87  * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details).
 88  * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing
 89  * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}.
 90  *
 91  * @see Test
 92  * @see Check
 93  * @see Run
 94  * @see IR
 95  */
 96 public class TestFramework {
 97     /**
 98      * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only
 99      * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags
100      * by user code) are whitelisted.
101      *
102      * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of
103      * the entries of this list as a substring (partial match).
104      */
105     public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>(
106             Arrays.asList(
107                     // The following substrings are part of more than one VM flag
108                     "RAM",
109                     "Heap",
110                     "Trace",
111                     "Print",
112                     "Verify",
113                     "TLAB",
114                     "UseNewCode",
115                     "Xmn",
116                     "Xms",
117                     "Xmx",
118                     "Xss",
119                     // The following substrings are only part of one VM flag (=exact match)
120                     "CreateCoredumpOnCrash",
121                     "IgnoreUnrecognizedVMOptions",
122                     "UnlockDiagnosticVMOptions",
123                     "UnlockExperimentalVMOptions",
124                     "BackgroundCompilation",
125                     "Xbatch",
126                     "TieredCompilation",
127                     "CompileThreshold",
128                     "Xmixed",
129                     "server",
130                     "Xlog",
131                     "LogCompilation"
132             )
133     );
134 
135     public static final boolean VERBOSE = Boolean.getBoolean("Verbose");
136     public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty();
137     public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty();
138     private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout");
139     // Only used for internal testing and should not be used for normal user testing.
140     private static final boolean SKIP_WHITEBOX_INSTALL = Boolean.getBoolean("SkipWhiteBoxInstall");
141 
142     private static final String RERUN_HINT = """
143                                                #############################################################
144                                                 - To only run the failed tests use -DTest, -DExclude,
145                                                   and/or -DScenarios.
146                                                 - To also get the standard output of the test VM run with
147                                                   -DReportStdout=true or for even more fine-grained logging
148                                                   use -DVerbose=true.
149                                                #############################################################
150                                              """ + System.lineSeparator();
151 
152     private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true"));
153     private boolean shouldVerifyIR; // Should we perform IR matching?
154     private static boolean toggleBool;
155 
156     private final Class<?> testClass;
157     private Set<Class<?>> helperClasses;
158     private List<Scenario> scenarios;
159     private Set<Integer> scenarioIndices;
160     private List<String> flags;
161     private int defaultWarmup = -1;
162 
163     /*
164      * Public interface methods
165      */
166 
167     /**
168      * Creates an instance acting as a builder to test the class from which this constructor was invoked from.
169      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
170      * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
171      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
172      */
173     public TestFramework() {
174         this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass());
175     }
176 
177     /**
178      * Creates an instance acting as a builder to test {@code testClass}.
179      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
180      * Use the associated add methods ({@link #addFlags(String...)}, @link #addScenarios(Scenario...)},
181      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
182      *
183      * @param testClass the class to be tested by the framework.
184      * @see #TestFramework()
185      */
186     public TestFramework(Class<?> testClass) {
187         TestRun.check(testClass != null, "Test class cannot be null");
188         this.testClass = testClass;
189         if (VERBOSE) {
190             System.out.println("Test class: " + testClass);
191         }
192     }
193 
194     /**
195      * Tests the class from which this method was invoked from.
196      */
197     public static void run() {
198         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
199         run(walker.getCallerClass());
200     }
201 
202     /**
203      * Tests {@code testClass}.
204      *
205      * @param testClass the class to be tested by the framework.
206      * @see #run()
207      */
208     public static void run(Class<?> testClass) {
209         TestFramework framework = new TestFramework(testClass);
210         framework.start();
211     }
212 
213     /**
214      * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}.
215      * <ul>
216      *     <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p>
217      *            Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and  Javaoptions flags over
218      *            the specified {@code flags} of this method.</li>
219      *     <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li>
220      *     <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one
221      *            from which this method was called from, use {@link #addFlags(String...)}, use this method.</li>
222      *     <li><p>If you want to run your JTreg test with multiple flag combinations, use
223      *            {@link #addScenarios(Scenario...)}</li>
224      * </ul>
225      *
226      * @param flags VM flags to be used for the test VM.
227      */
228     public static void runWithFlags(String... flags) {
229         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
230         TestFramework framework = new TestFramework(walker.getCallerClass());
231         framework.addFlags(flags);
232         framework.start();
233     }
234 
235     /**
236      * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p>
237      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags.
238      *
239      * <p>
240      * The testing can be started by invoking {@link #start()}
241      *
242      * @param flags VM options to be applied to the test VM.
243      * @return the same framework instance.
244      */
245     public TestFramework addFlags(String... flags) {
246         TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null");
247         if (this.flags == null) {
248             this.flags = new ArrayList<>();
249         }
250         this.flags.addAll(Arrays.asList(flags));
251         return this;
252     }
253 
254     /**
255      * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile},
256      * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied
257      * while testing {@code testClass} (also see description of {@link TestFramework}).
258      *
259      * <p>
260      * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any
261      * compile command annotations, you do not need to include it with this method. If no helper class specifies any
262      * compile commands, you do not need to call this method at all.
263      *
264      * <p>
265      * The testing can be started by invoking {@link #start()}.
266      *
267      * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile},
268      *                      {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied
269      *                      while testing {@code testClass} (also see description of {@link TestFramework}).
270      * @return the same framework instance.
271      */
272     public TestFramework addHelperClasses(Class<?>... helperClasses) {
273         TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull),
274                       "A Helper class cannot be null");
275         if (this.helperClasses == null) {
276             this.helperClasses = new HashSet<>();
277         }
278 
279         this.helperClasses.addAll(Arrays.asList(helperClasses));
280         return this;
281     }
282 
283     /**
284      * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the
285      * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)}
286      * and thus also override any VM or Javaoptions set by JTreg by default.<p>
287      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags.
288      *
289      * <p>
290      * The testing can be started by invoking {@link #start()}
291      *
292      * @param scenarios scenarios which specify specific flags for the test VM.
293      * @return the same framework instance.
294      */
295     public TestFramework addScenarios(Scenario... scenarios) {
296         TestFormat.check(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull),
297                          "A scenario cannot be null");
298         if (this.scenarios == null) {
299             this.scenarios = new ArrayList<>();
300             this.scenarioIndices = new HashSet<>();
301         }
302 
303         for (Scenario scenario : scenarios) {
304             int scenarioIndex = scenario.getIndex();
305             TestFormat.check(scenarioIndices.add(scenarioIndex),
306                              "Cannot define two scenarios with the same index " + scenarioIndex);
307             this.scenarios.add(scenario);
308         }
309         return this;
310     }
311 
312     /**
313      * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)})
314      * set test class.
315      */
316     public void start() {
317         if (!SKIP_WHITEBOX_INSTALL) {
318             installWhiteBox();
319         }
320         disableIRVerificationIfNotFeasible();
321 
322         if (scenarios == null) {
323             try {
324                 start(null);
325             } catch (TestVMException e) {
326                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT);
327                 throw e;
328             } catch (IRViolationException e) {
329                 System.out.println("Compilation(s) of failed match(es):");
330                 System.out.println(e.getCompilations());
331                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT);
332                 throw e;
333             }
334         } else {
335             startWithScenarios();
336         }
337     }
338 
339     /**
340      * Set a new default warm-up (overriding the framework default of 2000 at
341      * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit
342      * warm-up with {@link Warmup @Warmup}.
343      *
344      * @param defaultWarmup a new non-negative default warm-up.
345      * @return the same framework instance.
346      */
347     public TestFramework setDefaultWarmup(int defaultWarmup) {
348         TestFormat.check(defaultWarmup >= 0, "Cannot specify a negative default warm-up");
349         this.defaultWarmup = defaultWarmup;
350         return this;
351     }
352 
353     /**
354      * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios
355      * were run, use {@link Scenario#getTestVMOutput()}.
356      *
357      * @return the last test VM output.
358      */
359     public static String getLastTestVMOutput() {
360         return TestVMProcess.getLastTestVMOutput();
361     }
362 
363     /*
364      * The following methods are only intended to be called from actual @Test methods and not from the main() method of
365      * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled).
366      */
367 
368     /**
369      * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled,
370      * yet, upon returning from this method.
371      *
372      * @param m the method to be compiled.
373      * @param compLevel the (valid) compilation level at which the method should be compiled.
374      * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}.
375      */
376     public static void compile(Method m, CompLevel compLevel) {
377         TestVM.compile(m, compLevel);
378     }
379 
380     /**
381      * Deoptimize {@code m}.
382      *
383      * @param m the method to be deoptimized.
384      */
385     public static void deoptimize(Method m) {
386         TestVM.deoptimize(m);
387     }
388 
389     /**
390      * Returns a boolean indicating if {@code m} is compiled at any level.
391      *
392      * @param m the method to be checked.
393      * @return {@code true} if {@code m} is compiled at any level;
394      *         {@code false} otherwise.
395      */
396     public static boolean isCompiled(Method m) {
397         return TestVM.isCompiled(m);
398     }
399 
400     /**
401      * Returns a boolean indicating if {@code m} is compiled with C1.
402      *
403      * @param m the method to be checked.
404      * @return {@code true} if {@code m} is compiled with C1;
405      *         {@code false} otherwise.
406      */
407     public static boolean isC1Compiled(Method m) {
408         return TestVM.isC1Compiled(m);
409     }
410 
411     /**
412      * Returns a boolean indicating if {@code m} is compiled with C2.
413      *
414      * @param m the method to be checked.
415      * @return {@code true} if {@code m} is compiled with C2;
416      *         {@code false} otherwise.
417      */
418     public static boolean isC2Compiled(Method m) {
419         return TestVM.isC2Compiled(m);
420     }
421 
422     /**
423      * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}.
424      *
425      * @param m the method to be checked.
426      * @param compLevel the compilation level.
427      * @return {@code true} if {@code m} is compiled at {@code compLevel};
428      *         {@code false} otherwise.
429      */
430     public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
431         return TestVM.isCompiledAtLevel(m, compLevel);
432     }
433 
434     /**
435      * Checks if {@code m} is compiled at any level.
436      *
437      * @param m the method to be checked.
438      * @throws TestRunException if {@code m} is not compiled at any level.
439      */
440     public static void assertCompiled(Method m) {
441         TestVM.assertCompiled(m);
442     }
443 
444     /**
445      * Checks if {@code m} is not compiled at any level.
446      *
447      * @param m the method to be checked.
448      * @throws TestRunException if {@code m} is compiled at any level.
449      */
450     public static void assertNotCompiled(Method m) {
451         TestVM.assertNotCompiled(m);
452     }
453 
454     /**
455      * Verifies that {@code m} is compiled with C1.
456      *
457      * @param m the method to be verified.
458      * @throws TestRunException if {@code m} is not compiled with C1.
459      */
460     public static void assertCompiledByC1(Method m) {
461         TestVM.assertCompiledByC1(m);
462     }
463 
464     /**
465      * Verifies that {@code m} is compiled with C2.
466      *
467      * @param m the method to be checked.
468      * @throws TestRunException if {@code m} is not compiled with C2.
469      */
470     public static void assertCompiledByC2(Method m) {
471         TestVM.assertCompiledByC2(m);
472     }
473 
474     /**
475      * Verifies that {@code m} is compiled at the specified {@code compLevel}.
476      *
477      * @param m the method to be checked.
478      * @param compLevel the compilation level.
479      * @throws TestRunException if {@code m} is not compiled at {@code compLevel}.
480      */
481     public static void assertCompiledAtLevel(Method m, CompLevel compLevel) {
482         TestVM.assertCompiledAtLevel(m, compLevel);
483     }
484 
485     /**
486      * Verifies that {@code m} was deoptimized after being C1 compiled.
487      *
488      * @param m the method to be checked.
489      * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled.
490      */
491     public static void assertDeoptimizedByC1(Method m) {
492         TestVM.assertDeoptimizedByC1(m);
493     }
494 
495     /**
496      * Verifies that {@code m} was deoptimized after being C2 compiled.
497      *
498      * @param m the method to be checked.
499      * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled.
500      */
501     public static void assertDeoptimizedByC2(Method m) {
502         TestVM.assertDeoptimizedByC2(m);
503     }
504 
505     /**
506      * Checks if deopt of {@code m} is stable at the specified {@code compLevel}.
507      *
508      * @param m the method to be checked.
509      * @param compLevel the compilation level.
510      * @return {@code true} if deopt of {@code m} is stable at {@code compLevel};
511      *         {@code false} otherwise.
512      */
513     public static boolean isStableDeopt(Method m, CompLevel compLevel) {
514         return TestVM.isStableDeopt(m, compLevel);
515     }
516 
517     /**
518      * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}).
519      * The very first invocation returns {@code false}. Note that this method could be used by different tests and
520      * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times
521      * other tests have already invoked this method.
522      *
523      * @return an inverted boolean of the result of the last invocation of this method.
524      */
525     public static boolean toggleBoolean() {
526         toggleBool = !toggleBool;
527         return toggleBool;
528     }
529 
530     /*
531      * End of public interface methods
532      */
533 
534     /**
535      * Used to move Whitebox class to the right folder in the JTreg test
536      */
537     private void installWhiteBox() {
538         try {
539             ClassFileInstaller.main(WhiteBox.class.getName());
540         } catch (Exception e) {
541             throw new Error("failed to install whitebox classes", e);
542         }
543     }
544 
545     /**
546      * Disable IR verification completely in certain cases.
547      */
548     private void disableIRVerificationIfNotFeasible() {
549         if (irVerificationPossible) {
550             irVerificationPossible = Platform.isDebugBuild() && !Platform.isInt() && !Platform.isComp();
551             if (!irVerificationPossible) {
552                 System.out.println("IR verification disabled due to not running a debug build (required for PrintIdeal" +
553                                    "and PrintOptoAssembly), running with -Xint, or -Xcomp (use warm-up of 0 instead)");
554                 return;
555             }
556 
557             irVerificationPossible = hasIRAnnotations();
558             if (!irVerificationPossible) {
559                 System.out.println("IR verification disabled due to test " + testClass + " not specifying any @IR annotations");
560                 return;
561             }
562 
563             // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified.
564             irVerificationPossible = onlyWhitelistedJTregVMAndJavaOptsFlags();
565             if (!irVerificationPossible) {
566                 System.out.println("IR verification disabled due to using non-whitelisted JTreg VM or Javaoptions flag(s)."
567                                    + System.lineSeparator());
568             }
569         }
570     }
571 
572     /**
573      * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all
574      * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios
575      * and thus is reported immediately on the first scenario execution.
576      */
577     private void startWithScenarios() {
578         Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex));
579         for (Scenario scenario : scenarios) {
580             try {
581                 start(scenario);
582             } catch (TestFormatException e) {
583                 // Test format violation is wrong for all the scenarios. Only report once.
584                 throw e;
585             } catch (Exception e) {
586                 exceptionMap.put(scenario, e);
587             }
588         }
589         if (!exceptionMap.isEmpty()) {
590             reportScenarioFailures(exceptionMap);
591         }
592     }
593 
594     private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) {
595         String failedScenarios = "The following scenarios have failed: #"
596                                  + exceptionMap.keySet().stream()
597                                                .map(s -> String.valueOf(s.getIndex()))
598                                                .collect(Collectors.joining(", #"));
599         StringBuilder builder = new StringBuilder(failedScenarios);
600         builder.append(System.lineSeparator()).append(System.lineSeparator());
601         for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) {
602             Exception e = entry.getValue();
603             Scenario scenario = entry.getKey();
604             String errorMsg = "";
605             if (scenario != null) {
606                 errorMsg = getScenarioTitleAndFlags(scenario);
607             }
608             if (e instanceof IRViolationException irException) {
609                 // For IR violations, only show the actual violations and not the (uninteresting) stack trace.
610                 System.out.println((scenario != null ? "Scenario #" + scenario.getIndex() + " - " : "")
611                                    + "Compilation(s) of failed matche(s):");
612                 System.out.println(irException.getCompilations());
613                 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo());
614             } else if (e instanceof TestVMException testVMException) {
615                 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo());
616             } else {
617                 // Print stack trace otherwise
618                 StringWriter errors = new StringWriter();
619                 e.printStackTrace(new PrintWriter(errors));
620                 builder.append(errors);
621             }
622             builder.append(System.lineSeparator());
623         }
624         System.err.println(builder);
625         if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) {
626             // Provide a hint to the user how to get additional output/debugging information.
627             System.err.println(RERUN_HINT);
628         }
629         throw new TestRunException(failedScenarios + ". Please check stderr for more information.");
630     }
631 
632     private static String getScenarioTitleAndFlags(Scenario scenario) {
633         StringBuilder builder = new StringBuilder();
634         String title = "Scenario #" + scenario.getIndex();
635         builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length()))
636                .append(System.lineSeparator());
637         builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]")
638                .append(System.lineSeparator());
639         return builder.toString();
640     }
641 
642     /**
643      * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of
644      * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the
645      * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests.
646      */
647     private void start(Scenario scenario) {
648         if (scenario != null && !scenario.isEnabled()) {
649             System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " +
650                                "-DScenarios and is therefore not executed.");
651             return;
652         }
653         shouldVerifyIR = irVerificationPossible;
654         try {
655             // Use TestFramework flags and scenario flags for new VMs.
656             List<String> additionalFlags = new ArrayList<>();
657             if (flags != null) {
658                 additionalFlags.addAll(flags);
659             }
660             if (scenario != null) {
661                 List<String> scenarioFlags = scenario.getFlags();
662                 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]";
663                 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":");
664                 additionalFlags.addAll(scenarioFlags);
665             }
666             String frameworkAndScenarioFlags = additionalFlags.isEmpty() ?
667                     "" : " - [" + String.join(", ", additionalFlags) + "]";
668 
669             if (shouldVerifyIR) {
670                 // Only need to use flag VM if an IR verification is possibly done.
671                 System.out.println("Run Flag VM:");
672                 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags);
673                 shouldVerifyIR = flagVMProcess.shouldVerifyIR();
674                 if (shouldVerifyIR) {
675                     // Add more flags for the test VM which are required to do IR verification.
676                     additionalFlags.addAll(flagVMProcess.getTestVMFlags());
677                 } // else: Flag VM found a reason to not do IR verification.
678             } else {
679                 System.out.println("Skip Flag VM due to not performing IR verification.");
680             }
681 
682             System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":");
683             runTestVM(additionalFlags);
684         } finally {
685             if (scenario != null) {
686                 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput());
687             }
688             System.out.println();
689         }
690     }
691 
692     private boolean hasIRAnnotations() {
693         return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class) != null);
694     }
695 
696     private boolean onlyWhitelistedJTregVMAndJavaOptsFlags() {
697         List<String> flags = Arrays.stream(Utils.getTestJavaOpts())
698                                    .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", ""))
699                                    .collect(Collectors.toList());
700         for (String flag : flags) {
701             // Property flags (prefix -D), -ea and -esa are whitelisted.
702             if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) {
703                 // Found VM flag that is not whitelisted
704                 return false;
705             }
706         }
707         return true;
708     }
709 
710     private void runTestVM(List<String> additionalFlags) {
711         TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup);
712         if (shouldVerifyIR) {
713             try {
714                 new IRMatcher(testVMProcess.getHotspotPidFileName(), testVMProcess.getIrEncoding(), testClass);
715             } catch (IRViolationException e) {
716                 e.addCommandLine(testVMProcess.getCommandLine());
717                 throw e;
718             }
719         } else {
720             System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " +
721                                "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " +
722                                "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " +
723                                "that make the IR verification impossible (e.g. -XX:-UseCompile, " +
724                                "-XX:TieredStopAtLevel=[1,2,3], etc.).");
725         }
726     }
727 
728     public static void check(boolean test, String failureMessage) {
729         if (!test) {
730             throw new TestFrameworkException(failureMessage);
731         }
732     }
733 }