1 /*
  2  * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 package compiler.lib.ir_framework;
 25 
 26 import compiler.lib.ir_framework.driver.*;
 27 import compiler.lib.ir_framework.driver.irmatching.IRMatcher;
 28 import compiler.lib.ir_framework.driver.irmatching.IRViolationException;
 29 import compiler.lib.ir_framework.shared.*;
 30 import compiler.lib.ir_framework.test.*;
 31 import jdk.test.lib.Platform;
 32 import jdk.test.lib.Utils;
 33 import jdk.test.lib.helpers.ClassFileInstaller;
 34 import sun.hotspot.WhiteBox;
 35 
 36 import java.io.PrintWriter;
 37 import java.io.StringWriter;
 38 import java.lang.reflect.Method;
 39 import java.util.*;
 40 import java.util.stream.Collectors;
 41 
 42 /**
 43  * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on
 44  * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can
 45  * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used
 46  * testing patterns and compiler control flags.
 47  * <p>
 48  * The framework offers various annotations to control how your test code should be invoked and being checked. There are
 49  * three kinds of tests depending on how much control is needed over the test invocation:
 50  * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b>
 51  * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test}
 52  * annotation which represents the test code that is eventually executed by the test framework. More information about
 53  * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}.
 54  * <p>
 55  * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which
 56  * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}.
 57  * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage
 58  * and how to write different IR rules can be found at {@link IR}.
 59  * <p>
 60  * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>:
 61  * <pre>
 62  * {@literal @}library /test/lib
 63  * {@literal @}run driver some.package.Test
 64  * </pre>
 65  * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the
 66  * JTreg test if the test itself is not utilizing any Whitebox features directly.
 67  * <p>
 68  * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or
 69  * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations
 70  * (instead of specifying multiple JTreg {@code @run} entries).
 71  * <p>
 72  * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the
 73  * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various
 74  * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new
 75  * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing.
 76  * <p>
 77  * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code
 78  * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions.
 79  * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are
 80  * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}
 81  * for IR matching).
 82  * <p>
 83  * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified
 84  * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}.
 85  * <p>
 86  * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR
 87  * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details).
 88  * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing
 89  * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}.
 90  *
 91  * @see Test
 92  * @see Check
 93  * @see Run
 94  * @see IR
 95  */
 96 public class TestFramework {
 97     /**
 98      * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only
 99      * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags
100      * by user code) are whitelisted.
101      *
102      * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of
103      * the entries of this list as a substring (partial match).
104      */
105     public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>(
106             Arrays.asList(
107                     // The following substrings are part of more than one VM flag
108                     "RAM",
109                     "Heap",
110                     "Trace",
111                     "Print",
112                     "Verify",
113                     "TLAB",
114                     "UseNewCode",
115                     "Xmn",
116                     "Xms",
117                     "Xmx",
118                     "Xss",
119                     // The following substrings are only part of one VM flag (=exact match)
120                     "CreateCoredumpOnCrash",
121                     "IgnoreUnrecognizedVMOptions",
122                     "UnlockDiagnosticVMOptions",
123                     "UnlockExperimentalVMOptions",
124                     "BackgroundCompilation",
125                     "Xbatch",
126                     "TieredCompilation",
127                     "CompileThreshold",
128                     "Xmixed",
129                     "server",
130                     "Xlog",
131                     "LogCompilation"
132             )
133     );
134 
135     public static final boolean VERBOSE = Boolean.getBoolean("Verbose");
136     public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty();
137     public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty();
138     private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout");
139     // Only used for internal testing and should not be used for normal user testing.
140     private static final boolean SKIP_WHITEBOX_INSTALL = Boolean.getBoolean("SkipWhiteBoxInstall");
141 
142     private static final String RERUN_HINT = """
143                                                #############################################################
144                                                 - To only run the failed tests use -DTest, -DExclude,
145                                                   and/or -DScenarios.
146                                                 - To also get the standard output of the test VM run with
147                                                   -DReportStdout=true or for even more fine-grained logging
148                                                   use -DVerbose=true.
149                                                #############################################################
150                                              """ + System.lineSeparator();
151 
152     private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true"));
153     private boolean shouldVerifyIR; // Should we perform IR matching?
154     private static boolean toggleBool;
155 
156     private final Class<?> testClass;
157     private Set<Class<?>> helperClasses;
158     private List<Scenario> scenarios;
159     private Set<Integer> scenarioIndices;
160     private List<String> flags;
161     private int defaultWarmup = -1;
162 
163     /*
164      * Public interface methods
165      */
166 
167     /**
168      * Creates an instance acting as a builder to test the class from which this constructor was invoked from.
169      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
170      * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
171      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
172      */
173     public TestFramework() {
174         this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass());
175     }
176 
177     /**
178      * Creates an instance acting as a builder to test {@code testClass}.
179      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
180      * Use the associated add methods ({@link #addFlags(String...)}, @link #addScenarios(Scenario...)},
181      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
182      *
183      * @param testClass the class to be tested by the framework.
184      * @see #TestFramework()
185      */
186     public TestFramework(Class<?> testClass) {
187         TestRun.check(testClass != null, "Test class cannot be null");
188         this.testClass = testClass;
189         if (VERBOSE) {
190             System.out.println("Test class: " + testClass);
191         }
192     }
193 
194     /**
195      * Tests the class from which this method was invoked from.
196      */
197     public static void run() {
198         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
199         run(walker.getCallerClass());
200     }
201 
202     /**
203      * Tests {@code testClass}.
204      *
205      * @param testClass the class to be tested by the framework.
206      * @see #run()
207      */
208     public static void run(Class<?> testClass) {
209         TestFramework framework = new TestFramework(testClass);
210         framework.start();
211     }
212 
213     /**
214      * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}.
215      * <ul>
216      *     <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p>
217      *            Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and  Javaoptions flags over
218      *            the specified {@code flags} of this method.</li>
219      *     <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li>
220      *     <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one
221      *            from which this method was called from, use {@link #addFlags(String...)}, use this method.</li>
222      *     <li><p>If you want to run your JTreg test with multiple flag combinations, use
223      *            {@link #addScenarios(Scenario...)}</li>
224      * </ul>
225      *
226      * @param flags VM flags to be used for the test VM.
227      */
228     public static void runWithFlags(String... flags) {
229         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
230         TestFramework framework = new TestFramework(walker.getCallerClass());
231         framework.addFlags(flags);
232         framework.start();
233     }
234 
235     /**
236      * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p>
237      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags.
238      *
239      * <p>
240      * The testing can be started by invoking {@link #start()}
241      *
242      * @param flags VM options to be applied to the test VM.
243      * @return the same framework instance.
244      */
245     public TestFramework addFlags(String... flags) {
246         TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null");
247         if (this.flags == null) {
248             this.flags = new ArrayList<>();
249         }
250         this.flags.addAll(Arrays.asList(flags));
251         return this;
252     }
253 
254     /**
255      * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile},
256      * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied
257      * while testing {@code testClass} (also see description of {@link TestFramework}).
258      *
259      * <p>
260      * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any
261      * compile command annotations, you do not need to include it with this method. If no helper class specifies any
262      * compile commands, you do not need to call this method at all.
263      *
264      * <p>
265      * The testing can be started by invoking {@link #start()}.
266      *
267      * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile},
268      *                      {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied
269      *                      while testing {@code testClass} (also see description of {@link TestFramework}).
270      * @return the same framework instance.
271      */
272     public TestFramework addHelperClasses(Class<?>... helperClasses) {
273         TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull),
274                       "A Helper class cannot be null");
275         if (this.helperClasses == null) {
276             this.helperClasses = new HashSet<>();
277         }
278 
279         this.helperClasses.addAll(Arrays.asList(helperClasses));
280         return this;
281     }
282 
283     /**
284      * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the
285      * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)}
286      * and thus also override any VM or Javaoptions set by JTreg by default.<p>
287      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags.
288      *
289      * <p>
290      * The testing can be started by invoking {@link #start()}
291      *
292      * @param scenarios scenarios which specify specific flags for the test VM.
293      * @return the same framework instance.
294      */
295     public TestFramework addScenarios(Scenario... scenarios) {
296         TestFormat.check(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull),
297                          "A scenario cannot be null");
298         if (this.scenarios == null) {
299             this.scenarios = new ArrayList<>();
300             this.scenarioIndices = new HashSet<>();
301         }
302 
303         for (Scenario scenario : scenarios) {
304             int scenarioIndex = scenario.getIndex();
305             TestFormat.check(scenarioIndices.add(scenarioIndex),
306                              "Cannot define two scenarios with the same index " + scenarioIndex);
307             this.scenarios.add(scenario);
308         }
309         return this;
310     }
311 
312     /**
313      * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)})
314      * set test class.
315      */
316     public void start() {
317         if (!SKIP_WHITEBOX_INSTALL) {
318             installWhiteBox();
319         }
320         disableIRVerificationIfNotFeasible();
321 
322         if (scenarios == null) {
323             try {
324                 start(null);
325             } catch (TestVMException e) {
326                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT);
327                 throw e;
328             } catch (IRViolationException e) {
329                 System.out.println("Compilation(s) of failed match(es):");
330                 System.out.println(e.getCompilations());
331                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT);
332                 throw e;
333             }
334         } else {
335             startWithScenarios();
336         }
337     }
338 
339     /**
340      * Set a new default warm-up (overriding the framework default of 2000 at
341      * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit
342      * warm-up with {@link Warmup @Warmup}.
343      *
344      * @param defaultWarmup a new non-negative default warm-up.
345      * @return the same framework instance.
346      */
347     public TestFramework setDefaultWarmup(int defaultWarmup) {
348         TestFormat.check(defaultWarmup >= 0, "Cannot specify a negative default warm-up");
349         this.defaultWarmup = defaultWarmup;
350         return this;
351     }
352 
353     /**
354      * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios
355      * were run, use {@link Scenario#getTestVMOutput()}.
356      *
357      * @return the last test VM output.
358      */
359     public static String getLastTestVMOutput() {
360         return TestVMProcess.getLastTestVMOutput();
361     }
362 
363     /*
364      * The following methods are only intended to be called from actual @Test methods and not from the main() method of
365      * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled).
366      */
367 
368     /**
369      * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled,
370      * yet, upon returning from this method.
371      *
372      * @param m the method to be compiled.
373      * @param compLevel the (valid) compilation level at which the method should be compiled.
374      * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}.
375      */
376     public static void compile(Method m, CompLevel compLevel) {
377         TestVM.compile(m, compLevel);
378     }
379 
380     /**
381      * Deoptimize {@code m}.
382      *
383      * @param m the method to be deoptimized.
384      */
385     public static void deoptimize(Method m) {
386         TestVM.deoptimize(m);
387     }
388 
389     /**
390      * Returns a boolean indicating if {@code m} is compiled at any level.
391      *
392      * @param m the method to be checked.
393      * @return {@code true} if {@code m} is compiled at any level;
394      *         {@code false} otherwise.
395      */
396     public static boolean isCompiled(Method m) {
397         return TestVM.isCompiled(m);
398     }
399 
400     /**
401      * Returns a boolean indicating if {@code m} is compiled with C1.
402      *
403      * @param m the method to be checked.
404      * @return {@code true} if {@code m} is compiled with C1;
405      *         {@code false} otherwise.
406      */
407     public static boolean isC1Compiled(Method m) {
408         return TestVM.isC1Compiled(m);
409     }
410 
411     /**
412      * Returns a boolean indicating if {@code m} is compiled with C2.
413      *
414      * @param m the method to be checked.
415      * @return {@code true} if {@code m} is compiled with C2;
416      *         {@code false} otherwise.
417      */
418     public static boolean isC2Compiled(Method m) {
419         return TestVM.isC2Compiled(m);
420     }
421 
422     /**
423      * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}.
424      *
425      * @param m the method to be checked.
426      * @param compLevel the compilation level.
427      * @return {@code true} if {@code m} is compiled at {@code compLevel};
428      *         {@code false} otherwise.
429      */
430     public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
431         return TestVM.isCompiledAtLevel(m, compLevel);
432     }
433 
434     /**
435      * Checks if {@code m} is compiled at any level.
436      *
437      * @param m the method to be checked.
438      * @throws TestRunException if {@code m} is not compiled at any level.
439      */
440     public static void assertCompiled(Method m) {
441         TestVM.assertCompiled(m);
442     }
443 
444     /**
445      * Checks if {@code m} is not compiled at any level.
446      *
447      * @param m the method to be checked.
448      * @throws TestRunException if {@code m} is compiled at any level.
449      */
450     public static void assertNotCompiled(Method m) {
451         TestVM.assertNotCompiled(m);
452     }
453 
454     /**
455      * Verifies that {@code m} is compiled with C1.
456      *
457      * @param m the method to be verified.
458      * @throws TestRunException if {@code m} is not compiled with C1.
459      */
460     public static void assertCompiledByC1(Method m) {
461         TestVM.assertCompiledByC1(m);
462     }
463 
464     /**
465      * Verifies that {@code m} is compiled with C2.
466      *
467      * @param m the method to be checked.
468      * @throws TestRunException if {@code m} is not compiled with C2.
469      */
470     public static void assertCompiledByC2(Method m) {
471         TestVM.assertCompiledByC2(m);
472     }
473 
474     /**
475      * Verifies that {@code m} is compiled at the specified {@code compLevel}.
476      *
477      * @param m the method to be checked.
478      * @param compLevel the compilation level.
479      * @throws TestRunException if {@code m} is not compiled at {@code compLevel}.
480      */
481     public static void assertCompiledAtLevel(Method m, CompLevel compLevel) {
482         TestVM.assertCompiledAtLevel(m, compLevel);
483     }
484 
485     /**
486      * Verifies that {@code m} was deoptimized after being C1 compiled.
487      *
488      * @param m the method to be checked.
489      * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled.
490      */
491     public static void assertDeoptimizedByC1(Method m) {
492         TestVM.assertDeoptimizedByC1(m);
493     }
494 
495     /**
496      * Verifies that {@code m} was deoptimized after being C2 compiled.
497      *
498      * @param m the method to be checked.
499      * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled.
500      */
501     public static void assertDeoptimizedByC2(Method m) {
502         TestVM.assertDeoptimizedByC2(m);
503     }
504 
505     /**
506      * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}).
507      * The very first invocation returns {@code false}. Note that this method could be used by different tests and
508      * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times
509      * other tests have already invoked this method.
510      *
511      * @return an inverted boolean of the result of the last invocation of this method.
512      */
513     public static boolean toggleBoolean() {
514         toggleBool = !toggleBool;
515         return toggleBool;
516     }
517 
518     /*
519      * End of public interface methods
520      */
521 
522     /**
523      * Used to move Whitebox class to the right folder in the JTreg test
524      */
525     private void installWhiteBox() {
526         try {
527             ClassFileInstaller.main(WhiteBox.class.getName());
528         } catch (Exception e) {
529             throw new Error("failed to install whitebox classes", e);
530         }
531     }
532 
533     /**
534      * Disable IR verification completely in certain cases.
535      */
536     private void disableIRVerificationIfNotFeasible() {
537         if (irVerificationPossible) {
538             irVerificationPossible = Platform.isDebugBuild() && !Platform.isInt() && !Platform.isComp();
539             if (!irVerificationPossible) {
540                 System.out.println("IR verification disabled due to not running a debug build (required for PrintIdeal" +
541                                    "and PrintOptoAssembly), running with -Xint, or -Xcomp (use warm-up of 0 instead)");
542                 return;
543             }
544 
545             irVerificationPossible = hasIRAnnotations();
546             if (!irVerificationPossible) {
547                 System.out.println("IR verification disabled due to test " + testClass + " not specifying any @IR annotations");
548                 return;
549             }
550 
551             // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified.
552             irVerificationPossible = onlyWhitelistedJTregVMAndJavaOptsFlags();
553             if (!irVerificationPossible) {
554                 System.out.println("IR verification disabled due to using non-whitelisted JTreg VM or Javaoptions flag(s)."
555                                    + System.lineSeparator());
556             }
557         }
558     }
559 
560     /**
561      * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all
562      * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios
563      * and thus is reported immediately on the first scenario execution.
564      */
565     private void startWithScenarios() {
566         Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex));
567         for (Scenario scenario : scenarios) {
568             try {
569                 start(scenario);
570             } catch (TestFormatException e) {
571                 // Test format violation is wrong for all the scenarios. Only report once.
572                 throw e;
573             } catch (Exception e) {
574                 exceptionMap.put(scenario, e);
575             }
576         }
577         if (!exceptionMap.isEmpty()) {
578             reportScenarioFailures(exceptionMap);
579         }
580     }
581 
582     private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) {
583         String failedScenarios = "The following scenarios have failed: #"
584                                  + exceptionMap.keySet().stream()
585                                                .map(s -> String.valueOf(s.getIndex()))
586                                                .collect(Collectors.joining(", #"));
587         StringBuilder builder = new StringBuilder(failedScenarios);
588         builder.append(System.lineSeparator()).append(System.lineSeparator());
589         for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) {
590             Exception e = entry.getValue();
591             Scenario scenario = entry.getKey();
592             String errorMsg = "";
593             if (scenario != null) {
594                 errorMsg = getScenarioTitleAndFlags(scenario);
595             }
596             if (e instanceof IRViolationException irException) {
597                 // For IR violations, only show the actual violations and not the (uninteresting) stack trace.
598                 System.out.println((scenario != null ? "Scenario #" + scenario.getIndex() + " - " : "")
599                                    + "Compilation(s) of failed matche(s):");
600                 System.out.println(irException.getCompilations());
601                 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo());
602             } else if (e instanceof TestVMException testVMException) {
603                 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo());
604             } else {
605                 // Print stack trace otherwise
606                 StringWriter errors = new StringWriter();
607                 e.printStackTrace(new PrintWriter(errors));
608                 builder.append(errors);
609             }
610             builder.append(System.lineSeparator());
611         }
612         System.err.println(builder);
613         if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) {
614             // Provide a hint to the user how to get additional output/debugging information.
615             System.err.println(RERUN_HINT);
616         }
617         throw new TestRunException(failedScenarios + ". Please check stderr for more information.");
618     }
619 
620     private static String getScenarioTitleAndFlags(Scenario scenario) {
621         StringBuilder builder = new StringBuilder();
622         String title = "Scenario #" + scenario.getIndex();
623         builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length()))
624                .append(System.lineSeparator());
625         builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]")
626                .append(System.lineSeparator());
627         return builder.toString();
628     }
629 
630     /**
631      * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of
632      * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the
633      * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests.
634      */
635     private void start(Scenario scenario) {
636         if (scenario != null && !scenario.isEnabled()) {
637             System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " +
638                                "-DScenarios and is therefore not executed.");
639             return;
640         }
641         shouldVerifyIR = irVerificationPossible;
642         try {
643             // Use TestFramework flags and scenario flags for new VMs.
644             List<String> additionalFlags = new ArrayList<>();
645             if (flags != null) {
646                 additionalFlags.addAll(flags);
647             }
648             if (scenario != null) {
649                 List<String> scenarioFlags = scenario.getFlags();
650                 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]";
651                 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":");
652                 additionalFlags.addAll(scenarioFlags);
653             }
654             String frameworkAndScenarioFlags = additionalFlags.isEmpty() ?
655                     "" : " - [" + String.join(", ", additionalFlags) + "]";
656 
657             if (shouldVerifyIR) {
658                 // Only need to use flag VM if an IR verification is possibly done.
659                 System.out.println("Run Flag VM:");
660                 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags);
661                 shouldVerifyIR = flagVMProcess.shouldVerifyIR();
662                 if (shouldVerifyIR) {
663                     // Add more flags for the test VM which are required to do IR verification.
664                     additionalFlags.addAll(flagVMProcess.getTestVMFlags());
665                 } // else: Flag VM found a reason to not do IR verification.
666             } else {
667                 System.out.println("Skip Flag VM due to not performing IR verification.");
668             }
669 
670             System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":");
671             runTestVM(additionalFlags);
672         } finally {
673             if (scenario != null) {
674                 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput());
675             }
676             System.out.println();
677         }
678     }
679 
680     private boolean hasIRAnnotations() {
681         return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class) != null);
682     }
683 
684     private boolean onlyWhitelistedJTregVMAndJavaOptsFlags() {
685         List<String> flags = Arrays.stream(Utils.getTestJavaOpts())
686                                    .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", ""))
687                                    .collect(Collectors.toList());
688         for (String flag : flags) {
689             // Property flags (prefix -D), -ea and -esa are whitelisted.
690             if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) {
691                 // Found VM flag that is not whitelisted
692                 return false;
693             }
694         }
695         return true;
696     }
697 
698     private void runTestVM(List<String> additionalFlags) {
699         TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup);
700         if (shouldVerifyIR) {
701             try {
702                 new IRMatcher(testVMProcess.getHotspotPidFileName(), testVMProcess.getIrEncoding(), testClass);
703             } catch (IRViolationException e) {
704                 e.addCommandLine(testVMProcess.getCommandLine());
705                 throw e;
706             }
707         } else {
708             System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " +
709                                "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " +
710                                "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " +
711                                "that make the IR verification impossible (e.g. -XX:-UseCompile, " +
712                                "-XX:TieredStopAtLevel=[1,2,3], etc.).");
713         }
714     }
715 
716     public static void check(boolean test, String failureMessage) {
717         if (!test) {
718             throw new TestFrameworkException(failureMessage);
719         }
720     }
721 }