1 /*
  2  * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 package compiler.lib.ir_framework;
 25 
 26 import compiler.lib.ir_framework.driver.FlagVMProcess;
 27 import compiler.lib.ir_framework.driver.TestVMException;
 28 import compiler.lib.ir_framework.driver.TestVMProcess;
 29 import compiler.lib.ir_framework.driver.irmatching.IRMatcher;
 30 import compiler.lib.ir_framework.driver.irmatching.IRViolationException;
 31 import compiler.lib.ir_framework.shared.*;
 32 import compiler.lib.ir_framework.test.TestVM;
 33 import jdk.test.lib.Platform;
 34 import jdk.test.lib.Utils;
 35 import jdk.test.lib.helpers.ClassFileInstaller;
 36 import jdk.test.whitebox.WhiteBox;
 37 
 38 import java.io.PrintWriter;
 39 import java.io.StringWriter;
 40 import java.lang.reflect.Method;
 41 import java.net.MalformedURLException;
 42 import java.net.URL;
 43 import java.net.URLClassLoader;
 44 import java.nio.file.Path;
 45 import java.util.*;
 46 import java.util.stream.Collectors;
 47 
 48 /**
 49  * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on
 50  * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can
 51  * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used
 52  * testing patterns and compiler control flags.
 53  * <p>
 54  * The framework offers various annotations to control how your test code should be invoked and being checked. There are
 55  * three kinds of tests depending on how much control is needed over the test invocation:
 56  * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b>
 57  * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test}
 58  * annotation which represents the test code that is eventually executed by the test framework. More information about
 59  * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}.
 60  * <p>
 61  * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which
 62  * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}.
 63  * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage
 64  * and how to write different IR rules can be found at {@link IR}.
 65  * <p>
 66  * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>:
 67  * <pre>
 68  * {@literal @}library /test/lib
 69  * {@literal @}run driver some.package.Test
 70  * </pre>
 71  * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the
 72  * JTreg test if the test itself is not utilizing any Whitebox features directly.
 73  * <p>
 74  * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or
 75  * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations
 76  * (instead of specifying multiple JTreg {@code @run} entries).
 77  * <p>
 78  * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the
 79  * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various
 80  * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new
 81  * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing.
 82  * <p>
 83  * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code
 84  * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions.
 85  * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are
 86  * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}
 87  * for IR matching).
 88  * <p>
 89  * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified
 90  * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}.
 91  * <p>
 92  * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR
 93  * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details).
 94  * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing
 95  * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}.
 96  *
 97  * @see Test
 98  * @see Check
 99  * @see Run
100  * @see IR
101  */
102 public class TestFramework {
103     /**
104      * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only
105      * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags
106      * by user code) are whitelisted.
107      *
108      * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of
109      * the entries of this list as a substring (partial match).
110      */
111     public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>(
112             Arrays.asList(
113                     // The following substrings are part of more than one VM flag
114                     "RAM",
115                     "Heap",
116                     "Trace",
117                     "Print",
118                     "Verify",
119                     "TLAB",
120                     "UseNewCode",
121                     "Xmn",
122                     "Xms",
123                     "Xmx",
124                     "Xss",
125                     // The following substrings are only part of one VM flag (=exact match)
126                     "CreateCoredumpOnCrash",
127                     "IgnoreUnrecognizedVMOptions",
128                     "UnlockDiagnosticVMOptions",
129                     "UnlockExperimentalVMOptions",
130                     "BackgroundCompilation",
131                     "Xbatch",
132                     "TieredCompilation",
133                     "CompileThreshold",
134                     "Xmixed",
135                     "server",
136                     "UseAVX",
137                     "UseSSE",
138                     "UseSVE",
139                     "Xlog",
140                     "LogCompilation"
141             )
142     );
143 
144     public static final boolean VERBOSE = Boolean.getBoolean("Verbose");
145     public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty();
146     public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty();
147     private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout");
148     // Only used for internal testing and should not be used for normal user testing.
149 
150     private static final String RERUN_HINT = """
151                                                #############################################################
152                                                 - To only run the failed tests use -DTest, -DExclude,
153                                                   and/or -DScenarios.
154                                                 - To also get the standard output of the test VM run with
155                                                   -DReportStdout=true or for even more fine-grained logging
156                                                   use -DVerbose=true.
157                                                #############################################################
158                                              """ + System.lineSeparator();
159 
160     private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true"));
161     private boolean shouldVerifyIR; // Should we perform IR matching?
162     private static boolean toggleBool;
163 
164     private final Class<?> testClass;
165     private Set<Class<?>> helperClasses;
166     private List<Scenario> scenarios;
167     private Set<Integer> scenarioIndices;
168     private List<String> flags;
169     private int defaultWarmup = -1;
170 
171     /*
172      * Public interface methods
173      */
174 
175     /**
176      * Creates an instance acting as a builder to test the class from which this constructor was invoked from.
177      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
178      * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
179      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
180      */
181     public TestFramework() {
182         this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass());
183     }
184 
185     /**
186      * Creates an instance acting as a builder to test {@code testClass}.
187      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
188      * Use the associated add methods ({@link #addFlags(String...)}, @link #addScenarios(Scenario...)},
189      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
190      *
191      * @param testClass the class to be tested by the framework.
192      * @see #TestFramework()
193      */
194     public TestFramework(Class<?> testClass) {
195         TestRun.check(testClass != null, "Test class cannot be null");
196         this.testClass = testClass;
197         if (VERBOSE) {
198             System.out.println("Test class: " + testClass);
199         }
200     }
201 
202     /**
203      * Tests the class from which this method was invoked from.
204      */
205     public static void run() {
206         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
207         run(walker.getCallerClass());
208     }
209 
210     /**
211      * Tests {@code testClass}.
212      *
213      * @param testClass the class to be tested by the framework.
214      * @see #run()
215      */
216     public static void run(Class<?> testClass) {
217         TestFramework framework = new TestFramework(testClass);
218         framework.start();
219     }
220 
221     /**
222      * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}.
223      * <ul>
224      *     <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p>
225      *            Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and  Javaoptions flags over
226      *            the specified {@code flags} of this method.</li>
227      *     <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li>
228      *     <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one
229      *            from which this method was called from, use {@link #addFlags(String...)}, use this method.</li>
230      *     <li><p>If you want to run your JTreg test with multiple flag combinations, use
231      *            {@link #addScenarios(Scenario...)}</li>
232      * </ul>
233      *
234      * @param flags VM flags to be used for the test VM.
235      */
236     public static void runWithFlags(String... flags) {
237         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
238         TestFramework framework = new TestFramework(walker.getCallerClass());
239         framework.addFlags(flags);
240         framework.start();
241     }
242 
243     /**
244      * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p>
245      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags.
246      *
247      * <p>
248      * The testing can be started by invoking {@link #start()}
249      *
250      * @param flags VM options to be applied to the test VM.
251      * @return the same framework instance.
252      */
253     public TestFramework addFlags(String... flags) {
254         TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null");
255         if (this.flags == null) {
256             this.flags = new ArrayList<>();
257         }
258         this.flags.addAll(Arrays.asList(flags));
259         return this;
260     }
261 
262     /**
263      * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile},
264      * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied
265      * while testing {@code testClass} (also see description of {@link TestFramework}).
266      *
267      * <p>
268      * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any
269      * compile command annotations, you do not need to include it with this method. If no helper class specifies any
270      * compile commands, you do not need to call this method at all.
271      *
272      * <p>
273      * The testing can be started by invoking {@link #start()}.
274      *
275      * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile},
276      *                      {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied
277      *                      while testing {@code testClass} (also see description of {@link TestFramework}).
278      * @return the same framework instance.
279      */
280     public TestFramework addHelperClasses(Class<?>... helperClasses) {
281         TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull),
282                       "A Helper class cannot be null");
283         if (this.helperClasses == null) {
284             this.helperClasses = new HashSet<>();
285         }
286 
287         this.helperClasses.addAll(Arrays.asList(helperClasses));
288         return this;
289     }
290 
291     /**
292      * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the
293      * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)}
294      * and thus also override any VM or Javaoptions set by JTreg by default.<p>
295      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags.
296      *
297      * <p>
298      * The testing can be started by invoking {@link #start()}
299      *
300      * @param scenarios scenarios which specify specific flags for the test VM.
301      * @return the same framework instance.
302      */
303     public TestFramework addScenarios(Scenario... scenarios) {
304         TestFormat.check(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull),
305                          "A scenario cannot be null");
306         if (this.scenarios == null) {
307             this.scenarios = new ArrayList<>();
308             this.scenarioIndices = new HashSet<>();
309         }
310 
311         for (Scenario scenario : scenarios) {
312             int scenarioIndex = scenario.getIndex();
313             TestFormat.check(scenarioIndices.add(scenarioIndex),
314                              "Cannot define two scenarios with the same index " + scenarioIndex);
315             this.scenarios.add(scenario);
316         }
317         return this;
318     }
319 
320     /**
321      * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)})
322      * set test class.
323      */
324     public void start() {
325         if (shouldInstallWhiteBox()) {
326             installWhiteBox();
327         }
328         disableIRVerificationIfNotFeasible();
329 
330         if (scenarios == null) {
331             try {
332                 start(null);
333             } catch (TestVMException e) {
334                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT);
335                 throw e;
336             } catch (IRViolationException e) {
337                 System.out.println("Compilation(s) of failed match(es):");
338                 System.out.println(e.getCompilations());
339                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT);
340                 throw e;
341             }
342         } else {
343             startWithScenarios();
344         }
345     }
346 
347     /**
348      * Try to load the Whitebox class from the user directory with a custom class loader. If the user has already built the
349      * Whitebox, we can load it. Otherwise, the framework needs to install it.
350      *
351      * @return true if the framework needs to install the Whitebox
352      */
353     private boolean shouldInstallWhiteBox() {
354         try {
355             URL url = Path.of(System.getProperty("user.dir")).toUri().toURL();
356             URLClassLoader userDirClassLoader =
357                     URLClassLoader.newInstance(new URL[] {url}, TestFramework.class.getClassLoader().getParent());
358             Class.forName(WhiteBox.class.getName(), false, userDirClassLoader);
359         } catch (MalformedURLException e) {
360             throw new TestFrameworkException("corrupted user.dir property", e);
361         } catch (ClassNotFoundException e) {
362             // We need to manually install the WhiteBox if we cannot load the WhiteBox class from the user directory.
363             // This happens when the user test does not explicitly install the WhiteBox as part of the test.
364             return true;
365         }
366         return false;
367     }
368 
369     /**
370      * Set a new default warm-up (overriding the framework default of 2000 at
371      * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit
372      * warm-up with {@link Warmup @Warmup}.
373      *
374      * @param defaultWarmup a new non-negative default warm-up.
375      * @return the same framework instance.
376      */
377     public TestFramework setDefaultWarmup(int defaultWarmup) {
378         TestFormat.check(defaultWarmup >= 0, "Cannot specify a negative default warm-up");
379         this.defaultWarmup = defaultWarmup;
380         return this;
381     }
382 
383     /**
384      * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios
385      * were run, use {@link Scenario#getTestVMOutput()}.
386      *
387      * @return the last test VM output.
388      */
389     public static String getLastTestVMOutput() {
390         return TestVMProcess.getLastTestVMOutput();
391     }
392 
393     /*
394      * The following methods are only intended to be called from actual @Test methods and not from the main() method of
395      * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled).
396      */
397 
398     /**
399      * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled,
400      * yet, upon returning from this method.
401      *
402      * @param m the method to be compiled.
403      * @param compLevel the (valid) compilation level at which the method should be compiled.
404      * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}.
405      */
406     public static void compile(Method m, CompLevel compLevel) {
407         TestVM.compile(m, compLevel);
408     }
409 
410     /**
411      * Deoptimize {@code m}.
412      *
413      * @param m the method to be deoptimized.
414      */
415     public static void deoptimize(Method m) {
416         TestVM.deoptimize(m);
417     }
418 
419     /**
420      * Returns a boolean indicating if {@code m} is compiled at any level.
421      *
422      * @param m the method to be checked.
423      * @return {@code true} if {@code m} is compiled at any level;
424      *         {@code false} otherwise.
425      */
426     public static boolean isCompiled(Method m) {
427         return TestVM.isCompiled(m);
428     }
429 
430     /**
431      * Returns a boolean indicating if {@code m} is compiled with C1.
432      *
433      * @param m the method to be checked.
434      * @return {@code true} if {@code m} is compiled with C1;
435      *         {@code false} otherwise.
436      */
437     public static boolean isC1Compiled(Method m) {
438         return TestVM.isC1Compiled(m);
439     }
440 
441     /**
442      * Returns a boolean indicating if {@code m} is compiled with C2.
443      *
444      * @param m the method to be checked.
445      * @return {@code true} if {@code m} is compiled with C2;
446      *         {@code false} otherwise.
447      */
448     public static boolean isC2Compiled(Method m) {
449         return TestVM.isC2Compiled(m);
450     }
451 
452     /**
453      * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}.
454      *
455      * @param m the method to be checked.
456      * @param compLevel the compilation level.
457      * @return {@code true} if {@code m} is compiled at {@code compLevel};
458      *         {@code false} otherwise.
459      */
460     public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
461         return TestVM.isCompiledAtLevel(m, compLevel);
462     }
463 
464     /**
465      * Checks if {@code m} is compiled at any level.
466      *
467      * @param m the method to be checked.
468      * @throws TestRunException if {@code m} is not compiled at any level.
469      */
470     public static void assertCompiled(Method m) {
471         TestVM.assertCompiled(m);
472     }
473 
474     /**
475      * Checks if {@code m} is not compiled at any level.
476      *
477      * @param m the method to be checked.
478      * @throws TestRunException if {@code m} is compiled at any level.
479      */
480     public static void assertNotCompiled(Method m) {
481         TestVM.assertNotCompiled(m);
482     }
483 
484     /**
485      * Verifies that {@code m} is compiled with C1.
486      *
487      * @param m the method to be verified.
488      * @throws TestRunException if {@code m} is not compiled with C1.
489      */
490     public static void assertCompiledByC1(Method m) {
491         TestVM.assertCompiledByC1(m);
492     }
493 
494     /**
495      * Verifies that {@code m} is compiled with C2.
496      *
497      * @param m the method to be checked.
498      * @throws TestRunException if {@code m} is not compiled with C2.
499      */
500     public static void assertCompiledByC2(Method m) {
501         TestVM.assertCompiledByC2(m);
502     }
503 
504     /**
505      * Verifies that {@code m} is compiled at the specified {@code compLevel}.
506      *
507      * @param m the method to be checked.
508      * @param compLevel the compilation level.
509      * @throws TestRunException if {@code m} is not compiled at {@code compLevel}.
510      */
511     public static void assertCompiledAtLevel(Method m, CompLevel compLevel) {
512         TestVM.assertCompiledAtLevel(m, compLevel);
513     }
514 
515     /**
516      * Verifies that {@code m} was deoptimized after being C1 compiled.
517      *
518      * @param m the method to be checked.
519      * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled.
520      */
521     public static void assertDeoptimizedByC1(Method m) {
522         TestVM.assertDeoptimizedByC1(m);
523     }
524 
525     /**
526      * Verifies that {@code m} was deoptimized after being C2 compiled.
527      *
528      * @param m the method to be checked.
529      * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled.
530      */
531     public static void assertDeoptimizedByC2(Method m) {
532         TestVM.assertDeoptimizedByC2(m);
533     }
534 
535     /**
536      * Checks if deopt of {@code m} is stable at the specified {@code compLevel}.
537      *
538      * @param m the method to be checked.
539      * @param compLevel the compilation level.
540      * @return {@code true} if deopt of {@code m} is stable at {@code compLevel};
541      *         {@code false} otherwise.
542      */
543     public static boolean isStableDeopt(Method m, CompLevel compLevel) {
544         return TestVM.isStableDeopt(m, compLevel);
545     }
546 
547     /**
548      * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}).
549      * The very first invocation returns {@code false}. Note that this method could be used by different tests and
550      * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times
551      * other tests have already invoked this method.
552      *
553      * @return an inverted boolean of the result of the last invocation of this method.
554      */
555     public static boolean toggleBoolean() {
556         toggleBool = !toggleBool;
557         return toggleBool;
558     }
559 
560     /*
561      * End of public interface methods
562      */
563 
564     /**
565      * Used to move Whitebox class to the right folder in the JTreg test
566      */
567     private void installWhiteBox() {
568         try {
569             ClassFileInstaller.main(WhiteBox.class.getName());
570         } catch (Exception e) {
571             throw new Error("failed to install whitebox classes", e);
572         }
573     }
574 
575     /**
576      * Disable IR verification completely in certain cases.
577      */
578     private void disableIRVerificationIfNotFeasible() {
579         if (irVerificationPossible) {
580             irVerificationPossible = Platform.isDebugBuild() && !Platform.isInt() && !Platform.isComp();
581             if (!irVerificationPossible) {
582                 System.out.println("IR verification disabled due to not running a debug build (required for PrintIdeal" +
583                                    "and PrintOptoAssembly), running with -Xint, or -Xcomp (use warm-up of 0 instead)");
584                 return;
585             }
586 
587             irVerificationPossible = hasIRAnnotations();
588             if (!irVerificationPossible) {
589                 System.out.println("IR verification disabled due to test " + testClass + " not specifying any @IR annotations");
590                 return;
591             }
592 
593             // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified.
594             irVerificationPossible = onlyWhitelistedJTregVMAndJavaOptsFlags();
595             if (!irVerificationPossible) {
596                 System.out.println("IR verification disabled due to using non-whitelisted JTreg VM or Javaoptions flag(s)."
597                                    + System.lineSeparator());
598             }
599         }
600     }
601 
602     /**
603      * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all
604      * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios
605      * and thus is reported immediately on the first scenario execution.
606      */
607     private void startWithScenarios() {
608         Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex));
609         for (Scenario scenario : scenarios) {
610             try {
611                 start(scenario);
612             } catch (TestFormatException e) {
613                 // Test format violation is wrong for all the scenarios. Only report once.
614                 throw e;
615             } catch (Exception e) {
616                 exceptionMap.put(scenario, e);
617             }
618         }
619         if (!exceptionMap.isEmpty()) {
620             reportScenarioFailures(exceptionMap);
621         }
622     }
623 
624     private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) {
625         String failedScenarios = "The following scenarios have failed: #"
626                                  + exceptionMap.keySet().stream()
627                                                .map(s -> String.valueOf(s.getIndex()))
628                                                .collect(Collectors.joining(", #"));
629         StringBuilder builder = new StringBuilder(failedScenarios);
630         builder.append(System.lineSeparator()).append(System.lineSeparator());
631         for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) {
632             Exception e = entry.getValue();
633             Scenario scenario = entry.getKey();
634             String errorMsg = "";
635             if (scenario != null) {
636                 errorMsg = getScenarioTitleAndFlags(scenario);
637             }
638             if (e instanceof IRViolationException irException) {
639                 // For IR violations, only show the actual violations and not the (uninteresting) stack trace.
640                 System.out.println((scenario != null ? "Scenario #" + scenario.getIndex() + " - " : "")
641                                    + "Compilation(s) of failed matche(s):");
642                 System.out.println(irException.getCompilations());
643                 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo());
644             } else if (e instanceof TestVMException testVMException) {
645                 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo());
646             } else {
647                 // Print stack trace otherwise
648                 StringWriter errors = new StringWriter();
649                 e.printStackTrace(new PrintWriter(errors));
650                 builder.append(errors);
651             }
652             builder.append(System.lineSeparator());
653         }
654         System.err.println(builder);
655         if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) {
656             // Provide a hint to the user how to get additional output/debugging information.
657             System.err.println(RERUN_HINT);
658         }
659         throw new TestRunException(failedScenarios + ". Please check stderr for more information.");
660     }
661 
662     private static String getScenarioTitleAndFlags(Scenario scenario) {
663         StringBuilder builder = new StringBuilder();
664         String title = "Scenario #" + scenario.getIndex();
665         builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length()))
666                .append(System.lineSeparator());
667         builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]")
668                .append(System.lineSeparator());
669         return builder.toString();
670     }
671 
672     /**
673      * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of
674      * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the
675      * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests.
676      */
677     private void start(Scenario scenario) {
678         if (scenario != null && !scenario.isEnabled()) {
679             System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " +
680                                "-DScenarios and is therefore not executed.");
681             return;
682         }
683         shouldVerifyIR = irVerificationPossible;
684         try {
685             // Use TestFramework flags and scenario flags for new VMs.
686             List<String> additionalFlags = new ArrayList<>();
687             if (flags != null) {
688                 additionalFlags.addAll(flags);
689             }
690             if (scenario != null) {
691                 List<String> scenarioFlags = scenario.getFlags();
692                 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]";
693                 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":");
694                 additionalFlags.addAll(scenarioFlags);
695             }
696             String frameworkAndScenarioFlags = additionalFlags.isEmpty() ?
697                     "" : " - [" + String.join(", ", additionalFlags) + "]";
698 
699             if (shouldVerifyIR) {
700                 // Only need to use flag VM if an IR verification is possibly done.
701                 System.out.println("Run Flag VM:");
702                 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags);
703                 shouldVerifyIR = flagVMProcess.shouldVerifyIR();
704                 if (shouldVerifyIR) {
705                     // Add more flags for the test VM which are required to do IR verification.
706                     additionalFlags.addAll(flagVMProcess.getTestVMFlags());
707                 } // else: Flag VM found a reason to not do IR verification.
708             } else {
709                 System.out.println("Skip Flag VM due to not performing IR verification.");
710             }
711 
712             System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":");
713             runTestVM(additionalFlags);
714         } finally {
715             if (scenario != null) {
716                 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput());
717             }
718             System.out.println();
719         }
720     }
721 
722     private boolean hasIRAnnotations() {
723         return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class) != null);
724     }
725 
726     private boolean onlyWhitelistedJTregVMAndJavaOptsFlags() {
727         List<String> flags = Arrays.stream(Utils.getTestJavaOpts())
728                                    .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", ""))
729                                    .collect(Collectors.toList());
730         for (String flag : flags) {
731             // Property flags (prefix -D), -ea and -esa are whitelisted.
732             if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) {
733                 // Found VM flag that is not whitelisted
734                 return false;
735             }
736         }
737         return true;
738     }
739 
740     private void runTestVM(List<String> additionalFlags) {
741         TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup);
742         if (shouldVerifyIR) {
743             try {
744                 new IRMatcher(testVMProcess.getHotspotPidFileName(), testVMProcess.getIrEncoding(), testClass);
745             } catch (IRViolationException e) {
746                 e.addCommandLine(testVMProcess.getCommandLine());
747                 throw e;
748             }
749         } else {
750             System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " +
751                                "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " +
752                                "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " +
753                                "that make the IR verification impossible (e.g. -XX:-UseCompile, " +
754                                "-XX:TieredStopAtLevel=[1,2,3], etc.).");
755         }
756     }
757 
758     public static void check(boolean test, String failureMessage) {
759         if (!test) {
760             throw new TestFrameworkException(failureMessage);
761         }
762     }
763 }