1 /*
  2  * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 package compiler.lib.ir_framework;
 25 
 26 import compiler.lib.ir_framework.driver.FlagVMProcess;
 27 import compiler.lib.ir_framework.driver.TestVMException;
 28 import compiler.lib.ir_framework.driver.TestVMProcess;
 29 import compiler.lib.ir_framework.driver.irmatching.IRMatcher;
 30 import compiler.lib.ir_framework.driver.irmatching.IRViolationException;
 31 import compiler.lib.ir_framework.driver.irmatching.Matchable;
 32 import compiler.lib.ir_framework.driver.irmatching.parser.TestClassParser;
 33 import compiler.lib.ir_framework.shared.*;
 34 import compiler.lib.ir_framework.test.TestVM;
 35 import jdk.test.lib.Platform;
 36 import jdk.test.lib.Utils;
 37 import jdk.test.lib.helpers.ClassFileInstaller;
 38 import jdk.test.whitebox.WhiteBox;
 39 
 40 import java.io.PrintWriter;
 41 import java.io.StringWriter;
 42 import java.lang.reflect.Method;
 43 import java.net.MalformedURLException;
 44 import java.net.URL;
 45 import java.net.URLClassLoader;
 46 import java.nio.file.Path;
 47 import java.util.*;
 48 import java.util.stream.Collectors;
 49 
 50 /**
 51  * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on
 52  * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can
 53  * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used
 54  * testing patterns and compiler control flags.
 55  * <p>
 56  * The framework offers various annotations to control how your test code should be invoked and being checked. There are
 57  * three kinds of tests depending on how much control is needed over the test invocation:
 58  * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b>
 59  * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test}
 60  * annotation which represents the test code that is eventually executed by the test framework. More information about
 61  * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}.
 62  * <p>
 63  * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which
 64  * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}.
 65  * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage
 66  * and how to write different IR rules can be found at {@link IR}.
 67  * <p>
 68  * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>:
 69  * <pre>
 70  * {@literal @}library /test/lib
 71  * {@literal @}run driver some.package.Test
 72  * </pre>
 73  * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the
 74  * JTreg test if the test itself is not utilizing any Whitebox features directly.
 75  * <p>
 76  * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or
 77  * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations
 78  * (instead of specifying multiple JTreg {@code @run} entries).
 79  * <p>
 80  * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the
 81  * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various
 82  * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new
 83  * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing.
 84  * <p>
 85  * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code
 86  * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions.
 87  * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are
 88  * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}
 89  * for IR matching).
 90  * <p>
 91  * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified
 92  * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}.
 93  * <p>
 94  * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR
 95  * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details).
 96  * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing
 97  * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}.
 98  *
 99  * @see Test
100  * @see Check
101  * @see Run
102  * @see IR
103  */
104 public class TestFramework {
105     /**
106      * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only
107      * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags
108      * by user code) are whitelisted.
109      *
110      * <p>
111      * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of
112      * the entries of this list as a substring (partial match).
113      */
114     public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>(
115             Arrays.asList(
116                     // The following substrings are part of more than one VM flag
117                     "RAM",
118                     "Heap",
119                     "Trace",
120                     "Print",
121                     "Verify",
122                     "UseNewCode",
123                     "Xmn",
124                     "Xms",
125                     "Xmx",
126                     "Xss",
127                     // The following substrings are only part of one VM flag (=exact match)
128                     "CreateCoredumpOnCrash",
129                     "IgnoreUnrecognizedVMOptions",
130                     "UnlockDiagnosticVMOptions",
131                     "UnlockExperimentalVMOptions",
132                     "BackgroundCompilation",
133                     "Xbatch",
134                     "TieredCompilation",
135                     "CompileThreshold",
136                     "Xmixed",
137                     "server",
138                     "AlignVector",
139                     "UseAVX",
140                     "UseSSE",
141                     "UseSVE",
142                     "Xlog",
143                     "LogCompilation",
144                     "UseCompactObjectHeaders",
145                     // Riscv
146                     "UseRVV",
147                     "UseZbb",
148                     "UseZfh",
149                     "UseZicond",
150                     "UseZvbb"
151             )
152     );
153 
154     public static final boolean VERBOSE = Boolean.getBoolean("Verbose");
155     public static final boolean PRINT_RULE_MATCHING_TIME = Boolean.getBoolean("PrintRuleMatchingTime");
156     public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty();
157     public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty();
158     private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout");
159     // Only used for internal testing and should not be used for normal user testing.
160 
161     private static final String RERUN_HINT = """
162                                                #############################################################
163                                                 - To only run the failed tests use -DTest, -DExclude,
164                                                   and/or -DScenarios.
165                                                 - To also get the standard output of the test VM run with
166                                                   -DReportStdout=true or for even more fine-grained logging
167                                                   use -DVerbose=true.
168                                                #############################################################
169                                              """ + System.lineSeparator();
170 
171     private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true"));
172     private boolean shouldVerifyIR; // Should we perform IR matching?
173     private static boolean toggleBool;
174 
175     private final Class<?> testClass;
176     private Set<Class<?>> helperClasses;
177     private List<Scenario> scenarios;
178     private Set<Integer> scenarioIndices;
179     private List<String> flags;
180     private int defaultWarmup = -1;
181     private boolean testClassesOnBootClassPath;
182     private boolean isAllowNotCompilable = false;
183 
184     /*
185      * Public interface methods
186      */
187 
188     /**
189      * Creates an instance acting as a builder to test the class from which this constructor was invoked from.
190      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
191      * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
192      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
193      */
194     public TestFramework() {
195         this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass());
196     }
197 
198     /**
199      * Creates an instance acting as a builder to test {@code testClass}.
200      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
201      * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
202      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
203      *
204      * @param testClass the class to be tested by the framework.
205      * @see #TestFramework()
206      */
207     public TestFramework(Class<?> testClass) {
208         TestRun.check(testClass != null, "Test class cannot be null");
209         this.testClass = testClass;
210         if (VERBOSE) {
211             System.out.println("Test class: " + testClass);
212         }
213     }
214 
215     /**
216      * Tests the class from which this method was invoked from.
217      */
218     public static void run() {
219         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
220         run(walker.getCallerClass());
221     }
222 
223     /**
224      * Tests {@code testClass}.
225      *
226      * @param testClass the class to be tested by the framework.
227      * @see #run()
228      */
229     public static void run(Class<?> testClass) {
230         TestFramework framework = new TestFramework(testClass);
231         framework.start();
232     }
233 
234     /**
235      * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}.
236      * <ul>
237      *     <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p>
238      *            Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and  Javaoptions flags over
239      *            the specified {@code flags} of this method.</li>
240      *     <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li>
241      *     <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one
242      *            from which this method was called from, use {@link #addFlags(String...)}, use this method.</li>
243      *     <li><p>If you want to run your JTreg test with multiple flag combinations, use
244      *            {@link #addScenarios(Scenario...)}</li>
245      * </ul>
246      *
247      * @param flags VM flags to be used for the test VM.
248      */
249     public static void runWithFlags(String... flags) {
250         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
251         TestFramework framework = new TestFramework(walker.getCallerClass());
252         framework.addFlags(flags);
253         framework.start();
254     }
255 
256     /**
257      * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p>
258      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags.
259      *
260      * <p>
261      * The testing can be started by invoking {@link #start()}
262      *
263      * @param flags VM options to be applied to the test VM.
264      * @return the same framework instance.
265      */
266     public TestFramework addFlags(String... flags) {
267         TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null");
268         if (this.flags == null) {
269             this.flags = new ArrayList<>();
270         }
271         this.flags.addAll(Arrays.asList(flags));
272         return this;
273     }
274 
275     /**
276      * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile},
277      * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied
278      * while testing {@code testClass} (also see description of {@link TestFramework}).
279      *
280      * <p>
281      * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any
282      * compile command annotations, you do not need to include it with this method. If no helper class specifies any
283      * compile commands, you do not need to call this method at all.
284      *
285      * <p>
286      * The testing can be started by invoking {@link #start()}.
287      *
288      * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile},
289      *                      {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied
290      *                      while testing {@code testClass} (also see description of {@link TestFramework}).
291      * @return the same framework instance.
292      */
293     public TestFramework addHelperClasses(Class<?>... helperClasses) {
294         TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull),
295                       "A Helper class cannot be null");
296         if (this.helperClasses == null) {
297             this.helperClasses = new HashSet<>();
298         }
299 
300         this.helperClasses.addAll(Arrays.asList(helperClasses));
301         return this;
302     }
303 
304     /**
305      * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the
306      * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)}
307      * and thus also override any VM or Javaoptions set by JTreg by default.<p>
308      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags.
309      *
310      * <p>
311      * The testing can be started by invoking {@link #start()}
312      *
313      * @param scenarios scenarios which specify specific flags for the test VM.
314      * @return the same framework instance.
315      */
316     public TestFramework addScenarios(Scenario... scenarios) {
317         TestFormat.checkAndReport(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull),
318                                   "A scenario cannot be null");
319         if (this.scenarios == null) {
320             this.scenarios = new ArrayList<>();
321             this.scenarioIndices = new HashSet<>();
322         }
323 
324         for (Scenario scenario : scenarios) {
325             int scenarioIndex = scenario.getIndex();
326             TestFormat.checkNoThrow(scenarioIndices.add(scenarioIndex),
327                              "Cannot define two scenarios with the same index " + scenarioIndex);
328             this.scenarios.add(scenario);
329         }
330         TestFormat.throwIfAnyFailures();
331         return this;
332     }
333 
334     /**
335      * Add test classes to boot classpath. This adds all classes found on path {@link jdk.test.lib.Utils#TEST_CLASSES}
336      * to the boot classpath with "-Xbootclasspath/a". This is useful when trying to run tests in a privileged mode.
337      */
338     public TestFramework addTestClassesToBootClassPath() {
339         this.testClassesOnBootClassPath = true;
340         return this;
341     }
342 
343     /**
344      * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)})
345      * set test class.
346      */
347     public void start() {
348         if (shouldInstallWhiteBox()) {
349             installWhiteBox();
350         }
351         checkIRRuleCompilePhasesFormat();
352         disableIRVerificationIfNotFeasible();
353 
354         if (scenarios == null) {
355             try {
356                 start(null);
357             } catch (TestVMException e) {
358                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT);
359                 throw e;
360             } catch (IRViolationException e) {
361                 System.out.println(e.getCompilations());
362                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT);
363                 throw e;
364             }
365         } else {
366             startWithScenarios();
367         }
368     }
369 
370     private void checkIRRuleCompilePhasesFormat() {
371         for (Method method : testClass.getDeclaredMethods()) {
372             for (IR irAnno : method.getAnnotationsByType(IR.class)) {
373                 TestFormat.checkNoThrow(irAnno.phase().length > 0,
374                                         "@IR rule " + irAnno + " must specify a non-empty list of compile " +
375                                         "phases \"phase\" at " + method);
376             }
377         }
378         TestFormat.throwIfAnyFailures();
379     }
380 
381     /**
382      * Try to load the Whitebox class from the user directory with a custom class loader. If the user has already built the
383      * Whitebox, we can load it. Otherwise, the framework needs to install it.
384      *
385      * @return true if the framework needs to install the Whitebox
386      */
387     private boolean shouldInstallWhiteBox() {
388         try {
389             URL url = Path.of(System.getProperty("user.dir")).toUri().toURL();
390             URLClassLoader userDirClassLoader =
391                     URLClassLoader.newInstance(new URL[] {url}, TestFramework.class.getClassLoader().getParent());
392             Class.forName(WhiteBox.class.getName(), false, userDirClassLoader);
393         } catch (MalformedURLException e) {
394             throw new TestFrameworkException("corrupted user.dir property", e);
395         } catch (ClassNotFoundException e) {
396             // We need to manually install the WhiteBox if we cannot load the WhiteBox class from the user directory.
397             // This happens when the user test does not explicitly install the WhiteBox as part of the test.
398             return true;
399         }
400         return false;
401     }
402 
403     /**
404      * Set a new default warm-up (overriding the framework default of 2000 at
405      * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit
406      * warm-up with {@link Warmup @Warmup}.
407      *
408      * @param defaultWarmup a new non-negative default warm-up.
409      * @return the same framework instance.
410      */
411     public TestFramework setDefaultWarmup(int defaultWarmup) {
412         TestFormat.checkAndReport(defaultWarmup >= 0, "Cannot specify a negative default warm-up");
413         this.defaultWarmup = defaultWarmup;
414         return this;
415     }
416 
417     /**
418      * In rare cases, methods may not be compilable because of a compilation bailout. By default, this leads to a
419      * test failure. However, if such cases are expected in multiple methods in a test class, this flag can be set to
420      * true, which allows any test to pass even if there is a compilation bailout. If only selected methods are prone
421      * to bail out, it is preferred to use {@link Test#allowNotCompilable()} instead for more fine-grained control.
422      * By setting this flag, any associated {@link IR} rule of a test is only executed if the test method was compiled,
423      * and else it is ignored silently.
424      */
425     public TestFramework allowNotCompilable() {
426         this.isAllowNotCompilable = true;
427         return this;
428     }
429 
430     /**
431      * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios
432      * were run, use {@link Scenario#getTestVMOutput()}.
433      *
434      * @return the last test VM output.
435      */
436     public static String getLastTestVMOutput() {
437         return TestVMProcess.getLastTestVMOutput();
438     }
439 
440     /*
441      * The following methods are only intended to be called from actual @Test methods and not from the main() method of
442      * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled).
443      */
444 
445     /**
446      * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled,
447      * yet, upon returning from this method.
448      *
449      * @param m the method to be compiled.
450      * @param compLevel the (valid) compilation level at which the method should be compiled.
451      * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}.
452      */
453     public static void compile(Method m, CompLevel compLevel) {
454         TestVM.compile(m, compLevel);
455     }
456 
457     /**
458      * Deoptimize {@code m}.
459      *
460      * @param m the method to be deoptimized.
461      */
462     public static void deoptimize(Method m) {
463         TestVM.deoptimize(m);
464     }
465 
466     /**
467      * Returns a boolean indicating if {@code m} is compiled at any level.
468      *
469      * @param m the method to be checked.
470      * @return {@code true} if {@code m} is compiled at any level;
471      *         {@code false} otherwise.
472      */
473     public static boolean isCompiled(Method m) {
474         return TestVM.isCompiled(m);
475     }
476 
477     /**
478      * Returns a boolean indicating if {@code m} is compiled with C1.
479      *
480      * @param m the method to be checked.
481      * @return {@code true} if {@code m} is compiled with C1;
482      *         {@code false} otherwise.
483      */
484     public static boolean isC1Compiled(Method m) {
485         return TestVM.isC1Compiled(m);
486     }
487 
488     /**
489      * Returns a boolean indicating if {@code m} is compiled with C2.
490      *
491      * @param m the method to be checked.
492      * @return {@code true} if {@code m} is compiled with C2;
493      *         {@code false} otherwise.
494      */
495     public static boolean isC2Compiled(Method m) {
496         return TestVM.isC2Compiled(m);
497     }
498 
499     /**
500      * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}.
501      *
502      * @param m the method to be checked.
503      * @param compLevel the compilation level.
504      * @return {@code true} if {@code m} is compiled at {@code compLevel};
505      *         {@code false} otherwise.
506      */
507     public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
508         return TestVM.isCompiledAtLevel(m, compLevel);
509     }
510 
511     /**
512      * Checks if {@code m} is compiled at any level.
513      *
514      * @param m the method to be checked.
515      * @throws TestRunException if {@code m} is not compiled at any level.
516      */
517     public static void assertCompiled(Method m) {
518         TestVM.assertCompiled(m);
519     }
520 
521     /**
522      * Checks if {@code m} is not compiled at any level.
523      *
524      * @param m the method to be checked.
525      * @throws TestRunException if {@code m} is compiled at any level.
526      */
527     public static void assertNotCompiled(Method m) {
528         TestVM.assertNotCompiled(m);
529     }
530 
531     /**
532      * Verifies that {@code m} is compiled with C1.
533      *
534      * @param m the method to be verified.
535      * @throws TestRunException if {@code m} is not compiled with C1.
536      */
537     public static void assertCompiledByC1(Method m) {
538         TestVM.assertCompiledByC1(m);
539     }
540 
541     /**
542      * Verifies that {@code m} is compiled with C2.
543      *
544      * @param m the method to be checked.
545      * @throws TestRunException if {@code m} is not compiled with C2.
546      */
547     public static void assertCompiledByC2(Method m) {
548         TestVM.assertCompiledByC2(m);
549     }
550 
551     /**
552      * Verifies that {@code m} is compiled at the specified {@code compLevel}.
553      *
554      * @param m the method to be checked.
555      * @param compLevel the compilation level.
556      * @throws TestRunException if {@code m} is not compiled at {@code compLevel}.
557      */
558     public static void assertCompiledAtLevel(Method m, CompLevel compLevel) {
559         TestVM.assertCompiledAtLevel(m, compLevel);
560     }
561 
562     /**
563      * Verifies that {@code m} was deoptimized after being C1 compiled.
564      *
565      * @param m the method to be checked.
566      * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled.
567      */
568     public static void assertDeoptimizedByC1(Method m) {
569         TestVM.assertDeoptimizedByC1(m);
570     }
571 
572     /**
573      * Verifies that {@code m} was deoptimized after being C2 compiled.
574      *
575      * @param m the method to be checked.
576      * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled.
577      */
578     public static void assertDeoptimizedByC2(Method m) {
579         TestVM.assertDeoptimizedByC2(m);
580     }
581 
582     /**
583      * Checks if deopt of {@code m} is stable at the specified {@code compLevel}.
584      *
585      * @param m the method to be checked.
586      * @param compLevel the compilation level.
587      * @return {@code true} if deopt of {@code m} is stable at {@code compLevel};
588      *         {@code false} otherwise.
589      */
590     public static boolean isStableDeopt(Method m, CompLevel compLevel) {
591         return TestVM.isStableDeopt(m, compLevel);
592     }
593 
594     /**
595      * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}).
596      * The very first invocation returns {@code false}. Note that this method could be used by different tests and
597      * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times
598      * other tests have already invoked this method.
599      *
600      * @return an inverted boolean of the result of the last invocation of this method.
601      */
602     public static boolean toggleBoolean() {
603         toggleBool = !toggleBool;
604         return toggleBool;
605     }
606 
607     /*
608      * End of public interface methods
609      */
610 
611     /**
612      * Used to move Whitebox class to the right folder in the JTreg test
613      */
614     private void installWhiteBox() {
615         try {
616             ClassFileInstaller.main(WhiteBox.class.getName());
617         } catch (Exception e) {
618             throw new Error("failed to install whitebox classes", e);
619         }
620     }
621 
622     /**
623      * Disable IR verification completely in certain cases.
624      */
625     private void disableIRVerificationIfNotFeasible() {
626         if (!irVerificationPossible) {
627             return;
628         }
629 
630         boolean debugTest = Platform.isDebugBuild();
631         boolean intTest = !Platform.isInt();
632         boolean compTest = !Platform.isComp();
633         boolean irTest = hasIRAnnotations();
634         // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified.
635         List<String> nonWhiteListedFlags = anyNonWhitelistedJTregVMAndJavaOptsFlags();
636         boolean nonWhiteListedTest = nonWhiteListedFlags.isEmpty();
637 
638         irVerificationPossible = debugTest && intTest && compTest && irTest && nonWhiteListedTest;
639         if (irVerificationPossible) {
640             return;
641         }
642 
643         System.out.println("IR verification disabled due to the following reason(s):");
644         if (!debugTest) {
645             System.out.println("- Not running a debug build (required for PrintIdeal and PrintOptoAssembly)");
646         }
647         if (!intTest) {
648             System.out.println("- Running with -Xint (no compilations)");
649         }
650         if (!compTest) {
651             System.out.println("- Running with -Xcomp (use warm-up of 0 instead)");
652         }
653         if (!irTest) {
654             System.out.println("- Test " + testClass + " not specifying any @IR annotations");
655         }
656         if (!nonWhiteListedTest) {
657             System.out.println("- Using non-whitelisted JTreg VM or Javaoptions flag(s):");
658             nonWhiteListedFlags.forEach((f) -> System.out.println("  - " + f));
659         }
660 
661         System.out.println("");
662     }
663 
664     /**
665      * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all
666      * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios
667      * and thus is reported immediately on the first scenario execution.
668      */
669     private void startWithScenarios() {
670         Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex));
671         for (Scenario scenario : scenarios) {
672             try {
673                 start(scenario);
674             } catch (TestFormatException e) {
675                 // Test format violation is wrong for all the scenarios. Only report once.
676                 throw e;
677             } catch (Exception e) {
678                 exceptionMap.put(scenario, e);
679             }
680         }
681         if (!exceptionMap.isEmpty()) {
682             reportScenarioFailures(exceptionMap);
683         }
684     }
685 
686     private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) {
687         String failedScenarios = "The following scenarios have failed: #"
688                                  + exceptionMap.keySet().stream()
689                                                .map(s -> String.valueOf(s.getIndex()))
690                                                .collect(Collectors.joining(", #"));
691         StringBuilder builder = new StringBuilder(failedScenarios);
692         builder.append(System.lineSeparator()).append(System.lineSeparator());
693         for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) {
694             Exception e = entry.getValue();
695             Scenario scenario = entry.getKey();
696             String errorMsg = "";
697             if (scenario != null) {
698                 errorMsg = getScenarioTitleAndFlags(scenario);
699             }
700             if (e instanceof IRViolationException irException) {
701                 // For IR violations, only show the actual violations and not the (uninteresting) stack trace.
702                 if (scenario != null) {
703                     System.out.println("Scenario #" + scenario.getIndex());
704                 }
705                 System.out.println(irException.getCompilations());
706                 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo());
707             } else if (e instanceof TestVMException testVMException) {
708                 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo());
709             } else {
710                 // Print stack trace otherwise
711                 StringWriter errors = new StringWriter();
712                 e.printStackTrace(new PrintWriter(errors));
713                 builder.append(errors);
714             }
715             builder.append(System.lineSeparator());
716         }
717         System.err.println(builder);
718         if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) {
719             // Provide a hint to the user how to get additional output/debugging information.
720             System.err.println(RERUN_HINT);
721         }
722         throw new TestRunException(failedScenarios + ". Please check stderr for more information.");
723     }
724 
725     private static String getScenarioTitleAndFlags(Scenario scenario) {
726         StringBuilder builder = new StringBuilder();
727         String title = "Scenario #" + scenario.getIndex();
728         builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length()))
729                .append(System.lineSeparator());
730         builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]")
731                .append(System.lineSeparator());
732         return builder.toString();
733     }
734 
735     /**
736      * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of
737      * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the
738      * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests.
739      */
740     private void start(Scenario scenario) {
741         if (scenario != null && !scenario.isEnabled()) {
742             System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " +
743                                "-DScenarios and is therefore not executed.");
744             return;
745         }
746         shouldVerifyIR = irVerificationPossible;
747         try {
748             // Use TestFramework flags and scenario flags for new VMs.
749             List<String> additionalFlags = new ArrayList<>();
750             if (flags != null) {
751                 additionalFlags.addAll(flags);
752             }
753             if (scenario != null) {
754                 List<String> scenarioFlags = scenario.getFlags();
755                 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]";
756                 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":");
757                 additionalFlags.addAll(scenarioFlags);
758             }
759             String frameworkAndScenarioFlags = additionalFlags.isEmpty() ?
760                     "" : " - [" + String.join(", ", additionalFlags) + "]";
761 
762             if (shouldVerifyIR) {
763                 // Only need to use flag VM if an IR verification is possibly done.
764                 System.out.println("Run Flag VM:");
765                 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags);
766                 shouldVerifyIR = flagVMProcess.shouldVerifyIR();
767                 if (shouldVerifyIR) {
768                     // Add more flags for the test VM which are required to do IR verification.
769                     additionalFlags.addAll(flagVMProcess.getTestVMFlags());
770                 } // else: Flag VM found a reason to not do IR verification.
771             } else {
772                 System.out.println("Skip Flag VM due to not performing IR verification.");
773             }
774 
775             System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":");
776             runTestVM(additionalFlags);
777         } finally {
778             if (scenario != null) {
779                 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput());
780             }
781             System.out.println();
782         }
783     }
784 
785     private boolean hasIRAnnotations() {
786         return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class).length > 0);
787     }
788 
789     private List<String> anyNonWhitelistedJTregVMAndJavaOptsFlags() {
790         List<String> flags = Arrays.stream(Utils.getTestJavaOpts())
791                                    .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", ""))
792                                    .collect(Collectors.toList());
793         List<String> nonWhiteListedFlags = new ArrayList();
794         for (String flag : flags) {
795             // Property flags (prefix -D), -ea and -esa are whitelisted.
796             if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) {
797                 // Found VM flag that is not whitelisted
798                 nonWhiteListedFlags.add(flag);
799             }
800         }
801         return nonWhiteListedFlags;
802     }
803 
804     private void runTestVM(List<String> additionalFlags) {
805         TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup,
806                                                         isAllowNotCompilable, testClassesOnBootClassPath);
807         if (shouldVerifyIR) {
808             try {
809                 TestClassParser testClassParser = new TestClassParser(testClass, isAllowNotCompilable);
810                 Matchable testClassMatchable = testClassParser.parse(testVMProcess.getHotspotPidFileName(),
811                                                                      testVMProcess.getIrEncoding());
812                 IRMatcher matcher = new IRMatcher(testClassMatchable);
813                 matcher.match();
814             } catch (IRViolationException e) {
815                 e.addCommandLine(testVMProcess.getCommandLine());
816                 throw e;
817             }
818         } else {
819             System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " +
820                                "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " +
821                                "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " +
822                                "that make the IR verification impossible (e.g. -XX:-UseCompile, " +
823                                "-XX:TieredStopAtLevel=[1,2,3], etc.).");
824         }
825     }
826 
827     public static void check(boolean test, String failureMessage) {
828         if (!test) {
829             throw new TestFrameworkException(failureMessage);
830         }
831     }
832 }