1 /*
  2  * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 package compiler.lib.ir_framework;
 25 
 26 import compiler.lib.ir_framework.driver.FlagVMProcess;
 27 import compiler.lib.ir_framework.driver.TestVMException;
 28 import compiler.lib.ir_framework.driver.TestVMProcess;
 29 import compiler.lib.ir_framework.driver.irmatching.IRMatcher;
 30 import compiler.lib.ir_framework.driver.irmatching.IRViolationException;
 31 import compiler.lib.ir_framework.driver.irmatching.Matchable;
 32 import compiler.lib.ir_framework.driver.irmatching.parser.TestClassParser;
 33 import compiler.lib.ir_framework.shared.*;
 34 import compiler.lib.ir_framework.test.TestVM;
 35 import jdk.test.lib.Platform;
 36 import jdk.test.lib.Utils;
 37 import jdk.test.lib.helpers.ClassFileInstaller;
 38 import jdk.test.whitebox.WhiteBox;
 39 import jtreg.SkippedException;
 40 
 41 import java.io.PrintWriter;
 42 import java.io.StringWriter;
 43 import java.lang.reflect.Method;
 44 import java.net.MalformedURLException;
 45 import java.net.URL;
 46 import java.net.URLClassLoader;
 47 import java.nio.file.Path;
 48 import java.util.*;
 49 import java.util.stream.Collectors;
 50 
 51 /**
 52  * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on
 53  * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can
 54  * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used
 55  * testing patterns and compiler control flags.
 56  * <p>
 57  * The framework offers various annotations to control how your test code should be invoked and being checked. There are
 58  * three kinds of tests depending on how much control is needed over the test invocation:
 59  * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b>
 60  * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test}
 61  * annotation which represents the test code that is eventually executed by the test framework. More information about
 62  * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}.
 63  * <p>
 64  * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which
 65  * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}.
 66  * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage
 67  * and how to write different IR rules can be found at {@link IR}.
 68  * <p>
 69  * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>:
 70  * <pre>
 71  * {@literal @}library /test/lib
 72  * {@literal @}run driver some.package.Test
 73  * </pre>
 74  * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the
 75  * JTreg test if the test itself is not utilizing any Whitebox features directly.
 76  * <p>
 77  * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or
 78  * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations
 79  * (instead of specifying multiple JTreg {@code @run} entries).
 80  * <p>
 81  * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the
 82  * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various
 83  * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new
 84  * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing.
 85  * <p>
 86  * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code
 87  * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions.
 88  * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are
 89  * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}
 90  * for IR matching).
 91  * <p>
 92  * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified
 93  * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}.
 94  * <p>
 95  * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR
 96  * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details).
 97  * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing
 98  * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}.
 99  *
100  * @see Test
101  * @see Check
102  * @see Run
103  * @see IR
104  */
105 public class TestFramework {
106     /**
107      * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only
108      * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags
109      * by user code) are whitelisted.
110      *
111      * <p>
112      * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of
113      * the entries of this list as a substring (partial match).
114      */
115     public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>(
116             Arrays.asList(
117                     // The following substrings are part of more than one VM flag
118                     "RAM",
119                     "Heap",
120                     "Trace",
121                     "Print",
122                     "Verify",
123                     "UseNewCode",
124                     "Xmn",
125                     "Xms",
126                     "Xmx",
127                     "Xss",
128                     // The following substrings are only part of one VM flag (=exact match)
129                     "CreateCoredumpOnCrash",
130                     "IgnoreUnrecognizedVMOptions",
131                     "UnlockDiagnosticVMOptions",
132                     "UnlockExperimentalVMOptions",
133                     "BackgroundCompilation",
134                     "Xbatch",
135                     "TieredCompilation",
136                     "CompileThreshold",
137                     "Xmixed",
138                     "server",
139                     "AlignVector",
140                     "UseAVX",
141                     "UseSSE",
142                     "UseSVE",
143                     "Xlog",
144                     "LogCompilation",
145                     "UseCompactObjectHeaders",
146                     "UseFMA",
147                     // Riscv
148                     "UseRVV",
149                     "UseZbb",
150                     "UseZfh",
151                     "UseZicond",
152                     "UseZvbb"
153             )
154     );
155 
156     public static final boolean VERBOSE = Boolean.getBoolean("Verbose");
157     public static final boolean PRINT_RULE_MATCHING_TIME = Boolean.getBoolean("PrintRuleMatchingTime");
158     public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty();
159     public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty();
160     private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout");
161     // Only used for internal testing and should not be used for normal user testing.
162 
163     private static final String RERUN_HINT = """
164                                                #############################################################
165                                                 - To only run the failed tests use -DTest, -DExclude,
166                                                   and/or -DScenarios.
167                                                 - To also get the standard output of the test VM run with
168                                                   -DReportStdout=true or for even more fine-grained logging
169                                                   use -DVerbose=true.
170                                                #############################################################
171                                              """ + System.lineSeparator();
172 
173     private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true"));
174     private boolean shouldVerifyIR; // Should we perform IR matching?
175     private static boolean toggleBool;
176 
177     private final Class<?> testClass;
178     private Set<Class<?>> helperClasses;
179     private List<Scenario> scenarios;
180     private Set<Integer> scenarioIndices;
181     private List<String> flags;
182     private int defaultWarmup = -1;
183     private boolean testClassesOnBootClassPath;
184     private boolean isAllowNotCompilable = false;
185 
186     /*
187      * Public interface methods
188      */
189 
190     /**
191      * Creates an instance acting as a builder to test the class from which this constructor was invoked from.
192      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
193      * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
194      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
195      */
196     public TestFramework() {
197         this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass());
198     }
199 
200     /**
201      * Creates an instance acting as a builder to test {@code testClass}.
202      * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
203      * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
204      * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
205      *
206      * @param testClass the class to be tested by the framework.
207      * @see #TestFramework()
208      */
209     public TestFramework(Class<?> testClass) {
210         TestRun.check(testClass != null, "Test class cannot be null");
211         this.testClass = testClass;
212         if (VERBOSE) {
213             System.out.println("Test class: " + testClass);
214         }
215     }
216 
217     /**
218      * Tests the class from which this method was invoked from.
219      */
220     public static void run() {
221         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
222         run(walker.getCallerClass());
223     }
224 
225     /**
226      * Tests {@code testClass}.
227      *
228      * @param testClass the class to be tested by the framework.
229      * @see #run()
230      */
231     public static void run(Class<?> testClass) {
232         TestFramework framework = new TestFramework(testClass);
233         framework.start();
234     }
235 
236     /**
237      * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}.
238      * <ul>
239      *     <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p>
240      *            Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and  Javaoptions flags over
241      *            the specified {@code flags} of this method.</li>
242      *     <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li>
243      *     <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one
244      *            from which this method was called from, use {@link #addFlags(String...)}, use this method.</li>
245      *     <li><p>If you want to run your JTreg test with multiple flag combinations, use
246      *            {@link #addScenarios(Scenario...)}</li>
247      * </ul>
248      *
249      * @param flags VM flags to be used for the test VM.
250      */
251     public static void runWithFlags(String... flags) {
252         StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
253         TestFramework framework = new TestFramework(walker.getCallerClass());
254         framework.addFlags(flags);
255         framework.start();
256     }
257 
258     /**
259      * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p>
260      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags.
261      *
262      * <p>
263      * The testing can be started by invoking {@link #start()}
264      *
265      * @param flags VM options to be applied to the test VM.
266      * @return the same framework instance.
267      */
268     public TestFramework addFlags(String... flags) {
269         TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null");
270         if (this.flags == null) {
271             this.flags = new ArrayList<>();
272         }
273         this.flags.addAll(Arrays.asList(flags));
274         return this;
275     }
276 
277     /**
278      * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile},
279      * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied
280      * while testing {@code testClass} (also see description of {@link TestFramework}).
281      *
282      * <p>
283      * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any
284      * compile command annotations, you do not need to include it with this method. If no helper class specifies any
285      * compile commands, you do not need to call this method at all.
286      *
287      * <p>
288      * The testing can be started by invoking {@link #start()}.
289      *
290      * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile},
291      *                      {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied
292      *                      while testing {@code testClass} (also see description of {@link TestFramework}).
293      * @return the same framework instance.
294      */
295     public TestFramework addHelperClasses(Class<?>... helperClasses) {
296         TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull),
297                       "A Helper class cannot be null");
298         if (this.helperClasses == null) {
299             this.helperClasses = new HashSet<>();
300         }
301 
302         this.helperClasses.addAll(Arrays.asList(helperClasses));
303         return this;
304     }
305 
306     /**
307      * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the
308      * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)}
309      * and thus also override any VM or Javaoptions set by JTreg by default.<p>
310      * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags.
311      *
312      * <p>
313      * The testing can be started by invoking {@link #start()}
314      *
315      * @param scenarios scenarios which specify specific flags for the test VM.
316      * @return the same framework instance.
317      */
318     public TestFramework addScenarios(Scenario... scenarios) {
319         TestFormat.checkAndReport(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull),
320                                   "A scenario cannot be null");
321         if (this.scenarios == null) {
322             this.scenarios = new ArrayList<>();
323             this.scenarioIndices = new HashSet<>();
324         }
325 
326         for (Scenario scenario : scenarios) {
327             int scenarioIndex = scenario.getIndex();
328             TestFormat.checkNoThrow(scenarioIndices.add(scenarioIndex),
329                              "Cannot define two scenarios with the same index " + scenarioIndex);
330             this.scenarios.add(scenario);
331         }
332         TestFormat.throwIfAnyFailures();
333         return this;
334     }
335 
336     /**
337      * Add test classes to boot classpath. This adds all classes found on path {@link jdk.test.lib.Utils#TEST_CLASSES}
338      * to the boot classpath with "-Xbootclasspath/a". This is useful when trying to run tests in a privileged mode.
339      */
340     public TestFramework addTestClassesToBootClassPath() {
341         this.testClassesOnBootClassPath = true;
342         return this;
343     }
344 
345     /**
346      * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)})
347      * set test class.
348      */
349     public void start() {
350         if (shouldInstallWhiteBox()) {
351             installWhiteBox();
352         }
353         checkCompatibleFlags();
354         checkIRRuleCompilePhasesFormat();
355         disableIRVerificationIfNotFeasible();
356 
357         if (scenarios == null) {
358             try {
359                 start(null);
360             } catch (TestVMException e) {
361                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT);
362                 throw e;
363             } catch (IRViolationException e) {
364                 System.out.println(e.getCompilations());
365                 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT);
366                 throw e;
367             }
368         } else {
369             startWithScenarios();
370         }
371     }
372 
373     private void checkIRRuleCompilePhasesFormat() {
374         for (Method method : testClass.getDeclaredMethods()) {
375             for (IR irAnno : method.getAnnotationsByType(IR.class)) {
376                 TestFormat.checkNoThrow(irAnno.phase().length > 0,
377                                         "@IR rule " + irAnno + " must specify a non-empty list of compile " +
378                                         "phases \"phase\" at " + method);
379             }
380         }
381         TestFormat.throwIfAnyFailures();
382     }
383 
384     /**
385      * Try to load the Whitebox class from the user directory with a custom class loader. If the user has already built the
386      * Whitebox, we can load it. Otherwise, the framework needs to install it.
387      *
388      * @return true if the framework needs to install the Whitebox
389      */
390     private boolean shouldInstallWhiteBox() {
391         try {
392             URL url = Path.of(System.getProperty("user.dir")).toUri().toURL();
393             URLClassLoader userDirClassLoader =
394                     URLClassLoader.newInstance(new URL[] {url}, TestFramework.class.getClassLoader().getParent());
395             Class.forName(WhiteBox.class.getName(), false, userDirClassLoader);
396         } catch (MalformedURLException e) {
397             throw new TestFrameworkException("corrupted user.dir property", e);
398         } catch (ClassNotFoundException e) {
399             // We need to manually install the WhiteBox if we cannot load the WhiteBox class from the user directory.
400             // This happens when the user test does not explicitly install the WhiteBox as part of the test.
401             return true;
402         }
403         return false;
404     }
405 
406     /**
407      * Set a new default warm-up (overriding the framework default of 2000 at
408      * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit
409      * warm-up with {@link Warmup @Warmup}.
410      *
411      * @param defaultWarmup a new non-negative default warm-up.
412      * @return the same framework instance.
413      */
414     public TestFramework setDefaultWarmup(int defaultWarmup) {
415         TestFormat.checkAndReport(defaultWarmup >= 0, "Cannot specify a negative default warm-up");
416         this.defaultWarmup = defaultWarmup;
417         return this;
418     }
419 
420     /**
421      * In rare cases, methods may not be compilable because of a compilation bailout. By default, this leads to a
422      * test failure. However, if such cases are expected in multiple methods in a test class, this flag can be set to
423      * true, which allows any test to pass even if there is a compilation bailout. If only selected methods are prone
424      * to bail out, it is preferred to use {@link Test#allowNotCompilable()} instead for more fine-grained control.
425      * By setting this flag, any associated {@link IR} rule of a test is only executed if the test method was compiled,
426      * and else it is ignored silently.
427      */
428     public TestFramework allowNotCompilable() {
429         this.isAllowNotCompilable = true;
430         return this;
431     }
432 
433     /**
434      * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios
435      * were run, use {@link Scenario#getTestVMOutput()}.
436      *
437      * @return the last test VM output.
438      */
439     public static String getLastTestVMOutput() {
440         return TestVMProcess.getLastTestVMOutput();
441     }
442 
443     /*
444      * The following methods are only intended to be called from actual @Test methods and not from the main() method of
445      * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled).
446      */
447 
448     /**
449      * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled,
450      * yet, upon returning from this method.
451      *
452      * @param m the method to be compiled.
453      * @param compLevel the (valid) compilation level at which the method should be compiled.
454      * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}.
455      */
456     public static void compile(Method m, CompLevel compLevel) {
457         TestVM.compile(m, compLevel);
458     }
459 
460     /**
461      * Deoptimize {@code m}.
462      *
463      * @param m the method to be deoptimized.
464      */
465     public static void deoptimize(Method m) {
466         TestVM.deoptimize(m);
467     }
468 
469     /**
470      * Returns a boolean indicating if {@code m} is compiled at any level.
471      *
472      * @param m the method to be checked.
473      * @return {@code true} if {@code m} is compiled at any level;
474      *         {@code false} otherwise.
475      */
476     public static boolean isCompiled(Method m) {
477         return TestVM.isCompiled(m);
478     }
479 
480     /**
481      * Returns a boolean indicating if {@code m} is compiled with C1.
482      *
483      * @param m the method to be checked.
484      * @return {@code true} if {@code m} is compiled with C1;
485      *         {@code false} otherwise.
486      */
487     public static boolean isC1Compiled(Method m) {
488         return TestVM.isC1Compiled(m);
489     }
490 
491     /**
492      * Returns a boolean indicating if {@code m} is compiled with C2.
493      *
494      * @param m the method to be checked.
495      * @return {@code true} if {@code m} is compiled with C2;
496      *         {@code false} otherwise.
497      */
498     public static boolean isC2Compiled(Method m) {
499         return TestVM.isC2Compiled(m);
500     }
501 
502     /**
503      * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}.
504      *
505      * @param m the method to be checked.
506      * @param compLevel the compilation level.
507      * @return {@code true} if {@code m} is compiled at {@code compLevel};
508      *         {@code false} otherwise.
509      */
510     public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
511         return TestVM.isCompiledAtLevel(m, compLevel);
512     }
513 
514     /**
515      * Checks if {@code m} is compiled at any level.
516      *
517      * @param m the method to be checked.
518      * @throws TestRunException if {@code m} is not compiled at any level.
519      */
520     public static void assertCompiled(Method m) {
521         TestVM.assertCompiled(m);
522     }
523 
524     /**
525      * Checks if {@code m} is not compiled at any level.
526      *
527      * @param m the method to be checked.
528      * @throws TestRunException if {@code m} is compiled at any level.
529      */
530     public static void assertNotCompiled(Method m) {
531         TestVM.assertNotCompiled(m);
532     }
533 
534     /**
535      * Verifies that {@code m} is compiled with C1.
536      *
537      * @param m the method to be verified.
538      * @throws TestRunException if {@code m} is not compiled with C1.
539      */
540     public static void assertCompiledByC1(Method m) {
541         TestVM.assertCompiledByC1(m);
542     }
543 
544     /**
545      * Verifies that {@code m} is compiled with C2.
546      *
547      * @param m the method to be checked.
548      * @throws TestRunException if {@code m} is not compiled with C2.
549      */
550     public static void assertCompiledByC2(Method m) {
551         TestVM.assertCompiledByC2(m);
552     }
553 
554     /**
555      * Verifies that {@code m} is compiled at the specified {@code compLevel}.
556      *
557      * @param m the method to be checked.
558      * @param compLevel the compilation level.
559      * @throws TestRunException if {@code m} is not compiled at {@code compLevel}.
560      */
561     public static void assertCompiledAtLevel(Method m, CompLevel compLevel) {
562         TestVM.assertCompiledAtLevel(m, compLevel);
563     }
564 
565     /**
566      * Verifies that {@code m} was deoptimized after being C1 compiled.
567      *
568      * @param m the method to be checked.
569      * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled.
570      */
571     public static void assertDeoptimizedByC1(Method m) {
572         TestVM.assertDeoptimizedByC1(m);
573     }
574 
575     /**
576      * Verifies that {@code m} was deoptimized after being C2 compiled.
577      *
578      * @param m the method to be checked.
579      * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled.
580      */
581     public static void assertDeoptimizedByC2(Method m) {
582         TestVM.assertDeoptimizedByC2(m);
583     }
584 
585     /**
586      * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}).
587      * The very first invocation returns {@code false}. Note that this method could be used by different tests and
588      * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times
589      * other tests have already invoked this method.
590      *
591      * @return an inverted boolean of the result of the last invocation of this method.
592      */
593     public static boolean toggleBoolean() {
594         toggleBool = !toggleBool;
595         return toggleBool;
596     }
597 
598     /*
599      * End of public interface methods
600      */
601 
602     /**
603      * Used to move Whitebox class to the right folder in the JTreg test
604      */
605     private void installWhiteBox() {
606         try {
607             ClassFileInstaller.main(WhiteBox.class.getName());
608         } catch (Exception e) {
609             throw new Error("failed to install whitebox classes", e);
610         }
611     }
612 
613     /**
614      * Disable IR verification completely in certain cases.
615      */
616     private void disableIRVerificationIfNotFeasible() {
617         if (!irVerificationPossible) {
618             return;
619         }
620 
621         boolean debugTest = Platform.isDebugBuild();
622         boolean intTest = !Platform.isInt();
623         boolean compTest = !Platform.isComp();
624         boolean irTest = hasIRAnnotations();
625         // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified.
626         List<String> nonWhiteListedFlags = anyNonWhitelistedJTregVMAndJavaOptsFlags();
627         boolean nonWhiteListedTest = nonWhiteListedFlags.isEmpty();
628 
629         irVerificationPossible = debugTest && intTest && compTest && irTest && nonWhiteListedTest;
630         if (irVerificationPossible) {
631             return;
632         }
633 
634         System.out.println("IR verification disabled due to the following reason(s):");
635         if (!debugTest) {
636             System.out.println("- Not running a debug build (required for PrintIdeal and PrintOptoAssembly)");
637         }
638         if (!intTest) {
639             System.out.println("- Running with -Xint (no compilations)");
640         }
641         if (!compTest) {
642             System.out.println("- Running with -Xcomp (use warm-up of 0 instead)");
643         }
644         if (!irTest) {
645             System.out.println("- Test " + testClass + " not specifying any @IR annotations");
646         }
647         if (!nonWhiteListedTest) {
648             System.out.println("- Using non-whitelisted JTreg VM or Javaoptions flag(s):");
649             nonWhiteListedFlags.forEach((f) -> System.out.println("  - " + f));
650         }
651 
652         System.out.println("");
653     }
654 
655     /**
656      * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all
657      * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios
658      * and thus is reported immediately on the first scenario execution.
659      */
660     private void startWithScenarios() {
661         Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex));
662         for (Scenario scenario : scenarios) {
663             try {
664                 start(scenario);
665             } catch (TestFormatException e) {
666                 // Test format violation is wrong for all the scenarios. Only report once.
667                 throw e;
668             } catch (Exception e) {
669                 exceptionMap.put(scenario, e);
670             }
671         }
672         if (!exceptionMap.isEmpty()) {
673             reportScenarioFailures(exceptionMap);
674         }
675     }
676 
677     private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) {
678         String failedScenarios = "The following scenarios have failed: #"
679                                  + exceptionMap.keySet().stream()
680                                                .map(s -> String.valueOf(s.getIndex()))
681                                                .collect(Collectors.joining(", #"));
682         StringBuilder builder = new StringBuilder(failedScenarios);
683         builder.append(System.lineSeparator()).append(System.lineSeparator());
684         for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) {
685             Exception e = entry.getValue();
686             Scenario scenario = entry.getKey();
687             String errorMsg = "";
688             if (scenario != null) {
689                 errorMsg = getScenarioTitleAndFlags(scenario);
690             }
691             if (e instanceof IRViolationException irException) {
692                 // For IR violations, only show the actual violations and not the (uninteresting) stack trace.
693                 if (scenario != null) {
694                     System.out.println("Scenario #" + scenario.getIndex());
695                 }
696                 System.out.println(irException.getCompilations());
697                 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo());
698             } else if (e instanceof TestVMException testVMException) {
699                 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo());
700             } else {
701                 // Print stack trace otherwise
702                 StringWriter errors = new StringWriter();
703                 e.printStackTrace(new PrintWriter(errors));
704                 builder.append(errors);
705             }
706             builder.append(System.lineSeparator());
707         }
708         System.err.println(builder);
709         if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) {
710             // Provide a hint to the user how to get additional output/debugging information.
711             System.err.println(RERUN_HINT);
712         }
713         throw new TestRunException(failedScenarios + ". Please check stderr for more information.");
714     }
715 
716     private static String getScenarioTitleAndFlags(Scenario scenario) {
717         StringBuilder builder = new StringBuilder();
718         String title = "Scenario #" + scenario.getIndex();
719         builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length()))
720                .append(System.lineSeparator());
721         builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]")
722                .append(System.lineSeparator());
723         return builder.toString();
724     }
725 
726     /**
727      * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of
728      * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the
729      * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests.
730      */
731     private void start(Scenario scenario) {
732         if (scenario != null && !scenario.isEnabled()) {
733             System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " +
734                                "-DScenarios and is therefore not executed.");
735             return;
736         }
737         shouldVerifyIR = irVerificationPossible;
738         try {
739             // Use TestFramework flags and scenario flags for new VMs.
740             List<String> additionalFlags = new ArrayList<>();
741             if (flags != null) {
742                 additionalFlags.addAll(flags);
743             }
744             if (scenario != null) {
745                 List<String> scenarioFlags = scenario.getFlags();
746                 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]";
747                 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":");
748                 additionalFlags.addAll(scenarioFlags);
749             }
750             String frameworkAndScenarioFlags = additionalFlags.isEmpty() ?
751                     "" : " - [" + String.join(", ", additionalFlags) + "]";
752 
753             if (shouldVerifyIR) {
754                 // Only need to use flag VM if an IR verification is possibly done.
755                 System.out.println("Run Flag VM:");
756                 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags);
757                 shouldVerifyIR = flagVMProcess.shouldVerifyIR();
758                 if (shouldVerifyIR) {
759                     // Add more flags for the test VM which are required to do IR verification.
760                     additionalFlags.addAll(flagVMProcess.getTestVMFlags());
761                 } // else: Flag VM found a reason to not do IR verification.
762             } else {
763                 System.out.println("Skip Flag VM due to not performing IR verification.");
764             }
765 
766             System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":");
767             runTestVM(additionalFlags);
768         } finally {
769             if (scenario != null) {
770                 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput());
771             }
772             System.out.println();
773         }
774     }
775 
776     private boolean hasIRAnnotations() {
777         return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class).length > 0);
778     }
779 
780     private void checkCompatibleFlags() {
781         for (String flag : Utils.getTestJavaOpts()) {
782             if (flag.contains("-agentpath")) {
783                 throw new SkippedException("Can't run test with agent.");
784             }
785         }
786     }
787 
788     private List<String> anyNonWhitelistedJTregVMAndJavaOptsFlags() {
789         List<String> flags = Arrays.stream(Utils.getTestJavaOpts())
790                                    .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", ""))
791                                    .collect(Collectors.toList());
792         List<String> nonWhiteListedFlags = new ArrayList();
793         for (String flag : flags) {
794             if (flag.contains("agentpath")) {
795                 throw new SkippedException("Can't run test with -javaagent");
796             }
797             // Property flags (prefix -D), -ea and -esa are whitelisted.
798             if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) {
799                 // Found VM flag that is not whitelisted
800                 nonWhiteListedFlags.add(flag);
801             }
802         }
803         return nonWhiteListedFlags;
804     }
805 
806     private void runTestVM(List<String> additionalFlags) {
807         TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup,
808                                                         isAllowNotCompilable, testClassesOnBootClassPath);
809         if (shouldVerifyIR) {
810             try {
811                 TestClassParser testClassParser = new TestClassParser(testClass, isAllowNotCompilable);
812                 Matchable testClassMatchable = testClassParser.parse(testVMProcess.getHotspotPidFileName(),
813                                                                      testVMProcess.getIrEncoding());
814                 IRMatcher matcher = new IRMatcher(testClassMatchable);
815                 matcher.match();
816             } catch (IRViolationException e) {
817                 e.addCommandLine(testVMProcess.getCommandLine());
818                 throw e;
819             }
820         } else {
821             System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " +
822                                "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " +
823                                "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " +
824                                "that make the IR verification impossible (e.g. -XX:-UseCompile, " +
825                                "-XX:TieredStopAtLevel=[1,2,3], etc.).");
826         }
827     }
828 
829     public static void check(boolean test, String failureMessage) {
830         if (!test) {
831             throw new TestFrameworkException(failureMessage);
832         }
833     }
834 }