1 /*
2 * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 package compiler.lib.ir_framework;
25
26 import compiler.lib.ir_framework.driver.FlagVMProcess;
27 import compiler.lib.ir_framework.driver.TestVMException;
28 import compiler.lib.ir_framework.driver.TestVMProcess;
29 import compiler.lib.ir_framework.driver.irmatching.IRMatcher;
30 import compiler.lib.ir_framework.driver.irmatching.IRViolationException;
31 import compiler.lib.ir_framework.driver.irmatching.Matchable;
32 import compiler.lib.ir_framework.driver.irmatching.parser.TestClassParser;
33 import compiler.lib.ir_framework.shared.*;
34 import compiler.lib.ir_framework.test.TestVM;
35 import jdk.test.lib.Platform;
36 import jdk.test.lib.Utils;
37 import jdk.test.lib.helpers.ClassFileInstaller;
38 import jdk.test.whitebox.WhiteBox;
39
40 import java.io.PrintWriter;
41 import java.io.StringWriter;
42 import java.lang.reflect.Method;
43 import java.net.MalformedURLException;
44 import java.net.URL;
45 import java.net.URLClassLoader;
46 import java.nio.file.Path;
47 import java.util.*;
48 import java.util.stream.Collectors;
49
50 /**
51 * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on
52 * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can
53 * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used
54 * testing patterns and compiler control flags.
55 * <p>
56 * The framework offers various annotations to control how your test code should be invoked and being checked. There are
57 * three kinds of tests depending on how much control is needed over the test invocation:
58 * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b>
59 * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test}
60 * annotation which represents the test code that is eventually executed by the test framework. More information about
61 * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}.
62 * <p>
63 * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which
64 * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}.
65 * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage
66 * and how to write different IR rules can be found at {@link IR}.
67 * <p>
68 * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>:
69 * <pre>
70 * {@literal @}library /test/lib
71 * {@literal @}run driver some.package.Test
72 * </pre>
73 * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the
74 * JTreg test if the test itself is not utilizing any Whitebox features directly.
75 * <p>
76 * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or
77 * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations
78 * (instead of specifying multiple JTreg {@code @run} entries).
79 * <p>
80 * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the
81 * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various
82 * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new
83 * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing.
84 * <p>
85 * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code
86 * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions.
87 * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are
88 * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}
89 * for IR matching).
90 * <p>
91 * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified
92 * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}.
93 * <p>
94 * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR
95 * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details).
96 * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing
97 * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}.
98 *
99 * @see Test
100 * @see Check
101 * @see Run
102 * @see IR
103 */
104 public class TestFramework {
105 /**
106 * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only
107 * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags
108 * by user code) are whitelisted.
109 *
110 * <p>
111 * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of
112 * the entries of this list as a substring (partial match).
113 */
114 public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>(
115 Arrays.asList(
116 // The following substrings are part of more than one VM flag
117 "RAM",
118 "Heap",
119 "Trace",
120 "Print",
121 "Verify",
122 "UseNewCode",
123 "Xmn",
124 "Xms",
125 "Xmx",
126 "Xss",
127 // The following substrings are only part of one VM flag (=exact match)
128 "CreateCoredumpOnCrash",
129 "IgnoreUnrecognizedVMOptions",
130 "UnlockDiagnosticVMOptions",
131 "UnlockExperimentalVMOptions",
132 "BackgroundCompilation",
133 "Xbatch",
134 "TieredCompilation",
135 "CompileThreshold",
136 "Xmixed",
137 "server",
138 "AlignVector",
139 "UseAVX",
140 "UseSSE",
141 "UseSVE",
142 "UseZbb",
143 "UseRVV",
144 "Xlog",
145 "LogCompilation",
146 "UseCompactObjectHeaders"
147 )
148 );
149
150 public static final boolean VERBOSE = Boolean.getBoolean("Verbose");
151 public static final boolean PRINT_RULE_MATCHING_TIME = Boolean.getBoolean("PrintRuleMatchingTime");
152 public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty();
153 public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty();
154 private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout");
155 // Only used for internal testing and should not be used for normal user testing.
156
157 private static final String RERUN_HINT = """
158 #############################################################
159 - To only run the failed tests use -DTest, -DExclude,
160 and/or -DScenarios.
161 - To also get the standard output of the test VM run with
162 -DReportStdout=true or for even more fine-grained logging
163 use -DVerbose=true.
164 #############################################################
165 """ + System.lineSeparator();
166
167 private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true"));
168 private boolean shouldVerifyIR; // Should we perform IR matching?
169 private static boolean toggleBool;
170
171 private final Class<?> testClass;
172 private Set<Class<?>> helperClasses;
173 private List<Scenario> scenarios;
174 private Set<Integer> scenarioIndices;
175 private List<String> flags;
176 private int defaultWarmup = -1;
177 private boolean testClassesOnBootClassPath;
178 private boolean isAllowNotCompilable = false;
179
180 /*
181 * Public interface methods
182 */
183
184 /**
185 * Creates an instance acting as a builder to test the class from which this constructor was invoked from.
186 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
187 * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
188 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
189 */
190 public TestFramework() {
191 this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass());
192 }
193
194 /**
195 * Creates an instance acting as a builder to test {@code testClass}.
196 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios).
197 * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)},
198 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}.
199 *
200 * @param testClass the class to be tested by the framework.
201 * @see #TestFramework()
202 */
203 public TestFramework(Class<?> testClass) {
204 TestRun.check(testClass != null, "Test class cannot be null");
205 this.testClass = testClass;
206 if (VERBOSE) {
207 System.out.println("Test class: " + testClass);
208 }
209 }
210
211 /**
212 * Tests the class from which this method was invoked from.
213 */
214 public static void run() {
215 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
216 run(walker.getCallerClass());
217 }
218
219 /**
220 * Tests {@code testClass}.
221 *
222 * @param testClass the class to be tested by the framework.
223 * @see #run()
224 */
225 public static void run(Class<?> testClass) {
226 TestFramework framework = new TestFramework(testClass);
227 framework.start();
228 }
229
230 /**
231 * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}.
232 * <ul>
233 * <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p>
234 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and Javaoptions flags over
235 * the specified {@code flags} of this method.</li>
236 * <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li>
237 * <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one
238 * from which this method was called from, use {@link #addFlags(String...)}, use this method.</li>
239 * <li><p>If you want to run your JTreg test with multiple flag combinations, use
240 * {@link #addScenarios(Scenario...)}</li>
241 * </ul>
242 *
243 * @param flags VM flags to be used for the test VM.
244 */
245 public static void runWithFlags(String... flags) {
246 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
247 TestFramework framework = new TestFramework(walker.getCallerClass());
248 framework.addFlags(flags);
249 framework.start();
250 }
251
252 /**
253 * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p>
254 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags.
255 *
256 * <p>
257 * The testing can be started by invoking {@link #start()}
258 *
259 * @param flags VM options to be applied to the test VM.
260 * @return the same framework instance.
261 */
262 public TestFramework addFlags(String... flags) {
263 TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null");
264 if (this.flags == null) {
265 this.flags = new ArrayList<>();
266 }
267 this.flags.addAll(Arrays.asList(flags));
268 return this;
269 }
270
271 /**
272 * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile},
273 * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied
274 * while testing {@code testClass} (also see description of {@link TestFramework}).
275 *
276 * <p>
277 * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any
278 * compile command annotations, you do not need to include it with this method. If no helper class specifies any
279 * compile commands, you do not need to call this method at all.
280 *
281 * <p>
282 * The testing can be started by invoking {@link #start()}.
283 *
284 * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile},
285 * {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied
286 * while testing {@code testClass} (also see description of {@link TestFramework}).
287 * @return the same framework instance.
288 */
289 public TestFramework addHelperClasses(Class<?>... helperClasses) {
290 TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull),
291 "A Helper class cannot be null");
292 if (this.helperClasses == null) {
293 this.helperClasses = new HashSet<>();
294 }
295
296 this.helperClasses.addAll(Arrays.asList(helperClasses));
297 return this;
298 }
299
300 /**
301 * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the
302 * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)}
303 * and thus also override any VM or Javaoptions set by JTreg by default.<p>
304 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags.
305 *
306 * <p>
307 * The testing can be started by invoking {@link #start()}
308 *
309 * @param scenarios scenarios which specify specific flags for the test VM.
310 * @return the same framework instance.
311 */
312 public TestFramework addScenarios(Scenario... scenarios) {
313 TestFormat.checkAndReport(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull),
314 "A scenario cannot be null");
315 if (this.scenarios == null) {
316 this.scenarios = new ArrayList<>();
317 this.scenarioIndices = new HashSet<>();
318 }
319
320 for (Scenario scenario : scenarios) {
321 int scenarioIndex = scenario.getIndex();
322 TestFormat.checkNoThrow(scenarioIndices.add(scenarioIndex),
323 "Cannot define two scenarios with the same index " + scenarioIndex);
324 this.scenarios.add(scenario);
325 }
326 TestFormat.throwIfAnyFailures();
327 return this;
328 }
329
330 /**
331 * Add test classes to boot classpath. This adds all classes found on path {@link jdk.test.lib.Utils#TEST_CLASSES}
332 * to the boot classpath with "-Xbootclasspath/a". This is useful when trying to run tests in a privileged mode.
333 */
334 public TestFramework addTestClassesToBootClassPath() {
335 this.testClassesOnBootClassPath = true;
336 return this;
337 }
338
339 /**
340 * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)})
341 * set test class.
342 */
343 public void start() {
344 if (shouldInstallWhiteBox()) {
345 installWhiteBox();
346 }
347 checkIRRuleCompilePhasesFormat();
348 disableIRVerificationIfNotFeasible();
349
350 if (scenarios == null) {
351 try {
352 start(null);
353 } catch (TestVMException e) {
354 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT);
355 throw e;
356 } catch (IRViolationException e) {
357 System.out.println(e.getCompilations());
358 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT);
359 throw e;
360 }
361 } else {
362 startWithScenarios();
363 }
364 }
365
366 private void checkIRRuleCompilePhasesFormat() {
367 for (Method method : testClass.getDeclaredMethods()) {
368 for (IR irAnno : method.getAnnotationsByType(IR.class)) {
369 TestFormat.checkNoThrow(irAnno.phase().length > 0,
370 "@IR rule " + irAnno + " must specify a non-empty list of compile " +
371 "phases \"phase\" at " + method);
372 }
373 }
374 TestFormat.throwIfAnyFailures();
375 }
376
377 /**
378 * Try to load the Whitebox class from the user directory with a custom class loader. If the user has already built the
379 * Whitebox, we can load it. Otherwise, the framework needs to install it.
380 *
381 * @return true if the framework needs to install the Whitebox
382 */
383 private boolean shouldInstallWhiteBox() {
384 try {
385 URL url = Path.of(System.getProperty("user.dir")).toUri().toURL();
386 URLClassLoader userDirClassLoader =
387 URLClassLoader.newInstance(new URL[] {url}, TestFramework.class.getClassLoader().getParent());
388 Class.forName(WhiteBox.class.getName(), false, userDirClassLoader);
389 } catch (MalformedURLException e) {
390 throw new TestFrameworkException("corrupted user.dir property", e);
391 } catch (ClassNotFoundException e) {
392 // We need to manually install the WhiteBox if we cannot load the WhiteBox class from the user directory.
393 // This happens when the user test does not explicitly install the WhiteBox as part of the test.
394 return true;
395 }
396 return false;
397 }
398
399 /**
400 * Set a new default warm-up (overriding the framework default of 2000 at
401 * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit
402 * warm-up with {@link Warmup @Warmup}.
403 *
404 * @param defaultWarmup a new non-negative default warm-up.
405 * @return the same framework instance.
406 */
407 public TestFramework setDefaultWarmup(int defaultWarmup) {
408 TestFormat.checkAndReport(defaultWarmup >= 0, "Cannot specify a negative default warm-up");
409 this.defaultWarmup = defaultWarmup;
410 return this;
411 }
412
413 /**
414 * In rare cases, methods may not be compilable because of a compilation bailout. By default, this leads to a
415 * test failure. However, if such cases are expected in multiple methods in a test class, this flag can be set to
416 * true, which allows any test to pass even if there is a compilation bailout. If only selected methods are prone
417 * to bail out, it is preferred to use {@link Test#allowNotCompilable()} instead for more fine-grained control.
418 * By setting this flag, any associated {@link IR} rule of a test is only executed if the test method was compiled,
419 * and else it is ignored silently.
420 */
421 public TestFramework allowNotCompilable() {
422 this.isAllowNotCompilable = true;
423 return this;
424 }
425
426 /**
427 * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios
428 * were run, use {@link Scenario#getTestVMOutput()}.
429 *
430 * @return the last test VM output.
431 */
432 public static String getLastTestVMOutput() {
433 return TestVMProcess.getLastTestVMOutput();
434 }
435
436 /*
437 * The following methods are only intended to be called from actual @Test methods and not from the main() method of
438 * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled).
439 */
440
441 /**
442 * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled,
443 * yet, upon returning from this method.
444 *
445 * @param m the method to be compiled.
446 * @param compLevel the (valid) compilation level at which the method should be compiled.
447 * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}.
448 */
449 public static void compile(Method m, CompLevel compLevel) {
450 TestVM.compile(m, compLevel);
451 }
452
453 /**
454 * Deoptimize {@code m}.
455 *
456 * @param m the method to be deoptimized.
457 */
458 public static void deoptimize(Method m) {
459 TestVM.deoptimize(m);
460 }
461
462 /**
463 * Returns a boolean indicating if {@code m} is compiled at any level.
464 *
465 * @param m the method to be checked.
466 * @return {@code true} if {@code m} is compiled at any level;
467 * {@code false} otherwise.
468 */
469 public static boolean isCompiled(Method m) {
470 return TestVM.isCompiled(m);
471 }
472
473 /**
474 * Returns a boolean indicating if {@code m} is compiled with C1.
475 *
476 * @param m the method to be checked.
477 * @return {@code true} if {@code m} is compiled with C1;
478 * {@code false} otherwise.
479 */
480 public static boolean isC1Compiled(Method m) {
481 return TestVM.isC1Compiled(m);
482 }
483
484 /**
485 * Returns a boolean indicating if {@code m} is compiled with C2.
486 *
487 * @param m the method to be checked.
488 * @return {@code true} if {@code m} is compiled with C2;
489 * {@code false} otherwise.
490 */
491 public static boolean isC2Compiled(Method m) {
492 return TestVM.isC2Compiled(m);
493 }
494
495 /**
496 * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}.
497 *
498 * @param m the method to be checked.
499 * @param compLevel the compilation level.
500 * @return {@code true} if {@code m} is compiled at {@code compLevel};
501 * {@code false} otherwise.
502 */
503 public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
504 return TestVM.isCompiledAtLevel(m, compLevel);
505 }
506
507 /**
508 * Checks if {@code m} is compiled at any level.
509 *
510 * @param m the method to be checked.
511 * @throws TestRunException if {@code m} is not compiled at any level.
512 */
513 public static void assertCompiled(Method m) {
514 TestVM.assertCompiled(m);
515 }
516
517 /**
518 * Checks if {@code m} is not compiled at any level.
519 *
520 * @param m the method to be checked.
521 * @throws TestRunException if {@code m} is compiled at any level.
522 */
523 public static void assertNotCompiled(Method m) {
524 TestVM.assertNotCompiled(m);
525 }
526
527 /**
528 * Verifies that {@code m} is compiled with C1.
529 *
530 * @param m the method to be verified.
531 * @throws TestRunException if {@code m} is not compiled with C1.
532 */
533 public static void assertCompiledByC1(Method m) {
534 TestVM.assertCompiledByC1(m);
535 }
536
537 /**
538 * Verifies that {@code m} is compiled with C2.
539 *
540 * @param m the method to be checked.
541 * @throws TestRunException if {@code m} is not compiled with C2.
542 */
543 public static void assertCompiledByC2(Method m) {
544 TestVM.assertCompiledByC2(m);
545 }
546
547 /**
548 * Verifies that {@code m} is compiled at the specified {@code compLevel}.
549 *
550 * @param m the method to be checked.
551 * @param compLevel the compilation level.
552 * @throws TestRunException if {@code m} is not compiled at {@code compLevel}.
553 */
554 public static void assertCompiledAtLevel(Method m, CompLevel compLevel) {
555 TestVM.assertCompiledAtLevel(m, compLevel);
556 }
557
558 /**
559 * Verifies that {@code m} was deoptimized after being C1 compiled.
560 *
561 * @param m the method to be checked.
562 * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled.
563 */
564 public static void assertDeoptimizedByC1(Method m) {
565 TestVM.assertDeoptimizedByC1(m);
566 }
567
568 /**
569 * Verifies that {@code m} was deoptimized after being C2 compiled.
570 *
571 * @param m the method to be checked.
572 * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled.
573 */
574 public static void assertDeoptimizedByC2(Method m) {
575 TestVM.assertDeoptimizedByC2(m);
576 }
577
578 /**
579 * Checks if deopt of {@code m} is stable at the specified {@code compLevel}.
580 *
581 * @param m the method to be checked.
582 * @param compLevel the compilation level.
583 * @return {@code true} if deopt of {@code m} is stable at {@code compLevel};
584 * {@code false} otherwise.
585 */
586 public static boolean isStableDeopt(Method m, CompLevel compLevel) {
587 return TestVM.isStableDeopt(m, compLevel);
588 }
589
590 /**
591 * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}).
592 * The very first invocation returns {@code false}. Note that this method could be used by different tests and
593 * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times
594 * other tests have already invoked this method.
595 *
596 * @return an inverted boolean of the result of the last invocation of this method.
597 */
598 public static boolean toggleBoolean() {
599 toggleBool = !toggleBool;
600 return toggleBool;
601 }
602
603 /*
604 * End of public interface methods
605 */
606
607 /**
608 * Used to move Whitebox class to the right folder in the JTreg test
609 */
610 private void installWhiteBox() {
611 try {
612 ClassFileInstaller.main(WhiteBox.class.getName());
613 } catch (Exception e) {
614 throw new Error("failed to install whitebox classes", e);
615 }
616 }
617
618 /**
619 * Disable IR verification completely in certain cases.
620 */
621 private void disableIRVerificationIfNotFeasible() {
622 if (!irVerificationPossible) {
623 return;
624 }
625
626 boolean debugTest = Platform.isDebugBuild();
627 boolean intTest = !Platform.isInt();
628 boolean compTest = !Platform.isComp();
629 boolean irTest = hasIRAnnotations();
630 // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified.
631 List<String> nonWhiteListedFlags = anyNonWhitelistedJTregVMAndJavaOptsFlags();
632 boolean nonWhiteListedTest = nonWhiteListedFlags.isEmpty();
633
634 irVerificationPossible = debugTest && intTest && compTest && irTest && nonWhiteListedTest;
635 if (irVerificationPossible) {
636 return;
637 }
638
639 System.out.println("IR verification disabled due to the following reason(s):");
640 if (!debugTest) {
641 System.out.println("- Not running a debug build (required for PrintIdeal and PrintOptoAssembly)");
642 }
643 if (!intTest) {
644 System.out.println("- Running with -Xint (no compilations)");
645 }
646 if (!compTest) {
647 System.out.println("- Running with -Xcomp (use warm-up of 0 instead)");
648 }
649 if (!irTest) {
650 System.out.println("- Test " + testClass + " not specifying any @IR annotations");
651 }
652 if (!nonWhiteListedTest) {
653 System.out.println("- Using non-whitelisted JTreg VM or Javaoptions flag(s):");
654 nonWhiteListedFlags.forEach((f) -> System.out.println(" - " + f));
655 }
656
657 System.out.println("");
658 }
659
660 /**
661 * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all
662 * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios
663 * and thus is reported immediately on the first scenario execution.
664 */
665 private void startWithScenarios() {
666 Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex));
667 for (Scenario scenario : scenarios) {
668 try {
669 start(scenario);
670 } catch (TestFormatException e) {
671 // Test format violation is wrong for all the scenarios. Only report once.
672 throw e;
673 } catch (Exception e) {
674 exceptionMap.put(scenario, e);
675 }
676 }
677 if (!exceptionMap.isEmpty()) {
678 reportScenarioFailures(exceptionMap);
679 }
680 }
681
682 private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) {
683 String failedScenarios = "The following scenarios have failed: #"
684 + exceptionMap.keySet().stream()
685 .map(s -> String.valueOf(s.getIndex()))
686 .collect(Collectors.joining(", #"));
687 StringBuilder builder = new StringBuilder(failedScenarios);
688 builder.append(System.lineSeparator()).append(System.lineSeparator());
689 for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) {
690 Exception e = entry.getValue();
691 Scenario scenario = entry.getKey();
692 String errorMsg = "";
693 if (scenario != null) {
694 errorMsg = getScenarioTitleAndFlags(scenario);
695 }
696 if (e instanceof IRViolationException irException) {
697 // For IR violations, only show the actual violations and not the (uninteresting) stack trace.
698 if (scenario != null) {
699 System.out.println("Scenario #" + scenario.getIndex());
700 }
701 System.out.println(irException.getCompilations());
702 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo());
703 } else if (e instanceof TestVMException testVMException) {
704 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo());
705 } else {
706 // Print stack trace otherwise
707 StringWriter errors = new StringWriter();
708 e.printStackTrace(new PrintWriter(errors));
709 builder.append(errors);
710 }
711 builder.append(System.lineSeparator());
712 }
713 System.err.println(builder);
714 if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) {
715 // Provide a hint to the user how to get additional output/debugging information.
716 System.err.println(RERUN_HINT);
717 }
718 throw new TestRunException(failedScenarios + ". Please check stderr for more information.");
719 }
720
721 private static String getScenarioTitleAndFlags(Scenario scenario) {
722 StringBuilder builder = new StringBuilder();
723 String title = "Scenario #" + scenario.getIndex();
724 builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length()))
725 .append(System.lineSeparator());
726 builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]")
727 .append(System.lineSeparator());
728 return builder.toString();
729 }
730
731 /**
732 * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of
733 * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the
734 * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests.
735 */
736 private void start(Scenario scenario) {
737 if (scenario != null && !scenario.isEnabled()) {
738 System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " +
739 "-DScenarios and is therefore not executed.");
740 return;
741 }
742 shouldVerifyIR = irVerificationPossible;
743 try {
744 // Use TestFramework flags and scenario flags for new VMs.
745 List<String> additionalFlags = new ArrayList<>();
746 if (flags != null) {
747 additionalFlags.addAll(flags);
748 }
749 if (scenario != null) {
750 List<String> scenarioFlags = scenario.getFlags();
751 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]";
752 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":");
753 additionalFlags.addAll(scenarioFlags);
754 }
755 String frameworkAndScenarioFlags = additionalFlags.isEmpty() ?
756 "" : " - [" + String.join(", ", additionalFlags) + "]";
757
758 if (shouldVerifyIR) {
759 // Only need to use flag VM if an IR verification is possibly done.
760 System.out.println("Run Flag VM:");
761 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags);
762 shouldVerifyIR = flagVMProcess.shouldVerifyIR();
763 if (shouldVerifyIR) {
764 // Add more flags for the test VM which are required to do IR verification.
765 additionalFlags.addAll(flagVMProcess.getTestVMFlags());
766 } // else: Flag VM found a reason to not do IR verification.
767 } else {
768 System.out.println("Skip Flag VM due to not performing IR verification.");
769 }
770
771 System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":");
772 runTestVM(additionalFlags);
773 } finally {
774 if (scenario != null) {
775 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput());
776 }
777 System.out.println();
778 }
779 }
780
781 private boolean hasIRAnnotations() {
782 return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class).length > 0);
783 }
784
785 private List<String> anyNonWhitelistedJTregVMAndJavaOptsFlags() {
786 List<String> flags = Arrays.stream(Utils.getTestJavaOpts())
787 .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", ""))
788 .collect(Collectors.toList());
789 List<String> nonWhiteListedFlags = new ArrayList();
790 for (String flag : flags) {
791 // Property flags (prefix -D), -ea and -esa are whitelisted.
792 if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) {
793 // Found VM flag that is not whitelisted
794 nonWhiteListedFlags.add(flag);
795 }
796 }
797 return nonWhiteListedFlags;
798 }
799
800 private void runTestVM(List<String> additionalFlags) {
801 TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup,
802 isAllowNotCompilable, testClassesOnBootClassPath);
803 if (shouldVerifyIR) {
804 try {
805 TestClassParser testClassParser = new TestClassParser(testClass, isAllowNotCompilable);
806 Matchable testClassMatchable = testClassParser.parse(testVMProcess.getHotspotPidFileName(),
807 testVMProcess.getIrEncoding());
808 IRMatcher matcher = new IRMatcher(testClassMatchable);
809 matcher.match();
810 } catch (IRViolationException e) {
811 e.addCommandLine(testVMProcess.getCommandLine());
812 throw e;
813 }
814 } else {
815 System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " +
816 "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " +
817 "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " +
818 "that make the IR verification impossible (e.g. -XX:-UseCompile, " +
819 "-XX:TieredStopAtLevel=[1,2,3], etc.).");
820 }
821 }
822
823 public static void check(boolean test, String failureMessage) {
824 if (!test) {
825 throw new TestFrameworkException(failureMessage);
826 }
827 }
828 }