1 /* 2 * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 package compiler.lib.ir_framework; 25 26 import compiler.lib.ir_framework.driver.FlagVMProcess; 27 import compiler.lib.ir_framework.driver.TestVMException; 28 import compiler.lib.ir_framework.driver.TestVMProcess; 29 import compiler.lib.ir_framework.driver.irmatching.IRMatcher; 30 import compiler.lib.ir_framework.driver.irmatching.IRViolationException; 31 import compiler.lib.ir_framework.driver.irmatching.Matchable; 32 import compiler.lib.ir_framework.driver.irmatching.parser.TestClassParser; 33 import compiler.lib.ir_framework.shared.*; 34 import compiler.lib.ir_framework.test.TestVM; 35 import jdk.test.lib.Platform; 36 import jdk.test.lib.Utils; 37 import jdk.test.lib.helpers.ClassFileInstaller; 38 import jdk.test.whitebox.WhiteBox; 39 import jtreg.SkippedException; 40 41 import java.io.PrintWriter; 42 import java.io.StringWriter; 43 import java.lang.reflect.Method; 44 import java.net.MalformedURLException; 45 import java.net.URL; 46 import java.net.URLClassLoader; 47 import java.nio.file.Path; 48 import java.util.*; 49 import java.util.stream.Collectors; 50 51 /** 52 * This class represents the main entry point to the test framework whose main purpose is to perform regex-based checks on 53 * the C2 IR shape emitted by the VM flags {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. The framework can 54 * also be used for other non-IR matching (and non-compiler) tests by providing easy to use annotations for commonly used 55 * testing patterns and compiler control flags. 56 * <p> 57 * The framework offers various annotations to control how your test code should be invoked and being checked. There are 58 * three kinds of tests depending on how much control is needed over the test invocation: 59 * <b>Base tests</b> (see {@link Test}), <b>checked tests</b> (see {@link Check}), and <b>custom run tests</b> 60 * (see {@link Run}). Each type of test needs to define a unique <i>test method</i> that specifies a {@link Test @Test} 61 * annotation which represents the test code that is eventually executed by the test framework. More information about 62 * the usage and how to write different tests can be found in {@link Test}, {@link Check}, and {@link Run}. 63 * <p> 64 * Each test method can specify an arbitrary number of IR rules. This is done by using {@link IR @IR} annotations which 65 * can define regex strings that are matched on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly}. 66 * The matching is done after the test method was (optionally) warmed up and compiled. More information about the usage 67 * and how to write different IR rules can be found at {@link IR}. 68 * <p> 69 * This framework should be used with the following JTreg setup in your Test.java file in package <i>some.package</i>: 70 * <pre> 71 * {@literal @}library /test/lib 72 * {@literal @}run driver some.package.Test 73 * </pre> 74 * Note that even though the framework uses the Whitebox API internally, it is not required to build and enabel it in the 75 * JTreg test if the test itself is not utilizing any Whitebox features directly. 76 * <p> 77 * To specify additional flags, use {@link #runWithFlags(String...)}, {@link #addFlags(String...)}, or 78 * {@link #addScenarios(Scenario...)} where the scenarios can also be used to run different flag combinations 79 * (instead of specifying multiple JTreg {@code @run} entries). 80 * <p> 81 * After annotating your test code with the framework specific annotations, the framework needs to be invoked from the 82 * {@code main()} method of your JTreg test. There are two ways to do so. The first way is by calling the various 83 * {@code runXX()} methods of {@link TestFramework}. The second way, which gives more control, is to create a new 84 * {@code TestFramework} builder object on which {@link #start()} needs to be eventually called to start the testing. 85 * <p> 86 * The framework is called from the <i>driver VM</i> in which the JTreg test is initially run by specifying {@code 87 * @run driver} in the JTreg header. This strips all additionally specified JTreg VM and Javaoptions. 88 * The framework creates a new <i>flag VM</i> with all these flags added again in order to figure out which flags are 89 * required to run the tests specified in the test class (e.g. {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} 90 * for IR matching). 91 * <p> 92 * After the flag VM terminates, it starts a new <i>test VM</i> which performs the execution of the specified 93 * tests in the test class as described in {@link Test}, {@link Check}, and {@link Run}. 94 * <p> 95 * In a last step, once the test VM has terminated without exceptions, IR matching is performed if there are any IR 96 * rules and if no VM flags disable it (e.g. not running with {@code -Xint}, see {@link IR} for more details). 97 * The IR regex matching is done on the output of {@code -XX:+PrintIdeal} and {@code -XX:+PrintOptoAssembly} by parsing 98 * the hotspot_pid file of the test VM. Failing IR rules are reported by throwing a {@link IRViolationException}. 99 * 100 * @see Test 101 * @see Check 102 * @see Run 103 * @see IR 104 */ 105 public class TestFramework { 106 /** 107 * JTreg can define additional VM (-Dtest.vm.opts) and Javaoptions (-Dtest.java.opts) flags. IR verification is only 108 * performed when all these additional JTreg flags (does not include additionally added framework and scenario flags 109 * by user code) are whitelisted. 110 * 111 * <p> 112 * A flag is whitelisted if it is a property flag (starting with -D), -ea, -esa, or if the flag name contains any of 113 * the entries of this list as a substring (partial match). 114 */ 115 public static final Set<String> JTREG_WHITELIST_FLAGS = new HashSet<>( 116 Arrays.asList( 117 // The following substrings are part of more than one VM flag 118 "RAM", 119 "Heap", 120 "Trace", 121 "Print", 122 "Verify", 123 "UseNewCode", 124 "Xmn", 125 "Xms", 126 "Xmx", 127 "Xss", 128 // The following substrings are only part of one VM flag (=exact match) 129 "CreateCoredumpOnCrash", 130 "IgnoreUnrecognizedVMOptions", 131 "UnlockDiagnosticVMOptions", 132 "UnlockExperimentalVMOptions", 133 "BackgroundCompilation", 134 "Xbatch", 135 "TieredCompilation", 136 "CompileThreshold", 137 "Xmixed", 138 "server", 139 "AlignVector", 140 "UseAVX", 141 "UseSSE", 142 "UseSVE", 143 "Xlog", 144 "LogCompilation", 145 "UseCompactObjectHeaders", 146 "UseFMA", 147 // Riscv 148 "UseRVV", 149 "UseZbb", 150 "UseZfh", 151 "UseZicond", 152 "UseZvbb" 153 ) 154 ); 155 156 public static final boolean VERBOSE = Boolean.getBoolean("Verbose"); 157 public static final boolean PRINT_RULE_MATCHING_TIME = Boolean.getBoolean("PrintRuleMatchingTime"); 158 public static final boolean TESTLIST = !System.getProperty("Test", "").isEmpty(); 159 public static final boolean EXCLUDELIST = !System.getProperty("Exclude", "").isEmpty(); 160 private static final boolean REPORT_STDOUT = Boolean.getBoolean("ReportStdout"); 161 // Only used for internal testing and should not be used for normal user testing. 162 163 private static final String RERUN_HINT = """ 164 ############################################################# 165 - To only run the failed tests use -DTest, -DExclude, 166 and/or -DScenarios. 167 - To also get the standard output of the test VM run with 168 -DReportStdout=true or for even more fine-grained logging 169 use -DVerbose=true. 170 ############################################################# 171 """ + System.lineSeparator(); 172 173 private boolean irVerificationPossible = Boolean.parseBoolean(System.getProperty("VerifyIR", "true")); 174 private boolean shouldVerifyIR; // Should we perform IR matching? 175 private static boolean toggleBool; 176 177 private final Class<?> testClass; 178 private Set<Class<?>> helperClasses; 179 private List<Scenario> scenarios; 180 private Set<Integer> scenarioIndices; 181 private List<String> flags; 182 private int defaultWarmup = -1; 183 private boolean testClassesOnBootClassPath; 184 private boolean isAllowNotCompilable = false; 185 186 /* 187 * Public interface methods 188 */ 189 190 /** 191 * Creates an instance acting as a builder to test the class from which this constructor was invoked from. 192 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios). 193 * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)}, 194 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}. 195 */ 196 public TestFramework() { 197 this(StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).getCallerClass()); 198 } 199 200 /** 201 * Creates an instance acting as a builder to test {@code testClass}. 202 * Use this constructor if you want to use multiple run options (flags, helper classes, scenarios). 203 * Use the associated add methods ({@link #addFlags(String...)}, {@link #addScenarios(Scenario...)}, 204 * {@link #addHelperClasses(Class...)}) to set up everything and then start the testing by invoking {@link #start()}. 205 * 206 * @param testClass the class to be tested by the framework. 207 * @see #TestFramework() 208 */ 209 public TestFramework(Class<?> testClass) { 210 TestRun.check(testClass != null, "Test class cannot be null"); 211 this.testClass = testClass; 212 if (VERBOSE) { 213 System.out.println("Test class: " + testClass); 214 } 215 } 216 217 /** 218 * Tests the class from which this method was invoked from. 219 */ 220 public static void run() { 221 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE); 222 run(walker.getCallerClass()); 223 } 224 225 /** 226 * Tests {@code testClass}. 227 * 228 * @param testClass the class to be tested by the framework. 229 * @see #run() 230 */ 231 public static void run(Class<?> testClass) { 232 TestFramework framework = new TestFramework(testClass); 233 framework.start(); 234 } 235 236 /** 237 * Tests the class from which this method was invoked from. The test VM is called with the specified {@code flags}. 238 * <ul> 239 * <li><p>The {@code flags} override any set VM or Javaoptions flags by JTreg by default.<p> 240 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the JTreg VM and Javaoptions flags over 241 * the specified {@code flags} of this method.</li> 242 * <li><p>If you want to run your entire JTreg test with additional flags, use this method.</li> 243 * <li><p>If you want to run your entire JTreg test with additional flags but for another test class then the one 244 * from which this method was called from, use {@link #addFlags(String...)}, use this method.</li> 245 * <li><p>If you want to run your JTreg test with multiple flag combinations, use 246 * {@link #addScenarios(Scenario...)}</li> 247 * </ul> 248 * 249 * @param flags VM flags to be used for the test VM. 250 */ 251 public static void runWithFlags(String... flags) { 252 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE); 253 TestFramework framework = new TestFramework(walker.getCallerClass()); 254 framework.addFlags(flags); 255 framework.start(); 256 } 257 258 /** 259 * Add VM flags to be used for the test VM. These flags override any VM or Javaoptions set by JTreg by default.<p> 260 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM or Javaoptions over the scenario flags. 261 * 262 * <p> 263 * The testing can be started by invoking {@link #start()} 264 * 265 * @param flags VM options to be applied to the test VM. 266 * @return the same framework instance. 267 */ 268 public TestFramework addFlags(String... flags) { 269 TestRun.check(flags != null && Arrays.stream(flags).noneMatch(Objects::isNull), "A flag cannot be null"); 270 if (this.flags == null) { 271 this.flags = new ArrayList<>(); 272 } 273 this.flags.addAll(Arrays.asList(flags)); 274 return this; 275 } 276 277 /** 278 * Add helper classes that can specify additional compile command annotations ({@link ForceCompile @ForceCompile}, 279 * {@link DontCompile @DontCompile}, {@link ForceInline @ForceInline}, {@link DontInline @DontInline}) to be applied 280 * while testing {@code testClass} (also see description of {@link TestFramework}). 281 * 282 * <p> 283 * Duplicates in {@code helperClasses} are ignored. If a class is used by the test class that does not specify any 284 * compile command annotations, you do not need to include it with this method. If no helper class specifies any 285 * compile commands, you do not need to call this method at all. 286 * 287 * <p> 288 * The testing can be started by invoking {@link #start()}. 289 * 290 * @param helperClasses helper classes containing compile command annotations ({@link ForceCompile}, 291 * {@link DontCompile}, {@link ForceInline}, {@link DontInline}) to be applied 292 * while testing {@code testClass} (also see description of {@link TestFramework}). 293 * @return the same framework instance. 294 */ 295 public TestFramework addHelperClasses(Class<?>... helperClasses) { 296 TestRun.check(helperClasses != null && Arrays.stream(helperClasses).noneMatch(Objects::isNull), 297 "A Helper class cannot be null"); 298 if (this.helperClasses == null) { 299 this.helperClasses = new HashSet<>(); 300 } 301 302 this.helperClasses.addAll(Arrays.asList(helperClasses)); 303 return this; 304 } 305 306 /** 307 * Add scenarios to be used for the test VM. A test VM is called for each scenario in {@code scenarios} by using the 308 * specified VM flags in the scenario. The scenario flags override any flags set by {@link #addFlags(String...)} 309 * and thus also override any VM or Javaoptions set by JTreg by default.<p> 310 * Use {@code -DPreferCommandLineFlags=true} if you want to prefer the VM and Javaoptions over the scenario flags. 311 * 312 * <p> 313 * The testing can be started by invoking {@link #start()} 314 * 315 * @param scenarios scenarios which specify specific flags for the test VM. 316 * @return the same framework instance. 317 */ 318 public TestFramework addScenarios(Scenario... scenarios) { 319 TestFormat.checkAndReport(scenarios != null && Arrays.stream(scenarios).noneMatch(Objects::isNull), 320 "A scenario cannot be null"); 321 if (this.scenarios == null) { 322 this.scenarios = new ArrayList<>(); 323 this.scenarioIndices = new HashSet<>(); 324 } 325 326 for (Scenario scenario : scenarios) { 327 int scenarioIndex = scenario.getIndex(); 328 TestFormat.checkNoThrow(scenarioIndices.add(scenarioIndex), 329 "Cannot define two scenarios with the same index " + scenarioIndex); 330 this.scenarios.add(scenario); 331 } 332 TestFormat.throwIfAnyFailures(); 333 return this; 334 } 335 336 /** 337 * Add test classes to boot classpath. This adds all classes found on path {@link jdk.test.lib.Utils#TEST_CLASSES} 338 * to the boot classpath with "-Xbootclasspath/a". This is useful when trying to run tests in a privileged mode. 339 */ 340 public TestFramework addTestClassesToBootClassPath() { 341 this.testClassesOnBootClassPath = true; 342 return this; 343 } 344 345 /** 346 * Start the testing of the implicitly (by {@link #TestFramework()}) or explicitly (by {@link #TestFramework(Class)}) 347 * set test class. 348 */ 349 public void start() { 350 if (shouldInstallWhiteBox()) { 351 installWhiteBox(); 352 } 353 checkCompatibleFlags(); 354 checkIRRuleCompilePhasesFormat(); 355 disableIRVerificationIfNotFeasible(); 356 357 if (scenarios == null) { 358 try { 359 start(null); 360 } catch (TestVMException e) { 361 System.err.println(System.lineSeparator() + e.getExceptionInfo() + RERUN_HINT); 362 throw e; 363 } catch (IRViolationException e) { 364 System.out.println(e.getCompilations()); 365 System.err.println(System.lineSeparator() + e.getExceptionInfo() + System.lineSeparator() + RERUN_HINT); 366 throw e; 367 } 368 } else { 369 startWithScenarios(); 370 } 371 } 372 373 private void checkIRRuleCompilePhasesFormat() { 374 for (Method method : testClass.getDeclaredMethods()) { 375 for (IR irAnno : method.getAnnotationsByType(IR.class)) { 376 TestFormat.checkNoThrow(irAnno.phase().length > 0, 377 "@IR rule " + irAnno + " must specify a non-empty list of compile " + 378 "phases \"phase\" at " + method); 379 } 380 } 381 TestFormat.throwIfAnyFailures(); 382 } 383 384 /** 385 * Try to load the Whitebox class from the user directory with a custom class loader. If the user has already built the 386 * Whitebox, we can load it. Otherwise, the framework needs to install it. 387 * 388 * @return true if the framework needs to install the Whitebox 389 */ 390 private boolean shouldInstallWhiteBox() { 391 try { 392 URL url = Path.of(System.getProperty("user.dir")).toUri().toURL(); 393 URLClassLoader userDirClassLoader = 394 URLClassLoader.newInstance(new URL[] {url}, TestFramework.class.getClassLoader().getParent()); 395 Class.forName(WhiteBox.class.getName(), false, userDirClassLoader); 396 } catch (MalformedURLException e) { 397 throw new TestFrameworkException("corrupted user.dir property", e); 398 } catch (ClassNotFoundException e) { 399 // We need to manually install the WhiteBox if we cannot load the WhiteBox class from the user directory. 400 // This happens when the user test does not explicitly install the WhiteBox as part of the test. 401 return true; 402 } 403 return false; 404 } 405 406 /** 407 * Set a new default warm-up (overriding the framework default of 2000 at 408 * {@link TestVM#WARMUP_ITERATIONS}) to be applied for all tests that do not specify an explicit 409 * warm-up with {@link Warmup @Warmup}. 410 * 411 * @param defaultWarmup a new non-negative default warm-up. 412 * @return the same framework instance. 413 */ 414 public TestFramework setDefaultWarmup(int defaultWarmup) { 415 TestFormat.checkAndReport(defaultWarmup >= 0, "Cannot specify a negative default warm-up"); 416 this.defaultWarmup = defaultWarmup; 417 return this; 418 } 419 420 /** 421 * In rare cases, methods may not be compilable because of a compilation bailout. By default, this leads to a 422 * test failure. However, if such cases are expected in multiple methods in a test class, this flag can be set to 423 * true, which allows any test to pass even if there is a compilation bailout. If only selected methods are prone 424 * to bail out, it is preferred to use {@link Test#allowNotCompilable()} instead for more fine-grained control. 425 * By setting this flag, any associated {@link IR} rule of a test is only executed if the test method was compiled, 426 * and else it is ignored silently. 427 */ 428 public TestFramework allowNotCompilable() { 429 this.isAllowNotCompilable = true; 430 return this; 431 } 432 433 /** 434 * Get the VM output of the test VM. Use {@code -DVerbose=true} to enable more debug information. If scenarios 435 * were run, use {@link Scenario#getTestVMOutput()}. 436 * 437 * @return the last test VM output. 438 */ 439 public static String getLastTestVMOutput() { 440 return TestVMProcess.getLastTestVMOutput(); 441 } 442 443 /* 444 * The following methods are only intended to be called from actual @Test methods and not from the main() method of 445 * a JTreg test. Calling these methods from main() results in a linking exception (Whitebox not yet loaded and enabled). 446 */ 447 448 /** 449 * Compile {@code m} at compilation level {@code compLevel}. {@code m} is first enqueued and might not be compiled, 450 * yet, upon returning from this method. 451 * 452 * @param m the method to be compiled. 453 * @param compLevel the (valid) compilation level at which the method should be compiled. 454 * @throws TestRunException if compilation level is {@link CompLevel#SKIP} or {@link CompLevel#WAIT_FOR_COMPILATION}. 455 */ 456 public static void compile(Method m, CompLevel compLevel) { 457 TestVM.compile(m, compLevel); 458 } 459 460 /** 461 * Deoptimize {@code m}. 462 * 463 * @param m the method to be deoptimized. 464 */ 465 public static void deoptimize(Method m) { 466 TestVM.deoptimize(m); 467 } 468 469 /** 470 * Returns a boolean indicating if {@code m} is compiled at any level. 471 * 472 * @param m the method to be checked. 473 * @return {@code true} if {@code m} is compiled at any level; 474 * {@code false} otherwise. 475 */ 476 public static boolean isCompiled(Method m) { 477 return TestVM.isCompiled(m); 478 } 479 480 /** 481 * Returns a boolean indicating if {@code m} is compiled with C1. 482 * 483 * @param m the method to be checked. 484 * @return {@code true} if {@code m} is compiled with C1; 485 * {@code false} otherwise. 486 */ 487 public static boolean isC1Compiled(Method m) { 488 return TestVM.isC1Compiled(m); 489 } 490 491 /** 492 * Returns a boolean indicating if {@code m} is compiled with C2. 493 * 494 * @param m the method to be checked. 495 * @return {@code true} if {@code m} is compiled with C2; 496 * {@code false} otherwise. 497 */ 498 public static boolean isC2Compiled(Method m) { 499 return TestVM.isC2Compiled(m); 500 } 501 502 /** 503 * Returns a boolean indicating if {@code m} is compiled at the specified {@code compLevel}. 504 * 505 * @param m the method to be checked. 506 * @param compLevel the compilation level. 507 * @return {@code true} if {@code m} is compiled at {@code compLevel}; 508 * {@code false} otherwise. 509 */ 510 public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) { 511 return TestVM.isCompiledAtLevel(m, compLevel); 512 } 513 514 /** 515 * Checks if {@code m} is compiled at any level. 516 * 517 * @param m the method to be checked. 518 * @throws TestRunException if {@code m} is not compiled at any level. 519 */ 520 public static void assertCompiled(Method m) { 521 TestVM.assertCompiled(m); 522 } 523 524 /** 525 * Checks if {@code m} is not compiled at any level. 526 * 527 * @param m the method to be checked. 528 * @throws TestRunException if {@code m} is compiled at any level. 529 */ 530 public static void assertNotCompiled(Method m) { 531 TestVM.assertNotCompiled(m); 532 } 533 534 /** 535 * Verifies that {@code m} is compiled with C1. 536 * 537 * @param m the method to be verified. 538 * @throws TestRunException if {@code m} is not compiled with C1. 539 */ 540 public static void assertCompiledByC1(Method m) { 541 TestVM.assertCompiledByC1(m); 542 } 543 544 /** 545 * Verifies that {@code m} is compiled with C2. 546 * 547 * @param m the method to be checked. 548 * @throws TestRunException if {@code m} is not compiled with C2. 549 */ 550 public static void assertCompiledByC2(Method m) { 551 TestVM.assertCompiledByC2(m); 552 } 553 554 /** 555 * Verifies that {@code m} is compiled at the specified {@code compLevel}. 556 * 557 * @param m the method to be checked. 558 * @param compLevel the compilation level. 559 * @throws TestRunException if {@code m} is not compiled at {@code compLevel}. 560 */ 561 public static void assertCompiledAtLevel(Method m, CompLevel compLevel) { 562 TestVM.assertCompiledAtLevel(m, compLevel); 563 } 564 565 /** 566 * Verifies that {@code m} was deoptimized after being C1 compiled. 567 * 568 * @param m the method to be checked. 569 * @throws TestRunException if {@code m} is was not deoptimized after being C1 compiled. 570 */ 571 public static void assertDeoptimizedByC1(Method m) { 572 TestVM.assertDeoptimizedByC1(m); 573 } 574 575 /** 576 * Verifies that {@code m} was deoptimized after being C2 compiled. 577 * 578 * @param m the method to be checked. 579 * @throws TestRunException if {@code m} is was not deoptimized after being C2 compiled. 580 */ 581 public static void assertDeoptimizedByC2(Method m) { 582 TestVM.assertDeoptimizedByC2(m); 583 } 584 585 /** 586 * Returns a different boolean each time this method is invoked (switching between {@code false} and {@code true}). 587 * The very first invocation returns {@code false}. Note that this method could be used by different tests and 588 * thus the first invocation for a test could be {@code true} or {@code false} depending on how many times 589 * other tests have already invoked this method. 590 * 591 * @return an inverted boolean of the result of the last invocation of this method. 592 */ 593 public static boolean toggleBoolean() { 594 toggleBool = !toggleBool; 595 return toggleBool; 596 } 597 598 /* 599 * End of public interface methods 600 */ 601 602 /** 603 * Used to move Whitebox class to the right folder in the JTreg test 604 */ 605 private void installWhiteBox() { 606 try { 607 ClassFileInstaller.main(WhiteBox.class.getName()); 608 } catch (Exception e) { 609 throw new Error("failed to install whitebox classes", e); 610 } 611 } 612 613 /** 614 * Disable IR verification completely in certain cases. 615 */ 616 private void disableIRVerificationIfNotFeasible() { 617 if (!irVerificationPossible) { 618 return; 619 } 620 621 boolean debugTest = Platform.isDebugBuild(); 622 boolean intTest = !Platform.isInt(); 623 boolean compTest = !Platform.isComp(); 624 boolean irTest = hasIRAnnotations(); 625 // No IR verification is done if additional non-whitelisted JTreg VM or Javaoptions flag is specified. 626 List<String> nonWhiteListedFlags = anyNonWhitelistedJTregVMAndJavaOptsFlags(); 627 boolean nonWhiteListedTest = nonWhiteListedFlags.isEmpty(); 628 629 irVerificationPossible = debugTest && intTest && compTest && irTest && nonWhiteListedTest; 630 if (irVerificationPossible) { 631 return; 632 } 633 634 System.out.println("IR verification disabled due to the following reason(s):"); 635 if (!debugTest) { 636 System.out.println("- Not running a debug build (required for PrintIdeal and PrintOptoAssembly)"); 637 } 638 if (!intTest) { 639 System.out.println("- Running with -Xint (no compilations)"); 640 } 641 if (!compTest) { 642 System.out.println("- Running with -Xcomp (use warm-up of 0 instead)"); 643 } 644 if (!irTest) { 645 System.out.println("- Test " + testClass + " not specifying any @IR annotations"); 646 } 647 if (!nonWhiteListedTest) { 648 System.out.println("- Using non-whitelisted JTreg VM or Javaoptions flag(s):"); 649 nonWhiteListedFlags.forEach((f) -> System.out.println(" - " + f)); 650 } 651 652 System.out.println(""); 653 } 654 655 /** 656 * For scenarios: Run the tests with the scenario settings and collect all exceptions to be able to run all 657 * scenarios without prematurely throwing an exception. Format violations, however, are wrong for all scenarios 658 * and thus is reported immediately on the first scenario execution. 659 */ 660 private void startWithScenarios() { 661 Map<Scenario, Exception> exceptionMap = new TreeMap<>(Comparator.comparingInt(Scenario::getIndex)); 662 for (Scenario scenario : scenarios) { 663 try { 664 start(scenario); 665 } catch (TestFormatException e) { 666 // Test format violation is wrong for all the scenarios. Only report once. 667 throw e; 668 } catch (Exception e) { 669 exceptionMap.put(scenario, e); 670 } 671 } 672 if (!exceptionMap.isEmpty()) { 673 reportScenarioFailures(exceptionMap); 674 } 675 } 676 677 private void reportScenarioFailures(Map<Scenario, Exception> exceptionMap) { 678 String failedScenarios = "The following scenarios have failed: #" 679 + exceptionMap.keySet().stream() 680 .map(s -> String.valueOf(s.getIndex())) 681 .collect(Collectors.joining(", #")); 682 StringBuilder builder = new StringBuilder(failedScenarios); 683 builder.append(System.lineSeparator()).append(System.lineSeparator()); 684 for (Map.Entry<Scenario, Exception> entry : exceptionMap.entrySet()) { 685 Exception e = entry.getValue(); 686 Scenario scenario = entry.getKey(); 687 String errorMsg = ""; 688 if (scenario != null) { 689 errorMsg = getScenarioTitleAndFlags(scenario); 690 } 691 if (e instanceof IRViolationException irException) { 692 // For IR violations, only show the actual violations and not the (uninteresting) stack trace. 693 if (scenario != null) { 694 System.out.println("Scenario #" + scenario.getIndex()); 695 } 696 System.out.println(irException.getCompilations()); 697 builder.append(errorMsg).append(System.lineSeparator()).append(irException.getExceptionInfo()); 698 } else if (e instanceof TestVMException testVMException) { 699 builder.append(errorMsg).append(System.lineSeparator()).append(testVMException.getExceptionInfo()); 700 } else { 701 // Print stack trace otherwise 702 StringWriter errors = new StringWriter(); 703 e.printStackTrace(new PrintWriter(errors)); 704 builder.append(errors); 705 } 706 builder.append(System.lineSeparator()); 707 } 708 System.err.println(builder); 709 if (!VERBOSE && !REPORT_STDOUT && !TESTLIST && !EXCLUDELIST) { 710 // Provide a hint to the user how to get additional output/debugging information. 711 System.err.println(RERUN_HINT); 712 } 713 throw new TestRunException(failedScenarios + ". Please check stderr for more information."); 714 } 715 716 private static String getScenarioTitleAndFlags(Scenario scenario) { 717 StringBuilder builder = new StringBuilder(); 718 String title = "Scenario #" + scenario.getIndex(); 719 builder.append(title).append(System.lineSeparator()).append("=".repeat(title.length())) 720 .append(System.lineSeparator()); 721 builder.append("Scenario flags: [").append(String.join(", ", scenario.getFlags())).append("]") 722 .append(System.lineSeparator()); 723 return builder.toString(); 724 } 725 726 /** 727 * Execute a separate "flag" VM with White Box access to determine all test VM flags. The flag VM sends an encoding of 728 * all required flags for the test VM to the driver VM over a socket. Once the flag VM exits, this driver VM parses the 729 * test VM flags, which also determine if IR matching should be done, and then starts the test VM to execute all tests. 730 */ 731 private void start(Scenario scenario) { 732 if (scenario != null && !scenario.isEnabled()) { 733 System.out.println("Disabled scenario #" + scenario.getIndex() + "! This scenario is not present in set flag " + 734 "-DScenarios and is therefore not executed."); 735 return; 736 } 737 shouldVerifyIR = irVerificationPossible; 738 try { 739 // Use TestFramework flags and scenario flags for new VMs. 740 List<String> additionalFlags = new ArrayList<>(); 741 if (flags != null) { 742 additionalFlags.addAll(flags); 743 } 744 if (scenario != null) { 745 List<String> scenarioFlags = scenario.getFlags(); 746 String scenarioFlagsString = scenarioFlags.isEmpty() ? "" : " - [" + String.join(", ", scenarioFlags) + "]"; 747 System.out.println("Scenario #" + scenario.getIndex() + scenarioFlagsString + ":"); 748 additionalFlags.addAll(scenarioFlags); 749 } 750 String frameworkAndScenarioFlags = additionalFlags.isEmpty() ? 751 "" : " - [" + String.join(", ", additionalFlags) + "]"; 752 753 if (shouldVerifyIR) { 754 // Only need to use flag VM if an IR verification is possibly done. 755 System.out.println("Run Flag VM:"); 756 FlagVMProcess flagVMProcess = new FlagVMProcess(testClass, additionalFlags); 757 shouldVerifyIR = flagVMProcess.shouldVerifyIR(); 758 if (shouldVerifyIR) { 759 // Add more flags for the test VM which are required to do IR verification. 760 additionalFlags.addAll(flagVMProcess.getTestVMFlags()); 761 } // else: Flag VM found a reason to not do IR verification. 762 } else { 763 System.out.println("Skip Flag VM due to not performing IR verification."); 764 } 765 766 System.out.println("Run Test VM" + frameworkAndScenarioFlags + ":"); 767 runTestVM(additionalFlags); 768 } finally { 769 if (scenario != null) { 770 scenario.setTestVMOutput(TestVMProcess.getLastTestVMOutput()); 771 } 772 System.out.println(); 773 } 774 } 775 776 private boolean hasIRAnnotations() { 777 return Arrays.stream(testClass.getDeclaredMethods()).anyMatch(m -> m.getAnnotationsByType(IR.class).length > 0); 778 } 779 780 private void checkCompatibleFlags() { 781 for (String flag : Utils.getTestJavaOpts()) { 782 if (flag.contains("-agentpath")) { 783 throw new SkippedException("Can't run test with agent."); 784 } 785 } 786 } 787 788 private List<String> anyNonWhitelistedJTregVMAndJavaOptsFlags() { 789 List<String> flags = Arrays.stream(Utils.getTestJavaOpts()) 790 .map(s -> s.replaceFirst("-XX:[+|-]?|-(?=[^D|^e])", "")) 791 .collect(Collectors.toList()); 792 List<String> nonWhiteListedFlags = new ArrayList(); 793 for (String flag : flags) { 794 if (flag.contains("agentpath")) { 795 throw new SkippedException("Can't run test with -javaagent"); 796 } 797 // Property flags (prefix -D), -ea and -esa are whitelisted. 798 if (!flag.startsWith("-D") && !flag.startsWith("-e") && JTREG_WHITELIST_FLAGS.stream().noneMatch(flag::contains)) { 799 // Found VM flag that is not whitelisted 800 nonWhiteListedFlags.add(flag); 801 } 802 } 803 return nonWhiteListedFlags; 804 } 805 806 private void runTestVM(List<String> additionalFlags) { 807 TestVMProcess testVMProcess = new TestVMProcess(additionalFlags, testClass, helperClasses, defaultWarmup, 808 isAllowNotCompilable, testClassesOnBootClassPath); 809 if (shouldVerifyIR) { 810 try { 811 TestClassParser testClassParser = new TestClassParser(testClass, isAllowNotCompilable); 812 Matchable testClassMatchable = testClassParser.parse(testVMProcess.getHotspotPidFileName(), 813 testVMProcess.getIrEncoding()); 814 IRMatcher matcher = new IRMatcher(testClassMatchable); 815 matcher.match(); 816 } catch (IRViolationException e) { 817 e.addCommandLine(testVMProcess.getCommandLine()); 818 throw e; 819 } 820 } else { 821 System.out.println("IR verification disabled either due to no @IR annotations, through explicitly setting " + 822 "-DVerify=false, due to not running a debug build, using a non-whitelisted JTreg VM or " + 823 "Javaopts flag like -Xint, or running the test VM with other VM flags added by user code " + 824 "that make the IR verification impossible (e.g. -XX:-UseCompile, " + 825 "-XX:TieredStopAtLevel=[1,2,3], etc.)."); 826 } 827 } 828 829 public static void check(boolean test, String failureMessage) { 830 if (!test) { 831 throw new TestFrameworkException(failureMessage); 832 } 833 } 834 }