1 /*
2 * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 package compiler.lib.ir_framework.test;
25
26 import compiler.lib.ir_framework.*;
27 import compiler.lib.ir_framework.Compiler;
28 import compiler.lib.ir_framework.shared.*;
29 import jdk.test.lib.Platform;
30 import jdk.test.lib.Utils;
31 import jdk.test.whitebox.WhiteBox;
32
33 import java.io.PrintWriter;
34 import java.io.StringWriter;
35 import java.lang.annotation.Annotation;
36 import java.lang.reflect.*;
37 import java.util.*;
38 import java.util.stream.Collectors;
39 import java.util.stream.Stream;
40
41 import static compiler.lib.ir_framework.shared.TestFrameworkSocket.PRINT_TIMES_TAG;
42
43 /**
44 * This class' main method is called from {@link TestFramework} and represents the so-called "test VM". The class is
45 * the heart of the framework and is responsible for executing all the specified tests in the test class. It uses the
46 * Whitebox API and reflection to achieve this task.
47 */
48 public class TestVM {
49 private static final WhiteBox WHITE_BOX;
50
51 static {
52 try {
53 WHITE_BOX = WhiteBox.getWhiteBox();
54 } catch (UnsatisfiedLinkError e) {
55 System.err.println(System.lineSeparator() + """
56 ##########################################################
57 - Did you call a test-related interface method from
58 TestFramework in main() of your test? Make sure to
59 only call setup/run methods and no checks or
60 assertions from main() of your test!
61 - Are you rerunning the test VM (TestVM class)
62 directly after a JTreg run? Make sure to start it
63 from within JTwork/scratch and with the flag
64 -DReproduce=true!
65 ##########################################################
66 """);
67 throw e;
68 }
69 }
70
71 /**
72 * The default number of warm-up iterations used to warm up a {@link Test} annotated test method.
73 * Use {@code -DWarmup=XY} to specify a different default value. An individual warm-up can also be
74 * set by specifying a {@link Warmup} iteration for a test.
75 */
76 public static final int WARMUP_ITERATIONS = Integer.parseInt(System.getProperty("Warmup", "2000"));
77
78 private static final boolean ALLOW_METHOD_NOT_COMPILABLE = Boolean.getBoolean("AllowNotCompilable");
79 private static final boolean TIERED_COMPILATION = (Boolean)WHITE_BOX.getVMFlag("TieredCompilation");
80 private static final CompLevel TIERED_COMPILATION_STOP_AT_LEVEL;
81 private static final boolean CLIENT_VM = Platform.isClient();
82
83 static {
84 CompLevel level = CompLevel.forValue(((Long)WHITE_BOX.getVMFlag("TieredStopAtLevel")).intValue());
85 if (CLIENT_VM && level == CompLevel.C2) {
86 // No C2 available, use C1 level without profiling.
87 level = CompLevel.C1_SIMPLE;
88 }
89 TIERED_COMPILATION_STOP_AT_LEVEL = level;
90 }
91 public static final boolean TEST_C1 = (TIERED_COMPILATION && TIERED_COMPILATION_STOP_AT_LEVEL.getValue() < CompLevel.C2.getValue()) || CLIENT_VM;
92
93 static final boolean XCOMP = Platform.isComp();
94 static final boolean VERBOSE = Boolean.getBoolean("Verbose");
95 private static final boolean PRINT_TIMES = Boolean.getBoolean("PrintTimes");
96 public static final boolean USE_COMPILER = WHITE_BOX.getBooleanVMFlag("UseCompiler");
97 static final boolean EXCLUDE_RANDOM = Boolean.getBoolean("ExcludeRandom");
98 private static final String TESTLIST = System.getProperty("Test", "");
99 private static final String EXCLUDELIST = System.getProperty("Exclude", "");
100 private static final boolean DUMP_REPLAY = Boolean.getBoolean("DumpReplay");
101 private static final boolean GC_AFTER = Boolean.getBoolean("GCAfter");
102 private static final boolean SHUFFLE_TESTS = Boolean.parseBoolean(System.getProperty("ShuffleTests", "true"));
103 // Use separate flag as VERIFY_IR could have been set by user but due to other flags it was disabled by flag VM.
104 private static final boolean PRINT_VALID_IR_RULES = Boolean.getBoolean("ShouldDoIRVerification");
105 protected static final long PER_METHOD_TRAP_LIMIT = (Long)WHITE_BOX.getVMFlag("PerMethodTrapLimit");
106 protected static final boolean PROFILE_INTERPRETER = (Boolean)WHITE_BOX.getVMFlag("ProfileInterpreter");
107 protected static final boolean DEOPT_BARRIERS_ALOT = (Boolean)WHITE_BOX.getVMFlag("DeoptimizeNMethodBarriersALot");
108 private static final boolean FLIP_C1_C2 = Boolean.getBoolean("FlipC1C2");
109 private static final boolean IGNORE_COMPILER_CONTROLS = Boolean.getBoolean("IgnoreCompilerControls");
110
111 private final HashMap<Method, DeclaredTest> declaredTests = new HashMap<>();
112 private final List<AbstractTest> allTests = new ArrayList<>();
113 private final HashMap<String, Method> testMethodMap = new HashMap<>();
114 private final HashMap<String, Method> setupMethodMap = new HashMap<>();
115 private final List<String> excludeList;
116 private final List<String> testList;
117 private Set<Class<?>> helperClasses = null; // Helper classes that contain framework annotations to be processed.
118 private final IREncodingPrinter irMatchRulePrinter;
119 private final Class<?> testClass;
120 private final Map<Executable, CompLevel> forceCompileMap = new HashMap<>();
121
122 private TestVM(Class<?> testClass) {
123 TestRun.check(testClass != null, "Test class cannot be null");
124 this.testClass = testClass;
125 this.testList = createTestFilterList(TESTLIST, testClass);
126 this.excludeList = createTestFilterList(EXCLUDELIST, testClass);
127
128 if (PRINT_VALID_IR_RULES) {
129 irMatchRulePrinter = new IREncodingPrinter();
130 } else {
131 irMatchRulePrinter = null;
132 }
133 }
134
135 /**
136 * Parse "test1,test2,test3" into a list.
137 */
138 private static List<String> createTestFilterList(String list, Class<?> testClass) {
139 List<String> filterList = null;
140 if (!list.isEmpty()) {
141 String classPrefix = testClass.getSimpleName() + ".";
142 filterList = new ArrayList<>(Arrays.asList(list.split(",")));
143 for (int i = filterList.size() - 1; i >= 0; i--) {
144 String test = filterList.get(i);
145 if (test.indexOf(".") > 0) {
146 if (test.startsWith(classPrefix)) {
147 test = test.substring(classPrefix.length());
148 filterList.set(i, test);
149 } else {
150 filterList.remove(i);
151 }
152 }
153 }
154 }
155 return filterList;
156 }
157
158 /**
159 * Main entry point of the test VM.
160 */
161 public static void main(String[] args) {
162 try {
163 String testClassName = args[0];
164 System.out.println("TestVM main() called - about to run tests in class " + testClassName);
165 Class<?> testClass = getClassObject(testClassName, "test");
166
167 TestVM framework = new TestVM(testClass);
168 framework.addHelperClasses(args);
169 framework.start();
170 } finally {
171 TestFrameworkSocket.closeClientSocket();
172 }
173 }
174
175 protected static Class<?> getClassObject(String className, String classType) {
176 try {
177 return Class.forName(className);
178 } catch (Exception e) {
179 throw new TestRunException("Could not find " + classType + " class", e);
180 }
181 }
182
183 /**
184 * Set up all helper classes and verify they are specified correctly.
185 */
186 private void addHelperClasses(String[] args) {
187 Class<?>[] helperClassesList = getHelperClasses(args);
188 if (helperClassesList != null) {
189 TestRun.check(Arrays.stream(helperClassesList).noneMatch(Objects::isNull), "A Helper class cannot be null");
190 this.helperClasses = new HashSet<>();
191
192 for (Class<?> helperClass : helperClassesList) {
193 if (Arrays.stream(testClass.getDeclaredClasses()).anyMatch(c -> c == helperClass)) {
194 // Nested class of test class is automatically treated as helper class
195 TestFormat.failNoThrow("Nested " + helperClass + " inside test " + testClass + " is implicitly"
196 + " treated as helper class and does not need to be specified as such.");
197 continue;
198 }
199 TestRun.check(!this.helperClasses.contains(helperClass), "Cannot add the same class twice: " + helperClass);
200 this.helperClasses.add(helperClass);
201 }
202 }
203 }
204
205 private static Class<?>[] getHelperClasses(String[] args) {
206 if (args.length == 1) {
207 return null;
208 }
209 Class<?>[] helperClasses = new Class<?>[args.length - 1]; // First argument is test class
210 for (int i = 1; i < args.length; i++) {
211 String helperClassName = args[i];
212 helperClasses[i - 1] = getClassObject(helperClassName, "helper");
213 }
214 return helperClasses;
215 }
216
217 private void checkHelperClass(Class<?> clazz) {
218 checkAnnotationsInClass(clazz, "helper");
219 for (Class<?> c : clazz.getDeclaredClasses()) {
220 checkAnnotationsInClass(c, "nested (and helper)");
221 }
222 }
223
224 private void checkAnnotationsInClass(Class<?> c, String clazzType) {
225 Method[] methods = c.getDeclaredMethods();
226 for (Method m : methods) {
227 TestFormat.checkNoThrow(getAnnotation(m, Test.class) == null,
228 "Cannot use @Test annotation in " + clazzType + " " + c + " at " + m);
229 TestFormat.checkNoThrow(getAnnotation(m, Run.class) == null,
230 "Cannot use @Run annotation in " + clazzType + " " + c + " at " + m);
231 TestFormat.checkNoThrow(getAnnotation(m, Check.class) == null,
232 "Cannot use @Check annotation in " + clazzType + " " + c + " at " + m);
233 TestFormat.checkNoThrow(getAnnotation(m, Setup.class) == null,
234 "Cannot use @Setup annotation in " + clazzType + " " + c + " at " + m);
235 }
236 }
237
238 /**
239 * Only called by internal tests testing the framework itself. Accessed by reflection. Not exposed to normal users.
240 */
241 private static void runTestsOnSameVM(Class<?> testClass) {
242 if (testClass == null) {
243 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
244 testClass = walker.getCallerClass();
245 }
246 TestVM framework = new TestVM(testClass);
247 framework.start();
248 }
249
250 /**
251 * Once everything is initialized and set up, start collecting tests and executing them afterwards.
252 */
253 private void start() {
254 setupTests();
255 checkForcedCompilationsCompleted();
256 runTests();
257 }
258
259 private void setupTests() {
260 // TODO remove this once JDK-8273591 is fixed
261 if (!IGNORE_COMPILER_CONTROLS) {
262 for (Class<?> clazz : testClass.getDeclaredClasses()) {
263 checkAnnotationsInClass(clazz, "inner");
264 }
265 }
266 if (DUMP_REPLAY) {
267 addReplay();
268 }
269
270 // Collect the @Setup methods so we can reference them
271 // from the test methods
272 collectSetupMethods();
273
274 // Make sure to first setup test methods and make them non-inlineable and only then process compile commands.
275 setupDeclaredTests();
276 processControlAnnotations(testClass);
277 processHelperClasses();
278 setupCheckedAndCustomRunTests();
279
280 // All remaining tests are simple base tests without check or specific way to run them.
281 addBaseTests();
282 if (PRINT_VALID_IR_RULES) {
283 irMatchRulePrinter.emit();
284 VMInfoPrinter.emit();
285 }
286 TestFormat.throwIfAnyFailures();
287 declaredTests.clear();
288 testMethodMap.clear();
289 }
290
291 private void addBaseTests() {
292 declaredTests.forEach((m, test) -> {
293 if (test.getAttachedMethod() == null) {
294 try {
295 Arguments argumentsAnno = getAnnotation(m, Arguments.class);
296 TestFormat.check(argumentsAnno != null || m.getParameterCount() == 0, "Missing @Arguments annotation to define arguments of " + m);
297 BaseTest baseTest = new BaseTest(test, shouldExcludeTest(m.getName()));
298 allTests.add(baseTest);
299 if (PRINT_VALID_IR_RULES) {
300 irMatchRulePrinter.emitRuleEncoding(m, baseTest.isSkipped());
301 }
302 } catch (TestFormatException e) {
303 // Failure logged. Continue and report later.
304 }
305 }
306 });
307 }
308
309 /**
310 * Check if user wants to exclude this test by checking the -DTest and -DExclude lists.
311 */
312 private boolean shouldExcludeTest(String testName) {
313 boolean hasTestList = testList != null;
314 boolean hasExcludeList = excludeList != null;
315 if (hasTestList) {
316 return !testList.contains(testName) || (hasExcludeList && excludeList.contains(testName));
317 } else if (hasExcludeList) {
318 return excludeList.contains(testName);
319 }
320 return false;
321 }
322
323 /**
324 * Generate replay compilation files.
325 */
326 private void addReplay() {
327 String directive = "[{ match: \"*.*\", DumpReplay: true }]";
328 TestFramework.check(WHITE_BOX.addCompilerDirective(directive) == 1, "Failed to add DUMP_REPLAY directive");
329 }
330
331 private void processControlAnnotations(Class<?> clazz) {
332 if (IGNORE_COMPILER_CONTROLS) {
333 return;
334 }
335 // Also apply compile commands to all inner classes of 'clazz'.
336 ArrayList<Class<?>> classes = new ArrayList<>(Arrays.asList(clazz.getDeclaredClasses()));
337 classes.add(clazz);
338 for (Class<?> c : classes) {
339 applyClassAnnotations(c);
340 List<Executable> executables = new ArrayList<>(Arrays.asList(c.getDeclaredMethods()));
341 Collections.addAll(executables, c.getDeclaredConstructors());
342 for (Executable ex : executables) {
343 checkClassAnnotations(ex);
344 try {
345 applyIndependentCompilationCommands(ex);
346 } catch (TestFormatException e) {
347 // Failure logged. Continue and report later.
348 }
349 }
350
351 // Only force compilation now because above annotations affect inlining
352 for (Executable ex : executables) {
353 try {
354 applyForceCompileCommand(ex);
355 } catch (TestFormatException e) {
356 // Failure logged. Continue and report later.
357 }
358 }
359 }
360 }
361
362 private void applyClassAnnotations(Class<?> c) {
363 ForceCompileClassInitializer anno = getAnnotation(c, ForceCompileClassInitializer.class);
364 if (anno == null) {
365 return;
366 }
367
368 // Compile class initializer
369 CompLevel level = anno.value();
370 if (level == CompLevel.SKIP || level == CompLevel.WAIT_FOR_COMPILATION) {
371 TestFormat.failNoThrow("Cannot define compilation level SKIP or WAIT_FOR_COMPILATION in " +
372 "@ForceCompileClassInitializer at " + c);
373 return;
374 }
375 level = restrictCompLevel(anno.value());
376 if (level != CompLevel.SKIP) {
377 // Make sure class is initialized to avoid compilation bailout of <clinit>
378 getClassObject(c.getName(), "nested"); // calls Class.forName() to initialize 'c'
379 TestFormat.checkNoThrow(WHITE_BOX.enqueueInitializerForCompilation(c, level.getValue()),
380 "Failed to enqueue <clinit> of " + c + " for compilation. Did you specify "
381 + "@ForceCompileClassInitializer without providing a static class initialization? "
382 + "Make sure to provide any form of static initialization or remove the annotation. "
383 + "For debugging purposes, -DIgnoreCompilerControls=true can be used to temporarly "
384 + "ignore @ForceCompileClassInitializer annotations.");
385 }
386 }
387
388 private void checkClassAnnotations(Executable ex) {
389 TestFormat.checkNoThrow(getAnnotation(ex, ForceCompileClassInitializer.class) == null,
390 "@ForceCompileClassInitializer only allowed at classes but not at method " + ex);
391 }
392
393 /**
394 * Exclude a method from compilation with a compiler randomly. Return the compiler for which the method was made
395 * not compilable.
396 */
397 public static Compiler excludeRandomly(Executable ex) {
398 Compiler compiler = switch (Utils.getRandomInstance().nextInt() % 3) {
399 case 1 -> Compiler.C1;
400 case 2 -> Compiler.C2;
401 default -> Compiler.ANY;
402 };
403 WHITE_BOX.makeMethodNotCompilable(ex, compiler.getValue(), false);
404 WHITE_BOX.makeMethodNotCompilable(ex, compiler.getValue(), true);
405 System.out.println("Excluding from " + compiler.name() + " compilation: " + ex);
406 return compiler;
407 }
408
409 private void applyIndependentCompilationCommands(Executable ex) {
410 ForceInline forceInlineAnno = getAnnotation(ex, ForceInline.class);
411 DontInline dontInlineAnno = getAnnotation(ex, DontInline.class);
412 ForceCompile forceCompileAnno = getAnnotation(ex, ForceCompile.class);
413 DontCompile dontCompileAnno = getAnnotation(ex, DontCompile.class);
414 checkCompilationCommandAnnotations(ex, forceInlineAnno, dontInlineAnno, forceCompileAnno, dontCompileAnno);
415 // First handle inline annotations
416 if (dontInlineAnno != null) {
417 WHITE_BOX.testSetDontInlineMethod(ex, true);
418 } else if (forceInlineAnno != null) {
419 WHITE_BOX.testSetForceInlineMethod(ex, true);
420 }
421 if (dontCompileAnno != null) {
422 dontCompileWithCompiler(ex, dontCompileAnno.value());
423 }
424 if (EXCLUDE_RANDOM && getAnnotation(ex, Test.class) == null && forceCompileAnno == null && dontCompileAnno == null) {
425 // Randomly exclude helper methods from compilation
426 if (Utils.getRandomInstance().nextBoolean()) {
427 excludeRandomly(ex);
428 }
429 }
430 }
431
432 private void checkCompilationCommandAnnotations(Executable ex, ForceInline forceInlineAnno, DontInline dontInlineAnno, ForceCompile forceCompileAnno, DontCompile dontCompileAnno) {
433 Test testAnno = getAnnotation(ex, Test.class);
434 Run runAnno = getAnnotation(ex, Run.class);
435 Check checkAnno = getAnnotation(ex, Check.class);
436 TestFormat.check((testAnno == null && runAnno == null && checkAnno == null) || Stream.of(forceCompileAnno, dontCompileAnno, forceInlineAnno, dontInlineAnno).noneMatch(Objects::nonNull),
437 "Cannot use explicit compile command annotations (@ForceInline, @DontInline, " +
438 "@ForceCompile or @DontCompile) together with @Test, @Check or @Run: " + ex + ". Use compLevel in @Test for fine tuning.");
439 if (Stream.of(forceInlineAnno, dontCompileAnno, dontInlineAnno).filter(Objects::nonNull).count() > 1) {
440 // Failure
441 TestFormat.check(dontCompileAnno == null || dontInlineAnno == null,
442 "@DontInline is implicitely done with @DontCompile annotation at " + ex);
443 TestFormat.fail("Cannot mix @ForceInline, @DontInline and @DontCompile at the same time at " + ex);
444 }
445 TestFormat.check(forceInlineAnno == null || dontInlineAnno == null, "Cannot have @ForceInline and @DontInline at the same time at " + ex);
446 if (forceCompileAnno != null && dontCompileAnno != null) {
447 CompLevel forceCompileLevel = forceCompileAnno.value();
448 Compiler dontCompileCompiler = dontCompileAnno.value();
449 TestFormat.check(dontCompileCompiler != Compiler.ANY,
450 "Cannot have @DontCompile(Compiler.ANY) and @ForceCompile at the same time at " + ex);
451 TestFormat.check(forceCompileLevel != CompLevel.ANY,
452 "Cannot have @ForceCompile(CompLevel.ANY) and @DontCompile at the same time at " + ex);
453 TestFormat.check(forceCompileLevel.isNotCompilationLevelOfCompiler(dontCompileCompiler),
454 "Overlapping compilation level and compiler with @ForceCompile and @DontCompile at " + ex);
455 }
456 }
457
458 /**
459 * Exlude the method from compilation and make sure it is not inlined.
460 */
461 private void dontCompileAndDontInlineMethod(Method m) {
462 if (!IGNORE_COMPILER_CONTROLS) {
463 WHITE_BOX.makeMethodNotCompilable(m, CompLevel.ANY.getValue(), true);
464 WHITE_BOX.makeMethodNotCompilable(m, CompLevel.ANY.getValue(), false);
465 WHITE_BOX.testSetDontInlineMethod(m, true);
466 }
467 }
468
469 private void dontCompileWithCompiler(Executable ex, Compiler compiler) {
470 if (VERBOSE) {
471 System.out.println("dontCompileWithCompiler " + ex + " , compiler = " + compiler.name());
472 }
473 WHITE_BOX.makeMethodNotCompilable(ex, compiler.getValue(), true);
474 WHITE_BOX.makeMethodNotCompilable(ex, compiler.getValue(), false);
475 if (compiler == Compiler.ANY) {
476 WHITE_BOX.testSetDontInlineMethod(ex, true);
477 }
478 }
479
480 private void applyForceCompileCommand(Executable ex) {
481 ForceCompile forceCompileAnno = getAnnotation(ex, ForceCompile.class);
482 if (forceCompileAnno != null) {
483 CompLevel compLevel = forceCompileAnno.value();
484 TestFormat.check(compLevel != CompLevel.SKIP && compLevel != CompLevel.WAIT_FOR_COMPILATION,
485 "Cannot define compilation level SKIP or WAIT_FOR_COMPILATION in @ForceCompile at " + ex);
486 compLevel = restrictCompLevel(forceCompileAnno.value());
487 if (FLIP_C1_C2) {
488 compLevel = compLevel.flipCompLevel();
489 compLevel = restrictCompLevel(compLevel.flipCompLevel());
490 }
491 if (EXCLUDE_RANDOM) {
492 compLevel = compLevel.excludeCompilationRandomly(ex);
493 }
494 if (compLevel != CompLevel.SKIP) {
495 enqueueForCompilation(ex, compLevel);
496 forceCompileMap.put(ex, compLevel);
497 }
498 }
499 }
500
501 static void enqueueForCompilation(Executable ex, CompLevel requestedCompLevel) {
502 if (TestVM.VERBOSE) {
503 System.out.println("enqueueForCompilation " + ex + ", level = " + requestedCompLevel);
504 }
505 CompLevel compLevel = restrictCompLevel(requestedCompLevel);
506 if (compLevel != CompLevel.SKIP) {
507 WHITE_BOX.enqueueMethodForCompilation(ex, compLevel.getValue());
508 } else {
509 System.out.println("Skipped compilation on level " + requestedCompLevel + " due to VM flags not allowing it.");
510 }
511 }
512
513
514 /**
515 * Collect all @Setup annotated methods and add them to setupMethodMap, for convenience to reference later from
516 * tests with @Arguments(setup = "setupMethodName").
517 */
518 private void collectSetupMethods() {
519 for (Method m : testClass.getDeclaredMethods()) {
520 Setup setupAnnotation = getAnnotation(m, Setup.class);
521 if (setupAnnotation != null) {
522 addSetupMethod(m);
523 }
524 }
525 }
526
527 private void addSetupMethod(Method m) {
528 TestFormat.checkNoThrow(getAnnotation(m, Test.class) == null,
529 "@Setup method cannot have @Test annotation: " + m);
530 TestFormat.checkNoThrow(getAnnotation(m, Check.class) == null,
531 "@Setup method cannot have @Check annotation: " + m);
532 TestFormat.checkNoThrow(getAnnotation(m, Arguments.class) == null,
533 "@Setup method cannot have @Arguments annotation: " + m);
534 TestFormat.checkNoThrow(getAnnotation(m, Run.class) == null,
535 "@Setup method cannot have @Run annotation: " + m);
536 Method mOverloaded = setupMethodMap.put(m.getName(), m);
537 TestFormat.checkNoThrow(mOverloaded == null,
538 "@Setup method cannot be overloaded: " + mOverloaded + " with " + m);
539 m.setAccessible(true);
540 }
541
542 /**
543 * Setup @Test annotated method an add them to the declaredTests map to have a convenient way of accessing them
544 * once setting up a framework test (base checked, or custom run test).
545 */
546 private void setupDeclaredTests() {
547 for (Method m : testClass.getDeclaredMethods()) {
548 Test testAnno = getAnnotation(m, Test.class);
549 try {
550 if (testAnno != null) {
551 addDeclaredTest(m);
552 } else {
553 TestFormat.checkNoThrow(!m.isAnnotationPresent(IR.class) && !m.isAnnotationPresent(IRs.class),
554 "Found @IR annotation on non-@Test method " + m);
555 TestFormat.checkNoThrow(!m.isAnnotationPresent(Warmup.class) || getAnnotation(m, Run.class) != null,
556 "Found @Warmup annotation on non-@Test or non-@Run method " + m);
557 }
558 } catch (TestFormatException e) {
559 // Failure logged. Continue and report later.
560 }
561 }
562 TestFormat.checkNoThrow(!declaredTests.isEmpty(), "Did not specify any @Test methods in " + testClass);
563 }
564
565 private void addDeclaredTest(Method m) {
566 Test testAnno = getAnnotation(m, Test.class);
567 checkTestAnnotations(m, testAnno);
568 Warmup warmup = getAnnotation(m, Warmup.class);
569 int warmupIterations = WARMUP_ITERATIONS;
570 if (warmup != null) {
571 warmupIterations = warmup.value();
572 TestFormat.checkNoThrow(warmupIterations >= 0, "Cannot have negative value for @Warmup at " + m);
573 }
574
575 if (!IGNORE_COMPILER_CONTROLS) {
576 // Don't inline test methods by default. Do not apply this when -DIgnoreCompilerControls=true is set.
577 WHITE_BOX.testSetDontInlineMethod(m, true);
578 }
579 CompLevel compLevel = restrictCompLevel(testAnno.compLevel());
580 if (FLIP_C1_C2) {
581 compLevel = compLevel.flipCompLevel();
582 compLevel = restrictCompLevel(compLevel.flipCompLevel());
583 }
584 if (EXCLUDE_RANDOM) {
585 compLevel = compLevel.excludeCompilationRandomly(m);
586 }
587 boolean allowNotCompilable = testAnno.allowNotCompilable() || ALLOW_METHOD_NOT_COMPILABLE;
588 ArgumentsProvider argumentsProvider = ArgumentsProviderBuilder.build(m, setupMethodMap);
589 DeclaredTest test = new DeclaredTest(m, argumentsProvider, compLevel, warmupIterations, allowNotCompilable);
590 declaredTests.put(m, test);
591 testMethodMap.put(m.getName(), m);
592 }
593
594 private void checkTestAnnotations(Method m, Test testAnno) {
595 TestFormat.check(!testMethodMap.containsKey(m.getName()),
596 "Cannot overload two @Test methods: " + m + ", " + testMethodMap.get(m.getName()));
597 TestFormat.check(testAnno != null, m + " must be a method with a @Test annotation");
598
599 Check checkAnno = getAnnotation(m, Check.class);
600 Run runAnno = getAnnotation(m, Run.class);
601 TestFormat.check(checkAnno == null && runAnno == null,
602 m + " has invalid @Check or @Run annotation while @Test annotation is present.");
603
604 TestFormat.checkNoThrow(Arrays.stream(m.getParameterTypes()).noneMatch(AbstractInfo.class::isAssignableFrom),
605 "Cannot " + AbstractInfo.class + " or any of its subclasses as parameter type at " +
606 "@Test method " + m);
607
608 TestFormat.checkNoThrow(!AbstractInfo.class.isAssignableFrom(m.getReturnType()),
609 "Cannot " + AbstractInfo.class + " or any of its subclasses as return type at " +
610 "@Test method " + m);
611 }
612
613
614 /**
615 * Get the appropriate level as permitted by the test scenario and VM flags.
616 */
617 private static CompLevel restrictCompLevel(CompLevel compLevel) {
618 if (!USE_COMPILER) {
619 return CompLevel.SKIP;
620 }
621 if (compLevel == CompLevel.ANY) {
622 // Use highest available compilation level by default (usually C2).
623 compLevel = TIERED_COMPILATION_STOP_AT_LEVEL;
624 }
625 if (TEST_C1 && compLevel == CompLevel.C2) {
626 return CompLevel.SKIP;
627 }
628 if ((!TIERED_COMPILATION && !CLIENT_VM) && compLevel.getValue() < CompLevel.C2.getValue()) {
629 return CompLevel.SKIP;
630 }
631 if ((TIERED_COMPILATION || CLIENT_VM) && compLevel.getValue() > TIERED_COMPILATION_STOP_AT_LEVEL.getValue()) {
632 return CompLevel.SKIP;
633 }
634 return compLevel;
635 }
636
637 /**
638 * Verify that the helper classes do not contain illegal framework annotations and then apply the actions as
639 * specified by the different helper class annotations.
640 */
641 private void processHelperClasses() {
642 if (helperClasses != null) {
643 for (Class<?> helperClass : helperClasses) {
644 // Process the helper classes and apply the explicit compile commands
645 TestFormat.checkNoThrow(helperClass != testClass,
646 "Cannot specify test " + testClass + " as helper class, too.");
647 checkHelperClass(helperClass);
648 processControlAnnotations(helperClass);
649 }
650 }
651 }
652
653 /**
654 * First set up checked (with @Check) and custom run tests (with @Run). All remaining unmatched/unused @Test methods
655 * are treated as base tests and set up as such later.
656 */
657 private void setupCheckedAndCustomRunTests() {
658 for (Method m : testClass.getDeclaredMethods()) {
659 Check checkAnno = getAnnotation(m, Check.class);
660 Run runAnno = getAnnotation(m, Run.class);
661 Arguments argumentsAnno = getAnnotation(m, Arguments.class);
662 try {
663 TestFormat.check(argumentsAnno == null || (checkAnno == null && runAnno == null),
664 "Cannot have @Argument annotation in combination with @Run or @Check at " + m);
665 if (checkAnno != null) {
666 addCheckedTest(m, checkAnno, runAnno);
667 } else if (runAnno != null) {
668 addCustomRunTest(m, runAnno);
669 }
670 } catch (TestFormatException e) {
671 // Failure logged. Continue and report later.
672 }
673 }
674 }
675
676 /**
677 * Set up a checked test by first verifying the correct format of the @Test and @Check method and then adding it
678 * to the allTests list which keeps track of all framework tests that are eventually executed.
679 */
680 private void addCheckedTest(Method m, Check checkAnno, Run runAnno) {
681 Method testMethod = testMethodMap.get(checkAnno.test());
682 DeclaredTest test = declaredTests.get(testMethod);
683 checkCheckedTest(m, checkAnno, runAnno, testMethod, test);
684 test.setAttachedMethod(m);
685 TestFormat.check(getAnnotation(testMethod, Arguments.class) != null || testMethod.getParameterCount() == 0,
686 "Missing @Arguments annotation to define arguments of " + testMethod + " required by "
687 + "checked test " + m);
688 CheckedTest.Parameter parameter = getCheckedTestParameter(m, testMethod);
689 dontCompileAndDontInlineMethod(m);
690 CheckedTest checkedTest = new CheckedTest(test, m, checkAnno, parameter, shouldExcludeTest(testMethod.getName()));
691 allTests.add(checkedTest);
692 if (PRINT_VALID_IR_RULES) {
693 // Only need to emit IR verification information if IR verification is actually performed.
694 irMatchRulePrinter.emitRuleEncoding(testMethod, checkedTest.isSkipped());
695 }
696 }
697
698 private void checkCheckedTest(Method m, Check checkAnno, Run runAnno, Method testMethod, DeclaredTest test) {
699 TestFormat.check(runAnno == null, m + " has invalid @Run annotation while @Check annotation is present.");
700 TestFormat.check(testMethod != null, "Did not find associated test method \"" + m.getDeclaringClass().getName()
701 + "." + checkAnno.test() + "\" for @Check at " + m);
702 TestFormat.check(test != null, "Missing @Test annotation for associated test method " + testMethod + " for @Check at " + m);
703 Method attachedMethod = test.getAttachedMethod();
704 TestFormat.check(attachedMethod == null,
705 "Cannot use @Test " + testMethod + " for more than one @Run or one @Check method. Found: " + m + ", " + attachedMethod);
706 }
707
708 /**
709 * Only allow parameters as specified in {@link Check}.
710 */
711 private CheckedTest.Parameter getCheckedTestParameter(Method m, Method testMethod) {
712 boolean firstParameterTestInfo = m.getParameterCount() > 0 && m.getParameterTypes()[0].equals(TestInfo.class);
713 boolean secondParameterTestInfo = m.getParameterCount() > 1 && m.getParameterTypes()[1].equals(TestInfo.class);
714 CheckedTest.Parameter parameter = null;
715 Class<?> testReturnType = testMethod.getReturnType();
716 switch (m.getParameterCount()) {
717 case 0 -> parameter = CheckedTest.Parameter.NONE;
718 case 1 -> {
719 TestFormat.checkNoThrow(firstParameterTestInfo || m.getParameterTypes()[0] == testReturnType,
720 "Single-parameter version of @Check method " + m + " must match return type of @Test " + testMethod);
721 parameter = firstParameterTestInfo ? CheckedTest.Parameter.TEST_INFO_ONLY : CheckedTest.Parameter.RETURN_ONLY;
722 }
723 case 2 -> {
724 TestFormat.checkNoThrow(m.getParameterTypes()[0] == testReturnType && secondParameterTestInfo,
725 "Two-parameter version of @Check method " + m + " must provide as first parameter the same"
726 + " return type as @Test method " + testMethod + " and as second parameter an object of " + TestInfo.class);
727 parameter = CheckedTest.Parameter.BOTH;
728 }
729 default -> TestFormat.failNoThrow("@Check method " + m + " must provide either a none, single or two-parameter variant.");
730 }
731 return parameter;
732 }
733
734 /**
735 * Set up a custom run test by first verifying the correct format of the @Test and @Run method and then adding it
736 * to the allTests list which keeps track of all framework tests that are eventually executed.
737 */
738 private void addCustomRunTest(Method m, Run runAnno) {
739 checkRunMethod(m, runAnno);
740 List<DeclaredTest> tests = new ArrayList<>();
741 boolean shouldExcludeTest = true;
742 for (String testName : runAnno.test()) {
743 try {
744 Method testMethod = testMethodMap.get(testName);
745 DeclaredTest test = declaredTests.get(testMethod);
746 checkCustomRunTest(m, testName, testMethod, test, runAnno.mode());
747 test.setAttachedMethod(m);
748 tests.add(test);
749 // Only exclude custom run test if all test methods excluded
750 shouldExcludeTest &= shouldExcludeTest(testMethod.getName());
751 } catch (TestFormatException e) {
752 // Logged, continue.
753 }
754 }
755 if (tests.isEmpty()) {
756 return; // There was a format violation. Return.
757 }
758 dontCompileAndDontInlineMethod(m);
759 CustomRunTest customRunTest = new CustomRunTest(m, getAnnotation(m, Warmup.class), runAnno, tests, shouldExcludeTest);
760 allTests.add(customRunTest);
761 if (PRINT_VALID_IR_RULES) {
762 tests.forEach(test -> irMatchRulePrinter.emitRuleEncoding(test.getTestMethod(), customRunTest.isSkipped()));
763 }
764 }
765
766 /**
767 * Only allow parameters as specified in {@link Run}.
768 */
769 private void checkCustomRunTest(Method m, String testName, Method testMethod, DeclaredTest test, RunMode runMode) {
770 TestFormat.check(testMethod != null, "Did not find associated @Test method \"" + m.getDeclaringClass().getName()
771 + "." + testName + "\" specified in @Run at " + m);
772 TestFormat.check(test != null,
773 "Missing @Test annotation for associated test method " + testName + " for @Run at " + m);
774 Method attachedMethod = test.getAttachedMethod();
775 TestFormat.check(attachedMethod == null,
776 "Cannot use @Test " + testMethod + " for more than one @Run/@Check method. Found: "
777 + m + ", " + attachedMethod);
778 Arguments argumentsAnno = getAnnotation(testMethod, Arguments.class);
779 TestFormat.check(argumentsAnno == null,
780 "Cannot use @Arguments at test method " + testMethod + " in combination with @Run method " + m);
781 Warmup warmupAnno = getAnnotation(testMethod, Warmup.class);
782 TestFormat.checkNoThrow(warmupAnno == null,
783 "Cannot set @Warmup at @Test method " + testMethod + " when used with its @Run method "
784 + m + ". Use @Warmup at @Run method instead.");
785 Test testAnno = getAnnotation(testMethod, Test.class);
786 TestFormat.checkNoThrow(runMode != RunMode.STANDALONE || testAnno.compLevel() == CompLevel.ANY,
787 "Setting explicit compilation level for @Test method " + testMethod + " has no effect "
788 + "when used with STANDALONE @Run method " + m);
789 }
790
791 private void checkRunMethod(Method m, Run runAnno) {
792 TestFormat.check(runAnno.test().length > 0, "@Run method " + m + " must specify at least one test method");
793 TestFormat.checkNoThrow(m.getParameterCount() == 0 || (m.getParameterCount() == 1 && m.getParameterTypes()[0].equals(RunInfo.class)),
794 "@Run method " + m + " must specify either no parameter or exactly one " + RunInfo.class + " parameter.");
795 Warmup warmupAnno = getAnnotation(m, Warmup.class);
796 TestFormat.checkNoThrow(warmupAnno == null || runAnno.mode() != RunMode.STANDALONE,
797 "Cannot set @Warmup at @Run method " + m + " when used with RunMode.STANDALONE. The @Run method is only invoked once.");
798 }
799
800 private static <T extends Annotation> T getAnnotation(AnnotatedElement element, Class<T> c) {
801 T[] annos = element.getAnnotationsByType(c);
802 TestFormat.check(annos.length < 2, element + " has duplicated annotations");
803 return Arrays.stream(annos).findFirst().orElse(null);
804 }
805
806 /**
807 * Ensure that all compilations that were enforced (added to compilation queue) by framework annotations are
808 * completed. Wait if necessary for a short amount of time for their completion.
809 */
810 private void checkForcedCompilationsCompleted() {
811 if (forceCompileMap.isEmpty()) {
812 return;
813 }
814 final long started = System.currentTimeMillis();
815 long elapsed;
816 do {
817 forceCompileMap.entrySet().removeIf(entry -> WHITE_BOX.getMethodCompilationLevel(entry.getKey()) == entry.getValue().getValue());
818 if (forceCompileMap.isEmpty()) {
819 // All @ForceCompile methods are compiled at the requested level.
820 return;
821 }
822 // Retry again if not yet compiled.
823 forceCompileMap.forEach(TestVM::enqueueForCompilation);
824 elapsed = System.currentTimeMillis() - started;
825 } while (elapsed < 5000);
826 StringBuilder builder = new StringBuilder();
827 forceCompileMap.forEach((key, value) -> builder.append("- ").append(key).append(" at CompLevel.").append(value)
828 .append(System.lineSeparator()));
829 throw new TestRunException("Could not force compile the following @ForceCompile methods:"
830 + System.lineSeparator() + builder.toString());
831 }
832
833 /**
834 * Once all framework tests are collected, they are run in this method.
835 */
836 private void runTests() {
837 TreeMap<Long, String> durations = (PRINT_TIMES || VERBOSE) ? new TreeMap<>() : null;
838 long startTime = System.nanoTime();
839 List<AbstractTest> testList;
840 boolean testFilterPresent = testFilterPresent();
841 if (testFilterPresent) {
842 // Only run the specified tests by the user filters -DTest and/or -DExclude.
843 testList = allTests.stream().filter(test -> !test.isSkipped()).collect(Collectors.toList());
844 if (testList.isEmpty()) {
845 // Throw an exception to inform the user about an empty specified test set with -DTest and/or -DExclude
846 throw new NoTestsRunException();
847 }
848 } else {
849 testList = allTests;
850 }
851
852 if (SHUFFLE_TESTS) {
853 // Execute tests in random order (execution sequence affects profiling). This is done by default.
854 Collections.shuffle(testList, Utils.getRandomInstance());
855 }
856 StringBuilder builder = new StringBuilder();
857 int failures = 0;
858
859 // Execute all tests and keep track of each exception that is thrown. These are then reported once all tests
860 // are executing. This prevents a premature exit without running all tests.
861 for (AbstractTest test : testList) {
862 if (VERBOSE) {
863 System.out.println("Run " + test.toString());
864 }
865 if (testFilterPresent) {
866 TestFrameworkSocket.write("Run " + test.toString(), TestFrameworkSocket.TESTLIST_TAG, true);
867 }
868 try {
869 test.run();
870 } catch (TestRunException e) {
871 StringWriter sw = new StringWriter();
872 PrintWriter pw = new PrintWriter(sw);
873 e.printStackTrace(pw);
874 builder.append(test.toString()).append(":").append(System.lineSeparator()).append(sw.toString())
875 .append(System.lineSeparator()).append(System.lineSeparator());
876 failures++;
877 }
878 if (PRINT_TIMES || VERBOSE) {
879 long endTime = System.nanoTime();
880 long duration = (endTime - startTime);
881 durations.put(duration, test.getName());
882 if (VERBOSE) {
883 System.out.println("Done " + test.getName() + ": " + duration + " ns = " + (duration / 1000000) + " ms");
884 }
885 }
886 if (GC_AFTER) {
887 System.out.println("doing GC");
888 WHITE_BOX.fullGC();
889 }
890 }
891
892 // Print execution times
893 if (VERBOSE || PRINT_TIMES) {
894 TestFrameworkSocket.write("Test execution times:", PRINT_TIMES_TAG, true);
895 for (Map.Entry<Long, String> entry : durations.entrySet()) {
896 TestFrameworkSocket.write(String.format("%-25s%15d ns%n", entry.getValue() + ":", entry.getKey()),
897 PRINT_TIMES_TAG, true);
898 }
899 }
900
901 if (failures > 0) {
902 // Finally, report all occurred exceptions in a nice format.
903 String msg = System.lineSeparator() + System.lineSeparator() + "Test Failures (" + failures + ")"
904 + System.lineSeparator() + "----------------" + "-".repeat(String.valueOf(failures).length());
905 throw new TestRunException(msg + System.lineSeparator() + builder.toString());
906 }
907 }
908
909 private boolean testFilterPresent() {
910 return testList != null || excludeList != null;
911 }
912
913 enum TriState {
914 Maybe,
915 Yes,
916 No
917 }
918
919 public static void compile(Method m, CompLevel compLevel) {
920 TestRun.check(compLevel != CompLevel.SKIP && compLevel != CompLevel.WAIT_FOR_COMPILATION,
921 "Invalid compilation request with level " + compLevel);
922 enqueueForCompilation(m, compLevel);
923 }
924
925 public static void deoptimize(Method m) {
926 WHITE_BOX.deoptimizeMethod(m);
927 }
928
929 public static boolean isCompiled(Method m) {
930 return compiledAtLevel(m, CompLevel.ANY) == TriState.Yes;
931 }
932
933 public static boolean isC1Compiled(Method m) {
934 return compiledByC1(m) == TriState.Yes;
935 }
936
937 public static boolean isC2Compiled(Method m) {
938 return compiledByC2(m) == TriState.Yes;
939 }
940
941 public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
942 return compiledAtLevel(m, compLevel) == TriState.Yes;
943 }
944
945 public static void assertDeoptimizedByC1(Method m) {
946 if (isStableDeopt(m, CompLevel.C1_SIMPLE)) {
947 TestRun.check(compiledByC1(m) != TriState.Yes, m + " should have been deoptimized by C1");
948 }
949 }
950
951 public static void assertDeoptimizedByC2(Method m) {
952 if (isStableDeopt(m, CompLevel.C2)) {
953 TestRun.check(compiledByC2(m) != TriState.Yes, m + " should have been deoptimized by C2");
954 }
955 }
956
957 /**
958 * Some VM flags could make the deopt assertions unstable.
959 */
960 public static boolean isStableDeopt(Method m, CompLevel level) {
961 return (USE_COMPILER && !XCOMP && !IGNORE_COMPILER_CONTROLS && !TEST_C1 &&
962 PER_METHOD_TRAP_LIMIT != 0 && PROFILE_INTERPRETER && !DEOPT_BARRIERS_ALOT &&
963 (!EXCLUDE_RANDOM || WHITE_BOX.isMethodCompilable(m, level.getValue(), false)));
964 }
965
966 public static void assertCompiledByC1(Method m) {
967 TestRun.check(compiledByC1(m) != TriState.No, m + " should have been C1 compiled");
968 }
969
970 public static void assertCompiledByC2(Method m) {
971 TestRun.check(compiledByC2(m) != TriState.No, m + " should have been C2 compiled");
972 }
973
974 public static void assertCompiledAtLevel(Method m, CompLevel level) {
975 TestRun.check(compiledAtLevel(m, level) != TriState.No, m + " should have been compiled at level " + level.name());
976 }
977
978 public static void assertNotCompiled(Method m) {
979 TestRun.check(!isC1Compiled(m), m + " should not have been compiled by C1");
980 TestRun.check(!isC2Compiled(m), m + " should not have been compiled by C2");
981 }
982
983 public static void assertCompiled(Method m) {
984 TestRun.check(compiledByC1(m) != TriState.No || compiledByC2(m) != TriState.No, m + " should have been compiled");
985 }
986
987 private static TriState compiledByC1(Method m) {
988 TriState triState = compiledAtLevel(m, CompLevel.C1_SIMPLE);
989 if (triState != TriState.No) {
990 return triState;
991 }
992 triState = compiledAtLevel(m, CompLevel.C1_LIMITED_PROFILE);
993 if (triState != TriState.No) {
994 return triState;
995 }
996 triState = compiledAtLevel(m, CompLevel.C1_FULL_PROFILE);
997 return triState;
998 }
999
1000 private static TriState compiledByC2(Method m) {
1001 return compiledAtLevel(m, CompLevel.C2);
1002 }
1003
1004 private static TriState compiledAtLevel(Method m, CompLevel level) {
1005 if (WHITE_BOX.isMethodCompiled(m, false)) {
1006 switch (level) {
1007 case C1_SIMPLE, C1_LIMITED_PROFILE, C1_FULL_PROFILE, C2 -> {
1008 if (WHITE_BOX.getMethodCompilationLevel(m, false) == level.getValue()) {
1009 return TriState.Yes;
1010 }
1011 }
1012 case ANY -> {
1013 return TriState.Yes;
1014 }
1015 default -> throw new TestRunException("compiledAtLevel() should not be called with " + level);
1016 }
1017 }
1018 if (!USE_COMPILER || XCOMP || TEST_C1 || IGNORE_COMPILER_CONTROLS || FLIP_C1_C2 || DEOPT_BARRIERS_ALOT ||
1019 (EXCLUDE_RANDOM && !WHITE_BOX.isMethodCompilable(m, level.getValue(), false))) {
1020 return TriState.Maybe;
1021 }
1022 return TriState.No;
1023 }
1024 }