1 /*
2 * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 package compiler.lib.ir_framework.test;
25
26 import compiler.lib.ir_framework.*;
27 import compiler.lib.ir_framework.Compiler;
28 import compiler.lib.ir_framework.shared.*;
29 import jdk.test.lib.Platform;
30 import jdk.test.lib.Utils;
31 import jdk.test.whitebox.WhiteBox;
32
33 import java.io.PrintWriter;
34 import java.io.StringWriter;
35 import java.lang.annotation.Annotation;
36 import java.lang.reflect.*;
37 import java.util.*;
38 import java.util.stream.Collectors;
39 import java.util.stream.Stream;
40
41 /**
42 * This class' main method is called from {@link TestFramework} and represents the so-called "test VM". The class is
43 * the heart of the framework and is responsible for executing all the specified tests in the test class. It uses the
44 * Whitebox API and reflection to achieve this task.
45 */
46 public class TestVM {
47 private static final WhiteBox WHITE_BOX;
48
49 static {
50 try {
51 WHITE_BOX = WhiteBox.getWhiteBox();
52 } catch (UnsatisfiedLinkError e) {
53 System.err.println(System.lineSeparator() + """
54 ##########################################################
55 - Did you call a test-related interface method from
56 TestFramework in main() of your test? Make sure to
57 only call setup/run methods and no checks or
58 assertions from main() of your test!
59 - Are you rerunning the test VM (TestVM class)
60 directly after a JTreg run? Make sure to start it
61 from within JTwork/scratch and with the flag
62 -DReproduce=true!
63 ##########################################################
64 """);
65 throw e;
66 }
67 }
68
69 /**
70 * The default number of warm-up iterations used to warm up a {@link Test} annotated test method.
71 * Use {@code -DWarmup=XY} to specify a different default value. An individual warm-up can also be
72 * set by specifying a {@link Warmup} iteration for a test.
73 */
74 public static final int WARMUP_ITERATIONS = Integer.parseInt(System.getProperty("Warmup", "2000"));
75
76 private static final boolean TIERED_COMPILATION = (Boolean)WHITE_BOX.getVMFlag("TieredCompilation");
77 private static final CompLevel TIERED_COMPILATION_STOP_AT_LEVEL;
78 private static final boolean CLIENT_VM = Platform.isClient();
79
80 static {
81 CompLevel level = CompLevel.forValue(((Long)WHITE_BOX.getVMFlag("TieredStopAtLevel")).intValue());
82 if (CLIENT_VM && level == CompLevel.C2) {
83 // No C2 available, use C1 level without profiling.
84 level = CompLevel.C1_SIMPLE;
85 }
86 TIERED_COMPILATION_STOP_AT_LEVEL = level;
87 }
88 public static final boolean TEST_C1 = (TIERED_COMPILATION && TIERED_COMPILATION_STOP_AT_LEVEL.getValue() < CompLevel.C2.getValue()) || CLIENT_VM;
89
90 static final boolean XCOMP = Platform.isComp();
91 static final boolean VERBOSE = Boolean.getBoolean("Verbose");
92 private static final boolean PRINT_TIMES = Boolean.getBoolean("PrintTimes");
93 public static final boolean USE_COMPILER = WHITE_BOX.getBooleanVMFlag("UseCompiler");
94 static final boolean EXCLUDE_RANDOM = Boolean.getBoolean("ExcludeRandom");
95 private static final String TESTLIST = System.getProperty("Test", "");
96 private static final String EXCLUDELIST = System.getProperty("Exclude", "");
97 private static final boolean DUMP_REPLAY = Boolean.getBoolean("DumpReplay");
98 private static final boolean GC_AFTER = Boolean.getBoolean("GCAfter");
99 private static final boolean SHUFFLE_TESTS = Boolean.parseBoolean(System.getProperty("ShuffleTests", "true"));
100 // Use separate flag as VERIFY_IR could have been set by user but due to other flags it was disabled by flag VM.
101 private static final boolean PRINT_VALID_IR_RULES = Boolean.getBoolean("ShouldDoIRVerification");
102 protected static final long PER_METHOD_TRAP_LIMIT = (Long)WHITE_BOX.getVMFlag("PerMethodTrapLimit");
103 protected static final boolean PROFILE_INTERPRETER = (Boolean)WHITE_BOX.getVMFlag("ProfileInterpreter");
104 private static final boolean FLIP_C1_C2 = Boolean.getBoolean("FlipC1C2");
105 private static final boolean IGNORE_COMPILER_CONTROLS = Boolean.getBoolean("IgnoreCompilerControls");
106
107 private final HashMap<Method, DeclaredTest> declaredTests = new HashMap<>();
108 private final List<AbstractTest> allTests = new ArrayList<>();
109 private final HashMap<String, Method> testMethodMap = new HashMap<>();
110 private final HashMap<String, Method> setupMethodMap = new HashMap<>();
111 private final List<String> excludeList;
112 private final List<String> testList;
113 private Set<Class<?>> helperClasses = null; // Helper classes that contain framework annotations to be processed.
114 private final IREncodingPrinter irMatchRulePrinter;
115 private final Class<?> testClass;
116 private final Map<Executable, CompLevel> forceCompileMap = new HashMap<>();
117
118 private TestVM(Class<?> testClass) {
119 TestRun.check(testClass != null, "Test class cannot be null");
120 this.testClass = testClass;
121 this.testList = createTestFilterList(TESTLIST, testClass);
122 this.excludeList = createTestFilterList(EXCLUDELIST, testClass);
123
124 if (PRINT_VALID_IR_RULES) {
125 irMatchRulePrinter = new IREncodingPrinter();
126 } else {
127 irMatchRulePrinter = null;
128 }
129 }
130
131 /**
132 * Parse "test1,test2,test3" into a list.
133 */
134 private static List<String> createTestFilterList(String list, Class<?> testClass) {
135 List<String> filterList = null;
136 if (!list.isEmpty()) {
137 String classPrefix = testClass.getSimpleName() + ".";
138 filterList = new ArrayList<>(Arrays.asList(list.split(",")));
139 for (int i = filterList.size() - 1; i >= 0; i--) {
140 String test = filterList.get(i);
141 if (test.indexOf(".") > 0) {
142 if (test.startsWith(classPrefix)) {
143 test = test.substring(classPrefix.length());
144 filterList.set(i, test);
145 } else {
146 filterList.remove(i);
147 }
148 }
149 }
150 }
151 return filterList;
152 }
153
154 /**
155 * Main entry point of the test VM.
156 */
157 public static void main(String[] args) {
158 try {
159 String testClassName = args[0];
160 System.out.println("TestVM main() called - about to run tests in class " + testClassName);
161 Class<?> testClass = getClassObject(testClassName, "test");
162
163 TestVM framework = new TestVM(testClass);
164 framework.addHelperClasses(args);
165 framework.start();
166 } finally {
167 TestFrameworkSocket.closeClientSocket();
168 }
169 }
170
171 protected static Class<?> getClassObject(String className, String classType) {
172 try {
173 return Class.forName(className);
174 } catch (Exception e) {
175 throw new TestRunException("Could not find " + classType + " class", e);
176 }
177 }
178
179 /**
180 * Set up all helper classes and verify they are specified correctly.
181 */
182 private void addHelperClasses(String[] args) {
183 Class<?>[] helperClassesList = getHelperClasses(args);
184 if (helperClassesList != null) {
185 TestRun.check(Arrays.stream(helperClassesList).noneMatch(Objects::isNull), "A Helper class cannot be null");
186 this.helperClasses = new HashSet<>();
187
188 for (Class<?> helperClass : helperClassesList) {
189 if (Arrays.stream(testClass.getDeclaredClasses()).anyMatch(c -> c == helperClass)) {
190 // Nested class of test class is automatically treated as helper class
191 TestFormat.failNoThrow("Nested " + helperClass + " inside test " + testClass + " is implicitly"
192 + " treated as helper class and does not need to be specified as such.");
193 continue;
194 }
195 TestRun.check(!this.helperClasses.contains(helperClass), "Cannot add the same class twice: " + helperClass);
196 this.helperClasses.add(helperClass);
197 }
198 }
199 }
200
201 private static Class<?>[] getHelperClasses(String[] args) {
202 if (args.length == 1) {
203 return null;
204 }
205 Class<?>[] helperClasses = new Class<?>[args.length - 1]; // First argument is test class
206 for (int i = 1; i < args.length; i++) {
207 String helperClassName = args[i];
208 helperClasses[i - 1] = getClassObject(helperClassName, "helper");
209 }
210 return helperClasses;
211 }
212
213 private void checkHelperClass(Class<?> clazz) {
214 checkAnnotationsInClass(clazz, "helper");
215 for (Class<?> c : clazz.getDeclaredClasses()) {
216 checkAnnotationsInClass(c, "nested (and helper)");
217 }
218 }
219
220 private void checkAnnotationsInClass(Class<?> c, String clazzType) {
221 Method[] methods = c.getDeclaredMethods();
222 for (Method m : methods) {
223 TestFormat.checkNoThrow(getAnnotation(m, Test.class) == null,
224 "Cannot use @Test annotation in " + clazzType + " " + c + " at " + m);
225 TestFormat.checkNoThrow(getAnnotation(m, Run.class) == null,
226 "Cannot use @Run annotation in " + clazzType + " " + c + " at " + m);
227 TestFormat.checkNoThrow(getAnnotation(m, Check.class) == null,
228 "Cannot use @Check annotation in " + clazzType + " " + c + " at " + m);
229 TestFormat.checkNoThrow(getAnnotation(m, Setup.class) == null,
230 "Cannot use @Setup annotation in " + clazzType + " " + c + " at " + m);
231 }
232 }
233
234 /**
235 * Only called by internal tests testing the framework itself. Accessed by reflection. Not exposed to normal users.
236 */
237 private static void runTestsOnSameVM(Class<?> testClass) {
238 if (testClass == null) {
239 StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
240 testClass = walker.getCallerClass();
241 }
242 TestVM framework = new TestVM(testClass);
243 framework.start();
244 }
245
246 /**
247 * Once everything is initialized and set up, start collecting tests and executing them afterwards.
248 */
249 private void start() {
250 setupTests();
251 checkForcedCompilationsCompleted();
252 runTests();
253 }
254
255 private void setupTests() {
256 for (Class<?> clazz : testClass.getDeclaredClasses()) {
257 checkAnnotationsInClass(clazz, "inner");
258 }
259 if (DUMP_REPLAY) {
260 addReplay();
261 }
262
263 // Collect the @Setup methods so we can reference them
264 // from the test methods
265 collectSetupMethods();
266
267 // Make sure to first setup test methods and make them non-inlineable and only then process compile commands.
268 setupDeclaredTests();
269 processControlAnnotations(testClass);
270 processHelperClasses();
271 setupCheckedAndCustomRunTests();
272
273 // All remaining tests are simple base tests without check or specific way to run them.
274 addBaseTests();
275 if (PRINT_VALID_IR_RULES) {
276 irMatchRulePrinter.emit();
277 VMInfoPrinter.emit();
278 }
279 TestFormat.throwIfAnyFailures();
280 declaredTests.clear();
281 testMethodMap.clear();
282 }
283
284 private void addBaseTests() {
285 declaredTests.forEach((m, test) -> {
286 if (test.getAttachedMethod() == null) {
287 try {
288 Arguments argumentsAnno = getAnnotation(m, Arguments.class);
289 TestFormat.check(argumentsAnno != null || m.getParameterCount() == 0, "Missing @Arguments annotation to define arguments of " + m);
290 BaseTest baseTest = new BaseTest(test, shouldExcludeTest(m.getName()));
291 allTests.add(baseTest);
292 if (PRINT_VALID_IR_RULES) {
293 irMatchRulePrinter.emitRuleEncoding(m, baseTest.isSkipped());
294 }
295 } catch (TestFormatException e) {
296 // Failure logged. Continue and report later.
297 }
298 }
299 });
300 }
301
302 /**
303 * Check if user wants to exclude this test by checking the -DTest and -DExclude lists.
304 */
305 private boolean shouldExcludeTest(String testName) {
306 boolean hasTestList = testList != null;
307 boolean hasExcludeList = excludeList != null;
308 if (hasTestList) {
309 return !testList.contains(testName) || (hasExcludeList && excludeList.contains(testName));
310 } else if (hasExcludeList) {
311 return excludeList.contains(testName);
312 }
313 return false;
314 }
315
316 /**
317 * Generate replay compilation files.
318 */
319 private void addReplay() {
320 String directive = "[{ match: \"*.*\", DumpReplay: true }]";
321 TestFramework.check(WHITE_BOX.addCompilerDirective(directive) == 1, "Failed to add DUMP_REPLAY directive");
322 }
323
324 private void processControlAnnotations(Class<?> clazz) {
325 if (IGNORE_COMPILER_CONTROLS) {
326 return;
327 }
328 // Also apply compile commands to all inner classes of 'clazz'.
329 ArrayList<Class<?>> classes = new ArrayList<>(Arrays.asList(clazz.getDeclaredClasses()));
330 classes.add(clazz);
331 for (Class<?> c : classes) {
332 applyClassAnnotations(c);
333 List<Executable> executables = new ArrayList<>(Arrays.asList(c.getDeclaredMethods()));
334 Collections.addAll(executables, c.getDeclaredConstructors());
335 for (Executable ex : executables) {
336 checkClassAnnotations(ex);
337 try {
338 applyIndependentCompilationCommands(ex);
339 } catch (TestFormatException e) {
340 // Failure logged. Continue and report later.
341 }
342 }
343
344 // Only force compilation now because above annotations affect inlining
345 for (Executable ex : executables) {
346 try {
347 applyForceCompileCommand(ex);
348 } catch (TestFormatException e) {
349 // Failure logged. Continue and report later.
350 }
351 }
352 }
353 }
354
355 private void applyClassAnnotations(Class<?> c) {
356 ForceCompileClassInitializer anno = getAnnotation(c, ForceCompileClassInitializer.class);
357 if (anno == null) {
358 return;
359 }
360
361 // Compile class initializer
362 CompLevel level = anno.value();
363 if (level == CompLevel.SKIP || level == CompLevel.WAIT_FOR_COMPILATION) {
364 TestFormat.failNoThrow("Cannot define compilation level SKIP or WAIT_FOR_COMPILATION in " +
365 "@ForceCompileClassInitializer at " + c);
366 return;
367 }
368 level = restrictCompLevel(anno.value());
369 if (level != CompLevel.SKIP) {
370 // Make sure class is initialized to avoid compilation bailout of <clinit>
371 getClassObject(c.getName(), "nested"); // calls Class.forName() to initialize 'c'
372 TestFormat.checkNoThrow(WHITE_BOX.enqueueInitializerForCompilation(c, level.getValue()),
373 "Failed to enqueue <clinit> of " + c + " for compilation. Did you specify "
374 + "@ForceCompileClassInitializer without providing a static class initialization? "
375 + "Make sure to provide any form of static initialization or remove the annotation. "
376 + "For debugging purposes, -DIgnoreCompilerControls=true can be used to temporarly "
377 + "ignore @ForceCompileClassInitializer annotations.");
378 }
379 }
380
381 private void checkClassAnnotations(Executable ex) {
382 TestFormat.checkNoThrow(getAnnotation(ex, ForceCompileClassInitializer.class) == null,
383 "@ForceCompileClassInitializer only allowed at classes but not at method " + ex);
384 }
385
386 /**
387 * Exclude a method from compilation with a compiler randomly. Return the compiler for which the method was made
388 * not compilable.
389 */
390 public static Compiler excludeRandomly(Executable ex) {
391 Compiler compiler = switch (Utils.getRandomInstance().nextInt() % 3) {
392 case 1 -> Compiler.C1;
393 case 2 -> Compiler.C2;
394 default -> Compiler.ANY;
395 };
396 WHITE_BOX.makeMethodNotCompilable(ex, compiler.getValue(), false);
397 WHITE_BOX.makeMethodNotCompilable(ex, compiler.getValue(), true);
398 System.out.println("Excluding from " + compiler.name() + " compilation: " + ex);
399 return compiler;
400 }
401
402 private void applyIndependentCompilationCommands(Executable ex) {
403 ForceInline forceInlineAnno = getAnnotation(ex, ForceInline.class);
404 DontInline dontInlineAnno = getAnnotation(ex, DontInline.class);
405 ForceCompile forceCompileAnno = getAnnotation(ex, ForceCompile.class);
406 DontCompile dontCompileAnno = getAnnotation(ex, DontCompile.class);
407 checkCompilationCommandAnnotations(ex, forceInlineAnno, dontInlineAnno, forceCompileAnno, dontCompileAnno);
408 // First handle inline annotations
409 if (dontInlineAnno != null) {
410 WHITE_BOX.testSetDontInlineMethod(ex, true);
411 } else if (forceInlineAnno != null) {
412 WHITE_BOX.testSetForceInlineMethod(ex, true);
413 }
414 if (dontCompileAnno != null) {
415 dontCompileWithCompiler(ex, dontCompileAnno.value());
416 }
417 if (EXCLUDE_RANDOM && getAnnotation(ex, Test.class) == null && forceCompileAnno == null && dontCompileAnno == null) {
418 // Randomly exclude helper methods from compilation
419 if (Utils.getRandomInstance().nextBoolean()) {
420 excludeRandomly(ex);
421 }
422 }
423 }
424
425 private void checkCompilationCommandAnnotations(Executable ex, ForceInline forceInlineAnno, DontInline dontInlineAnno, ForceCompile forceCompileAnno, DontCompile dontCompileAnno) {
426 Test testAnno = getAnnotation(ex, Test.class);
427 Run runAnno = getAnnotation(ex, Run.class);
428 Check checkAnno = getAnnotation(ex, Check.class);
429 TestFormat.check((testAnno == null && runAnno == null && checkAnno == null) || Stream.of(forceCompileAnno, dontCompileAnno, forceInlineAnno, dontInlineAnno).noneMatch(Objects::nonNull),
430 "Cannot use explicit compile command annotations (@ForceInline, @DontInline, " +
431 "@ForceCompile or @DontCompile) together with @Test, @Check or @Run: " + ex + ". Use compLevel in @Test for fine tuning.");
432 if (Stream.of(forceInlineAnno, dontCompileAnno, dontInlineAnno).filter(Objects::nonNull).count() > 1) {
433 // Failure
434 TestFormat.check(dontCompileAnno == null || dontInlineAnno == null,
435 "@DontInline is implicitely done with @DontCompile annotation at " + ex);
436 TestFormat.fail("Cannot mix @ForceInline, @DontInline and @DontCompile at the same time at " + ex);
437 }
438 TestFormat.check(forceInlineAnno == null || dontInlineAnno == null, "Cannot have @ForceInline and @DontInline at the same time at " + ex);
439 if (forceCompileAnno != null && dontCompileAnno != null) {
440 CompLevel forceCompileLevel = forceCompileAnno.value();
441 Compiler dontCompileCompiler = dontCompileAnno.value();
442 TestFormat.check(dontCompileCompiler != Compiler.ANY,
443 "Cannot have @DontCompile(Compiler.ANY) and @ForceCompile at the same time at " + ex);
444 TestFormat.check(forceCompileLevel != CompLevel.ANY,
445 "Cannot have @ForceCompile(CompLevel.ANY) and @DontCompile at the same time at " + ex);
446 TestFormat.check(forceCompileLevel.isNotCompilationLevelOfCompiler(dontCompileCompiler),
447 "Overlapping compilation level and compiler with @ForceCompile and @DontCompile at " + ex);
448 }
449 }
450
451 /**
452 * Exlude the method from compilation and make sure it is not inlined.
453 */
454 private void dontCompileAndDontInlineMethod(Method m) {
455 if (!IGNORE_COMPILER_CONTROLS) {
456 WHITE_BOX.makeMethodNotCompilable(m, CompLevel.ANY.getValue(), true);
457 WHITE_BOX.makeMethodNotCompilable(m, CompLevel.ANY.getValue(), false);
458 WHITE_BOX.testSetDontInlineMethod(m, true);
459 }
460 }
461
462 private void dontCompileWithCompiler(Executable ex, Compiler compiler) {
463 if (VERBOSE) {
464 System.out.println("dontCompileWithCompiler " + ex + " , compiler = " + compiler.name());
465 }
466 WHITE_BOX.makeMethodNotCompilable(ex, compiler.getValue(), true);
467 WHITE_BOX.makeMethodNotCompilable(ex, compiler.getValue(), false);
468 if (compiler == Compiler.ANY) {
469 WHITE_BOX.testSetDontInlineMethod(ex, true);
470 }
471 }
472
473 private void applyForceCompileCommand(Executable ex) {
474 ForceCompile forceCompileAnno = getAnnotation(ex, ForceCompile.class);
475 if (forceCompileAnno != null) {
476 CompLevel compLevel = forceCompileAnno.value();
477 TestFormat.check(compLevel != CompLevel.SKIP && compLevel != CompLevel.WAIT_FOR_COMPILATION,
478 "Cannot define compilation level SKIP or WAIT_FOR_COMPILATION in @ForceCompile at " + ex);
479 compLevel = restrictCompLevel(forceCompileAnno.value());
480 if (FLIP_C1_C2) {
481 compLevel = compLevel.flipCompLevel();
482 compLevel = restrictCompLevel(compLevel.flipCompLevel());
483 }
484 if (EXCLUDE_RANDOM) {
485 compLevel = compLevel.excludeCompilationRandomly(ex);
486 }
487 if (compLevel != CompLevel.SKIP) {
488 enqueueForCompilation(ex, compLevel);
489 forceCompileMap.put(ex, compLevel);
490 }
491 }
492 }
493
494 static void enqueueForCompilation(Executable ex, CompLevel requestedCompLevel) {
495 if (TestVM.VERBOSE) {
496 System.out.println("enqueueForCompilation " + ex + ", level = " + requestedCompLevel);
497 }
498 CompLevel compLevel = restrictCompLevel(requestedCompLevel);
499 if (compLevel != CompLevel.SKIP) {
500 WHITE_BOX.enqueueMethodForCompilation(ex, compLevel.getValue());
501 } else {
502 System.out.println("Skipped compilation on level " + requestedCompLevel + " due to VM flags not allowing it.");
503 }
504 }
505
506
507 /**
508 * Collect all @Setup annotated methods and add them to setupMethodMap, for convenience to reference later from
509 * tests with @Arguments(setup = "setupMethodName").
510 */
511 private void collectSetupMethods() {
512 for (Method m : testClass.getDeclaredMethods()) {
513 Setup setupAnnotation = getAnnotation(m, Setup.class);
514 if (setupAnnotation != null) {
515 addSetupMethod(m);
516 }
517 }
518 }
519
520 private void addSetupMethod(Method m) {
521 TestFormat.checkNoThrow(getAnnotation(m, Test.class) == null,
522 "@Setup method cannot have @Test annotation: " + m);
523 TestFormat.checkNoThrow(getAnnotation(m, Check.class) == null,
524 "@Setup method cannot have @Check annotation: " + m);
525 TestFormat.checkNoThrow(getAnnotation(m, Arguments.class) == null,
526 "@Setup method cannot have @Arguments annotation: " + m);
527 TestFormat.checkNoThrow(getAnnotation(m, Run.class) == null,
528 "@Setup method cannot have @Run annotation: " + m);
529 Method mOverloaded = setupMethodMap.put(m.getName(), m);
530 TestFormat.checkNoThrow(mOverloaded == null,
531 "@Setup method cannot be overloaded: " + mOverloaded + " with " + m);
532 m.setAccessible(true);
533 }
534
535 /**
536 * Setup @Test annotated method an add them to the declaredTests map to have a convenient way of accessing them
537 * once setting up a framework test (base checked, or custom run test).
538 */
539 private void setupDeclaredTests() {
540 for (Method m : testClass.getDeclaredMethods()) {
541 Test testAnno = getAnnotation(m, Test.class);
542 try {
543 if (testAnno != null) {
544 addDeclaredTest(m);
545 } else {
546 TestFormat.checkNoThrow(!m.isAnnotationPresent(IR.class) && !m.isAnnotationPresent(IRs.class),
547 "Found @IR annotation on non-@Test method " + m);
548 TestFormat.checkNoThrow(!m.isAnnotationPresent(Warmup.class) || getAnnotation(m, Run.class) != null,
549 "Found @Warmup annotation on non-@Test or non-@Run method " + m);
550 }
551 } catch (TestFormatException e) {
552 // Failure logged. Continue and report later.
553 }
554 }
555 TestFormat.checkNoThrow(!declaredTests.isEmpty(), "Did not specify any @Test methods in " + testClass);
556 }
557
558 private void addDeclaredTest(Method m) {
559 Test testAnno = getAnnotation(m, Test.class);
560 checkTestAnnotations(m, testAnno);
561 Warmup warmup = getAnnotation(m, Warmup.class);
562 int warmupIterations = WARMUP_ITERATIONS;
563 if (warmup != null) {
564 warmupIterations = warmup.value();
565 TestFormat.checkNoThrow(warmupIterations >= 0, "Cannot have negative value for @Warmup at " + m);
566 }
567
568 if (!IGNORE_COMPILER_CONTROLS) {
569 // Don't inline test methods by default. Do not apply this when -DIgnoreCompilerControls=true is set.
570 WHITE_BOX.testSetDontInlineMethod(m, true);
571 }
572 CompLevel compLevel = restrictCompLevel(testAnno.compLevel());
573 if (FLIP_C1_C2) {
574 compLevel = compLevel.flipCompLevel();
575 compLevel = restrictCompLevel(compLevel.flipCompLevel());
576 }
577 if (EXCLUDE_RANDOM) {
578 compLevel = compLevel.excludeCompilationRandomly(m);
579 }
580 ArgumentsProvider argumentsProvider = ArgumentsProviderBuilder.build(m, setupMethodMap);
581 DeclaredTest test = new DeclaredTest(m, argumentsProvider, compLevel, warmupIterations);
582 declaredTests.put(m, test);
583 testMethodMap.put(m.getName(), m);
584 }
585
586 private void checkTestAnnotations(Method m, Test testAnno) {
587 TestFormat.check(!testMethodMap.containsKey(m.getName()),
588 "Cannot overload two @Test methods: " + m + ", " + testMethodMap.get(m.getName()));
589 TestFormat.check(testAnno != null, m + " must be a method with a @Test annotation");
590
591 Check checkAnno = getAnnotation(m, Check.class);
592 Run runAnno = getAnnotation(m, Run.class);
593 TestFormat.check(checkAnno == null && runAnno == null,
594 m + " has invalid @Check or @Run annotation while @Test annotation is present.");
595
596 TestFormat.checkNoThrow(Arrays.stream(m.getParameterTypes()).noneMatch(AbstractInfo.class::isAssignableFrom),
597 "Cannot " + AbstractInfo.class + " or any of its subclasses as parameter type at " +
598 "@Test method " + m);
599
600 TestFormat.checkNoThrow(!AbstractInfo.class.isAssignableFrom(m.getReturnType()),
601 "Cannot " + AbstractInfo.class + " or any of its subclasses as return type at " +
602 "@Test method " + m);
603 }
604
605
606 /**
607 * Get the appropriate level as permitted by the test scenario and VM flags.
608 */
609 private static CompLevel restrictCompLevel(CompLevel compLevel) {
610 if (!USE_COMPILER) {
611 return CompLevel.SKIP;
612 }
613 if (compLevel == CompLevel.ANY) {
614 // Use highest available compilation level by default (usually C2).
615 compLevel = TIERED_COMPILATION_STOP_AT_LEVEL;
616 }
617 if (TEST_C1 && compLevel == CompLevel.C2) {
618 return CompLevel.SKIP;
619 }
620 if ((!TIERED_COMPILATION && !CLIENT_VM) && compLevel.getValue() < CompLevel.C2.getValue()) {
621 return CompLevel.SKIP;
622 }
623 if ((TIERED_COMPILATION || CLIENT_VM) && compLevel.getValue() > TIERED_COMPILATION_STOP_AT_LEVEL.getValue()) {
624 return CompLevel.SKIP;
625 }
626 return compLevel;
627 }
628
629 /**
630 * Verify that the helper classes do not contain illegal framework annotations and then apply the actions as
631 * specified by the different helper class annotations.
632 */
633 private void processHelperClasses() {
634 if (helperClasses != null) {
635 for (Class<?> helperClass : helperClasses) {
636 // Process the helper classes and apply the explicit compile commands
637 TestFormat.checkNoThrow(helperClass != testClass,
638 "Cannot specify test " + testClass + " as helper class, too.");
639 checkHelperClass(helperClass);
640 processControlAnnotations(helperClass);
641 }
642 }
643 }
644
645 /**
646 * First set up checked (with @Check) and custom run tests (with @Run). All remaining unmatched/unused @Test methods
647 * are treated as base tests and set up as such later.
648 */
649 private void setupCheckedAndCustomRunTests() {
650 for (Method m : testClass.getDeclaredMethods()) {
651 Check checkAnno = getAnnotation(m, Check.class);
652 Run runAnno = getAnnotation(m, Run.class);
653 Arguments argumentsAnno = getAnnotation(m, Arguments.class);
654 try {
655 TestFormat.check(argumentsAnno == null || (checkAnno == null && runAnno == null),
656 "Cannot have @Argument annotation in combination with @Run or @Check at " + m);
657 if (checkAnno != null) {
658 addCheckedTest(m, checkAnno, runAnno);
659 } else if (runAnno != null) {
660 addCustomRunTest(m, runAnno);
661 }
662 } catch (TestFormatException e) {
663 // Failure logged. Continue and report later.
664 }
665 }
666 }
667
668 /**
669 * Set up a checked test by first verifying the correct format of the @Test and @Check method and then adding it
670 * to the allTests list which keeps track of all framework tests that are eventually executed.
671 */
672 private void addCheckedTest(Method m, Check checkAnno, Run runAnno) {
673 Method testMethod = testMethodMap.get(checkAnno.test());
674 DeclaredTest test = declaredTests.get(testMethod);
675 checkCheckedTest(m, checkAnno, runAnno, testMethod, test);
676 test.setAttachedMethod(m);
677 TestFormat.check(getAnnotation(testMethod, Arguments.class) != null || testMethod.getParameterCount() == 0,
678 "Missing @Arguments annotation to define arguments of " + testMethod + " required by "
679 + "checked test " + m);
680 CheckedTest.Parameter parameter = getCheckedTestParameter(m, testMethod);
681 dontCompileAndDontInlineMethod(m);
682 CheckedTest checkedTest = new CheckedTest(test, m, checkAnno, parameter, shouldExcludeTest(testMethod.getName()));
683 allTests.add(checkedTest);
684 if (PRINT_VALID_IR_RULES) {
685 // Only need to emit IR verification information if IR verification is actually performed.
686 irMatchRulePrinter.emitRuleEncoding(testMethod, checkedTest.isSkipped());
687 }
688 }
689
690 private void checkCheckedTest(Method m, Check checkAnno, Run runAnno, Method testMethod, DeclaredTest test) {
691 TestFormat.check(runAnno == null, m + " has invalid @Run annotation while @Check annotation is present.");
692 TestFormat.check(testMethod != null, "Did not find associated test method \"" + m.getDeclaringClass().getName()
693 + "." + checkAnno.test() + "\" for @Check at " + m);
694 TestFormat.check(test != null, "Missing @Test annotation for associated test method " + testMethod + " for @Check at " + m);
695 Method attachedMethod = test.getAttachedMethod();
696 TestFormat.check(attachedMethod == null,
697 "Cannot use @Test " + testMethod + " for more than one @Run or one @Check method. Found: " + m + ", " + attachedMethod);
698 }
699
700 /**
701 * Only allow parameters as specified in {@link Check}.
702 */
703 private CheckedTest.Parameter getCheckedTestParameter(Method m, Method testMethod) {
704 boolean firstParameterTestInfo = m.getParameterCount() > 0 && m.getParameterTypes()[0].equals(TestInfo.class);
705 boolean secondParameterTestInfo = m.getParameterCount() > 1 && m.getParameterTypes()[1].equals(TestInfo.class);
706 CheckedTest.Parameter parameter = null;
707 Class<?> testReturnType = testMethod.getReturnType();
708 switch (m.getParameterCount()) {
709 case 0 -> parameter = CheckedTest.Parameter.NONE;
710 case 1 -> {
711 TestFormat.checkNoThrow(firstParameterTestInfo || m.getParameterTypes()[0] == testReturnType,
712 "Single-parameter version of @Check method " + m + " must match return type of @Test " + testMethod);
713 parameter = firstParameterTestInfo ? CheckedTest.Parameter.TEST_INFO_ONLY : CheckedTest.Parameter.RETURN_ONLY;
714 }
715 case 2 -> {
716 TestFormat.checkNoThrow(m.getParameterTypes()[0] == testReturnType && secondParameterTestInfo,
717 "Two-parameter version of @Check method " + m + " must provide as first parameter the same"
718 + " return type as @Test method " + testMethod + " and as second parameter an object of " + TestInfo.class);
719 parameter = CheckedTest.Parameter.BOTH;
720 }
721 default -> TestFormat.failNoThrow("@Check method " + m + " must provide either a none, single or two-parameter variant.");
722 }
723 return parameter;
724 }
725
726 /**
727 * Set up a custom run test by first verifying the correct format of the @Test and @Run method and then adding it
728 * to the allTests list which keeps track of all framework tests that are eventually executed.
729 */
730 private void addCustomRunTest(Method m, Run runAnno) {
731 checkRunMethod(m, runAnno);
732 List<DeclaredTest> tests = new ArrayList<>();
733 boolean shouldExcludeTest = true;
734 for (String testName : runAnno.test()) {
735 try {
736 Method testMethod = testMethodMap.get(testName);
737 DeclaredTest test = declaredTests.get(testMethod);
738 checkCustomRunTest(m, testName, testMethod, test, runAnno.mode());
739 test.setAttachedMethod(m);
740 tests.add(test);
741 // Only exclude custom run test if all test methods excluded
742 shouldExcludeTest &= shouldExcludeTest(testMethod.getName());
743 } catch (TestFormatException e) {
744 // Logged, continue.
745 }
746 }
747 if (tests.isEmpty()) {
748 return; // There was a format violation. Return.
749 }
750 dontCompileAndDontInlineMethod(m);
751 CustomRunTest customRunTest = new CustomRunTest(m, getAnnotation(m, Warmup.class), runAnno, tests, shouldExcludeTest);
752 allTests.add(customRunTest);
753 if (PRINT_VALID_IR_RULES) {
754 tests.forEach(test -> irMatchRulePrinter.emitRuleEncoding(test.getTestMethod(), customRunTest.isSkipped()));
755 }
756 }
757
758 /**
759 * Only allow parameters as specified in {@link Run}.
760 */
761 private void checkCustomRunTest(Method m, String testName, Method testMethod, DeclaredTest test, RunMode runMode) {
762 TestFormat.check(testMethod != null, "Did not find associated @Test method \"" + m.getDeclaringClass().getName()
763 + "." + testName + "\" specified in @Run at " + m);
764 TestFormat.check(test != null,
765 "Missing @Test annotation for associated test method " + testName + " for @Run at " + m);
766 Method attachedMethod = test.getAttachedMethod();
767 TestFormat.check(attachedMethod == null,
768 "Cannot use @Test " + testMethod + " for more than one @Run/@Check method. Found: "
769 + m + ", " + attachedMethod);
770 Arguments argumentsAnno = getAnnotation(testMethod, Arguments.class);
771 TestFormat.check(argumentsAnno == null,
772 "Cannot use @Arguments at test method " + testMethod + " in combination with @Run method " + m);
773 Warmup warmupAnno = getAnnotation(testMethod, Warmup.class);
774 TestFormat.checkNoThrow(warmupAnno == null,
775 "Cannot set @Warmup at @Test method " + testMethod + " when used with its @Run method "
776 + m + ". Use @Warmup at @Run method instead.");
777 Test testAnno = getAnnotation(testMethod, Test.class);
778 TestFormat.checkNoThrow(runMode != RunMode.STANDALONE || testAnno.compLevel() == CompLevel.ANY,
779 "Setting explicit compilation level for @Test method " + testMethod + " has no effect "
780 + "when used with STANDALONE @Run method " + m);
781 }
782
783 private void checkRunMethod(Method m, Run runAnno) {
784 TestFormat.check(runAnno.test().length > 0, "@Run method " + m + " must specify at least one test method");
785 TestFormat.checkNoThrow(m.getParameterCount() == 0 || (m.getParameterCount() == 1 && m.getParameterTypes()[0].equals(RunInfo.class)),
786 "@Run method " + m + " must specify either no parameter or exactly one " + RunInfo.class + " parameter.");
787 Warmup warmupAnno = getAnnotation(m, Warmup.class);
788 TestFormat.checkNoThrow(warmupAnno == null || runAnno.mode() != RunMode.STANDALONE,
789 "Cannot set @Warmup at @Run method " + m + " when used with RunMode.STANDALONE. The @Run method is only invoked once.");
790 }
791
792 private static <T extends Annotation> T getAnnotation(AnnotatedElement element, Class<T> c) {
793 T[] annos = element.getAnnotationsByType(c);
794 TestFormat.check(annos.length < 2, element + " has duplicated annotations");
795 return Arrays.stream(annos).findFirst().orElse(null);
796 }
797
798 /**
799 * Ensure that all compilations that were enforced (added to compilation queue) by framework annotations are
800 * completed. Wait if necessary for a short amount of time for their completion.
801 */
802 private void checkForcedCompilationsCompleted() {
803 if (forceCompileMap.isEmpty()) {
804 return;
805 }
806 final long started = System.currentTimeMillis();
807 long elapsed;
808 do {
809 forceCompileMap.entrySet().removeIf(entry -> WHITE_BOX.getMethodCompilationLevel(entry.getKey()) == entry.getValue().getValue());
810 if (forceCompileMap.isEmpty()) {
811 // All @ForceCompile methods are compiled at the requested level.
812 return;
813 }
814 // Retry again if not yet compiled.
815 forceCompileMap.forEach(TestVM::enqueueForCompilation);
816 elapsed = System.currentTimeMillis() - started;
817 } while (elapsed < 5000);
818 StringBuilder builder = new StringBuilder();
819 forceCompileMap.forEach((key, value) -> builder.append("- ").append(key).append(" at CompLevel.").append(value)
820 .append(System.lineSeparator()));
821 throw new TestRunException("Could not force compile the following @ForceCompile methods:"
822 + System.lineSeparator() + builder.toString());
823 }
824
825 /**
826 * Once all framework tests are collected, they are run in this method.
827 */
828 private void runTests() {
829 TreeMap<Long, String> durations = (PRINT_TIMES || VERBOSE) ? new TreeMap<>() : null;
830 long startTime = System.nanoTime();
831 List<AbstractTest> testList;
832 boolean testFilterPresent = testFilterPresent();
833 if (testFilterPresent) {
834 // Only run the specified tests by the user filters -DTest and/or -DExclude.
835 testList = allTests.stream().filter(test -> !test.isSkipped()).collect(Collectors.toList());
836 if (testList.isEmpty()) {
837 // Throw an exception to inform the user about an empty specified test set with -DTest and/or -DExclude
838 throw new NoTestsRunException();
839 }
840 } else {
841 testList = allTests;
842 }
843
844 if (SHUFFLE_TESTS) {
845 // Execute tests in random order (execution sequence affects profiling). This is done by default.
846 Collections.shuffle(testList, Utils.getRandomInstance());
847 }
848 StringBuilder builder = new StringBuilder();
849 int failures = 0;
850
851 // Execute all tests and keep track of each exception that is thrown. These are then reported once all tests
852 // are executing. This prevents a premature exit without running all tests.
853 for (AbstractTest test : testList) {
854 if (VERBOSE) {
855 System.out.println("Run " + test.toString());
856 }
857 if (testFilterPresent) {
858 TestFrameworkSocket.write("Run " + test.toString(), TestFrameworkSocket.TESTLIST_TAG, true);
859 }
860 try {
861 test.run();
862 } catch (TestRunException e) {
863 StringWriter sw = new StringWriter();
864 PrintWriter pw = new PrintWriter(sw);
865 e.printStackTrace(pw);
866 builder.append(test.toString()).append(":").append(System.lineSeparator()).append(sw.toString())
867 .append(System.lineSeparator()).append(System.lineSeparator());
868 failures++;
869 }
870 if (PRINT_TIMES || VERBOSE) {
871 long endTime = System.nanoTime();
872 long duration = (endTime - startTime);
873 durations.put(duration, test.getName());
874 if (VERBOSE) {
875 System.out.println("Done " + test.getName() + ": " + duration + " ns = " + (duration / 1000000) + " ms");
876 }
877 }
878 if (GC_AFTER) {
879 System.out.println("doing GC");
880 WHITE_BOX.fullGC();
881 }
882 }
883
884 // Print execution times
885 if (VERBOSE || PRINT_TIMES) {
886 System.out.println(System.lineSeparator() + System.lineSeparator() + "Test execution times:");
887 for (Map.Entry<Long, String> entry : durations.entrySet()) {
888 System.out.format("%-10s%15d ns%n", entry.getValue() + ":", entry.getKey());
889 }
890 }
891
892 if (failures > 0) {
893 // Finally, report all occurred exceptions in a nice format.
894 String msg = System.lineSeparator() + System.lineSeparator() + "Test Failures (" + failures + ")"
895 + System.lineSeparator() + "----------------" + "-".repeat(String.valueOf(failures).length());
896 throw new TestRunException(msg + System.lineSeparator() + builder.toString());
897 }
898 }
899
900 private boolean testFilterPresent() {
901 return testList != null || excludeList != null;
902 }
903
904 enum TriState {
905 Maybe,
906 Yes,
907 No
908 }
909
910 public static void compile(Method m, CompLevel compLevel) {
911 TestRun.check(compLevel != CompLevel.SKIP && compLevel != CompLevel.WAIT_FOR_COMPILATION,
912 "Invalid compilation request with level " + compLevel);
913 enqueueForCompilation(m, compLevel);
914 }
915
916 public static void deoptimize(Method m) {
917 WHITE_BOX.deoptimizeMethod(m);
918 }
919
920 public static boolean isCompiled(Method m) {
921 return compiledAtLevel(m, CompLevel.ANY) == TriState.Yes;
922 }
923
924 public static boolean isC1Compiled(Method m) {
925 return compiledByC1(m) == TriState.Yes;
926 }
927
928 public static boolean isC2Compiled(Method m) {
929 return compiledByC2(m) == TriState.Yes;
930 }
931
932 public static boolean isCompiledAtLevel(Method m, CompLevel compLevel) {
933 return compiledAtLevel(m, compLevel) == TriState.Yes;
934 }
935
936 public static void assertDeoptimizedByC1(Method m) {
937 if (notUnstableDeoptAssertion(m, CompLevel.C1_SIMPLE)) {
938 TestRun.check(compiledByC1(m) != TriState.Yes || PER_METHOD_TRAP_LIMIT == 0 || !PROFILE_INTERPRETER,
939 m + " should have been deoptimized by C1");
940 }
941 }
942
943 public static void assertDeoptimizedByC2(Method m) {
944 if (notUnstableDeoptAssertion(m, CompLevel.C2)) {
945 TestRun.check(compiledByC2(m) != TriState.Yes || PER_METHOD_TRAP_LIMIT == 0 || !PROFILE_INTERPRETER,
946 m + " should have been deoptimized by C2");
947 }
948 }
949
950 /**
951 * Some VM flags could make the deopt assertions unstable.
952 */
953 private static boolean notUnstableDeoptAssertion(Method m, CompLevel level) {
954 return (USE_COMPILER && !XCOMP && !IGNORE_COMPILER_CONTROLS && !TEST_C1 &&
955 (!EXCLUDE_RANDOM || WHITE_BOX.isMethodCompilable(m, level.getValue(), false)));
956 }
957
958 public static void assertCompiledByC1(Method m) {
959 TestRun.check(compiledByC1(m) != TriState.No, m + " should have been C1 compiled");
960 }
961
962 public static void assertCompiledByC2(Method m) {
963 TestRun.check(compiledByC2(m) != TriState.No, m + " should have been C2 compiled");
964 }
965
966 public static void assertCompiledAtLevel(Method m, CompLevel level) {
967 TestRun.check(compiledAtLevel(m, level) != TriState.No, m + " should have been compiled at level " + level.name());
968 }
969
970 public static void assertNotCompiled(Method m) {
971 TestRun.check(!isC1Compiled(m), m + " should not have been compiled by C1");
972 TestRun.check(!isC2Compiled(m), m + " should not have been compiled by C2");
973 }
974
975 public static void assertCompiled(Method m) {
976 TestRun.check(compiledByC1(m) != TriState.No || compiledByC2(m) != TriState.No, m + " should have been compiled");
977 }
978
979 private static TriState compiledByC1(Method m) {
980 TriState triState = compiledAtLevel(m, CompLevel.C1_SIMPLE);
981 if (triState != TriState.No) {
982 return triState;
983 }
984 triState = compiledAtLevel(m, CompLevel.C1_LIMITED_PROFILE);
985 if (triState != TriState.No) {
986 return triState;
987 }
988 triState = compiledAtLevel(m, CompLevel.C1_FULL_PROFILE);
989 return triState;
990 }
991
992 private static TriState compiledByC2(Method m) {
993 return compiledAtLevel(m, CompLevel.C2);
994 }
995
996 private static TriState compiledAtLevel(Method m, CompLevel level) {
997 if (WHITE_BOX.isMethodCompiled(m, false)) {
998 switch (level) {
999 case C1_SIMPLE, C1_LIMITED_PROFILE, C1_FULL_PROFILE, C2 -> {
1000 if (WHITE_BOX.getMethodCompilationLevel(m, false) == level.getValue()) {
1001 return TriState.Yes;
1002 }
1003 }
1004 case ANY -> {
1005 return TriState.Yes;
1006 }
1007 default -> throw new TestRunException("compiledAtLevel() should not be called with " + level);
1008 }
1009 }
1010 if (!USE_COMPILER || XCOMP || TEST_C1 || IGNORE_COMPILER_CONTROLS || FLIP_C1_C2 ||
1011 (EXCLUDE_RANDOM && !WHITE_BOX.isMethodCompilable(m, level.getValue(), false))) {
1012 return TriState.Maybe;
1013 }
1014 return TriState.No;
1015 }
1016 }