1 /*
2 * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 package jdk.test.lib.combo;
25
26 import javax.tools.JavaCompiler;
27 import javax.tools.StandardJavaFileManager;
28 import javax.tools.ToolProvider;
29
30 import java.util.stream.Stream;
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.HashMap;
35 import java.util.Iterator;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Optional;
39 import java.util.Stack;
40 import java.util.function.Consumer;
41 import java.util.function.Predicate;
42 import java.util.function.Supplier;
43 import java.util.stream.Collectors;
44
45
46 /**
47 * An helper class for defining combinatorial (aka "combo" tests). A combo test is made up of one
48 * or more 'dimensions' - each of which represent a different axis of the test space. For instance,
49 * if we wanted to test class/interface declaration, one dimension could be the keyword used for
50 * the declaration (i.e. 'class' vs. 'interface') while another dimension could be the class/interface
51 * modifiers (i.e. 'public', 'pachake-private' etc.). A combo test consists in running a test instance
52 * for each point in the test space; that is, for any combination of the combo test dimension:
53 * <p>
54 * 'public' 'class'
55 * 'public' interface'
56 * 'package-private' 'class'
57 * 'package-private' 'interface'
58 * ...
59 * <p>
60 * A new test instance {@link ComboInstance} is created, and executed, after its dimensions have been
61 * initialized accordingly. Each instance can either pass, fail or throw an unexpected error; this helper
62 * class defines several policies for how failures should be handled during a combo test execution
63 * (i.e. should errors be ignored? Do we want the first failure to result in a failure of the whole
64 * combo test?).
65 * <p>
66 * Additionally, this helper class allows to specify filter methods that can be used to throw out
67 * illegal combinations of dimensions - for instance, in the example above, we might want to exclude
68 * all combinations involving 'protected' and 'private' modifiers, which are disallowed for toplevel
69 * declarations.
70 * <p>
71 * While combo tests can be used for a variety of workloads, typically their main task will consist
72 * in performing some kind of javac compilation. For this purpose, this framework defines an optimized
73 * javac context {@link ReusableContext} which can be shared across multiple combo instances,
74 * when the framework detects it's safe to do so. This allows to reduce the overhead associated with
75 * compiler initialization when the test space is big.
76 */
77 public class ComboTestHelper<X extends ComboInstance<X>> {
78
79 /** Failure mode. */
80 FailMode failMode = FailMode.FAIL_FAST;
81
82 /** Ignore mode. */
83 IgnoreMode ignoreMode = IgnoreMode.IGNORE_NONE;
84
85 /** Combo test instance filter. */
86 Optional<Predicate<X>> optFilter = Optional.empty();
87
88 /** Combo test dimensions. */
89 List<DimensionInfo<?>> dimensionInfos = new ArrayList<>();
90
91 /** Combo test stats. */
92 Info info = new Info();
93
94 /** Shared JavaCompiler used across all combo test instances. */
95 JavaCompiler comp = ToolProvider.getSystemJavaCompiler();
96
97 /** Shared file manager used across all combo test instances. */
98 StandardJavaFileManager fm = comp.getStandardFileManager(null, null, null);
99
100 /** Shared context used across all combo instances. */
101 ReusableContext context = new ReusableContext();
102
103 /**
104 * Set failure mode for this combo test.
105 */
106 public ComboTestHelper<X> withFailMode(FailMode failMode) {
107 this.failMode = failMode;
108 return this;
109 }
110
111 /**
112 * Set ignore mode for this combo test.
113 */
114 public ComboTestHelper<X> withIgnoreMode(IgnoreMode ignoreMode) {
115 this.ignoreMode = ignoreMode;
116 return this;
117 }
118
119 /**
120 * Set a filter for combo test instances to be ignored.
121 */
122 public ComboTestHelper<X> withFilter(Predicate<X> filter) {
123 optFilter = Optional.of(optFilter.map(filter::and).orElse(filter));
124 return this;
125 }
126
127 /**
128 * Adds a new dimension to this combo test, with a given name an array of values.
129 */
130 @SafeVarargs
131 public final <D> ComboTestHelper<X> withDimension(String name, D... dims) {
132 return withDimension(name, null, dims);
133 }
134
135 /**
136 * Adds a new dimension to this combo test, with a given name, an array of values and a
137 * coresponding setter to be called in order to set the dimension value on the combo test instance
138 * (before test execution).
139 */
140 @SuppressWarnings("unchecked")
141 @SafeVarargs
142 public final <D> ComboTestHelper<X> withDimension(String name, DimensionSetter<X, D> setter, D... dims) {
143 dimensionInfos.add(new BasicDimensionInfo<>(name, dims, setter));
144 return this;
145 }
146
147 public enum ArrayDimensionKind {
148 PERMUTATIONS,
149 COMBINATIONS;
150 }
151
152 /**
153 * Adds a new array dimension to this combo test, with a given base name. This allows to specify
154 * multiple dimensions at once; the names of the underlying dimensions will be generated from the
155 * base name, using standard array bracket notation - i.e. "DIM[0]", "DIM[1]", etc.
156 */
157 @SafeVarargs
158 public final <D> ComboTestHelper<X> withArrayDimension(String name, int size, D... dims) {
159 return withArrayDimension(name, null, size, ArrayDimensionKind.PERMUTATIONS, dims);
160 }
161
162 @SafeVarargs
163 public final <D> ComboTestHelper<X> withArrayDimension(String name, int size, ArrayDimensionKind kind, D... dims) {
164 return withArrayDimension(name, null, size, kind, dims);
165 }
166
167 /**
168 * Adds a new array dimension to this combo test, with a given base name, an array of values and a
169 * coresponding array setter to be called in order to set the dimension value on the combo test
170 * instance (before test execution). This allows to specify multiple dimensions at once; the names
171 * of the underlying dimensions will be generated from the base name, using standard array bracket
172 * notation - i.e. "DIM[0]", "DIM[1]", etc.
173 */
174 @SafeVarargs
175 public final <D> ComboTestHelper<X> withArrayDimension(String name, ArrayDimensionSetter<X, D> setter, int size, D... dims) {
176 return withArrayDimension(name, setter, size, ArrayDimensionKind.PERMUTATIONS, dims);
177 }
178
179 @SafeVarargs
180 public final <D> ComboTestHelper<X> withArrayDimension(String name, ArrayDimensionSetter<X, D> setter, int size, ArrayDimensionKind kind, D... dims) {
181 if (kind == ArrayDimensionKind.PERMUTATIONS) {
182 for (int i = 0 ; i < size ; i++) {
183 dimensionInfos.add(new ArrayDimensionInfo<>(name, dims, i, setter));
184 }
185 } else {
186 DependentArrayDimensionInfo<D> prev = null;
187 for (int i = 0 ; i < size ; i++) {
188 prev = new DependentArrayDimensionInfo<>(prev, name, dims, i, setter);
189 dimensionInfos.add(prev);
190 }
191 }
192 return this;
193 }
194
195 /**
196 * Returns the stat object associated with this combo test.
197 */
198 public Info info() {
199 return info;
200 }
201
202 /**
203 * Runs this combo test. This will generate the combinatorial explosion of all dimensions, and
204 * execute a new test instance (built using given supplier) for each such combination.
205 */
206 public void run(Supplier<X> instanceBuilder) {
207 run(instanceBuilder, null);
208 }
209
210 /**
211 * Runs this combo test. This will generate the combinatorial explosion of all dimensions, and
212 * execute a new test instance (built using given supplier) for each such combination. Before
213 * executing the test instance entry point, the supplied initialization method is called on
214 * the test instance; this is useful for ad-hoc test instance initialization once all the dimension
215 * values have been set.
216 */
217 public void run(Supplier<X> instanceBuilder, Consumer<X> initAction) {
218 runInternal(0, new Stack<>(), instanceBuilder, Optional.ofNullable(initAction));
219 end();
220 }
221
222 /**
223 * Generate combinatorial explosion of all dimension values and create a new test instance
224 * for each combination.
225 */
226 @SuppressWarnings({"unchecked", "rawtypes"})
227 private void runInternal(int index, Stack<DimensionBinding<?>> bindings, Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction) {
228 if (index == dimensionInfos.size()) {
229 runCombo(instanceBuilder, initAction, bindings);
230 } else {
231 DimensionInfo<?> dinfo = dimensionInfos.get(index);
232 for (Object d : dinfo.dimensions(bindings)) {
233 bindings.push(new DimensionBinding(d, dinfo));
234 runInternal(index + 1, bindings, instanceBuilder, initAction);
235 bindings.pop();
236 }
237 }
238 }
239
240 /**
241 * Run a new test instance using supplied dimension bindings. All required setters and initialization
242 * method are executed before calling the instance main entry point. Also checks if the instance
243 * is compatible with the specified test filters; if not, the test is simply skipped.
244 */
245 @SuppressWarnings("unchecked")
246 private void runCombo(Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction, List<DimensionBinding<?>> bindings) {
247 X x = instanceBuilder.get();
248 for (DimensionBinding<?> binding : bindings) {
249 binding.init(x);
250 }
251 initAction.ifPresent(action -> action.accept(x));
252 info.comboCount++;
253 if (!optFilter.isPresent() || optFilter.get().test(x)) {
254 x.run(new Env(bindings));
255 if (failMode.shouldStop(ignoreMode, info)) {
256 end();
257 }
258 } else {
259 info.skippedCount++;
260 }
261 }
262
263 /**
264 * This method is executed upon combo test completion (either normal or erroneous). Closes down
265 * all pending resources and dumps useful stats info.
266 */
267 private void end() {
268 try {
269 fm.close();
270 if (info.hasFailures()) {
271 throw new AssertionError("Failure when executing combo:" + info.lastFailure.orElse(""));
272 } else if (info.hasErrors()) {
273 throw new AssertionError("Unexpected exception while executing combo", info.lastError.get());
274 }
275 } catch (IOException ex) {
276 throw new AssertionError("Failure when closing down shared file manager; ", ex);
277 } finally {
278 info.dump();
279 }
280 }
281
282 /**
283 * Functional interface for specifying combo test instance setters.
284 */
285 public interface DimensionSetter<X extends ComboInstance<X>, D> {
286 void set(X x, D d);
287 }
288
289 /**
290 * Functional interface for specifying combo test instance array setters. The setter method
291 * receives an extra argument for the index of the array element to be set.
292 */
293 public interface ArrayDimensionSetter<X extends ComboInstance<X>, D> {
294 void set(X x, D d, int index);
295 }
296
297 /**
298 * Dimension descriptor; each dimension has a name, an array of value and an optional setter
299 * to be called on the associated combo test instance.
300 */
301 abstract class DimensionInfo<D> {
302 String name;
303 Optional<DimensionSetter<X, D>> optSetter;
304
305 DimensionInfo(String name, DimensionSetter<X, D> setter) {
306 this.name = name;
307 this.optSetter = Optional.ofNullable(setter);
308 }
309
310 abstract Iterable<D> dimensions(Stack<DimensionBinding<?>> bindings);
311 }
312
313 class BasicDimensionInfo<D> extends DimensionInfo<D> {
314 D[] dims;
315
316 BasicDimensionInfo(String name, D[] dims, DimensionSetter<X, D> setter) {
317 super(name, setter);
318 this.dims = dims;
319 }
320
321 @Override
322 public Iterable<D> dimensions(Stack<DimensionBinding<?>> bindings) {
323 return new ArrayIterable<D>(0, dims);
324 }
325 }
326
327 /**
328 * Array dimension descriptor. The dimension name is derived from a base name and an index using
329 * standard bracket notation; ; the setter accepts an additional 'index' argument to point
330 * to the array element to be initialized.
331 */
332 class ArrayDimensionInfo<D> extends BasicDimensionInfo<D> {
333 public ArrayDimensionInfo(String name, D[] dims, int index, ArrayDimensionSetter<X, D> setter) {
334 super(String.format("%s[%d]", name, index), dims,
335 setter != null ? (x, d) -> setter.set(x, d, index) : null);
336 }
337 }
338
339 /**
340 * Dependent array dimension descriptor. The dimension name is derived from a base name and an index using
341 * standard bracket notation; ; the setter accepts an additional 'index' argument to point
342 * to the array element to be initialized.
343 */
344 class DependentArrayDimensionInfo<D> extends BasicDimensionInfo<D> {
345
346 DependentArrayDimensionInfo<D> prev;
347
348 public DependentArrayDimensionInfo(DependentArrayDimensionInfo<D> prev, String name, D[] dims, int index, ArrayDimensionSetter<X, D> setter) {
349 super(String.format("%s[%d]", name, index), dims,
350 setter != null ? (x, d) -> setter.set(x, d, index) : null);
351 this.prev = prev;
352 }
353
354 @Override
355 public Iterable<D> dimensions(Stack<DimensionBinding<?>> bindings) {
356 if (bindings.peek().info == prev) {
357 Object d = bindings.peek().d;
358 for (int i = 0 ; i < dims.length ; i++) {
359 if (dims[i] == d) {
360 return new ArrayIterable<>(i, dims);
361 }
362 }
363 throw new IllegalStateException("BAD");
364 } else {
365 return super.dimensions(bindings);
366 }
367 }
368 }
369
370 static class ArrayIterable<D> implements Iterable<D> {
371 int start;
372 D[] dims;
373
374 public ArrayIterable(int start, D[] dims) {
375 this.start = start;
376 this.dims = dims;
377 }
378
379 public Iterator<D> iterator() {
380 return new Iterator<D>() {
381 private int curr = start;
382
383 public boolean hasNext() {
384 return dims.length > curr;
385 }
386
387 public D next() {
388 return dims[curr++];
389 }
390 };
391 }
392 }
393
394 /**
395 * Failure policies for a combo test run.
396 */
397 public enum FailMode {
398 /** Combo test fails when first failure is detected. */
399 FAIL_FAST,
400 /** Combo test fails after all instances have been executed. */
401 FAIL_AFTER;
402
403 boolean shouldStop(IgnoreMode ignoreMode, Info info) {
404 switch (this) {
405 case FAIL_FAST:
406 return !ignoreMode.canIgnore(info);
407 default:
408 return false;
409 }
410 }
411 }
412
413 /**
414 * Ignore policies for a combo test run.
415 */
416 public enum IgnoreMode {
417 /** No error or failure is ignored. */
418 IGNORE_NONE,
419 /** Only errors are ignored. */
420 IGNORE_ERRORS,
421 /** Only failures are ignored. */
422 IGNORE_FAILURES,
423 /** Both errors and failures are ignored. */
424 IGNORE_ALL;
425
426 boolean canIgnore(Info info) {
427 switch (this) {
428 case IGNORE_ERRORS:
429 return info.failCount == 0;
430 case IGNORE_FAILURES:
431 return info.errCount == 0;
432 case IGNORE_ALL:
433 return true;
434 default:
435 return info.failCount == 0 && info.errCount == 0;
436 }
437 }
438 }
439
440 /**
441 * A dimension binding. This is essentially a pair of a dimension value and its corresponding
442 * dimension info.
443 */
444 class DimensionBinding<D> {
445 D d;
446 DimensionInfo<D> info;
447
448 DimensionBinding(D d, DimensionInfo<D> info) {
449 this.d = d;
450 this.info = info;
451 }
452
453 void init(X x) {
454 info.optSetter.ifPresent(setter -> setter.set(x, d));
455 }
456
457 public String toString() {
458 return String.format("(%s -> %s)", info.name, d);
459 }
460 }
461
462 /**
463 * This class is used to keep track of combo tests stats; info such as numbero of failures/errors,
464 * number of times a context has been shared/dropped are all recorder here.
465 */
466 public static class Info {
467 int failCount;
468 int errCount;
469 int passCount;
470 int comboCount;
471 int skippedCount;
472 int ctxReusedCount;
473 int ctxDroppedCount;
474 Optional<String> lastFailure = Optional.empty();
475 Optional<Throwable> lastError = Optional.empty();
476
477 void dump() {
478 System.err.println(String.format("%d total checks executed", comboCount));
479 System.err.println(String.format("%d successes found", passCount));
480 System.err.println(String.format("%d failures found", failCount));
481 System.err.println(String.format("%d errors found", errCount));
482 System.err.println(String.format("%d skips found", skippedCount));
483 System.err.println(String.format("%d contexts shared", ctxReusedCount));
484 System.err.println(String.format("%d contexts dropped", ctxDroppedCount));
485 }
486
487 public int getComboCount() { return comboCount; }
488
489 public boolean hasFailures() {
490 return failCount != 0;
491 }
492
493 public boolean hasErrors() {
494 return errCount != 0;
495 }
496 }
497
498 /**
499 * THe execution environment for a given combo test instance. An environment contains the
500 * bindings for all the dimensions, along with the combo parameter cache (this is non-empty
501 * only if one or more dimensions are subclasses of the {@code ComboParameter} interface).
502 */
503 class Env {
504 List<DimensionBinding<?>> bindings;
505 Map<String, ComboParameter> parametersCache = new HashMap<>();
506
507 @SuppressWarnings({"Unchecked", "rawtypes"})
508 Env(List<DimensionBinding<?>> bindings) {
509 this.bindings = bindings;
510 for (DimensionBinding<?> binding : bindings) {
511 if (binding.d instanceof ComboParameter) {
512 parametersCache.put(binding.info.name, (ComboParameter)binding.d);
513 };
514 }
515 }
516
517 Info info() {
518 return ComboTestHelper.this.info();
519 }
520
521 StandardJavaFileManager fileManager() {
522 return fm;
523 }
524
525 JavaCompiler javaCompiler() {
526 return comp;
527 }
528
529 ReusableContext context() {
530 return context;
531 }
532
533 ReusableContext setContext(ReusableContext context) {
534 return ComboTestHelper.this.context = context;
535 }
536 }
537 }
538
539
540