1 /*
   2  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 package combo;
  25 
  26 import javax.tools.JavaCompiler;
  27 import javax.tools.StandardJavaFileManager;
  28 import javax.tools.ToolProvider;
  29 
  30 import java.io.IOException;
  31 import java.io.Writer;
  32 import java.util.ArrayList;
  33 import java.util.HashMap;
  34 import java.util.List;
  35 import java.util.Map;
  36 import java.util.Optional;
  37 import java.util.Stack;
  38 import java.util.function.Consumer;
  39 import java.util.function.Predicate;
  40 import java.util.function.Supplier;
  41 
  42 import javax.tools.DiagnosticListener;
  43 import javax.tools.JavaFileManager;
  44 import javax.tools.JavaFileObject;
  45 
  46 import com.sun.source.util.JavacTask;
  47 import com.sun.tools.javac.api.JavacTaskPool;
  48 
  49 /**
  50  * An helper class for defining combinatorial (aka "combo" tests). A combo test is made up of one
  51  * or more 'dimensions' - each of which represent a different axis of the test space. For instance,
  52  * if we wanted to test class/interface declaration, one dimension could be the keyword used for
  53  * the declaration (i.e. 'class' vs. 'interface') while another dimension could be the class/interface
  54  * modifiers (i.e. 'public', 'pachake-private' etc.). A combo test consists in running a test instance
  55  * for each point in the test space; that is, for any combination of the combo test dimension:
  56  * <p>
  57  * 'public' 'class'
  58  * 'public' interface'
  59  * 'package-private' 'class'
  60  * 'package-private' 'interface'
  61  * ...
  62  * <p>
  63  * A new test instance {@link ComboInstance} is created, and executed, after its dimensions have been
  64  * initialized accordingly. Each instance can either pass, fail or throw an unexpected error; this helper
  65  * class defines several policies for how failures should be handled during a combo test execution
  66  * (i.e. should errors be ignored? Do we want the first failure to result in a failure of the whole
  67  * combo test?).
  68  * <p>
  69  * Additionally, this helper class allows to specify filter methods that can be used to throw out
  70  * illegal combinations of dimensions - for instance, in the example above, we might want to exclude
  71  * all combinations involving 'protected' and 'private' modifiers, which are disallowed for toplevel
  72  * declarations.
  73  * <p>
  74  * While combo tests can be used for a variety of workloads, typically their main task will consist
  75  * in performing some kind of javac compilation. For this purpose, this framework defines an optimized
  76  * javac context {@link ReusableContext} which can be shared across multiple combo instances,
  77  * when the framework detects it's safe to do so. This allows to reduce the overhead associated with
  78  * compiler initialization when the test space is big.
  79  */
  80 public class ComboTestHelper<X extends ComboInstance<X>> {
  81 
  82     /** Failure mode. */
  83     FailMode failMode = FailMode.FAIL_FAST;
  84 
  85     /** Ignore mode. */
  86     IgnoreMode ignoreMode = IgnoreMode.IGNORE_NONE;
  87 
  88     /** Combo test instance filter. */
  89     Optional<Predicate<X>> optFilter = Optional.empty();
  90 
  91     /** Combo test dimensions. */
  92     List<DimensionInfo<?>> dimensionInfos = new ArrayList<>();
  93 
  94     /** Combo test stats. */
  95     Info info = new Info();
  96 
  97     /** Shared JavaCompiler used across all combo test instances. */
  98     JavaCompiler comp = ToolProvider.getSystemJavaCompiler();
  99 
 100     /** Shared file manager used across all combo test instances. */
 101     StandardJavaFileManager fm = comp.getStandardFileManager(null, null, null);
 102 
 103     /** JavacTask pool shared across all combo instances. */
 104     JavacTaskPool pool = new JavacTaskPool(1);
 105 
 106     /**
 107      * Set failure mode for this combo test.
 108      */
 109     public ComboTestHelper<X> withFailMode(FailMode failMode) {
 110         this.failMode = failMode;
 111         return this;
 112     }
 113 
 114     /**
 115      * Set ignore mode for this combo test.
 116      */
 117     public ComboTestHelper<X> withIgnoreMode(IgnoreMode ignoreMode) {
 118         this.ignoreMode = ignoreMode;
 119         return this;
 120     }
 121 
 122     /**
 123      * Set a filter for combo test instances to be ignored.
 124      */
 125     public ComboTestHelper<X> withFilter(Predicate<X> filter) {
 126         optFilter = Optional.of(optFilter.map(filter::and).orElse(filter));
 127         return this;
 128     }
 129 
 130     /**
 131      * Adds a new dimension to this combo test, with a given name an array of values.
 132      */
 133     @SafeVarargs
 134     public final <D> ComboTestHelper<X> withDimension(String name, D... dims) {
 135         return withDimension(name, null, dims);
 136     }
 137 
 138     /**
 139      * Adds a new dimension to this combo test, with a given name, an array of values and a
 140      * coresponding setter to be called in order to set the dimension value on the combo test instance
 141      * (before test execution).
 142      */
 143     @SuppressWarnings("unchecked")
 144     @SafeVarargs
 145     public final <D> ComboTestHelper<X> withDimension(String name, DimensionSetter<X, D> setter, D... dims) {
 146         dimensionInfos.add(new DimensionInfo<>(name, dims, setter));
 147         return this;
 148     }
 149 
 150     /**
 151      * Adds a new array dimension to this combo test, with a given base name. This allows to specify
 152      * multiple dimensions at once; the names of the underlying dimensions will be generated from the
 153      * base name, using standard array bracket notation - i.e. "DIM[0]", "DIM[1]", etc.
 154      */
 155     @SafeVarargs
 156     public final <D> ComboTestHelper<X> withArrayDimension(String name, int size, D... dims) {
 157         return withArrayDimension(name, null, size, dims);
 158     }
 159 
 160     /**
 161      * Adds a new array dimension to this combo test, with a given base name, an array of values and a
 162      * coresponding array setter to be called in order to set the dimension value on the combo test
 163      * instance (before test execution). This allows to specify multiple dimensions at once; the names
 164      * of the underlying dimensions will be generated from the base name, using standard array bracket
 165      * notation - i.e. "DIM[0]", "DIM[1]", etc.
 166      */
 167     @SafeVarargs
 168     public final <D> ComboTestHelper<X> withArrayDimension(String name, ArrayDimensionSetter<X, D> setter, int size, D... dims) {
 169         for (int i = 0 ; i < size ; i++) {
 170             dimensionInfos.add(new ArrayDimensionInfo<>(name, dims, i, setter));
 171         }
 172         return this;
 173     }
 174 
 175     /**
 176      * Returns the stat object associated with this combo test.
 177      */
 178     public Info info() {
 179         return info;
 180     }
 181 
 182     /**
 183      * Runs this combo test. This will generate the combinatorial explosion of all dimensions, and
 184      * execute a new test instance (built using given supplier) for each such combination.
 185      */
 186     public void run(Supplier<X> instanceBuilder) {
 187         run(instanceBuilder, null);
 188     }
 189 
 190     /**
 191      * Runs this combo test. This will generate the combinatorial explosion of all dimensions, and
 192      * execute a new test instance (built using given supplier) for each such combination. Before
 193      * executing the test instance entry point, the supplied initialization method is called on
 194      * the test instance; this is useful for ad-hoc test instance initialization once all the dimension
 195      * values have been set.
 196      */
 197     public void run(Supplier<X> instanceBuilder, Consumer<X> initAction) {
 198         runInternal(0, new Stack<>(), instanceBuilder, Optional.ofNullable(initAction));
 199         end();
 200     }
 201 
 202     /**
 203      * Generate combinatorial explosion of all dimension values and create a new test instance
 204      * for each combination.
 205      */
 206     @SuppressWarnings({"unchecked", "rawtypes"})
 207     private void runInternal(int index, Stack<DimensionBinding<?>> bindings, Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction) {
 208         if (index == dimensionInfos.size()) {
 209             runCombo(instanceBuilder, initAction, bindings);
 210         } else {
 211             DimensionInfo<?> dinfo = dimensionInfos.get(index);
 212             for (Object d : dinfo.dims) {
 213                 bindings.push(new DimensionBinding(d, dinfo));
 214                 runInternal(index + 1, bindings, instanceBuilder, initAction);
 215                 bindings.pop();
 216             }
 217         }
 218     }
 219 
 220     /**
 221      * Run a new test instance using supplied dimension bindings. All required setters and initialization
 222      * method are executed before calling the instance main entry point. Also checks if the instance
 223      * is compatible with the specified test filters; if not, the test is simply skipped.
 224      */
 225     @SuppressWarnings("unchecked")
 226     private void runCombo(Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction, List<DimensionBinding<?>> bindings) {
 227         X x = instanceBuilder.get();
 228         for (DimensionBinding<?> binding : bindings) {
 229             binding.init(x);
 230         }
 231         initAction.ifPresent(action -> action.accept(x));
 232         info.comboCount++;
 233         if (!optFilter.isPresent() || optFilter.get().test(x)) {
 234             x.run(new Env(bindings));
 235             if (failMode.shouldStop(ignoreMode, info)) {
 236                 end();
 237             }
 238         } else {
 239             info.skippedCount++;
 240         }
 241     }
 242 
 243     /**
 244      * This method is executed upon combo test completion (either normal or erroneous). Closes down
 245      * all pending resources and dumps useful stats info.
 246      */
 247     private void end() {
 248         try {
 249             fm.close();
 250             if (info.hasFailures()) {
 251                 throw new AssertionError("Failure when executing combo:" + info.lastFailure.orElse(""));
 252             } else if (info.hasErrors()) {
 253                 throw new AssertionError("Unexpected exception while executing combo", info.lastError.get());
 254             }
 255         } catch (IOException ex) {
 256             throw new AssertionError("Failure when closing down shared file manager; ", ex);
 257         } finally {
 258             info.dump(this);
 259         }
 260     }
 261 
 262     /**
 263      * Functional interface for specifying combo test instance setters.
 264      */
 265     public interface DimensionSetter<X extends ComboInstance<X>, D> {
 266         void set(X x, D d);
 267     }
 268 
 269     /**
 270      * Functional interface for specifying combo test instance array setters. The setter method
 271      * receives an extra argument for the index of the array element to be set.
 272      */
 273     public interface ArrayDimensionSetter<X extends ComboInstance<X>, D> {
 274         void set(X x, D d, int index);
 275     }
 276 
 277     /**
 278      * Dimension descriptor; each dimension has a name, an array of value and an optional setter
 279      * to be called on the associated combo test instance.
 280      */
 281     class DimensionInfo<D> {
 282         String name;
 283         D[] dims;
 284         boolean isParameter;
 285         Optional<DimensionSetter<X, D>> optSetter;
 286 
 287         DimensionInfo(String name, D[] dims, DimensionSetter<X, D> setter) {
 288             this.name = name;
 289             this.dims = dims;
 290             this.optSetter = Optional.ofNullable(setter);
 291             this.isParameter = dims[0] instanceof ComboParameter;
 292         }
 293     }
 294 
 295     /**
 296      * Array dimension descriptor. The dimension name is derived from a base name and an index using
 297      * standard bracket notation; ; the setter accepts an additional 'index' argument to point
 298      * to the array element to be initialized.
 299      */
 300     class ArrayDimensionInfo<D> extends DimensionInfo<D> {
 301         public ArrayDimensionInfo(String name, D[] dims, int index, ArrayDimensionSetter<X, D> setter) {
 302             super(String.format("%s[%d]", name, index), dims,
 303                     setter != null ? (x, d) -> setter.set(x, d, index) : null);
 304         }
 305     }
 306 
 307     /**
 308      * Failure policies for a combo test run.
 309      */
 310     public enum FailMode {
 311         /** Combo test fails when first failure is detected. */
 312         FAIL_FAST,
 313         /** Combo test fails after all instances have been executed. */
 314         FAIL_AFTER;
 315 
 316         boolean shouldStop(IgnoreMode ignoreMode, Info info) {
 317             switch (this) {
 318                 case FAIL_FAST:
 319                     return !ignoreMode.canIgnore(info);
 320                 default:
 321                     return false;
 322             }
 323         }
 324     }
 325 
 326     /**
 327      * Ignore policies for a combo test run.
 328      */
 329     public enum IgnoreMode {
 330         /** No error or failure is ignored. */
 331         IGNORE_NONE,
 332         /** Only errors are ignored. */
 333         IGNORE_ERRORS,
 334         /** Only failures are ignored. */
 335         IGNORE_FAILURES,
 336         /** Both errors and failures are ignored. */
 337         IGNORE_ALL;
 338 
 339         boolean canIgnore(Info info) {
 340             switch (this) {
 341                 case IGNORE_ERRORS:
 342                     return info.failCount == 0;
 343                 case IGNORE_FAILURES:
 344                     return info.errCount == 0;
 345                 case IGNORE_ALL:
 346                     return true;
 347                 default:
 348                     return info.failCount == 0 && info.errCount == 0;
 349             }
 350         }
 351     }
 352 
 353     /**
 354      * A dimension binding. This is essentially a pair of a dimension value and its corresponding
 355      * dimension info.
 356      */
 357     class DimensionBinding<D> {
 358         D d;
 359         DimensionInfo<D> info;
 360 
 361         DimensionBinding(D d, DimensionInfo<D> info) {
 362             this.d = d;
 363             this.info = info;
 364         }
 365 
 366         void init(X x) {
 367             info.optSetter.ifPresent(setter -> setter.set(x, d));
 368         }
 369 
 370         public String toString() {
 371             return String.format("(%s -> %s)", info.name, d);
 372         }
 373     }
 374 
 375     /**
 376      * This class is used to keep track of combo tests stats; info such as numbero of failures/errors,
 377      * number of times a context has been shared/dropped are all recorder here.
 378      */
 379     public static class Info {
 380         int failCount;
 381         int errCount;
 382         int passCount;
 383         int comboCount;
 384         int skippedCount;
 385         Optional<String> lastFailure = Optional.empty();
 386         Optional<Throwable> lastError = Optional.empty();
 387 
 388         void dump(ComboTestHelper<?> helper) {
 389             System.err.println(String.format("%d total checks executed", comboCount));
 390             System.err.println(String.format("%d successes found", passCount));
 391             System.err.println(String.format("%d failures found", failCount));
 392             System.err.println(String.format("%d errors found", errCount));
 393             System.err.println(String.format("%d skips found", skippedCount));
 394             helper.pool.printStatistics(System.err);
 395         }
 396 
 397         public boolean hasFailures() {
 398             return failCount != 0;
 399         }
 400 
 401         public boolean hasErrors() {
 402             return errCount != 0;
 403         }
 404     }
 405 
 406     /**
 407      * The execution environment for a given combo test instance. An environment contains the
 408      * bindings for all the dimensions, along with the combo parameter cache (this is non-empty
 409      * only if one or more dimensions are subclasses of the {@code ComboParameter} interface).
 410      */
 411     class Env {
 412         List<DimensionBinding<?>> bindings;
 413         Map<String, ComboParameter> parametersCache = new HashMap<>();
 414 
 415         @SuppressWarnings({"Unchecked", "rawtypes"})
 416         Env(List<DimensionBinding<?>> bindings) {
 417             this.bindings = bindings;
 418             for (DimensionBinding<?> binding : bindings) {
 419                 if (binding.info.isParameter) {
 420                     parametersCache.put(binding.info.name, (ComboParameter)binding.d);
 421                 };
 422             }
 423         }
 424 
 425         Info info() {
 426             return ComboTestHelper.this.info();
 427         }
 428 
 429         StandardJavaFileManager fileManager() {
 430             return fm;
 431         }
 432 
 433         JavaCompiler javaCompiler() {
 434             return comp;
 435         }
 436 
 437         JavacTaskPool pool() {
 438             return pool;
 439         }
 440     }
 441 }
 442 
 443 
 444