1 /*
   2  * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.jfr.event.metadata;
  26 
  27 import java.io.IOException;
  28 import java.lang.reflect.Field;
  29 import java.nio.file.Files;
  30 import java.nio.file.Path;
  31 import java.nio.file.Paths;
  32 import java.util.Arrays;
  33 import java.util.HashSet;
  34 import java.util.List;
  35 import java.util.Set;
  36 import java.util.stream.Collectors;
  37 import java.util.stream.Stream;
  38 
  39 import jdk.jfr.EventType;
  40 import jdk.jfr.Experimental;
  41 import jdk.jfr.FlightRecorder;
  42 import jdk.test.lib.Utils;
  43 import jdk.test.lib.jfr.EventNames;
  44 
  45 /**
  46  * @test Check for JFR events not covered by tests
  47  * @key jfr
  48  * @library /lib /
  49  * @run main jdk.jfr.event.metadata.TestLookForUntestedEvents
  50  */
  51 public class TestLookForUntestedEvents {
  52     private static final Path jfrTestRoot = Paths.get(Utils.TEST_SRC).getParent().getParent();
  53     private static final String MSG_SEPARATOR = "==========================";
  54     private static Set<String> jfrEventTypes = new HashSet<>();
  55 
  56     private static final Set<String> hardToTestEvents = new HashSet<>(
  57         Arrays.asList(
  58             "DataLoss", "IntFlag", "ReservedStackActivation",
  59             "DoubleFlag", "UnsignedLongFlagChanged", "IntFlagChanged",
  60             "UnsignedIntFlag", "UnsignedIntFlagChanged", "DoubleFlagChanged")
  61     );
  62 
  63     // GC uses specific framework to test the events, instead of using event names literally.
  64     // GC tests were inspected, as well as runtime output of GC tests.
  65     // The following events below are know to be covered based on that inspection.
  66     private static final Set<String> coveredGcEvents = new HashSet<>(
  67         Arrays.asList(
  68             "MetaspaceGCThreshold", "MetaspaceAllocationFailure", "MetaspaceOOM",
  69             "MetaspaceChunkFreeListSummary", "G1HeapSummary", "ParallelOldGarbageCollection",
  70             "OldGarbageCollection", "G1GarbageCollection", "GCPhasePause",
  71             "GCPhasePauseLevel1", "GCPhasePauseLevel2", "GCPhasePauseLevel3",
  72             "GCPhasePauseLevel4", "GCPhaseConcurrent")
  73     );
  74 
  75     // This is a "known failure list" for this test.
  76     // NOTE: if the event is not covered, a bug should be open, and bug number
  77     // noted in the comments for this set.
  78     private static final Set<String> knownNotCoveredEvents = new HashSet<>(
  79         Arrays.asList(
  80                 // DumpReason: JDK-8213918
  81                 "DumpReason",
  82                 // No G1 IHOP in JDK 8
  83                 "G1BasicIHOP",
  84                 "G1AdaptiveIHOP",
  85                 // Missing CI in JDK 8
  86                 "CodeSweeperStatistics"
  87         )
  88     );
  89 
  90 
  91     public static void main(String[] args) throws Exception {
  92         for (EventType type : FlightRecorder.getFlightRecorder().getEventTypes()) {
  93             if (type.getAnnotation(Experimental.class) == null) {
  94                 jfrEventTypes.add(type.getName().replace("jdk.", ""));
  95             }
  96         }
  97 
  98         checkEventNamesClass();
  99         lookForEventsNotCoveredByTests();
 100     }
 101 
 102     // Look thru JFR tests to make sure JFR events are referenced in the tests
 103     private static void lookForEventsNotCoveredByTests() throws Exception {
 104         List<Path> paths = Files.walk(jfrTestRoot)
 105             .filter(Files::isRegularFile)
 106             .filter(path -> isJavaFile(path))
 107             .collect(Collectors.toList());
 108 
 109         Set<String> eventsNotCoveredByTest = new HashSet<>(jfrEventTypes);
 110         for (String event : jfrEventTypes) {
 111             for (Path p : paths) {
 112                 if (findStringInFile(p, event)) {
 113                     eventsNotCoveredByTest.remove(event);
 114                     break;
 115                 }
 116             }
 117         }
 118 
 119         // Account for hard-to-test, experimental and GC tested events
 120         eventsNotCoveredByTest.removeAll(hardToTestEvents);
 121         eventsNotCoveredByTest.removeAll(coveredGcEvents);
 122 
 123         eventsNotCoveredByTest.removeAll(knownNotCoveredEvents);
 124 
 125         if (!eventsNotCoveredByTest.isEmpty()) {
 126             print(MSG_SEPARATOR + " Events not covered by test");
 127             for (String event: eventsNotCoveredByTest) {
 128                 print(event);
 129             }
 130             print(MSG_SEPARATOR);
 131             throw new RuntimeException("Found JFR events not covered by tests");
 132         }
 133     }
 134 
 135     // Make sure all the JFR events are accounted for in jdk.test.lib.jfr.EventNames
 136     private static void checkEventNamesClass() throws Exception {
 137         // jdk.test.lib.jfr.EventNames
 138         Set<String> eventsFromEventNamesClass = new HashSet<>();
 139         for (Field f : EventNames.class.getFields()) {
 140             String name = f.getName();
 141             if (!name.equals("PREFIX")) {
 142                 String eventName = (String) f.get(null);
 143                 eventName = eventName.replace(EventNames.PREFIX, "");
 144                 eventsFromEventNamesClass.add(eventName);
 145             }
 146         }
 147 
 148         if (!jfrEventTypes.equals(eventsFromEventNamesClass)) {
 149             String exceptionMsg = "Events declared in jdk.test.lib.jfr.EventNames differ " +
 150                          "from events returned by FlightRecorder.getEventTypes()";
 151             print(MSG_SEPARATOR);
 152             print(exceptionMsg);
 153             print("");
 154             printSetDiff(jfrEventTypes, eventsFromEventNamesClass,
 155                         "jfrEventTypes", "eventsFromEventNamesClass");
 156             print("");
 157 
 158             print("This could be because:");
 159             print("1) You forgot to write a unit test. Please do so in test/jdk/jdk/jfr/event/");
 160             print("2) You wrote a unit test, but you didn't reference the event in");
 161             print("   test/lib/jdk/test/lib/jfr/EventNames.java. ");
 162             print("3) It is not feasible to test the event, not even a sanity test. ");
 163             print("   Add the event name to test/lib/jdk/test/lib/jfr/EventNames.java ");
 164             print("   and a short comment why it can't be tested");
 165             print("4) The event is experimental. Please add 'experimental=\"true\"' to <Event> ");
 166             print("   element in metadata.xml if it is a native event, or @Experimental if it is a ");
 167             print("   Java event. The event will now not show up in JMC");
 168             System.out.println(MSG_SEPARATOR);
 169             throw new RuntimeException(exceptionMsg);
 170         }
 171     }
 172 
 173     // ================ Helper methods
 174     private static boolean isJavaFile(Path p) {
 175         String fileName = p.getFileName().toString();
 176         int i = fileName.lastIndexOf('.');
 177         if ( (i < 0) || (i > fileName.length()) ) {
 178             return false;
 179         }
 180         return "java".equals(fileName.substring(i+1));
 181     }
 182 
 183     private static boolean findStringInFile(Path p, String searchTerm) throws IOException {
 184         long c = 0;
 185         try (Stream<String> stream = Files.lines(p)) {
 186             c = stream
 187                 .filter(line -> line.contains(searchTerm))
 188                 .count();
 189         }
 190         return (c != 0);
 191     }
 192 
 193     private static void printSetDiff(Set<String> a, Set<String> b,
 194         String setAName, String setBName) {
 195         if (a.size() > b.size()) {
 196             a.removeAll(b);
 197             System.out.printf("Set %s has more elements than set %s:", setAName, setBName);
 198             System.out.println();
 199             printSet(a);
 200         } else {
 201             b.removeAll(a);
 202             System.out.printf("Set %s has more elements than set %s:", setBName, setAName);
 203             System.out.println();
 204             printSet(b);
 205         }
 206     }
 207 
 208     private static void printSet(Set<String> set) {
 209         for (String e : set) {
 210             System.out.println(e);
 211         }
 212     }
 213 
 214     private static void print(String s) {
 215         System.out.println(s);
 216     }
 217 }