1 /*
  2  * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2018, 2020 SAP SE. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  */
 24 
 25 /**
 26  * @test
 27  * @summary Validate and test -?, -h and --help flags. All tools in the jdk
 28  *          should take the same flags to display the help message. These
 29  *          flags should be documented in the printed help message. The
 30  *          tool should quit without error code after displaying the
 31  *          help message (if there  is no other problem with the command
 32  *          line).
 33  *          Also check that tools that used to accept -help still do
 34  *          so. Test that tools that never accepted -help don't do so
 35  *          in future. I.e., check that the tool returns with the same
 36  *          return code as called with an invalid flag, and does not
 37  *          print anything containing '-help' in that case.
 38  * @compile HelpFlagsTest.java
 39  * @run main HelpFlagsTest
 40  */
 41 
 42 import java.io.File;
 43 
 44 public class HelpFlagsTest extends TestHelper {
 45 
 46     // Tools that should not be tested because a usage message is pointless.
 47     static final String[] TOOLS_NOT_TO_TEST = {
 48         "appletviewer",     // deprecated, don't test
 49         "jaccessinspector", // gui, don't test, win only
 50         "jaccessinspector-32", // gui, don't test, win-32 only
 51         "jaccesswalker",    // gui, don't test, win only
 52         "jaccesswalker-32", // gui, don't test, win-32 only
 53         "jconsole",         // gui, don't test
 54         "servertool",       // none. Shell, don't test.
 55         "javaw",            // don't test, win only
 56         // These shall have a help message that resembles that of
 57         // MIT's tools. Thus -?, -h and --help are supported, but not
 58         // mentioned in the help text.
 59         "kinit",
 60         "klist",
 61         "ktab",
 62         // Oracle proprietary tools without help message.
 63         "javacpl",
 64         "jmc",
 65         "jweblauncher",
 66         "jcontrol",
 67         "ssvagent"
 68     };
 69 
 70     // Lists which tools support which flags.
 71     private static class ToolHelpSpec {
 72         String toolname;
 73 
 74         // How the flags supposed to be supported are handled.
 75         //
 76         // These flags are supported, i.e.,
 77         // * the tool accepts the flag
 78         // * the tool prints a help message if the flag is specified
 79         // * this help message lists the flag
 80         // * the tool exits with exit code '0'.
 81         boolean supportsQuestionMark;
 82         boolean supportsH;
 83         boolean supportsHelp;
 84 
 85         // One tool returns with exit code != '0'.
 86         int exitcodeOfHelp;
 87 
 88         // How legacy -help is handled.
 89         //
 90         // Tools that so far support -help should still do so, but
 91         // not print documentation about it. Tools that do not
 92         // support -help should not do so in future.
 93         //
 94         // The tools accepts legacy -help. -help should not be
 95         // documented in the usage message.
 96         boolean supportsLegacyHelp;
 97 
 98         // Java itself documents -help. -help prints to stderr,
 99         // while --help prints to stdout. Leave as is.
100         boolean documentsLegacyHelp;
101 
102         // The exit code of the tool if an invalid argument is passed to it.
103         // An exit code != 0 would be expected, but not all tools handle it
104         // that way.
105         int exitcodeOfWrongFlag;
106 
107         ToolHelpSpec(String n, int q, int h, int hp, int ex1, int l, int dl, int ex2) {
108             toolname = n;
109             supportsQuestionMark = ( q  == 1 ? true : false );
110             supportsH            = ( h  == 1 ? true : false );
111             supportsHelp         = ( hp == 1 ? true : false );
112             exitcodeOfHelp       = ex1;
113 
114             supportsLegacyHelp   = (  l == 1 ? true : false );
115             documentsLegacyHelp  = ( dl == 1 ? true : false );
116             exitcodeOfWrongFlag  = ex2;
117         }
118     }
119 
120     static ToolHelpSpec[] jdkTools = {
121         //               name          -?   -h --help exitcode   -help -help  exitcode
122         //                                            of help          docu   of wrong
123         //                                                             mented flag
124         new ToolHelpSpec("jabswitch",   0,   0,   0,   0,         0,    0,     0),     // /?, prints help message anyways, win only
125         new ToolHelpSpec("jar",         1,   1,   1,   0,         0,    0,     1),     // -?, -h, --help
126         new ToolHelpSpec("jarsigner",   1,   1,   1,   0,         1,    0,     1),     // -?, -h, --help, -help accepted but not documented.
127         new ToolHelpSpec("java",        1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
128         new ToolHelpSpec("javac",       1,   0,   1,   0,         1,    1,     2),     // -?,     --help -help, Documents -help, -h is already taken for "native header output directory".
129         new ToolHelpSpec("javadoc",     1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
130         new ToolHelpSpec("javap",       1,   1,   1,   0,         1,    1,     2),     // -?, -h, --help -help, Documents -help
131         new ToolHelpSpec("javaw",       1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help, win only
132         new ToolHelpSpec("jcmd",        1,   1,   1,   0,         1,    0,     1),     // -?, -h, --help, -help accepted but not documented.
133         new ToolHelpSpec("jdb",         1,   1,   1,   0,         1,    1,     0),     // -?, -h, --help -help, Documents -help
134         new ToolHelpSpec("jdeprscan",   1,   1,   1,   0,         0,    0,     1),     // -?, -h, --help
135         new ToolHelpSpec("jdeps",       1,   1,   1,   0,         1,    0,     2),     // -?, -h, --help, -help accepted but not documented.
136         new ToolHelpSpec("jfr",         1,   1,   1,   0,         0,    0,     2),     // -?, -h, --help
137         new ToolHelpSpec("jhsdb",       0,   0,   0,   0,         0,    0,     0),     // none, prints help message anyways.
138         new ToolHelpSpec("jimage",      1,   1,   1,   0,         0,    0,     2),     // -?, -h, --help
139         new ToolHelpSpec("jinfo",       1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
140         new ToolHelpSpec("jjs",         0,   1,   1, 100,         0,    0,   100),     //     -h, --help, return code 100
141         new ToolHelpSpec("jlink",       1,   1,   1,   0,         0,    0,     2),     // -?, -h, --help
142         new ToolHelpSpec("jmap",        1,   1,   1,   0,         1,    0,     1),     // -?, -h, --help, -help accepted but not documented.
143         new ToolHelpSpec("jmod",        1,   1,   1,   0,         1,    0,     2),     // -?, -h, --help, -help accepted but not documented.
144         new ToolHelpSpec("jps",         1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
145         new ToolHelpSpec("jrunscript",  1,   1,   1,   0,         1,    1,     7),     // -?, -h, --help -help, Documents -help
146         new ToolHelpSpec("jshell",      1,   1,   1,   0,         1,    0,     1),     // -?, -h, --help, -help accepted but not documented.
147         new ToolHelpSpec("jstack",      1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
148         new ToolHelpSpec("jstat",       1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
149         new ToolHelpSpec("jstatd",      1,   1,   1,   0,         0,    0,     1),     // -?, -h, --help
150         new ToolHelpSpec("keytool",     1,   1,   1,   0,         1,    0,     1),     // none, prints help message anyways.
151         new ToolHelpSpec("rmiregistry", 0,   0,   0,   0,         0,    0,     1),     // none, prints help message anyways.
152         new ToolHelpSpec("serialver",   0,   0,   0,   0,         0,    0,     1),     // none, prints help message anyways.
153         new ToolHelpSpec("jpackage",    0,   1,   1,   0,         0,    1,     1),     //     -h, --help,
154         new ToolHelpSpec("jwebserver",  1,   1,   1,   0,         0,    1,     1),     // -?, -h, --help
155         // Panama
156         new ToolHelpSpec("jextract"    ,1,   1,   1,   0,         1,    1,   1),       // -?, -h, --help -help, Documents -help
157     };
158 
159     // Returns corresponding object from jdkTools array.
160     static ToolHelpSpec getToolHelpSpec(String tool) {
161         for (ToolHelpSpec x : jdkTools) {
162             if (tool.toLowerCase().equals(x.toolname) ||
163                 tool.toLowerCase().equals(x.toolname + ".exe"))
164                 return x;
165         }
166         return null;
167     }
168 
169     // Check whether 'flag' appears in 'line' as a word of itself. It must not
170     // be a substring of a word, as then similar flags might be matched.
171     // E.g.: --help matches in the documentation of --help-extra.
172     // This works only with english locale, as some tools have translated
173     // usage messages.
174     static boolean findFlagInLine(String line, String flag) {
175         if (line.contains(flag) &&
176             !line.contains("nknown") &&                       // Some tools say 'Unknown option "<flag>"',
177             !line.contains("invalid flag") &&                 // 'invalid flag: <flag>'
178             !line.contains("invalid option") &&               // or 'invalid option: <flag>'. Skip that.
179             !line.contains("FileNotFoundException: -help") && // Special case for idlj.
180             !line.contains("-h requires an argument") &&      // Special case for javac.
181             !line.contains("port argument,")) {               // Special case for rmiregistry.
182             // There might be several appearances of 'flag' in
183             // 'line'. (-h as substring of --help).
184             int flagLen = flag.length();
185             int lineLen = line.length();
186             for (int i = line.indexOf(flag); i >= 0; i = line.indexOf(flag, i+1)) {
187                 // There should be a space before 'flag' in 'line', or it's right at the beginning.
188                 if (i > 0 &&
189                     line.charAt(i-1) != ' ' &&
190                     line.charAt(i-1) != '[' &&  // jarsigner
191                     line.charAt(i-1) != '|' &&  // jstatd
192                     line.charAt(i-1) != '\t') { // jjs
193                     continue;
194                 }
195                 // There should be a space or comma after 'flag' in 'line', or it's just at the end.
196                 int posAfter = i + flagLen;
197                 if (posAfter < lineLen &&
198                     line.charAt(posAfter) != ' ' &&
199                     line.charAt(posAfter) != ',' &&
200                     line.charAt(posAfter) != '[' && // jar
201                     line.charAt(posAfter) != ']' && // jarsigner
202                     line.charAt(posAfter) != ')' && // jfr
203                     line.charAt(posAfter) != '|' && // jstatd
204                     line.charAt(posAfter) != ':' && // jps
205                     line.charAt(posAfter) != '"') { // keytool
206                     continue;
207                 }
208                 return true;
209             }
210         }
211         return false;
212     }
213 
214     static TestResult runToolWithFlag(File f, String flag) {
215         String x = f.getAbsolutePath();
216         TestResult tr = doExec(x, flag);
217         System.out.println("Testing " + f.getName());
218         System.out.println("#> " + x + " " + flag);
219         tr.testOutput.forEach(System.out::println);
220         System.out.println("#> echo $?");
221         System.out.println(tr.exitValue);
222 
223         return tr;
224     }
225 
226     // Checks whether tool supports flag 'flag' and documents it
227     // in the help message.
228     static String testTool(File f, String flag, int exitcode) {
229         String result = "";
230         TestResult tr = runToolWithFlag(f, flag);
231 
232         // Check that the tool accepted the flag.
233         if (exitcode == 0 && !tr.isOK()) {
234             System.out.println("failed");
235             result = "failed: " + f.getName() + " " + flag + " has exit code " + tr.exitValue + ".\n";
236         }
237 
238         // Check there is a help message listing the flag.
239         boolean foundFlag = false;
240         for (String y : tr.testOutput) {
241             if (!foundFlag && findFlagInLine(y, flag)) { // javac
242                 foundFlag = true;
243                 System.out.println("Found documentation of '" + flag + "': '" + y.trim() +"'");
244             }
245         }
246         if (!foundFlag) {
247             result += "failed: " + f.getName() + " does not document " +
248                 flag + " in help message.\n";
249         }
250 
251         if (!result.isEmpty())
252             System.out.println(result);
253 
254         return result;
255     }
256 
257     // Test the tool supports legacy option -help, but does
258     // not document it.
259     static String testLegacyFlag(File f, int exitcode) {
260         String result = "";
261         TestResult tr = runToolWithFlag(f, "-help");
262 
263         // Check that the tool accepted the flag.
264         if (exitcode == 0 && !tr.isOK()) {
265             System.out.println("failed");
266             result = "failed: " + f.getName() + " -help has exit code " + tr.exitValue + ".\n";
267         }
268 
269         // Check there is _no_ documentation of -help.
270         boolean foundFlag = false;
271         for (String y : tr.testOutput) {
272             if (!foundFlag && findFlagInLine(y, "-help")) {  // javac
273                 foundFlag = true;
274                 System.out.println("Found documentation of '-help': '" + y.trim() +"'");
275             }
276         }
277         if (foundFlag) {
278             result += "failed: " + f.getName() + " does document -help " +
279                 "in help message. This legacy flag should not be documented.\n";
280         }
281 
282         if (!result.isEmpty())
283             System.out.println(result);
284 
285         return result;
286     }
287 
288     // Test that the tool exits with the exit code expected for
289     // invalid flags. In general, one would expect this to be != 0,
290     // but currently a row of tools exit with 0 in this case.
291     // The output should not ask to get help with flag '-help'.
292     static String testInvalidFlag(File f, String flag, int exitcode, boolean documentsLegacyHelp) {
293         String result = "";
294         TestResult tr = runToolWithFlag(f, flag);
295 
296         // Check that the tool did exit with the expected return code.
297         if (!((exitcode == tr.exitValue) ||
298               // Windows reports -1 where unix reports 255.
299               (tr.exitValue < 0 && exitcode == tr.exitValue + 256))) {
300             System.out.println("failed");
301             result = "failed: " + f.getName() + " " + flag + " should not be " +
302                      "accepted. But it has exit code " + tr.exitValue + ".\n";
303         }
304 
305         if (!documentsLegacyHelp) {
306             // Check there is _no_ documentation of -help.
307             boolean foundFlag = false;
308             for (String y : tr.testOutput) {
309                 if (!foundFlag && findFlagInLine(y, "-help")) {  // javac
310                     foundFlag = true;
311                     System.out.println("Found documentation of '-help': '" + y.trim() +"'");
312                 }
313             }
314             if (foundFlag) {
315                 result += "failed: " + f.getName() + " does document -help " +
316                     "in error message. This legacy flag should not be documented.\n";
317             }
318         }
319 
320         if (!result.isEmpty())
321             System.out.println(result);
322 
323         return result;
324     }
325 
326     public static void main(String[] args) {
327         String errorMessage = "";
328 
329         // The test analyses the help messages printed. It assumes englisch
330         // help messages. Thus it only works with english locale.
331         if (!isEnglishLocale()) { return; }
332 
333         for (File f : new File(JAVA_BIN).listFiles(new ToolFilter(TOOLS_NOT_TO_TEST))) {
334             String toolName = f.getName();
335 
336             ToolHelpSpec tool = getToolHelpSpec(toolName);
337             if (tool == null) {
338                 errorMessage += "Tool " + toolName + " not covered by this test. " +
339                     "Add specification to jdkTools array!\n";
340                 continue;
341             }
342 
343             // Test for help flags to be supported.
344             if (tool.supportsQuestionMark == true) {
345                 errorMessage += testTool(f, "-?", tool.exitcodeOfHelp);
346             } else {
347                 System.out.println("Skip " + tool.toolname + ". It does not support -?.");
348             }
349             if (tool.supportsH == true) {
350                 errorMessage += testTool(f, "-h", tool.exitcodeOfHelp);
351             } else {
352                 System.out.println("Skip " + tool.toolname + ". It does not support -h.");
353             }
354             if (tool.supportsHelp == true) {
355                 errorMessage += testTool(f, "--help", tool.exitcodeOfHelp);
356             } else {
357                 System.out.println("Skip " + tool.toolname + ". It does not support --help.");
358             }
359 
360             // Check that the return code listing in jdkTools[] is
361             // correct for an invalid flag.
362             errorMessage += testInvalidFlag(f, "-asdfxgr", tool.exitcodeOfWrongFlag, tool.documentsLegacyHelp);
363 
364             // Test for legacy -help flag.
365             if (!tool.documentsLegacyHelp) {
366                 if (tool.supportsLegacyHelp == true) {
367                     errorMessage += testLegacyFlag(f, tool.exitcodeOfHelp);
368                 } else {
369                     errorMessage += testInvalidFlag(f, "-help", tool.exitcodeOfWrongFlag, false);
370                 }
371             }
372         }
373 
374         if (errorMessage.isEmpty()) {
375             System.out.println("All help string tests: PASS");
376         } else {
377             throw new AssertionError("HelpFlagsTest failed:\n" + errorMessage);
378         }
379     }
380 }