1 /*
2 * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2020 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 */
24
25 /**
26 * @test
27 * @summary Validate and test -?, -h and --help flags. All tools in the jdk
28 * should take the same flags to display the help message. These
29 * flags should be documented in the printed help message. The
30 * tool should quit without error code after displaying the
31 * help message (if there is no other problem with the command
32 * line).
33 * Also check that tools that used to accept -help still do
34 * so. Test that tools that never accepted -help don't do so
35 * in future. I.e., check that the tool returns with the same
36 * return code as called with an invalid flag, and does not
37 * print anything containing '-help' in that case.
38 * @compile HelpFlagsTest.java
39 * @run main HelpFlagsTest
40 */
41
42 import java.io.File;
43
44 public class HelpFlagsTest extends TestHelper {
45
46 // Tools that should not be tested because a usage message is pointless.
47 static final String[] TOOLS_NOT_TO_TEST = {
48 "appletviewer", // deprecated, don't test
49 "jaccessinspector", // gui, don't test, win only
50 "jaccessinspector-32", // gui, don't test, win-32 only
51 "jaccesswalker", // gui, don't test, win only
52 "jaccesswalker-32", // gui, don't test, win-32 only
53 "jconsole", // gui, don't test
54 "servertool", // none. Shell, don't test.
55 "javaw", // don't test, win only
56 // These shall have a help message that resembles that of
57 // MIT's tools. Thus -?, -h and --help are supported, but not
58 // mentioned in the help text.
59 "kinit",
60 "klist",
61 "ktab",
62 // Oracle proprietary tools without help message.
63 "javacpl",
64 "jmc",
65 "jweblauncher",
66 "jcontrol",
67 "ssvagent"
68 };
69
70 // Lists which tools support which flags.
71 private static class ToolHelpSpec {
72 String toolname;
73
74 // How the flags supposed to be supported are handled.
75 //
76 // These flags are supported, i.e.,
77 // * the tool accepts the flag
78 // * the tool prints a help message if the flag is specified
79 // * this help message lists the flag
80 // * the tool exits with exit code '0'.
81 boolean supportsQuestionMark;
82 boolean supportsH;
83 boolean supportsHelp;
84
85 // One tool returns with exit code != '0'.
86 int exitcodeOfHelp;
87
88 // How legacy -help is handled.
89 //
90 // Tools that so far support -help should still do so, but
91 // not print documentation about it. Tools that do not
92 // support -help should not do so in future.
93 //
94 // The tools accepts legacy -help. -help should not be
95 // documented in the usage message.
96 boolean supportsLegacyHelp;
97
98 // Java itself documents -help. -help prints to stderr,
99 // while --help prints to stdout. Leave as is.
100 boolean documentsLegacyHelp;
101
102 // The exit code of the tool if an invalid argument is passed to it.
103 // An exit code != 0 would be expected, but not all tools handle it
104 // that way.
105 int exitcodeOfWrongFlag;
106
107 ToolHelpSpec(String n, int q, int h, int hp, int ex1, int l, int dl, int ex2) {
108 toolname = n;
109 supportsQuestionMark = ( q == 1 ? true : false );
110 supportsH = ( h == 1 ? true : false );
111 supportsHelp = ( hp == 1 ? true : false );
112 exitcodeOfHelp = ex1;
113
114 supportsLegacyHelp = ( l == 1 ? true : false );
115 documentsLegacyHelp = ( dl == 1 ? true : false );
116 exitcodeOfWrongFlag = ex2;
117 }
118 }
119
120 static ToolHelpSpec[] jdkTools = {
121 // name -? -h --help exitcode -help -help exitcode
122 // of help docu of wrong
123 // mented flag
124 new ToolHelpSpec("jabswitch", 0, 0, 0, 0, 0, 0, 0), // /?, prints help message anyways, win only
125 new ToolHelpSpec("jar", 1, 1, 1, 0, 0, 0, 1), // -?, -h, --help
126 new ToolHelpSpec("jarsigner", 1, 1, 1, 0, 1, 0, 1), // -?, -h, --help, -help accepted but not documented.
127 new ToolHelpSpec("java", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help
128 new ToolHelpSpec("javac", 1, 0, 1, 0, 1, 1, 2), // -?, --help -help, Documents -help, -h is already taken for "native header output directory".
129 new ToolHelpSpec("javadoc", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help
130 new ToolHelpSpec("javap", 1, 1, 1, 0, 1, 1, 2), // -?, -h, --help -help, Documents -help
131 new ToolHelpSpec("javaw", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help, win only
132 new ToolHelpSpec("jcmd", 1, 1, 1, 0, 1, 0, 1), // -?, -h, --help, -help accepted but not documented.
133 new ToolHelpSpec("jdb", 1, 1, 1, 0, 1, 1, 0), // -?, -h, --help -help, Documents -help
134 new ToolHelpSpec("jdeprscan", 1, 1, 1, 0, 0, 0, 1), // -?, -h, --help
135 new ToolHelpSpec("jdeps", 1, 1, 1, 0, 1, 0, 2), // -?, -h, --help, -help accepted but not documented.
136 new ToolHelpSpec("jfr", 1, 1, 1, 0, 0, 0, 2), // -?, -h, --help
137 new ToolHelpSpec("jhsdb", 0, 0, 0, 0, 0, 0, 0), // none, prints help message anyways.
138 new ToolHelpSpec("jimage", 1, 1, 1, 0, 0, 0, 2), // -?, -h, --help
139 new ToolHelpSpec("jinfo", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help
140 new ToolHelpSpec("jjs", 0, 1, 1, 100, 0, 0, 100), // -h, --help, return code 100
141 new ToolHelpSpec("jlink", 1, 1, 1, 0, 0, 0, 2), // -?, -h, --help
142 new ToolHelpSpec("jmap", 1, 1, 1, 0, 1, 0, 1), // -?, -h, --help, -help accepted but not documented.
143 new ToolHelpSpec("jmod", 1, 1, 1, 0, 1, 0, 2), // -?, -h, --help, -help accepted but not documented.
144 new ToolHelpSpec("jps", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help
145 new ToolHelpSpec("jrunscript", 1, 1, 1, 0, 1, 1, 7), // -?, -h, --help -help, Documents -help
146 new ToolHelpSpec("jshell", 1, 1, 1, 0, 1, 0, 1), // -?, -h, --help, -help accepted but not documented.
147 new ToolHelpSpec("jstack", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help
148 new ToolHelpSpec("jstat", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help
149 new ToolHelpSpec("jstatd", 1, 1, 1, 0, 0, 0, 1), // -?, -h, --help
150 new ToolHelpSpec("keytool", 1, 1, 1, 0, 1, 0, 1), // none, prints help message anyways.
151 new ToolHelpSpec("rmiregistry", 0, 0, 0, 0, 0, 0, 1), // none, prints help message anyways.
152 new ToolHelpSpec("serialver", 0, 0, 0, 0, 0, 0, 1), // none, prints help message anyways.
153 new ToolHelpSpec("jpackage", 0, 1, 1, 0, 0, 1, 1), // -h, --help,
154 new ToolHelpSpec("jwebserver", 1, 1, 1, 0, 0, 1, 1), // -?, -h, --help
155 // Panama
156 new ToolHelpSpec("jextract" ,1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help
157 };
158
159 // Returns corresponding object from jdkTools array.
160 static ToolHelpSpec getToolHelpSpec(String tool) {
161 for (ToolHelpSpec x : jdkTools) {
162 if (tool.toLowerCase().equals(x.toolname) ||
163 tool.toLowerCase().equals(x.toolname + ".exe"))
164 return x;
165 }
166 return null;
167 }
168
169 // Check whether 'flag' appears in 'line' as a word of itself. It must not
170 // be a substring of a word, as then similar flags might be matched.
171 // E.g.: --help matches in the documentation of --help-extra.
172 // This works only with english locale, as some tools have translated
173 // usage messages.
174 static boolean findFlagInLine(String line, String flag) {
175 if (line.contains(flag) &&
176 !line.contains("nknown") && // Some tools say 'Unknown option "<flag>"',
177 !line.contains("invalid flag") && // 'invalid flag: <flag>'
178 !line.contains("invalid option") && // or 'invalid option: <flag>'. Skip that.
179 !line.contains("FileNotFoundException: -help") && // Special case for idlj.
180 !line.contains("-h requires an argument") && // Special case for javac.
181 !line.contains("port argument,")) { // Special case for rmiregistry.
182 // There might be several appearances of 'flag' in
183 // 'line'. (-h as substring of --help).
184 int flagLen = flag.length();
185 int lineLen = line.length();
186 for (int i = line.indexOf(flag); i >= 0; i = line.indexOf(flag, i+1)) {
187 // There should be a space before 'flag' in 'line', or it's right at the beginning.
188 if (i > 0 &&
189 line.charAt(i-1) != ' ' &&
190 line.charAt(i-1) != '[' && // jarsigner
191 line.charAt(i-1) != '|' && // jstatd
192 line.charAt(i-1) != '\t') { // jjs
193 continue;
194 }
195 // There should be a space or comma after 'flag' in 'line', or it's just at the end.
196 int posAfter = i + flagLen;
197 if (posAfter < lineLen &&
198 line.charAt(posAfter) != ' ' &&
199 line.charAt(posAfter) != ',' &&
200 line.charAt(posAfter) != '[' && // jar
201 line.charAt(posAfter) != ']' && // jarsigner
202 line.charAt(posAfter) != ')' && // jfr
203 line.charAt(posAfter) != '|' && // jstatd
204 line.charAt(posAfter) != ':' && // jps
205 line.charAt(posAfter) != '"') { // keytool
206 continue;
207 }
208 return true;
209 }
210 }
211 return false;
212 }
213
214 static TestResult runToolWithFlag(File f, String flag) {
215 String x = f.getAbsolutePath();
216 TestResult tr = doExec(x, flag);
217 System.out.println("Testing " + f.getName());
218 System.out.println("#> " + x + " " + flag);
219 tr.testOutput.forEach(System.out::println);
220 System.out.println("#> echo $?");
221 System.out.println(tr.exitValue);
222
223 return tr;
224 }
225
226 // Checks whether tool supports flag 'flag' and documents it
227 // in the help message.
228 static String testTool(File f, String flag, int exitcode) {
229 String result = "";
230 TestResult tr = runToolWithFlag(f, flag);
231
232 // Check that the tool accepted the flag.
233 if (exitcode == 0 && !tr.isOK()) {
234 System.out.println("failed");
235 result = "failed: " + f.getName() + " " + flag + " has exit code " + tr.exitValue + ".\n";
236 }
237
238 // Check there is a help message listing the flag.
239 boolean foundFlag = false;
240 for (String y : tr.testOutput) {
241 if (!foundFlag && findFlagInLine(y, flag)) { // javac
242 foundFlag = true;
243 System.out.println("Found documentation of '" + flag + "': '" + y.trim() +"'");
244 }
245 }
246 if (!foundFlag) {
247 result += "failed: " + f.getName() + " does not document " +
248 flag + " in help message.\n";
249 }
250
251 if (!result.isEmpty())
252 System.out.println(result);
253
254 return result;
255 }
256
257 // Test the tool supports legacy option -help, but does
258 // not document it.
259 static String testLegacyFlag(File f, int exitcode) {
260 String result = "";
261 TestResult tr = runToolWithFlag(f, "-help");
262
263 // Check that the tool accepted the flag.
264 if (exitcode == 0 && !tr.isOK()) {
265 System.out.println("failed");
266 result = "failed: " + f.getName() + " -help has exit code " + tr.exitValue + ".\n";
267 }
268
269 // Check there is _no_ documentation of -help.
270 boolean foundFlag = false;
271 for (String y : tr.testOutput) {
272 if (!foundFlag && findFlagInLine(y, "-help")) { // javac
273 foundFlag = true;
274 System.out.println("Found documentation of '-help': '" + y.trim() +"'");
275 }
276 }
277 if (foundFlag) {
278 result += "failed: " + f.getName() + " does document -help " +
279 "in help message. This legacy flag should not be documented.\n";
280 }
281
282 if (!result.isEmpty())
283 System.out.println(result);
284
285 return result;
286 }
287
288 // Test that the tool exits with the exit code expected for
289 // invalid flags. In general, one would expect this to be != 0,
290 // but currently a row of tools exit with 0 in this case.
291 // The output should not ask to get help with flag '-help'.
292 static String testInvalidFlag(File f, String flag, int exitcode, boolean documentsLegacyHelp) {
293 String result = "";
294 TestResult tr = runToolWithFlag(f, flag);
295
296 // Check that the tool did exit with the expected return code.
297 if (!((exitcode == tr.exitValue) ||
298 // Windows reports -1 where unix reports 255.
299 (tr.exitValue < 0 && exitcode == tr.exitValue + 256))) {
300 System.out.println("failed");
301 result = "failed: " + f.getName() + " " + flag + " should not be " +
302 "accepted. But it has exit code " + tr.exitValue + ".\n";
303 }
304
305 if (!documentsLegacyHelp) {
306 // Check there is _no_ documentation of -help.
307 boolean foundFlag = false;
308 for (String y : tr.testOutput) {
309 if (!foundFlag && findFlagInLine(y, "-help")) { // javac
310 foundFlag = true;
311 System.out.println("Found documentation of '-help': '" + y.trim() +"'");
312 }
313 }
314 if (foundFlag) {
315 result += "failed: " + f.getName() + " does document -help " +
316 "in error message. This legacy flag should not be documented.\n";
317 }
318 }
319
320 if (!result.isEmpty())
321 System.out.println(result);
322
323 return result;
324 }
325
326 public static void main(String[] args) {
327 String errorMessage = "";
328
329 // The test analyses the help messages printed. It assumes englisch
330 // help messages. Thus it only works with english locale.
331 if (!isEnglishLocale()) { return; }
332
333 for (File f : new File(JAVA_BIN).listFiles(new ToolFilter(TOOLS_NOT_TO_TEST))) {
334 String toolName = f.getName();
335
336 ToolHelpSpec tool = getToolHelpSpec(toolName);
337 if (tool == null) {
338 errorMessage += "Tool " + toolName + " not covered by this test. " +
339 "Add specification to jdkTools array!\n";
340 continue;
341 }
342
343 // Test for help flags to be supported.
344 if (tool.supportsQuestionMark == true) {
345 errorMessage += testTool(f, "-?", tool.exitcodeOfHelp);
346 } else {
347 System.out.println("Skip " + tool.toolname + ". It does not support -?.");
348 }
349 if (tool.supportsH == true) {
350 errorMessage += testTool(f, "-h", tool.exitcodeOfHelp);
351 } else {
352 System.out.println("Skip " + tool.toolname + ". It does not support -h.");
353 }
354 if (tool.supportsHelp == true) {
355 errorMessage += testTool(f, "--help", tool.exitcodeOfHelp);
356 } else {
357 System.out.println("Skip " + tool.toolname + ". It does not support --help.");
358 }
359
360 // Check that the return code listing in jdkTools[] is
361 // correct for an invalid flag.
362 errorMessage += testInvalidFlag(f, "-asdfxgr", tool.exitcodeOfWrongFlag, tool.documentsLegacyHelp);
363
364 // Test for legacy -help flag.
365 if (!tool.documentsLegacyHelp) {
366 if (tool.supportsLegacyHelp == true) {
367 errorMessage += testLegacyFlag(f, tool.exitcodeOfHelp);
368 } else {
369 errorMessage += testInvalidFlag(f, "-help", tool.exitcodeOfWrongFlag, false);
370 }
371 }
372 }
373
374 if (errorMessage.isEmpty()) {
375 System.out.println("All help string tests: PASS");
376 } else {
377 throw new AssertionError("HelpFlagsTest failed:\n" + errorMessage);
378 }
379 }
380 }