1 /* 2 * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2018, 2020 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 */ 24 25 /** 26 * @test 27 * @summary Validate and test -?, -h and --help flags. All tools in the jdk 28 * should take the same flags to display the help message. These 29 * flags should be documented in the printed help message. The 30 * tool should quit without error code after displaying the 31 * help message (if there is no other problem with the command 32 * line). 33 * Also check that tools that used to accept -help still do 34 * so. Test that tools that never accepted -help don't do so 35 * in future. I.e., check that the tool returns with the same 36 * return code as called with an invalid flag, and does not 37 * print anything containing '-help' in that case. 38 * @compile HelpFlagsTest.java 39 * @run main HelpFlagsTest 40 */ 41 42 import java.io.File; 43 44 public class HelpFlagsTest extends TestHelper { 45 46 // Tools that should not be tested because a usage message is pointless. 47 static final String[] TOOLS_NOT_TO_TEST = { 48 "appletviewer", // deprecated, don't test 49 "jaccessinspector", // gui, don't test, win only 50 "jaccessinspector-32", // gui, don't test, win-32 only 51 "jaccesswalker", // gui, don't test, win only 52 "jaccesswalker-32", // gui, don't test, win-32 only 53 "jconsole", // gui, don't test 54 "servertool", // none. Shell, don't test. 55 "javaw", // don't test, win only 56 // These shall have a help message that resembles that of 57 // MIT's tools. Thus -?, -h and --help are supported, but not 58 // mentioned in the help text. 59 "kinit", 60 "klist", 61 "ktab", 62 // Oracle proprietary tools without help message. 63 "javacpl", 64 "jmc", 65 "jweblauncher", 66 "jcontrol", 67 "ssvagent" 68 }; 69 70 // Lists which tools support which flags. 71 private static class ToolHelpSpec { 72 String toolname; 73 74 // How the flags supposed to be supported are handled. 75 // 76 // These flags are supported, i.e., 77 // * the tool accepts the flag 78 // * the tool prints a help message if the flag is specified 79 // * this help message lists the flag 80 // * the tool exits with exit code '0'. 81 boolean supportsQuestionMark; 82 boolean supportsH; 83 boolean supportsHelp; 84 85 // One tool returns with exit code != '0'. 86 int exitcodeOfHelp; 87 88 // How legacy -help is handled. 89 // 90 // Tools that so far support -help should still do so, but 91 // not print documentation about it. Tools that do not 92 // support -help should not do so in future. 93 // 94 // The tools accepts legacy -help. -help should not be 95 // documented in the usage message. 96 boolean supportsLegacyHelp; 97 98 // Java itself documents -help. -help prints to stderr, 99 // while --help prints to stdout. Leave as is. 100 boolean documentsLegacyHelp; 101 102 // The exit code of the tool if an invalid argument is passed to it. 103 // An exit code != 0 would be expected, but not all tools handle it 104 // that way. 105 int exitcodeOfWrongFlag; 106 107 ToolHelpSpec(String n, int q, int h, int hp, int ex1, int l, int dl, int ex2) { 108 toolname = n; 109 supportsQuestionMark = ( q == 1 ? true : false ); 110 supportsH = ( h == 1 ? true : false ); 111 supportsHelp = ( hp == 1 ? true : false ); 112 exitcodeOfHelp = ex1; 113 114 supportsLegacyHelp = ( l == 1 ? true : false ); 115 documentsLegacyHelp = ( dl == 1 ? true : false ); 116 exitcodeOfWrongFlag = ex2; 117 } 118 } 119 120 static ToolHelpSpec[] jdkTools = { 121 // name -? -h --help exitcode -help -help exitcode 122 // of help docu of wrong 123 // mented flag 124 new ToolHelpSpec("jabswitch", 0, 0, 0, 0, 0, 0, 0), // /?, prints help message anyways, win only 125 new ToolHelpSpec("jar", 1, 1, 1, 0, 0, 0, 1), // -?, -h, --help 126 new ToolHelpSpec("jarsigner", 1, 1, 1, 0, 1, 0, 1), // -?, -h, --help, -help accepted but not documented. 127 new ToolHelpSpec("java", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help 128 new ToolHelpSpec("javac", 1, 0, 1, 0, 1, 1, 2), // -?, --help -help, Documents -help, -h is already taken for "native header output directory". 129 new ToolHelpSpec("javadoc", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help 130 new ToolHelpSpec("javap", 1, 1, 1, 0, 1, 1, 2), // -?, -h, --help -help, Documents -help 131 new ToolHelpSpec("javaw", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help, win only 132 new ToolHelpSpec("jcmd", 1, 1, 1, 0, 1, 0, 1), // -?, -h, --help, -help accepted but not documented. 133 new ToolHelpSpec("jdb", 1, 1, 1, 0, 1, 1, 0), // -?, -h, --help -help, Documents -help 134 new ToolHelpSpec("jdeprscan", 1, 1, 1, 0, 0, 0, 1), // -?, -h, --help 135 new ToolHelpSpec("jdeps", 1, 1, 1, 0, 1, 0, 2), // -?, -h, --help, -help accepted but not documented. 136 new ToolHelpSpec("jfr", 1, 1, 1, 0, 0, 0, 2), // -?, -h, --help 137 new ToolHelpSpec("jhsdb", 0, 0, 0, 0, 0, 0, 0), // none, prints help message anyways. 138 new ToolHelpSpec("jimage", 1, 1, 1, 0, 0, 0, 2), // -?, -h, --help 139 new ToolHelpSpec("jinfo", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help 140 new ToolHelpSpec("jjs", 0, 1, 1, 100, 0, 0, 100), // -h, --help, return code 100 141 new ToolHelpSpec("jlink", 1, 1, 1, 0, 0, 0, 2), // -?, -h, --help 142 new ToolHelpSpec("jmap", 1, 1, 1, 0, 1, 0, 1), // -?, -h, --help, -help accepted but not documented. 143 new ToolHelpSpec("jmod", 1, 1, 1, 0, 1, 0, 2), // -?, -h, --help, -help accepted but not documented. 144 new ToolHelpSpec("jps", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help 145 new ToolHelpSpec("jrunscript", 1, 1, 1, 0, 1, 1, 7), // -?, -h, --help -help, Documents -help 146 new ToolHelpSpec("jshell", 1, 1, 1, 0, 1, 0, 1), // -?, -h, --help, -help accepted but not documented. 147 new ToolHelpSpec("jstack", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help 148 new ToolHelpSpec("jstat", 1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help 149 new ToolHelpSpec("jstatd", 1, 1, 1, 0, 0, 0, 1), // -?, -h, --help 150 new ToolHelpSpec("keytool", 1, 1, 1, 0, 1, 0, 1), // none, prints help message anyways. 151 new ToolHelpSpec("rmiregistry", 0, 0, 0, 0, 0, 0, 1), // none, prints help message anyways. 152 new ToolHelpSpec("serialver", 0, 0, 0, 0, 0, 0, 1), // none, prints help message anyways. 153 new ToolHelpSpec("jpackage", 0, 1, 1, 0, 0, 1, 1), // -h, --help, 154 new ToolHelpSpec("jwebserver", 1, 1, 1, 0, 0, 1, 1), // -?, -h, --help 155 // Panama 156 new ToolHelpSpec("jextract" ,1, 1, 1, 0, 1, 1, 1), // -?, -h, --help -help, Documents -help 157 }; 158 159 // Returns corresponding object from jdkTools array. 160 static ToolHelpSpec getToolHelpSpec(String tool) { 161 for (ToolHelpSpec x : jdkTools) { 162 if (tool.toLowerCase().equals(x.toolname) || 163 tool.toLowerCase().equals(x.toolname + ".exe")) 164 return x; 165 } 166 return null; 167 } 168 169 // Check whether 'flag' appears in 'line' as a word of itself. It must not 170 // be a substring of a word, as then similar flags might be matched. 171 // E.g.: --help matches in the documentation of --help-extra. 172 // This works only with english locale, as some tools have translated 173 // usage messages. 174 static boolean findFlagInLine(String line, String flag) { 175 if (line.contains(flag) && 176 !line.contains("nknown") && // Some tools say 'Unknown option "<flag>"', 177 !line.contains("invalid flag") && // 'invalid flag: <flag>' 178 !line.contains("invalid option") && // or 'invalid option: <flag>'. Skip that. 179 !line.contains("FileNotFoundException: -help") && // Special case for idlj. 180 !line.contains("-h requires an argument") && // Special case for javac. 181 !line.contains("port argument,")) { // Special case for rmiregistry. 182 // There might be several appearances of 'flag' in 183 // 'line'. (-h as substring of --help). 184 int flagLen = flag.length(); 185 int lineLen = line.length(); 186 for (int i = line.indexOf(flag); i >= 0; i = line.indexOf(flag, i+1)) { 187 // There should be a space before 'flag' in 'line', or it's right at the beginning. 188 if (i > 0 && 189 line.charAt(i-1) != ' ' && 190 line.charAt(i-1) != '[' && // jarsigner 191 line.charAt(i-1) != '|' && // jstatd 192 line.charAt(i-1) != '\t') { // jjs 193 continue; 194 } 195 // There should be a space or comma after 'flag' in 'line', or it's just at the end. 196 int posAfter = i + flagLen; 197 if (posAfter < lineLen && 198 line.charAt(posAfter) != ' ' && 199 line.charAt(posAfter) != ',' && 200 line.charAt(posAfter) != '[' && // jar 201 line.charAt(posAfter) != ']' && // jarsigner 202 line.charAt(posAfter) != ')' && // jfr 203 line.charAt(posAfter) != '|' && // jstatd 204 line.charAt(posAfter) != ':' && // jps 205 line.charAt(posAfter) != '"') { // keytool 206 continue; 207 } 208 return true; 209 } 210 } 211 return false; 212 } 213 214 static TestResult runToolWithFlag(File f, String flag) { 215 String x = f.getAbsolutePath(); 216 TestResult tr = doExec(x, flag); 217 System.out.println("Testing " + f.getName()); 218 System.out.println("#> " + x + " " + flag); 219 tr.testOutput.forEach(System.out::println); 220 System.out.println("#> echo $?"); 221 System.out.println(tr.exitValue); 222 223 return tr; 224 } 225 226 // Checks whether tool supports flag 'flag' and documents it 227 // in the help message. 228 static String testTool(File f, String flag, int exitcode) { 229 String result = ""; 230 TestResult tr = runToolWithFlag(f, flag); 231 232 // Check that the tool accepted the flag. 233 if (exitcode == 0 && !tr.isOK()) { 234 System.out.println("failed"); 235 result = "failed: " + f.getName() + " " + flag + " has exit code " + tr.exitValue + ".\n"; 236 } 237 238 // Check there is a help message listing the flag. 239 boolean foundFlag = false; 240 for (String y : tr.testOutput) { 241 if (!foundFlag && findFlagInLine(y, flag)) { // javac 242 foundFlag = true; 243 System.out.println("Found documentation of '" + flag + "': '" + y.trim() +"'"); 244 } 245 } 246 if (!foundFlag) { 247 result += "failed: " + f.getName() + " does not document " + 248 flag + " in help message.\n"; 249 } 250 251 if (!result.isEmpty()) 252 System.out.println(result); 253 254 return result; 255 } 256 257 // Test the tool supports legacy option -help, but does 258 // not document it. 259 static String testLegacyFlag(File f, int exitcode) { 260 String result = ""; 261 TestResult tr = runToolWithFlag(f, "-help"); 262 263 // Check that the tool accepted the flag. 264 if (exitcode == 0 && !tr.isOK()) { 265 System.out.println("failed"); 266 result = "failed: " + f.getName() + " -help has exit code " + tr.exitValue + ".\n"; 267 } 268 269 // Check there is _no_ documentation of -help. 270 boolean foundFlag = false; 271 for (String y : tr.testOutput) { 272 if (!foundFlag && findFlagInLine(y, "-help")) { // javac 273 foundFlag = true; 274 System.out.println("Found documentation of '-help': '" + y.trim() +"'"); 275 } 276 } 277 if (foundFlag) { 278 result += "failed: " + f.getName() + " does document -help " + 279 "in help message. This legacy flag should not be documented.\n"; 280 } 281 282 if (!result.isEmpty()) 283 System.out.println(result); 284 285 return result; 286 } 287 288 // Test that the tool exits with the exit code expected for 289 // invalid flags. In general, one would expect this to be != 0, 290 // but currently a row of tools exit with 0 in this case. 291 // The output should not ask to get help with flag '-help'. 292 static String testInvalidFlag(File f, String flag, int exitcode, boolean documentsLegacyHelp) { 293 String result = ""; 294 TestResult tr = runToolWithFlag(f, flag); 295 296 // Check that the tool did exit with the expected return code. 297 if (!((exitcode == tr.exitValue) || 298 // Windows reports -1 where unix reports 255. 299 (tr.exitValue < 0 && exitcode == tr.exitValue + 256))) { 300 System.out.println("failed"); 301 result = "failed: " + f.getName() + " " + flag + " should not be " + 302 "accepted. But it has exit code " + tr.exitValue + ".\n"; 303 } 304 305 if (!documentsLegacyHelp) { 306 // Check there is _no_ documentation of -help. 307 boolean foundFlag = false; 308 for (String y : tr.testOutput) { 309 if (!foundFlag && findFlagInLine(y, "-help")) { // javac 310 foundFlag = true; 311 System.out.println("Found documentation of '-help': '" + y.trim() +"'"); 312 } 313 } 314 if (foundFlag) { 315 result += "failed: " + f.getName() + " does document -help " + 316 "in error message. This legacy flag should not be documented.\n"; 317 } 318 } 319 320 if (!result.isEmpty()) 321 System.out.println(result); 322 323 return result; 324 } 325 326 public static void main(String[] args) { 327 String errorMessage = ""; 328 329 // The test analyses the help messages printed. It assumes englisch 330 // help messages. Thus it only works with english locale. 331 if (!isEnglishLocale()) { return; } 332 333 for (File f : new File(JAVA_BIN).listFiles(new ToolFilter(TOOLS_NOT_TO_TEST))) { 334 String toolName = f.getName(); 335 336 ToolHelpSpec tool = getToolHelpSpec(toolName); 337 if (tool == null) { 338 errorMessage += "Tool " + toolName + " not covered by this test. " + 339 "Add specification to jdkTools array!\n"; 340 continue; 341 } 342 343 // Test for help flags to be supported. 344 if (tool.supportsQuestionMark == true) { 345 errorMessage += testTool(f, "-?", tool.exitcodeOfHelp); 346 } else { 347 System.out.println("Skip " + tool.toolname + ". It does not support -?."); 348 } 349 if (tool.supportsH == true) { 350 errorMessage += testTool(f, "-h", tool.exitcodeOfHelp); 351 } else { 352 System.out.println("Skip " + tool.toolname + ". It does not support -h."); 353 } 354 if (tool.supportsHelp == true) { 355 errorMessage += testTool(f, "--help", tool.exitcodeOfHelp); 356 } else { 357 System.out.println("Skip " + tool.toolname + ". It does not support --help."); 358 } 359 360 // Check that the return code listing in jdkTools[] is 361 // correct for an invalid flag. 362 errorMessage += testInvalidFlag(f, "-asdfxgr", tool.exitcodeOfWrongFlag, tool.documentsLegacyHelp); 363 364 // Test for legacy -help flag. 365 if (!tool.documentsLegacyHelp) { 366 if (tool.supportsLegacyHelp == true) { 367 errorMessage += testLegacyFlag(f, tool.exitcodeOfHelp); 368 } else { 369 errorMessage += testInvalidFlag(f, "-help", tool.exitcodeOfWrongFlag, false); 370 } 371 } 372 } 373 374 if (errorMessage.isEmpty()) { 375 System.out.println("All help string tests: PASS"); 376 } else { 377 throw new AssertionError("HelpFlagsTest failed:\n" + errorMessage); 378 } 379 } 380 }