1 /*
2 * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 import job.*;
27
28 static void logo(){
29 System.out.println("""
30 ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣀⣀⣀⣀⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
31 ⠀⠀⠀⠀⠀⠀⠀⢰⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣶⣤⣀⠀⠀⠀⠀⠀⠀⠀
32 ⠀⠀⠀⠀⠀⠀⠀ ⠙⠿⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠿⠀⠀⠀⠀⠀⠀⠀
33 ⠀⠀⠀⠀⠀⠀⠀⠀⣷⣶⣤⣄⣈⣉⣉⣉⣉⣉⣉⣉⣁⣤⡄⠀⠀⠀⠀⠀⠀⠀
34 ⠀⠀⠀⠀⠀⠀ ⠀⣿⣿⣿⣿⣿ HAT ⣿⣿⣿⣿⡇⠀⠀⠀⠀⠀⠀⠀
35 ⠀⠀⢀⣠⣶⣾⡏⢀⡈⠛⠻⠿⢿⣿⣿⣿⣿⣿⠿⠿⠟⠛⢁⠀⢶⣤⣀⠀⠀⠀
36 ⠀⢠⣿⣿⣿⣿⡇⠸⣿⣿⣶⣶⣤⣤⣤⣤⣤⣤⣤⣶⣶⣿⡿⠂⣸⣿⣿⣷⡄⠀
37 ⠀⢸⣿⣿⣿⣿⣿⣦⣄⡉⠛⠛⠛⠿⠿⠿⠿⠛⠛⠛⢉⣁⣤⣾⣿⣿⣿⣿⡷⠀
38 ⠀⠀⠙⢿⣿⣿⣿⣿⣿⣿⣿⣷⣶⣶⣶⣶⣶⣶⣾⣿⣿⣿⣿⣿⣿⣿⡿⠛⠁⠀
39 ⠀⠀⠀⠀⠈⠙⠛⠿⠿⢿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠿⠿⠿⠛⠛⠉⠁⠀⠀⠀⠀
40 """);
41 }
42 static void help(){
43 System.out.println("""
44 Usage bld|clean|run ...
45 bld:
46 Compile all buildable (based on capabilities) available jars and native code.
47
48 dot:
49 Create dot graph (bld.dot) of buildable dependencies (based on capabilities)
50 dot bld.dot -Tsvg > bld.svg && chrome bld.svg
51 clean:
52 Removes build directory entirely
53 conf dir and jextracted artifacts (opencl/cuda/opengl) remain
54
55
56 run: [ffi|my|seq]-[opencl|java|cuda|mock|hip] [-DXXX ...] runnable args
57 run ffi-opencl mandel
58 run ffi-opencl nbody 4096
59 run ffi-opencl -DHAT=SHOW_CODE nbody 4096
60 run ffi-opencl -DHAT=SHOW_KERNEL_MODEL heal
61 run ffi-opencl -DHAT=MINIMIZE_BUFFERS life
62
63 exp: [ffi|my|seq]-[opencl|java|cuda|mock|hip] [-DXXX ... ] experimentClassName args
64 exp ffi-opencl QuotedConstantArgs
65
66 test-suite: [ffi|my|seq]-[opencl|java|cuda|mock|hip]
67 test-suite ffi-opencl
68
69 test: [ffi|my|seq]-[opencl|java|cuda|mock|hip] classToTest (also classToTest#method)
70 test ffi-opencl hat.test.TestMatMul
71
72 sanity: Check source files for copyright and WS issues (tabs and trailing EOL WS)
73 """);
74 }
75
76 static void logoAndHelp(){
77 logo();
78 help();
79 }
80
81
82 public static void main(String[] argArr) throws IOException, InterruptedException {
83 var args = new ArrayList<>(List.of(argArr));
84 if (args.isEmpty()) {
85 help();
86 } else {
87 var hat = new Project(Path.of(System.getProperty("user.dir")), Reporter.progressAndErrors);
88 var cmake = hat.isAvailable("cmake", "--version");
89 if (!cmake.isAvailable()) {
90 System.out.println("We need cmake, to check the availability of opencl, cuda etc so we wont be able to build much ");
91 }
92 var jextract = hat.isAvailable("jextract", "--version");
93 if (!jextract.isAvailable()) {
94 System.out.println("We will need jextract to create jextracted backends and for examples requiring opengl ");
95 }
96
97 // This is an example of a user defined optional dependency.
98 // Anything depending on this will only build if (In this case) the property is true
99 // So in our case if the headless system property
100 var ui = new Opt(hat.id("ui"), !Boolean.getBoolean("headless")); //-Dheadless=true|false
101
102 // These dependencies are 'true' on the appropriate platform.
103 // So any target depending on one these, will only build on that platform
104 var mac = new Mac(hat.id("os-mac"));
105 var linux = new Linux(hat.id("os-linux"));
106
107 // These next three 'optional' dependencies use cmake to determine availability. We delegate to cmake which
108 // a) determines if capability is available,
109 // b) if they are, they extract from cmake vars (see conf/cmake-info/OpenCL/properties for example) information export headers and libs needed by JExtract
110 var openclCmakeInfo = new OpenCL(hat.id("cmake-info-opencl"), cmake);
111 var openglCmakeInfo = new OpenGL(hat.id("cmake-info-opengl"), cmake);
112 var cudaCmakeInfo = new Cuda(hat.id("cmake-info-cuda"), cmake);
113
114 // Now we just create jars and shared libs and declare dependencies
115 var core = hat.jar("core");
116 var tools = hat.jar("tools", core);
117 var tests = hat.jar("tests", core, tools);
118
119 var backend_ffi_native = hat.cmakeAndJar("backend{s}-ffi", core, cmake);
120 var ffiSharedBackend = hat.jar("backend{s}-ffi-shared", backend_ffi_native);
121 var backend_ffi_cuda = hat.jar("backend{s}-ffi-cuda", ffiSharedBackend);
122 var backend_ffi_opencl = hat.jar("backend{s}-ffi-opencl", ffiSharedBackend);
123 var backend_ffi_mock = hat.jar("backend{s}-ffi-mock", ffiSharedBackend);
124
125 // These examples just rely on core
126 var backend_mt_java = hat.jar("backend{s}-java-mt", core);
127 var backend_seq_java = hat.jar("backend{s}-java-seq", core);
128 var example_squares = hat.jar("example{s}-squares", core);
129 var example_matmul = hat.jar("example{s}-matmul", core);
130 var example_blackscholes = hat.jar("example{s}-blackscholes", core);
131 var example_view = hat.jar("example{s}-view", core);
132 var example_normmap = hat.jar("example{s}-normmap", core); // will probabvly need shared when we hatify
133
134 // example_shared allows us to break out common UI functions, views, even loops etc
135 var example_shared = hat.jar("example{s}-shared", ui, core);
136
137 // These examples use example_shared, so they are UI based
138 var example_mandel = hat.jar("example{s}-mandel", example_shared);
139 var example_life = hat.jar("example{s}-life", example_shared);
140 var example_heal = hat.jar("example{s}-heal", example_shared);
141 var example_violajones = hat.jar("example{s}-violajones", example_shared);
142
143 // experiments include code that expects an opencl backend, this is not idea, but we can accomodate
144 var example_experiments = hat.jar("example{s}-experiments", core);
145
146 // Now we have the more complex nonsense for nbody (which needs opengl and opencl extracted)
147 var wrapped_shared = hat.jar("wrap{s}-shared");
148 var jextracted_opencl = hat.jextract("extract{ions|ed}-opencl", jextract, openclCmakeInfo, core);
149 var wrapped_jextracted_opencl = hat.jar("wrap{s}-opencl", jextracted_opencl, wrapped_shared);
150 var backend_jextracted_shared = hat.jar("backend{s}-jextracted-shared", core);
151 var backend_jextracted_opencl = hat.jar("backend{s}-jextracted-opencl", wrapped_jextracted_opencl, backend_jextracted_shared);
152 var jextracted_opengl = hat.jextract("extract{ions|ed}-opengl", jextract, ui, openglCmakeInfo, core);
153
154 // Sigh... We have different src exclusions for wrapped opengl depending on the OS
155 var excludedOpenGLWrapSrc = hat.rootPath().resolve(
156 "wraps/opengl/src/main/java/wrap/opengl/GL" + (mac.isAvailable() ? "Callback" : "Func") + "EventHandler.java");
157
158 var wrapped_jextracted_opengl = hat.jar("wrap{s}-opengl", Set.of(excludedOpenGLWrapSrc), jextracted_opengl, wrapped_shared);
159
160 // Finally we have everything needed for nbody
161 var example_nbody = hat.jar("example{s}-nbody", ui, wrapped_jextracted_opengl, wrapped_jextracted_opencl);
162 class Stats {
163 int passed = 0;
164 int failed = 0;
165 int unsupported = 0;
166 }
167 var testEngine = "hat.test.engine.HATTestEngine";
168
169 while (!args.isEmpty()) {
170 var arg = args.removeFirst();
171 switch (arg) {
172 case "help" -> logoAndHelp();
173 case "clean" -> hat.clean();
174 case "dot" -> {
175 var dag = hat.all();
176 var available = dag.available();
177 Files.writeString(Path.of("bld.dot") , available.toDot());
178 System.out.println("Consider...\n dot bld.dot -Tsvg > bld.svg");
179 }
180 case "bld" -> {
181 var dag = hat.all();
182 var available = dag.available();
183 hat.build(available);
184 }
185 case "sanity" -> {
186 final var copyrightPattern = Pattern.compile("^.*Copyright.*202[0-9].*(Intel|Oracle).*$");
187 final var copyrightExemptPattern = Pattern.compile("^(robertograham|CMakeFiles|hip)");
188 final var tabOrEolWsPattern = Pattern.compile("^(.*\\t.*|.* )$");
189 final var textSuffix = Pattern.compile("^(.*\\.(java|cpp|h|hpp|md)|pom.xml)$");
190 final var sourceSuffix = Pattern.compile("^(.*\\.(java|cpp|h|hpp)|pom.xml)$");
191
192 Stream.of("hat","core","tools","examples","backends","docs","wraps")
193 .map(hat.rootPath()::resolve)
194 .forEach(dir-> {
195 System.out.println("Checking "+dir);
196 Util.recurse(dir,
197 (d)-> true, // we do this for all subdirs
198 (f)-> textSuffix.matcher(f.getFileName().toString()).matches() && Util.grepLines(tabOrEolWsPattern, f),
199 (c)-> System.out.println("File contains WS issue (TAB or EOLWs) " + c)
200 );
201 Util.recurse(dir,
202 (d)-> !copyrightExemptPattern.matcher(d.getFileName().toString()).matches(), // we skip these subdirs
203 (f)-> sourceSuffix.matcher(f.getFileName().toString()).matches() && !Util.grepLines(copyrightPattern, f),
204 (c)-> System.out.println("File does not contain copyright " + c)
205 );
206 });
207 args.clear();
208 }
209 case "run" -> {
210 if (args.size() > 1) {
211 var backendName = args.removeFirst();
212 var vmOpts = new ArrayList<String>(List.of());
213 while (args.getFirst() instanceof String possibleVmOpt && possibleVmOpt.startsWith("-")){
214 vmOpts.add(args.removeFirst());
215 }
216 var runnableName = args.removeFirst();
217 if (hat.get(backendName) instanceof Jar backend) {
218 if (hat.get(runnableName) instanceof Jar runnable) {
219 if (runnableName.equals("nbody") && mac.isAvailable()) { // nbody (anything on mac using OpenGL
220 vmOpts.add("-XstartOnFirstThread");
221 }
222 runnable.run(runnableName + ".Main", new job.Dag(runnable, backend).ordered(), vmOpts,args);
223 } else {
224 System.err.println("Found backend "+ backendName +" but failed to find runnable/example " + runnableName);
225 }
226 } else {
227 System.err.println("Failed to find backend " + backendName);
228 }
229 } else {
230 System.err.println("For run we expect 'run backend runnable' ");
231 }
232 args.clear(); //!! :)
233 }
234 case "test-suite" -> {
235 if (args.size() > 0) {
236 var backendName = args.removeFirst();
237
238 if (hat.get(backendName) instanceof Jar backend) {
239 var vmOpts = new ArrayList<String>(List.of());
240 while (!args.isEmpty() && args.getFirst() instanceof String possibleVmOpt && possibleVmOpt.startsWith("-")){
241 vmOpts.add(args.removeFirst());
242 }
243 var test_reports_txt = Paths.get("test_report.txt");
244 Files.deleteIfExists(test_reports_txt); // because we will append to it in the next loop
245 var suiteRe = Pattern.compile("(hat/test/Test[a-zA-Z0-9]*).class");
246 var jarFile = new JarFile(tests.jarFile().toString());
247 var entries = jarFile.entries();
248 var orderedDag = new job.Dag(tests, backend).ordered();
249 while (entries.hasMoreElements()) {
250 if (suiteRe.matcher(entries.nextElement().getName()) instanceof Matcher matched && matched.matches()){
251 tests.run(testEngine, orderedDag, vmOpts,List.of(matched.group(1).replace('/','.')));
252 }
253 }
254 System.out.println("\n\n");
255 System.out.println("*****************************************************************");
256 logo();
257 System.out.println(" HAT Test Report ");
258 System.out.println("*****************************************************************");
259 var pattern = Pattern.compile( "passed: (\\d+), failed: (\\d+), unsupported: (\\d+)");
260 var stats = new Stats();
261 Files.readAllLines(test_reports_txt).forEach(line->{
262 System.out.println(line);
263 if (pattern.matcher(line) instanceof Matcher matcher && matcher.find()){
264 stats.passed+=Integer.parseInt(matcher.group(1));
265 stats.failed+=Integer.parseInt(matcher.group(2));
266 stats.unsupported+=Integer.parseInt(matcher.group(3));
267 }
268 });
269 System.out.printf("Global passed: %d, failed: %d, unsupported: %d, pass-rate: %.2f%%\\n",
270 stats.passed, stats.failed, stats.unsupported, ((float)(stats.passed * 100 / (stats.passed + stats.failed + stats.unsupported))));
271 } else {
272 System.err.println("Failed to find backend " + backendName);
273 }
274 } else {
275 System.err.println("For test-suite we require a backend ");
276 }
277 args.clear(); //!! :)
278 }
279 case "test" -> {
280 if (args.size() >= 2) {
281 var backendName = args.removeFirst();
282 var classAndMethod = args.removeFirst();
283 if (hat.get(backendName) instanceof Jar backend) {
284 var vmOpts = new ArrayList<String>(List.of());
285 while (!args.isEmpty() && args.getFirst() instanceof String possibleVmOpt && possibleVmOpt.startsWith("-")){
286 vmOpts.add(args.removeFirst());
287 }
288
289 var orderedDag = new job.Dag(tests, backend).ordered();
290 tests.run(testEngine, orderedDag, vmOpts, List.of(classAndMethod));
291
292 } else {
293 System.err.println("Failed to find backend " + backendName);
294 }
295 } else {
296 System.err.println("For test we require a backend and a TestClass.");
297 System.err.println("Examples: ");
298 System.err.println("$ test ffi-opencl hat.test.TestMatMul");
299 System.err.println("$ test ffi-opencl hat.test.TestMatMul#method");
300 }
301 args.clear(); //!! :)
302 }
303 case "exp" -> {
304 if (args.size() > 1) {
305 var backendName = args.removeFirst();
306 var runnableName = "experiments";
307 var vmOpts = new ArrayList<String>(List.of());
308 while (args.getFirst() instanceof String possibleVmOpt && possibleVmOpt.startsWith("-")){
309 vmOpts.add(args.removeFirst());
310 }
311 var className = args.removeFirst();
312 if (hat.get(backendName) instanceof Jar backend) {
313 if (hat.get(runnableName) instanceof Jar runnable) {
314 runnable.run(runnableName + "."+className, new job.Dag(runnable, backend).ordered(), vmOpts,args);
315 } else {
316 System.err.println("Failed to find runnable " + runnableName);
317 }
318 } else {
319 System.err.println("Failed to find " + backendName);
320 }
321 } else {
322 System.err.println("For exp we expect 'exp backend [-DXXX ...] testclass' ");
323 }
324 args.clear(); //!! :)
325 }
326 default -> {
327 System.err.println("'" + arg + "' was unexpected ");
328 help();
329 args.clear();
330 }
331 }
332 }
333 }
334 }