1 /*
  2  * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.  Oracle designates this
  8  * particular file as subject to the "Classpath" exception as provided
  9  * by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  */
 25 
 26 package com.sun.tools.sjavac;
 27 
 28 import java.io.File;
 29 import java.net.URI;
 30 import java.util.ArrayList;
 31 import java.util.Arrays;
 32 import java.util.Collections;
 33 import java.util.HashMap;
 34 import java.util.List;
 35 import java.util.Map;
 36 import java.util.Random;
 37 import java.util.Set;
 38 import java.util.concurrent.Callable;
 39 import java.util.concurrent.ExecutionException;
 40 import java.util.concurrent.ExecutorService;
 41 import java.util.concurrent.Executors;
 42 import java.util.concurrent.Future;
 43 
 44 import com.sun.tools.javac.main.Main.Result;
 45 import com.sun.tools.sjavac.comp.CompilationService;
 46 import com.sun.tools.sjavac.options.Options;
 47 import com.sun.tools.sjavac.pubapi.PubApi;
 48 import com.sun.tools.sjavac.server.CompilationSubResult;
 49 import com.sun.tools.sjavac.server.SysInfo;
 50 
 51 /**
 52  * This transform compiles a set of packages containing Java sources.
 53  * The compile request is divided into separate sets of source files.
 54  * For each set a separate request thread is dispatched to a javac server
 55  * and the meta data is accumulated. The number of sets correspond more or
 56  * less to the number of cores. Less so now, than it will in the future.
 57  *
 58  * <p><b>This is NOT part of any supported API.
 59  * If you write code that depends on this, you do so at your own
 60  * risk.  This code and its internal interfaces are subject to change
 61  * or deletion without notice.</b></p>
 62  */
 63 public class CompileJavaPackages implements Transformer {
 64 
 65     // The current limited sharing of data between concurrent JavaCompilers
 66     // in the server will not give speedups above 3 cores. Thus this limit.
 67     // We hope to improve this in the future.
 68     static final int limitOnConcurrency = 3;
 69 
 70     Options args;
 71 
 72     public void setExtra(String e) {
 73     }
 74 
 75     public void setExtra(Options a) {
 76         args = a;
 77     }
 78 
 79     public boolean transform(final CompilationService sjavac,
 80                              Map<String,Set<URI>> pkgSrcs,
 81                              final Set<URI>             visibleSources,
 82                              Map<String,Set<String>> oldPackageDependents,
 83                              URI destRoot,
 84                              final Map<String,Set<URI>>    packageArtifacts,
 85                              final Map<String,Map<String, Set<String>>> packageDependencies,
 86                              final Map<String,Map<String, Set<String>>> packageCpDependencies,
 87                              final Map<String, PubApi> packagePubapis,
 88                              final Map<String, PubApi> dependencyPubapis,
 89                              int debugLevel,
 90                              boolean incremental,
 91                              int numCores) {
 92 
 93         Log.debug("Performing CompileJavaPackages transform...");
 94 
 95         boolean rc = true;
 96         boolean concurrentCompiles = true;
 97 
 98         // Fetch the id.
 99         final String id = String.valueOf(new Random().nextInt());
100         // Only keep portfile and sjavac settings..
101         //String psServerSettings = Util.cleanSubOptions(Util.set("portfile","sjavac","background","keepalive"), sjavac.serverSettings());
102 
103         SysInfo sysinfo = sjavac.getSysInfo();
104         int numMBytes = (int)(sysinfo.maxMemory / ((long)(1024*1024)));
105         Log.debug("Server reports "+numMBytes+"MiB of memory and "+sysinfo.numCores+" cores");
106 
107         if (numCores <= 0) {
108             // Set the requested number of cores to the number of cores on the server.
109             numCores = sysinfo.numCores;
110             Log.debug("Number of jobs not explicitly set, defaulting to "+sysinfo.numCores);
111         } else if (sysinfo.numCores < numCores) {
112             // Set the requested number of cores to the number of cores on the server.
113             Log.debug("Limiting jobs from explicitly set "+numCores+" to cores available on server: "+sysinfo.numCores);
114             numCores = sysinfo.numCores;
115         } else {
116             Log.debug("Number of jobs explicitly set to "+numCores);
117         }
118         // More than three concurrent cores does not currently give a speedup, at least for compiling the jdk
119         // in the OpenJDK. This will change in the future.
120         int numCompiles = numCores;
121         if (numCores > limitOnConcurrency) numCompiles = limitOnConcurrency;
122         // Split the work up in chunks to compiled.
123 
124         int numSources = 0;
125         for (String s : pkgSrcs.keySet()) {
126             Set<URI> ss = pkgSrcs.get(s);
127             numSources += ss.size();
128         }
129 
130         int sourcesPerCompile = numSources / numCompiles;
131 
132         // For 64 bit Java, it seems we can compile the OpenJDK 8800 files with a 1500M of heap
133         // in a single chunk, with reasonable performance.
134         // For 32 bit java, it seems we need 1G of heap.
135         // Number experimentally determined when compiling the OpenJDK.
136         // Includes space for reasonably efficient garbage collection etc,
137         // Calculating backwards gives us a requirement of
138         // 1500M/8800 = 175 KiB for 64 bit platforms
139         // and 1G/8800 = 119 KiB for 32 bit platform
140         // for each compile.....
141         int kbPerFile = 175;
142         String osarch = System.getProperty("os.arch");
143         String dataModel = System.getProperty("sun.arch.data.model");
144         if ("32".equals(dataModel)) {
145             // For 32 bit platforms, assume it is slightly smaller
146             // because of smaller object headers and pointers.
147             kbPerFile = 119;
148         }
149         int numRequiredMBytes = (kbPerFile*numSources)/1024;
150         Log.debug("For os.arch "+osarch+" the empirically determined heap required per file is "+kbPerFile+"KiB");
151         Log.debug("Server has "+numMBytes+"MiB of heap.");
152         Log.debug("Heuristics say that we need "+numRequiredMBytes+"MiB of heap for all source files.");
153         // Perform heuristics to see how many cores we can use,
154         // or if we have to the work serially in smaller chunks.
155         if (numMBytes < numRequiredMBytes) {
156             // Ouch, cannot fit even a single compile into the heap.
157             // Split it up into several serial chunks.
158             concurrentCompiles = false;
159             // Limit the number of sources for each compile to 500.
160             if (numSources < 500) {
161                 numCompiles = 1;
162                 sourcesPerCompile = numSources;
163                 Log.debug("Compiling as a single source code chunk to stay within heap size limitations!");
164             } else if (sourcesPerCompile > 500) {
165                 // This number is very low, and tuned to dealing with the OpenJDK
166                 // where the source is >very< circular! In normal application,
167                 // with less circularity the number could perhaps be increased.
168                 numCompiles = numSources / 500;
169                 sourcesPerCompile = numSources/numCompiles;
170                 Log.debug("Compiling source as "+numCompiles+" code chunks serially to stay within heap size limitations!");
171             }
172         } else {
173             if (numCompiles > 1) {
174                 // Ok, we can fit at least one full compilation on the heap.
175                 float usagePerCompile = (float)numRequiredMBytes / ((float)numCompiles * (float)0.7);
176                 int usage = (int)(usagePerCompile * (float)numCompiles);
177                 Log.debug("Heuristics say that for "+numCompiles+" concurrent compiles we need "+usage+"MiB");
178                 if (usage > numMBytes) {
179                     // Ouch it does not fit. Reduce to a single chunk.
180                     numCompiles = 1;
181                     sourcesPerCompile = numSources;
182                     // What if the relationship between number of compile_chunks and num_required_mbytes
183                     // is not linear? Then perhaps 2 chunks would fit where 3 does not. Well, this is
184                     // something to experiment upon in the future.
185                     Log.debug("Limiting compile to a single thread to stay within heap size limitations!");
186                 }
187             }
188         }
189 
190         Log.debug("Compiling sources in "+numCompiles+" chunk(s)");
191 
192         // Create the chunks to be compiled.
193         final CompileChunk[] compileChunks = createCompileChunks(pkgSrcs, oldPackageDependents,
194                 numCompiles, sourcesPerCompile);
195 
196         if (Log.isDebugging()) {
197             int cn = 1;
198             for (CompileChunk cc : compileChunks) {
199                 Log.debug("Chunk "+cn+" for "+id+" ---------------");
200                 cn++;
201                 for (URI u : cc.srcs) {
202                     Log.debug(""+u);
203                 }
204             }
205         }
206 
207         long start = System.currentTimeMillis();
208 
209         // Prepare compilation calls
210         List<Callable<CompilationSubResult>> compilationCalls = new ArrayList<>();
211         final Object lock = new Object();
212         for (int i = 0; i < numCompiles; i++) {
213             CompileChunk cc = compileChunks[i];
214             if (cc.srcs.isEmpty()) {
215                 continue;
216             }
217 
218             String chunkId = id + "-" + String.valueOf(i);
219             Log log = Log.get();
220             compilationCalls.add(() -> {
221                 Log.setLogForCurrentThread(log);
222                 CompilationSubResult result = sjavac.compile("n/a",
223                                                              chunkId,
224                                                              args.prepJavacArgs(),
225                                                              Collections.emptyList(),
226                                                              cc.srcs,
227                                                              visibleSources);
228                 synchronized (lock) {
229                     Util.getLines(result.stdout).forEach(Log::info);
230                     Util.getLines(result.stderr).forEach(Log::error);
231                 }
232                 return result;
233             });
234         }
235 
236         // Perform compilations and collect results
237         List<CompilationSubResult> subResults = new ArrayList<>();
238         List<Future<CompilationSubResult>> futs = new ArrayList<>();
239         ExecutorService exec = Executors.newFixedThreadPool(concurrentCompiles ? compilationCalls.size() : 1);
240         for (Callable<CompilationSubResult> compilationCall : compilationCalls) {
241             futs.add(exec.submit(compilationCall));
242         }
243         for (Future<CompilationSubResult> fut : futs) {
244             try {
245                 subResults.add(fut.get());
246             } catch (ExecutionException ee) {
247                 Log.error("Compilation failed: " + ee.getMessage());
248                 Log.error(ee);
249             } catch (InterruptedException ie) {
250                 Log.error("Compilation interrupted: " + ie.getMessage());
251                 Log.error(ie);
252                 Thread.currentThread().interrupt();
253             }
254         }
255         exec.shutdownNow();
256 
257         // Process each sub result
258         for (CompilationSubResult subResult : subResults) {
259             for (String pkg : subResult.packageArtifacts.keySet()) {
260                 Set<URI> pkgArtifacts = subResult.packageArtifacts.get(pkg);
261                 packageArtifacts.merge(pkg, pkgArtifacts, Util::union);
262             }
263 
264             for (String pkg : subResult.packageDependencies.keySet()) {
265                 packageDependencies.putIfAbsent(pkg, new HashMap<>());
266                 packageDependencies.get(pkg).putAll(subResult.packageDependencies.get(pkg));
267             }
268 
269             for (String pkg : subResult.packageCpDependencies.keySet()) {
270                 packageCpDependencies.putIfAbsent(pkg, new HashMap<>());
271                 packageCpDependencies.get(pkg).putAll(subResult.packageCpDependencies.get(pkg));
272             }
273 
274             for (String pkg : subResult.packagePubapis.keySet()) {
275                 packagePubapis.merge(pkg, subResult.packagePubapis.get(pkg), PubApi::mergeTypes);
276             }
277 
278             for (String pkg : subResult.dependencyPubapis.keySet()) {
279                 dependencyPubapis.merge(pkg, subResult.dependencyPubapis.get(pkg), PubApi::mergeTypes);
280             }
281 
282             // Check the return values.
283             if (subResult.result != Result.OK) {
284                 rc = false;
285             }
286         }
287 
288         long duration = System.currentTimeMillis() - start;
289         long minutes = duration/60000;
290         long seconds = (duration-minutes*60000)/1000;
291         Log.debug("Compilation of "+numSources+" source files took "+minutes+"m "+seconds+"s");
292 
293         return rc;
294     }
295 
296     /**
297      * Split up the sources into compile chunks. If old package dependents information
298      * is available, sort the order of the chunks into the most dependent first!
299      * (Typically that chunk contains the java.lang package.) In the future
300      * we could perhaps improve the heuristics to put the sources into even more sensible chunks.
301      * Now the package are simple sorted in alphabetical order and chunked, then the chunks
302      * are sorted on how dependent they are.
303      *
304      * @param pkgSrcs The sources to compile.
305      * @param oldPackageDependents Old package dependents, if non-empty, used to sort the chunks.
306      * @param numCompiles The number of chunks.
307      * @param sourcesPerCompile The number of sources per chunk.
308      * @return
309      */
310     CompileChunk[] createCompileChunks(Map<String,Set<URI>> pkgSrcs,
311                                        Map<String,Set<String>> oldPackageDependents,
312                                        int numCompiles,
313                                        int sourcesPerCompile) {
314 
315         CompileChunk[] compileChunks = new CompileChunk[numCompiles];
316         for (int i=0; i<compileChunks.length; ++i) {
317             compileChunks[i] = new CompileChunk();
318         }
319 
320         // Now go through the packages and spread out the source on the different chunks.
321         int ci = 0;
322         // Sort the packages
323         String[] packageNames = pkgSrcs.keySet().toArray(new String[0]);
324         Arrays.sort(packageNames);
325         String from = null;
326         for (String pkgName : packageNames) {
327             CompileChunk cc = compileChunks[ci];
328             Set<URI> s = pkgSrcs.get(pkgName);
329             if (cc.srcs.size()+s.size() > sourcesPerCompile && ci < numCompiles-1) {
330                 from = null;
331                 ci++;
332                 cc = compileChunks[ci];
333             }
334             cc.numPackages++;
335             cc.srcs.addAll(s);
336 
337             // Calculate nice package names to use as information when compiling.
338             String justPkgName = Util.justPackageName(pkgName);
339             // Fetch how many packages depend on this package from the old build state.
340             Set<String> ss = oldPackageDependents.get(pkgName);
341             if (ss != null) {
342                 // Accumulate this information onto this chunk.
343                 cc.numDependents += ss.size();
344             }
345             if (from == null || from.trim().equals("")) from = justPkgName;
346             cc.pkgNames.append(justPkgName+"("+s.size()+") ");
347             cc.pkgFromTos = from+" to "+justPkgName;
348         }
349         // If we are compiling serially, sort the chunks, so that the chunk (with the most dependents) (usually the chunk
350         // containing java.lang.Object, is to be compiled first!
351         // For concurrent compilation, this does not matter.
352         Arrays.sort(compileChunks);
353         return compileChunks;
354     }
355 }