1 # Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
2 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3 #
4 # This code is free software; you can redistribute it and/or modify it
5 # under the terms of the GNU General Public License version 2 only, as
6 # published by the Free Software Foundation.
7 #
8 # This code is distributed in the hope that it will be useful, but WITHOUT
9 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 # version 2 for more details (a copy is included in the LICENSE file that
12 # accompanied this code).
13 #
14 # You should have received a copy of the GNU General Public License version
15 # 2 along with this work; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 #
18 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 # or visit www.oracle.com if you need additional information or have any
20 # questions.
21
22 #======================================================================
23 # Overview
24 #
25 # This file provides the framework for running start-up benchmarks that have very
26 # short elapsed time.
27 #
28 # The goal is to be able measure very minor improvements (under 1%) even when there
29 # are system noises.
30 #
31 # - Measure the entire elapsed time with 'perf stat -r 16 bin/java .....'
32 # - Interleave the execution of the test JVMs. This way, the effect of system noises
33 # (CPU overheating, throttling, background tasks) is likely evenly distributed across
34 # the test JVMs.
35 # - Save the results in a CSV file so you can load it in a spreadsheet and manually
36 # check for noisy samples.
37 #
38 # NOTE: you can see the effect of CPU throttling in the sample data below. See the
39 # 2_xon column, which goes from 123ms when the CPU is cold to 128ms when the CPU is hot.
40
41 #======================================================================
42 # Config
43 #
44 # See ../javac_helloworld/run.sh for an example of using this file
45 #
46 # Before sourcing this file
47 # The caller script should set up the following variables
48
49 # required
50 # CMDLINE
51 # APP
52 # optional
53 # HEAP_SIZE
54 # RUNS
55 # REPEAT
56
57 # The number of the outer loop
58 if test "$RUNS" = ""; then
59 RUNS=10
60 fi
61
62 # The number passed to "perf stat -r"
63 if test "$REPEAT" = ""; then
64 REPEAT=16
65 fi
66
67 #======================================================================
68 # Example
69 #
70 # FYI: Results I got on 2023/12/21, Intel(R) Core(TM) i7-10700 CPU @ 2.90GHz, 32MB RAM
71 # With JDK mainline repo and leyden-premain branch that are pretty up to date.
72 #
73 # $ cd test/hotspot/jtreg/premain/javac_helloworld
74 # $ bash run.sh /bld/mainline/images/jdk/bin/java /bld/leyden/images/jdk/bin/java
75 #
76 # ===report.csv================================================
77 # Run,1_xoff,1_xon,2_xon,2_td,2_aot
78 # 1,313.86000,164.25000,123.031000,96.849000,92.38000
79 # 2,324.59000,163.745000,122.550000,97.730000,92.45000
80 # 3,313.61000,164.222000,122.91000,100.59000,91.18000
81 # 4,316.58000,163.779000,124.673000,101.085000,95.10000
82 # 5,313.002000,162.854000,126.240000,102.423000,91.81000
83 # 6,314.97000,166.020000,126.148000,101.159000,91.43000
84 # 7,313.30000,168.23000,128.19000,101.956000,88.13000
85 # 8,314.67000,169.04000,126.168000,102.220000,91.90000
86 # 9,313.26000,167.864000,125.856000,102.57000,90.75000
87 # 10,318.14000,169.075000,128.055000,103.523000,93.61000
88 # ==============================jvm1 /bld/mainline/images/jdk/bin/java
89 # [1_xoff] Mainline JDK (CDS disabled) 315.58 ms
90 # [1_xon ] Mainline JDK (CDS enabled) 165.89 ms
91 # ==============================jvm2 /bld/leyden/images/jdk/bin/java
92 # [2_xon ] Premain Prototype (CDS ) 125.37 ms
93 # [2_td ] Premain Prototype (CDS + Training Data) 100.99 ms
94 # [2_aot ] Premain Prototype (CDS + Training Data + AOT) 91.86 ms
95 #
96 #======================================================================
97
98
99 # "$@" is the command-line specified by the user. Each element specifies a
100 # JVM (which optionally can contain VM parameters).
101 #
102 # We automatically detect whether the JVM is from the mainline, or from the leyden
103 # repo.
104 function main () {
105 do_dump "$@"
106 do_exec "$@"
107 do_summary "$@"
108 }
109
110 function do_dump () {
111 if test "$NODUMP" != ""; then
112 return
113 fi
114
115 local n=0
116 for i in "$@"; do
117 n=$(expr $n + 1)
118 JAVA="$i"
119 if test "$HEAP_SIZE" != ""; then
120 JAVA="$JAVA $HEAP_SIZE"
121 fi
122 dump_one_jvm $n
123 done
124 }
125
126 function do_exec () {
127 if test "$NOEXEC" != ""; then
128 return
129 fi
130
131 rm -rf logs
132 mkdir -p logs
133 rm -f report.csv
134
135 REPORT_HEADER='Run'
136
137 local n=0
138 for i in "$@"; do
139 n=$(expr $n + 1)
140 JAVA="$i"
141 if test "$HEAP_SIZE" != ""; then
142 JAVA="$JAVA $HEAP_SIZE"
143 fi
144 local APPID=$APP-jvm$n
145 if test -f $APPID-mainline.classlist; then
146 REPORT_HEADER=${REPORT_HEADER},${n}_xoff
147 REPORT_HEADER=${REPORT_HEADER},${n}_xon
148 else
149 REPORT_HEADER=${REPORT_HEADER},${n}_xon
150 REPORT_HEADER=${REPORT_HEADER},${n}_td
151 REPORT_HEADER=${REPORT_HEADER},${n}_aot
152 fi
153 #report jvm$n = $JAVA
154 done
155
156 report $REPORT_HEADER
157
158 # The #0 run is to warm up the CPU and is not counted
159 for r in $(seq 0 $RUNS); do
160 if test $r = 0; then
161 echo "RUN $r (warmup - discarded)"
162 else
163 echo "RUN $r of $RUNS"
164 fi
165 RUNLOG=$r
166 local n=0
167 for i in "$@"; do
168 n=$(expr $n + 1)
169 JAVA="$i"
170 if test "$HEAP_SIZE" != ""; then
171 JAVA="$JAVA $HEAP_SIZE"
172 fi
173 echo === jvm$n
174 exec_one_jvm $n $r
175 done
176
177 if test $r -gt 0; then
178 report $RUNLOG
179 else
180 echo $RUNLOG
181 fi
182 done
183 }
184
185 function dump_one_jvm () {
186 local binjava=$(echo $JAVA | sed -e 's/ .*//g')
187 local APPID=$APP-jvm$1
188
189 if $JAVA --version > /dev/null; then
190 if $JAVA -XX:+PrintFlagsFinal --version | grep ArchiveInvokeDynamic > /dev/null; then
191 local is_premain=1
192 local type_msg="(premain 5 step)"
193 else
194 local is_premain=0
195 local type_msg="(mainline )"
196 fi
197 else
198 echo "$JAVA doesn't seem to be a real JVM"
199 exit 1
200 fi
201
202 echo ========================================
203 echo dumping archive "$type_msg" for $JAVA
204 echo ========================================
205
206 if test $is_premain = 0; then
207 echo "(Mainline) Dump classlist"
208 (set -x; $JAVA -Xshare:off -XX:DumpLoadedClassList=$APPID-mainline.classlist $CMDLINE) || exit 1
209
210 echo "(Mainline) Dump CDS archive"
211 rm -f $APPID-mainline-static.dump.log
212 (set -x; $JAVA -Xshare:dump -XX:SharedArchiveFile=$APPID-mainline.jsa -XX:SharedClassListFile=$APPID-mainline.classlist \
213 -Xlog:cds=debug,cds+class=debug:file=$APPID-mainline-static.dump.log::filesize=0 $CMDLINE) || exit 1
214
215 else
216 # FIXME -- add support for new workflow
217 LEYDEN_OPTS="-XX:+ArchiveInvokeDynamic"
218
219 echo "(Premain STEP 1 of 5) Dump classlist"
220 (set -x; $JAVA -Xshare:off -XX:DumpLoadedClassList=$APPID.classlist $CMDLINE) || exit 1
221
222 echo "(Premain STEP 2 of 5) Create Static $APPID-static.jsa"
223 rm -f $APPID-static.dump.log
224 (set -x; $JAVA -Xshare:dump $LEYDEN_OPTS -XX:SharedArchiveFile=$APPID-static.jsa -XX:SharedClassListFile=$APPID.classlist \
225 -Xlog:cds=debug,cds+class=debug,cds+resolve=debug:file=$APPID-static.dump.log::filesize=0 $CMDLINE) || exit 1
226
227
228 echo "(Premain STEP 3 of 5) Run with $APPID-static.jsa and dump profile in $APPID-dynamic.jsa (With Training Data Replay)"
229 rm -f $APPID-dynamic.dump.log
230 (set -x; $JAVA -XX:SharedArchiveFile=$APPID-static.jsa -XX:ArchiveClassesAtExit=$APPID-dynamic.jsa -XX:+RecordTraining \
231 -Xlog:cds=debug,cds+class=debug:file=$APPID-dynamic.dump.log::filesize=0 $CMDLINE) || exit 1
232
233 echo "(Premain 4 of 5) Run with $APPID-dynamic.jsa and generate AOT code"
234 rm -f $APPID-store-sc.log
235 (set -x; $JAVA -XX:SharedArchiveFile=$APPID-dynamic.jsa -XX:+ReplayTraining -XX:+StoreCachedCode \
236 -Xlog:scc*=warning:file=$APPID-store-sc.log::filesize=0 \
237 -XX:CachedCodeFile=$APPID-dynamic.jsa-sc -XX:CachedCodeMaxSize=100M $CMDLINE) || exit 1
238
239 fi
240 }
241
242 function exec_one_jvm () {
243 local APPID=$APP-jvm$1
244
245 if test -f $APPID-mainline.classlist; then
246 (set -x;
247 perf stat -r $REPEAT $JAVA -Xshare:off $CMDLINE 2> logs/${1}_xoff.$2
248 )
249 RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_xoff.$2)
250
251 (set -x;
252 perf stat -r $REPEAT $JAVA -XX:SharedArchiveFile=$APPID-mainline.jsa $CMDLINE 2> logs/${1}_xon.$2
253 )
254 RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_xon.$2)
255 fi
256
257 if test -f $APPID.classlist; then
258 (set -x;
259 perf stat -r $REPEAT $JAVA -XX:SharedArchiveFile=$APPID-static.jsa $CMDLINE 2> logs/${1}_xon.$2
260 )
261 RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_xon.$2)
262 (set -x;
263 perf stat -r $REPEAT $JAVA -XX:SharedArchiveFile=$APPID-dynamic.jsa -XX:+ReplayTraining \
264 $CMDLINE 2> logs/${1}_td.$2
265 )
266 RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_td.$2)
267 (set -x;
268 perf stat -r $REPEAT $JAVA -XX:SharedArchiveFile=$APPID-dynamic.jsa -XX:+ReplayTraining -XX:+LoadCachedCode \
269 -XX:CachedCodeFile=$APPID-dynamic.jsa-sc -Xlog:scc=error \
270 $CMDLINE 2> logs/${1}_aot.$2
271 )
272 RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_aot.$2)
273 fi
274 }
275
276 function get_elapsed () {
277 elapsed=$(bc <<< "scale=3; $(cat $1 | grep elapsed | sed -e 's/[+].*//') * 1000")
278 echo $elapsed >> $1.elapsed
279 echo $elapsed
280 }
281
282
283 function report () {
284 echo "$@"
285 echo "$@" >> report.csv
286 }
287
288 function geomean () {
289 printf "%6.2f ms" $(awk 'BEGIN{E = exp(1);} $1>0{tot+=log($1); c++} END{m=tot/c; printf "%.2f\n", E^m}' $*)
290 }
291
292
293
294 function do_summary () {
295 echo ===report.csv================================================
296 cat report.csv
297
298 mkdir -p logs/warmup
299 mv logs/*.0.elapsed logs/*.0 logs/warmup
300 local n=0
301 for i in "$@"; do
302 n=$(expr $n + 1)
303 local APPID=$APP-jvm$n
304 report ==============================jvm$n $i
305 if test -f $APPID-mainline.classlist; then
306 report "[$(printf %-6s ${n}_xoff)] Mainline JDK (CDS disabled) $(geomean logs/${n}_xoff.*.elapsed)"
307 report "[$(printf %-6s ${n}_xon )] Mainline JDK (CDS enabled) $(geomean logs/${n}_xon.*.elapsed)"
308 else
309 report "[$(printf %-6s ${n}_xon )] Premain Prototype (CDS ) $(geomean logs/${n}_xon.*.elapsed)"
310 report "[$(printf %-6s ${n}_td )] Premain Prototype (CDS + Training Data) $(geomean logs/${n}_td.*.elapsed)"
311 report "[$(printf %-6s ${n}_aot )] Premain Prototype (CDS + Training Data + AOT) $(geomean logs/${n}_aot.*.elapsed)"
312 fi
313 done
314 }
315
316 main "$@"