1 # Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
  2 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  3 #
  4 # This code is free software; you can redistribute it and/or modify it
  5 # under the terms of the GNU General Public License version 2 only, as
  6 # published by the Free Software Foundation.
  7 #
  8 # This code is distributed in the hope that it will be useful, but WITHOUT
  9 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 10 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 11 # version 2 for more details (a copy is included in the LICENSE file that
 12 # accompanied this code).
 13 #
 14 # You should have received a copy of the GNU General Public License version
 15 # 2 along with this work; if not, write to the Free Software Foundation,
 16 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 17 #
 18 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 19 # or visit www.oracle.com if you need additional information or have any
 20 # questions.
 21 
 22 #======================================================================
 23 # Overview
 24 #
 25 # This file provides the framework for running start-up benchmarks that have very
 26 # short elapsed time.
 27 #
 28 # The goal is to be able measure very minor improvements (under 1%) even when there
 29 # are system noises.
 30 #
 31 # - Measure the entire elapsed time with 'perf stat -r 16 bin/java .....'
 32 # - Interleave the execution of the test JVMs. This way, the effect of system noises 
 33 #   (CPU overheating, throttling, background tasks) is likely evenly distributed across
 34 #   the test JVMs.
 35 # - Save the results in a CSV file so you can load it in a spreadsheet and manually
 36 #   check for noisy samples.
 37 #
 38 # NOTE: you can see the effect of CPU throttling in the sample data below. See the
 39 # 2_xon column, which goes from 123ms when the CPU is cold to 128ms when the CPU is hot.
 40 
 41 #======================================================================
 42 # Config
 43 #
 44 # See ../javac_new_workflow/run_bench.sh for an example of using this file
 45 #
 46 # Before sourcing this file
 47 # The caller script should set up the following variables
 48 
 49 # required
 50 #     CMDLINE
 51 #     APP
 52 # optional
 53 #     HEAP_SIZE
 54 #     RUNS
 55 #     REPEAT
 56 #
 57 #     LEYDEN_CALLER_OPTS
 58 #       extra args to be added to Leyden JVM runs
 59 #       e.g. LEYDEN_CALLER_OPTS='-Xlog:scc=error'
 60 #     TASKSET
 61 #       allows pinning of perf runs to specific processors
 62 #       e.g. setting TASKSET='taskest 0xff0' will execute
 63 #       benchmark runs as
 64 #         "taskest 0xff0 perf stat -r $REPEAT $JAVA ... $APP $CMDLINE"
 65 
 66 # The number of the outer loop
 67 if test "$RUNS" = ""; then
 68     RUNS=10
 69 fi
 70 
 71 # The number passed to "perf stat -r"
 72 if test "$REPEAT" = ""; then
 73     REPEAT=16
 74 fi
 75 
 76 #======================================================================
 77 # Example
 78 #
 79 # FYI: Results I got on 2024/05/14, AArch64
 80 # With JDK mainline repo and leyden-premain branch that are pretty up to date.
 81 #
 82 # $ cd test/hotspot/jtreg/premain/javac_helloworld
 83 # $ bash run.sh .../mainline/images/jdk/bin/java .../leyden/images/jdk/bin/java
 84 # 
 85 # ===report.csv================================================
 86 # Run,1_xoff,1_xon,2_xoff,2_xon,2_td,2_aot
 87 # 1,245.95000,165.35000,253.3000,135.85000,114.08000,80.13000
 88 # 2,255.98000,172.27000,249.61000,131.03000,111.25000,92.39000
 89 # 3,244.63000,149.85000,247.29000,138.09000,113.11000,94.04000
 90 # 4,252.84000,150.57000,247.35000,144.17000,110.42000,93.87000
 91 # 5,247.72000,156.89000,249.38000,149.35000,111.93000,94.74000
 92 # 6,259.24000,167.54000,239.00000,147.69000,109.98000,84.89000
 93 # 7,241.83000,150.16000,248.35000,154.1000,110.58000,91.57000
 94 # 8,247.79000,143.51000,248.72000,144.88000,108.20000,78.43000
 95 # 9,254.68000,155.95000,253.50000,134.91000,114.17000,93.27000
 96 # 10,250.66000,166.34000,260.68000,142.91000,117.24000,87.14000
 97 # ==============================jvm1 /home/adinn/redhat/openjdk/jdkdev/image-jdk-master/jdk/bin/java
 98 # [1_xoff] Mainline JDK (CDS disabled)                   250.08 ms
 99 # [1_xon ] Mainline JDK (CDS enabled)                    157.58 ms
100 # ==============================jvm2 /home/adinn/redhat/openjdk/jdkdev/image-premain/jdk/bin/java
101 # [2_xoff] Premain JDK (CDS disabled)                    249.66 ms
102 # [2_xon ] Premain JDK (CDS enabled)                     142.13 ms
103 # [2_td  ] Premain Prototype (CDS + Training Data)        112.07 ms
104 # [2_aot ] Premain Prototype (CDS + Training Data + AOT)  88.86 ms
105 # 
106 #======================================================================
107 
108 
109 # "$@" is the command-line specified by the user. Each element specifies a 
110 # JVM (which optionally can contain VM parameters).
111 #
112 # We automatically detect whether the JVM is from the mainline, or from the leyden
113 # repo.
114 function main () {
115     do_dump "$@"
116     do_exec "$@"
117     do_summary "$@"
118 }
119 
120 function do_dump () {
121     if test "$NODUMP" != ""; then
122         return
123     fi
124 
125     local n=0
126     for i in "$@"; do
127         n=$(expr $n + 1)
128         JAVA="$i"
129         if test "$HEAP_SIZE" != ""; then
130             JAVA="$JAVA $HEAP_SIZE"
131         fi
132         dump_one_jvm $n
133     done
134 }
135 
136 function do_exec () {
137     if test "$NOEXEC" != ""; then
138         return
139     fi
140 
141     rm -rf logs
142     mkdir -p logs
143     rm -f report.csv
144 
145     REPORT_HEADER='Run'
146 
147     local n=0
148     for i in "$@"; do
149         n=$(expr $n + 1)
150         JAVA="$i"
151         if test "$HEAP_SIZE" != ""; then
152             JAVA="$JAVA $HEAP_SIZE"
153         fi
154         local APPID=$APP-jvm$n
155         if test -f $APPID-mainline.classlist; then
156             REPORT_HEADER=${REPORT_HEADER},${n}_xoff
157             REPORT_HEADER=${REPORT_HEADER},${n}_xon
158         else
159             REPORT_HEADER=${REPORT_HEADER},${n}_xoff
160             REPORT_HEADER=${REPORT_HEADER},${n}_xon
161             REPORT_HEADER=${REPORT_HEADER},${n}_td
162             REPORT_HEADER=${REPORT_HEADER},${n}_aot
163         fi
164         #report jvm$n = $JAVA
165     done
166 
167     report $REPORT_HEADER
168 
169     # The #0 run is to warm up the CPU and is not counted
170     for r in $(seq 0 $RUNS); do
171         if test $r = 0; then
172             echo "RUN $r (warmup - discarded)"
173         else
174             echo "RUN $r of $RUNS"
175         fi
176         RUNLOG=$r
177         local n=0
178         for i in "$@"; do
179             n=$(expr $n + 1)
180             JAVA="$i"
181             if test "$HEAP_SIZE" != ""; then
182                 JAVA="$JAVA $HEAP_SIZE"
183             fi
184             echo === jvm$n
185             exec_one_jvm $n $r
186         done
187 
188         if test $r -gt 0; then
189             report $RUNLOG
190         else
191             echo $RUNLOG
192         fi
193     done
194 }
195 
196 function dump_one_jvm () {
197     local binjava=$(echo $JAVA | sed -e 's/ .*//g')
198     local APPID=$APP-jvm$1
199 
200     if $JAVA --version > /dev/null; then
201         if $JAVA -XX:+PrintFlagsFinal --version | grep ArchiveInvokeDynamic > /dev/null; then
202             local is_premain=1
203             local type_msg="(premain )"
204         else
205             local is_premain=0
206             local type_msg="(mainline)"
207         fi
208     else
209         echo "$JAVA doesn't seem to be a real JVM"
210         exit 1
211     fi
212 
213     echo ========================================
214     echo dumping archive "$type_msg" for $JAVA
215     echo ========================================
216 
217     if test $is_premain = 0; then
218         echo "(Mainline) Dump classlist"
219         (set -x; $JAVA -Xshare:off -XX:DumpLoadedClassList=$APPID-mainline.classlist $CMDLINE) || exit 1
220 
221         echo "(Mainline) Dump CDS archive"
222         rm -f $APPID-mainline-static.dump.log
223         (set -x; $JAVA -Xshare:dump -XX:SharedArchiveFile=$APPID-mainline.jsa -XX:SharedClassListFile=$APPID-mainline.classlist \
224                        -Xlog:cds=debug,cds+class=debug:file=$APPID-mainline-static.dump.log::filesize=0 $CMDLINE) || exit 1
225 
226     else
227         echo "(Premain) Dump classlist"
228         (set -x; $JAVA -Xshare:off -XX:DumpLoadedClassList=$APPID-premain.classlist $CMDLINE) || exit 1
229 
230         echo "(Premain) Dump CDS archive"
231         rm -f $APPID-premain-static.dump.log
232         (set -x; $JAVA -Xshare:dump -XX:SharedArchiveFile=$APPID-premain.jsa -XX:SharedClassListFile=$APPID-premain.classlist \
233                        -Xlog:cds=debug,cds+class=debug:file=$APPID-premain-static.dump.log::filesize=0 $CMDLINE) || exit 1
234 
235 
236         LEYDEN_OPTS="-XX:+ArchiveInvokeDynamic $LEYDEN_CALLER_OPTS"
237 
238         echo "(Premain) Dump Leyden archive"
239         (set -x; $JAVA -XX:CacheDataStore=$APPID.cds $LEYDEN_OPTS $CMDLINE) || exit 1
240 
241     fi
242 }
243 
244 function exec_one_jvm () {
245     local APPID=$APP-jvm$1
246 
247     if test -f $APPID-mainline.classlist; then
248         (set -x;
249          $TASKSET perf stat -r $REPEAT $JAVA -Xshare:off $CMDLINE 2> logs/${1}_xoff.$2
250         )
251         RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_xoff.$2)
252 
253         (set -x;
254          $TASKSET perf stat -r $REPEAT $JAVA -XX:SharedArchiveFile=$APPID-mainline.jsa $CMDLINE 2> logs/${1}_xon.$2
255         )
256         RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_xon.$2)
257     fi
258 
259     if test -f $APPID.cds; then
260         LEYDEN_OPTS="$LEYDEN_CALLER_OPTS"
261         (set -x;
262          $TASKSET perf stat -r $REPEAT $JAVA -Xshare:off $CMDLINE 2> logs/${1}_xoff.$2
263         )
264         RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_xoff.$2)
265 
266         (set -x;
267          $TASKSET perf stat -r $REPEAT $JAVA -XX:SharedArchiveFile=$APPID-premain.jsa $CMDLINE 2> logs/${1}_xon.$2
268         )
269         RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_xon.$2)
270         (set -x;
271          $TASKSET perf stat -r $REPEAT $JAVA -XX:CacheDataStore=$APPID.cds -XX:-LoadCachedCode $LEYDEN_OPTS  $CMDLINE 2> logs/${1}_td.$2
272         )
273         RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_td.$2)
274         (set -x;
275          $TASKSET perf stat -r $REPEAT $JAVA -XX:CacheDataStore=$APPID.cds  -XX:+LoadCachedCode $LEYDEN_OPTS  $CMDLINE 2> logs/${1}_aot.$2
276         )
277         RUNLOG=$RUNLOG,$(get_elapsed logs/${1}_aot.$2)
278     fi
279 }
280 
281 function get_elapsed () {
282     elapsed=$(bc <<< "scale=3; $(cat $1 | grep elapsed | sed -e 's/[+].*//') * 1000")
283     echo $elapsed >> $1.elapsed
284     echo $elapsed
285 }
286 
287 
288 function report () {
289     echo "$@"
290     echo "$@" >> report.csv
291 }
292 
293 function geomean () {
294     printf "%6.2f ms" $(awk 'BEGIN{E = exp(1);} $1>0{tot+=log($1); c++} END{m=tot/c; printf "%.2f\n", E^m}' $*)
295 }
296 
297 
298 
299 function do_summary () {
300     echo ===report.csv================================================
301     cat report.csv
302 
303     mkdir -p logs/warmup
304     mv logs/*.0.elapsed logs/*.0 logs/warmup
305     local n=0
306     for i in "$@"; do
307         n=$(expr $n + 1)
308         local APPID=$APP-jvm$n
309         report ==============================jvm$n $i
310         if test -f $APPID-mainline.classlist; then
311             report "[$(printf %-6s ${n}_xoff)] Mainline JDK (CDS disabled)                   $(geomean logs/${n}_xoff.*.elapsed)"
312             report "[$(printf %-6s ${n}_xon )] Mainline JDK (CDS enabled)                    $(geomean logs/${n}_xon.*.elapsed)"
313         else
314             report "[$(printf %-6s ${n}_xoff)] Premain JDK (CDS disabled)                    $(geomean logs/${n}_xoff.*.elapsed)"
315             report "[$(printf %-6s ${n}_xon )] Premain JDK (CDS enabled)                     $(geomean logs/${n}_xon.*.elapsed)"
316             report "[$(printf %-6s ${n}_td )] Premain Prototype (CDS + Training Data)        $(geomean logs/${n}_td.*.elapsed)"
317             report "[$(printf %-6s ${n}_aot )] Premain Prototype (CDS + Training Data + AOT) $(geomean logs/${n}_aot.*.elapsed)"
318         fi
319     done
320 }
321 
322 main "$@"