1 /*
2 * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 /*
25 Used by DemoSupport.gmk:
26
27 - calculate the geomean and stdev of benchmark runs
28 - write markdown snippets for rendering benchmark results as a chart on GitHub .MD files
29
30 Example:
31
32 helidon-quickstart-se$ cat mainline_vs_premain.csv
33 run,mainline default,mainline custom static CDS,premain custom static CDS only,premain CDS + AOT
34 1,398,244,144,107
35 2,387,247,142,108
36 3,428,238,143,107
37 4,391,252,142,111
38 5,417,247,141,107
39 6,390,239,139,127
40 7,387,247,145,111
41 8,387,240,147,110
42 9,388,242,147,108
43 10,400,242,167,108
44
45 helidon-quickstart-se$ java ../lib/GithubMDChart.java mainline_vs_premain.md < mainline_vs_premain.csv
46 Geomean,397.08,243.76,145.52,110.26 (3.60x improvement)
47 Stdev,13.55,4.19,7.50,5.73
48
49 ```mermaid
50 ---
51 config:
52 xyChart:
53 chartOrientation: horizontal
54 height: 300
55 ---
56 xychart-beta
57 x-axis "variant" ["mainline default", "mainline custom static CDS", "premain custom static CDS only", "premain CDS + AOT"]
58 y-axis "Elapsed time (ms, smaller is better)" 0 --> 397
59 bar [397, 244, 146, 110]
60 ```
61
62 */
63
64 import java.io.PrintWriter;
65 import java.util.Scanner;
66 import java.util.ArrayList;
67 import java.util.Arrays;
68
69 public class GithubMDChart {
70 @SuppressWarnings("unchecked")
71 public static void main(String args[]) throws Exception {
72 Scanner input = new Scanner(System.in);
73 String line = input.nextLine();
74 //System.out.println(line);
75 String head[] = line.split(",");
76 Object[] groups = new Object[head.length];
77 String[] geomeans = new String[head.length];
78 for (int i = 0; i < head.length; i++) {
79 groups[i] = new ArrayList<Double>();
80 }
81 while (input.hasNext()) {
82 line = input.nextLine();
83 //System.out.println(line);
84 String parts[] = line.split(",");
85 for (int i = 0; i < head.length; i++) {
86 ArrayList<Double> list = (ArrayList<Double>)(groups[i]);
87 list.add(Double.valueOf(Double.parseDouble(parts[i])));
88 }
89 }
90
91 for (int i = 0; i < head.length; i++) {
92 ArrayList<Double> list = (ArrayList<Double>)(groups[i]);
93 if (i == 0) {
94 System.out.print("Geomean");
95 } else {
96 System.out.print(",");
97 geomeans[i] = geomean(list);
98 System.out.print(geomeans[i]);
99 }
100 }
101 if (head.length > 2) {
102 double old_elapsed = Double.parseDouble(geomeans[1]);
103 double new_elapsed = Double.parseDouble(geomeans[head.length-1]);
104 System.out.format(" (%.2fx improvement)", old_elapsed / new_elapsed);
105 }
106
107 System.out.println();
108
109 for (int i = 0; i < head.length; i++) {
110 ArrayList<Double> list = (ArrayList<Double>)(groups[i]);
111 if (i == 0) {
112 System.out.print("Stdev");
113 } else {
114 System.out.print(",");
115 System.out.print(stdev(list));
116 }
117 }
118 System.out.println();
119 System.out.println("Markdown snippets in " + args[0]);
120
121 String[] norm = new String[geomeans.length];
122 for (int i = 1; i < head.length; i++) {
123 double base = Double.parseDouble(geomeans[1]);
124 double me = Double.parseDouble(geomeans[i]);
125 norm[i] = String.format("%.0f", 1000.0 * me / base);
126 }
127
128 PrintWriter pw = new PrintWriter(args[0]);
129 // Horizontal avoids issues with long names overlapping each other,
130 // and setting a smaller-than-default height makes it easier to overlook and compare.
131 pw.println("""
132 ```mermaid
133 ---
134 config:
135 xyChart:
136 chartOrientation: horizontal
137 height: 300
138 ---
139 xychart-beta
140 x-axis "variant" [$names]
141 y-axis "Elapsed time (ms, smaller is better)" 0 --> $maxtime
142 bar [$geomeans]
143 ```
144
145 -----------------Normalized---------------------------------------------
146 ```mermaid
147 ---
148 config:
149 xyChart:
150 chartOrientation: horizontal
151 height: 300
152 ---
153 xychart-beta
154 x-axis "variant" [$names]
155 y-axis "Elapsed time (normalized, smaller is better)" 0 --> 1000
156 bar [$norms]
157 ```
158 """
159 .replace("$names", '"' + String.join("\", \"", Arrays.copyOfRange(head, 1, head.length)) + '"')
160 .replace("$maxtime", geomeans[1])
161 .replace("$geomeans", String.join(", ", Arrays.copyOfRange(geomeans, 1, geomeans.length)))
162 .replace("$norms", String.join(", ", Arrays.copyOfRange(norm, 1, norm.length)))
163 );
164 pw.close();
165 }
166
167
168 static String geomean(ArrayList<Double> list) {
169 double log = 0.0d;
170 for (Double d : list) {
171 double v = d.doubleValue();
172 if (v <= 0) {
173 v = 0.000001;
174 }
175 log += Math.log(v);
176 }
177
178 return String.format("%.2f", Math.exp(log / list.size()));
179 }
180
181 static String stdev(ArrayList<Double> list) {
182 double sum = 0.0;
183 for (Double d : list) {
184 sum += d.doubleValue();
185 }
186
187 double length = list.size();
188 double mean = sum / length;
189
190 double stdev = 0.0;
191 for (Double d : list) {
192 stdev += Math.pow(d.doubleValue() - mean, 2);
193 }
194
195 return String.format("%.2f", Math.sqrt(stdev / length));
196 }
197 }