1 /*
  2  * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 /*
 25  Used by DemoSupport.gmk:
 26  
 27  - calculate the geomean and stdev of benchmark runs
 28  - write markdown snippets for rendering benchmark results as a chart on GitHub .MD files
 29 
 30 Example:
 31 
 32 helidon-quickstart-se$ cat mainline_vs_premain.csv
 33 run,mainline default,mainline custom static CDS,premain custom static CDS only,premain CDS + AOT
 34 1,398,244,144,107
 35 2,387,247,142,108
 36 3,428,238,143,107
 37 4,391,252,142,111
 38 5,417,247,141,107
 39 6,390,239,139,127
 40 7,387,247,145,111
 41 8,387,240,147,110
 42 9,388,242,147,108
 43 10,400,242,167,108
 44 
 45 helidon-quickstart-se$ java ../lib/GithubMDChart.java mainline_vs_premain.md < mainline_vs_premain.csv
 46 Geomean,397.08,243.76,145.52,110.26 (3.60x improvement)
 47 Stdev,13.55,4.19,7.50,5.73
 48 
 49 ```mermaid
 50 ---
 51 config:
 52     xyChart:
 53         chartOrientation: horizontal
 54         height: 300
 55 ---
 56 xychart-beta
 57     x-axis "variant" ["mainline default", "mainline custom static CDS", "premain custom static CDS only", "premain CDS + AOT"]
 58     y-axis "Elapsed time (ms, smaller is better)" 0 --> 397
 59     bar [397, 244, 146, 110]
 60 ```
 61 
 62 */
 63 
 64 import java.io.PrintWriter;
 65 import java.util.Scanner;
 66 import java.util.ArrayList;
 67 import java.util.Arrays;
 68 
 69 public class GithubMDChart {
 70     @SuppressWarnings("unchecked")
 71     public static void main(String args[]) throws Exception {
 72         Scanner input = new Scanner(System.in);
 73         String line = input.nextLine();
 74         //System.out.println(line);
 75         String head[] = line.split(",");
 76         Object[] groups = new Object[head.length];
 77         String[] geomeans = new String[head.length];
 78         for (int i = 0; i < head.length; i++) {
 79             groups[i] = new ArrayList<Double>();
 80         }
 81         while (input.hasNext()) {
 82             line = input.nextLine();
 83             //System.out.println(line);
 84             String parts[] = line.split(",");
 85             for (int i = 0; i < head.length; i++) {
 86                 ArrayList<Double> list =  (ArrayList<Double>)(groups[i]);
 87                 list.add(Double.valueOf(Double.parseDouble(parts[i])));
 88             }
 89         }
 90 
 91         for (int i = 0; i < head.length; i++) {
 92             ArrayList<Double> list =  (ArrayList<Double>)(groups[i]);
 93             if (i == 0) {
 94                 System.out.print("Geomean");
 95             } else {
 96                 System.out.print(",");
 97                 geomeans[i] = geomean(list);
 98                 System.out.print(geomeans[i]);
 99             }
100         }
101         if (head.length > 2) {
102             double old_elapsed = Double.parseDouble(geomeans[1]);
103             double new_elapsed = Double.parseDouble(geomeans[head.length-1]);
104             System.out.format(" (%.2fx improvement)", old_elapsed / new_elapsed);
105         }
106 
107         System.out.println();
108 
109         for (int i = 0; i < head.length; i++) {
110             ArrayList<Double> list =  (ArrayList<Double>)(groups[i]);
111             if (i == 0) {
112                 System.out.print("Stdev");
113             } else {
114                 System.out.print(",");
115                 System.out.print(stdev(list));
116             }
117         }
118         System.out.println();
119         System.out.println("Markdown snippets in " + args[0]);
120 
121         String[] norm = new String[geomeans.length];
122         for (int i = 1; i < head.length; i++) {
123             double base = Double.parseDouble(geomeans[1]);
124             double me   = Double.parseDouble(geomeans[i]);
125             norm[i] = String.format("%.0f", 1000.0 * me / base);
126         }
127 
128         PrintWriter pw = new PrintWriter(args[0]);
129         // Horizontal avoids issues with long names overlapping each other,
130         // and setting a smaller-than-default height makes it easier to overlook and compare.
131         pw.println("""
132         ```mermaid
133         ---
134         config:
135             theme: "forest"
136             xyChart:
137                 chartOrientation: horizontal
138                 height: 300
139         ---
140         xychart-beta
141             x-axis "variant" [$names]
142             y-axis "Elapsed time (ms, smaller is better)" 0 --> $maxtime
143             bar [$geomeans]
144         ```
145         
146         -----------------Normalized---------------------------------------------
147         ```mermaid
148         ---
149         config:
150             theme: "forest"
151             xyChart:
152                 chartOrientation: horizontal
153                 height: 300
154         ---
155         xychart-beta
156             x-axis "variant" [$names]
157             y-axis "Elapsed time (normalized, smaller is better)" 0 --> 1000
158             bar [$norms]
159         ```
160         """
161         .replace("$names", '"' + String.join("\", \"", Arrays.copyOfRange(head, 1, head.length)) + '"')
162         .replace("$maxtime", geomeans[1])
163         .replace("$geomeans", String.join(", ", Arrays.copyOfRange(geomeans, 1, geomeans.length)))
164         .replace("$norms", String.join(", ", Arrays.copyOfRange(norm, 1, norm.length)))
165         );
166         pw.close();
167     }
168 
169 
170     static String geomean(ArrayList<Double> list) {
171         double log = 0.0d;
172         for (Double d : list) {
173             double v = d.doubleValue();
174             if (v <= 0) {
175                 v = 0.000001;
176             }
177             log += Math.log(v);
178         }
179 
180 	return String.format("%.2f", Math.exp(log / list.size()));
181     }
182 
183     static String stdev(ArrayList<Double> list) {
184         double sum = 0.0;
185         for (Double d : list) {
186             sum += d.doubleValue();
187         }
188 
189         double length = list.size();
190         double mean = sum / length;
191 
192         double stdev = 0.0;
193         for (Double d : list) {
194             stdev += Math.pow(d.doubleValue() - mean, 2);
195         }
196 
197 	return String.format("%.2f", Math.sqrt(stdev / length));
198     }
199 }