changeset 1025:40fcfac32653

7901089: CSV/SCSV/TeX formats miss the secondary results
author shade
date Fri, 07 Nov 2014 16:55:12 +0300
parents 9cc12a0ce88d
children 8ea7f1d0b202
files jmh-core/src/main/java/org/openjdk/jmh/results/format/LaTeXResultFormat.java jmh-core/src/main/java/org/openjdk/jmh/results/format/XSVResultFormat.java jmh-core/src/test/java/org/openjdk/jmh/results/format/ResultFormatTest.java jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.csv jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.latex jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.scsv
diffstat 6 files changed, 155 insertions(+), 60 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-core/src/main/java/org/openjdk/jmh/results/format/LaTeXResultFormat.java	Fri Nov 07 15:45:28 2014 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/results/format/LaTeXResultFormat.java	Fri Nov 07 16:55:12 2014 +0300
@@ -47,22 +47,42 @@
 
     @Override
     public void writeOut(Collection<RunResult> results) {
-
         SortedSet<String> params = new TreeSet<String>();
         Set<String> benchNames = new HashSet<String>();
 
         Set<String> units = new HashSet<String>();
-        for (RunResult rs : results) {
-            params.addAll(rs.getParams().getParamsKeys());
-            benchNames.add(rs.getParams().getBenchmark());
-            units.add(rs.getPrimaryResult().getScoreUnit());
+        for (RunResult rr : results) {
+            params.addAll(rr.getParams().getParamsKeys());
+            units.add(rr.getPrimaryResult().getScoreUnit());
+            benchNames.add(rr.getParams().getBenchmark());
+            for (String label : rr.getSecondaryResults().keySet()) {
+                benchNames.add(rr.getParams().getBenchmark() + ":" + label);
+            }
         }
 
         boolean singleUnit = (units.size() == 1);
         String unit = singleUnit ? units.iterator().next() : null;
 
-        Map<String, String> benchPrefixes = ClassUtils.denseClassNames(benchNames);
+        Map<String, String> prefixes = ClassUtils.denseClassNames(benchNames);
 
+        printHeader(params, singleUnit, unit);
+
+        for (RunResult rr : results) {
+            BenchmarkParams benchmarkParams = rr.getParams();
+            Result res = rr.getPrimaryResult();
+
+            printLine(benchmarkParams.getBenchmark(), benchmarkParams, params, prefixes, singleUnit, res);
+
+            for (String label : rr.getSecondaryResults().keySet()) {
+                Result subRes = rr.getSecondaryResults().get(label);
+                printLine(benchmarkParams.getBenchmark() + ":" + label, benchmarkParams, params, prefixes, singleUnit, subRes);
+            }
+        }
+
+        printFooter();
+    }
+
+    private void printHeader(SortedSet<String> params, boolean singleUnit, String unit) {
         pw.write("\\begin{tabular}{r|");
         for (String p : params) {
             pw.write("l|");
@@ -75,28 +95,25 @@
         }
         pw.write(" \\multicolumn{" + (singleUnit ? 2 : 3) + "}{c}{\\texttt{Score" + (singleUnit ? ", " + unit : "") + "}} \\\\\n");
         pw.write("\\hline\n");
+    }
 
-        for (RunResult runResult : results) {
-            BenchmarkParams benchmarkParams = runResult.getParams();
-            Result primaryResult = runResult.getPrimaryResult();
+    private void printFooter() {pw.write("\\end{tabular}");}
 
-            pw.printf("\\texttt{%s} & ", escape(benchPrefixes.get(benchmarkParams.getBenchmark())));
-            for (String p : params) {
-                pw.printf("\\texttt{%s}", escape(benchmarkParams.getParam(p)));
-                pw.write(" & ");
-            }
-            pw.printf("\\texttt{%5.3f} & ", primaryResult.getScore());
-            pw.printf("\\scriptsize $\\pm$ \\texttt{%5.3f}", primaryResult.getScoreError());
-            if (!singleUnit) {
-                pw.printf(" & \\texttt{%s}", primaryResult.getScoreUnit());
-            }
-            pw.write(" \\\\");
+    private void printLine(String label, BenchmarkParams benchParams, SortedSet<String> params,
+                           Map<String, String> prefixes, boolean singleUnit, Result res) {
+        pw.printf("\\texttt{%s} & ", escape(prefixes.get(label)));
+        for (String p : params) {
+            pw.printf("\\texttt{%s}", escape(benchParams.getParam(p)));
+            pw.write(" & ");
+        }
+        pw.printf("\\texttt{%5.3f} & ", res.getScore());
+        pw.printf("\\scriptsize $\\pm$ \\texttt{%5.3f}", res.getScoreError());
+        if (!singleUnit) {
+            pw.printf(" & \\texttt{%s}", res.getScoreUnit());
+        }
+        pw.write(" \\\\");
 
-            pw.write("\n");
-        }
-
-        pw.write("\\end{tabular}");
-
+        pw.write("\n");
     }
 
     private static String escape(String s) {
--- a/jmh-core/src/main/java/org/openjdk/jmh/results/format/XSVResultFormat.java	Fri Nov 07 15:45:28 2014 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/results/format/XSVResultFormat.java	Fri Nov 07 16:55:12 2014 +0300
@@ -46,10 +46,26 @@
     @Override
     public void writeOut(Collection<RunResult> results) {
         SortedSet<String> params = new TreeSet<String>();
-        for (RunResult rs : results) {
-            params.addAll(rs.getParams().getParamsKeys());
+        for (RunResult res : results) {
+            params.addAll(res.getParams().getParamsKeys());
         }
 
+        printHeader(params);
+
+        for (RunResult rr : results) {
+            BenchmarkParams benchParams = rr.getParams();
+            Result res = rr.getPrimaryResult();
+
+            printLine(benchParams.getBenchmark(), benchParams, params, res);
+
+            for (String label : rr.getSecondaryResults().keySet()) {
+                Result subRes = rr.getSecondaryResults().get(label);
+                printLine(benchParams.getBenchmark() + ":" + subRes.getLabel(), benchParams, params, subRes);
+            }
+        }
+    }
+
+    private void printHeader(SortedSet<String> params) {
         pw.write("\"Benchmark\"");
         pw.write(delimiter);
         pw.write("\"Mode\"");
@@ -68,44 +84,40 @@
             pw.write("\"Param: " + k + "\"");
         }
         pw.write("\r\n");
+    }
 
-        for (RunResult runResult : results) {
-            BenchmarkParams benchmarkParams = runResult.getParams();
-            Result primaryResult = runResult.getPrimaryResult();
+    private void printLine(String label, BenchmarkParams benchmarkParams, SortedSet<String> params, Result result) {
+        pw.write("\"");
+        pw.write(label);
+        pw.write("\"");
+        pw.write(delimiter);
+        pw.write("\"");
+        pw.write(benchmarkParams.getMode().shortLabel());
+        pw.write("\"");
+        pw.write(delimiter);
+        pw.write(String.valueOf(benchmarkParams.getThreads()));
+        pw.write(delimiter);
+        pw.write(String.valueOf(result.getSampleCount()));
+        pw.write(delimiter);
+        pw.write(String.valueOf(result.getScore()));
+        pw.write(delimiter);
+        pw.write(String.valueOf(result.getScoreError()));
+        pw.write(delimiter);
+        pw.write("\"");
+        pw.write(result.getScoreUnit());
+        pw.write("\"");
 
-            pw.write("\"");
-            pw.write(benchmarkParams.getBenchmark());
-            pw.write("\"");
+        for (String p : params) {
             pw.write(delimiter);
             pw.write("\"");
-            pw.write(benchmarkParams.getMode().shortLabel());
+            String v = benchmarkParams.getParam(p);
+            if (v != null) {
+                pw.write(v);
+            }
             pw.write("\"");
-            pw.write(delimiter);
-            pw.write(String.valueOf(benchmarkParams.getThreads()));
-            pw.write(delimiter);
-            pw.write(String.valueOf(primaryResult.getSampleCount()));
-            pw.write(delimiter);
-            pw.write(String.valueOf(primaryResult.getScore()));
-            pw.write(delimiter);
-            pw.write(String.valueOf(primaryResult.getScoreError()));
-            pw.write(delimiter);
-            pw.write("\"");
-            pw.write(primaryResult.getScoreUnit());
-            pw.write("\"");
-
-            for (String p : params) {
-                pw.write(delimiter);
-                pw.write("\"");
-                String v = benchmarkParams.getParam(p);
-                if (v != null) {
-                    pw.write(v);
-                }
-                pw.write("\"");
-            }
-
-            pw.write("\r\n");
         }
 
+        pw.write("\r\n");
     }
 
 }
--- a/jmh-core/src/test/java/org/openjdk/jmh/results/format/ResultFormatTest.java	Fri Nov 07 15:45:28 2014 +0300
+++ b/jmh-core/src/test/java/org/openjdk/jmh/results/format/ResultFormatTest.java	Fri Nov 07 16:55:12 2014 +0300
@@ -190,4 +190,31 @@
         compare(actualFile, "output-golden.scsv");
     }
 
+
+    @Test
+    public void latexTest() throws IOException {
+        String actualFile = FileUtils.tempFile("test").getAbsolutePath();
+
+        ResultFormatFactory.getInstance(
+                    ResultFormatType.LATEX,
+                    actualFile)
+                .writeOut(getStub());
+
+        compare(actualFile, "output-golden.latex");
+    }
+
+    @Test
+    public void latexTest_Stream() throws IOException {
+        String actualFile = FileUtils.tempFile("test").getAbsolutePath();
+
+        PrintWriter pw = new PrintWriter(actualFile);
+        ResultFormatFactory.getInstance(
+                    ResultFormatType.LATEX,
+                    actualFile)
+                .writeOut(getStub());
+        pw.close();
+
+        compare(actualFile, "output-golden.latex");
+    }
+
 }
--- a/jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.csv	Fri Nov 07 15:45:28 2014 +0300
+++ b/jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.csv	Fri Nov 07 16:55:12 2014 +0300
@@ -1,6 +1,16 @@
 "Benchmark","Mode","Threads","Samples","Score","Score Error (99.9%)","Unit","Param: param0","Param: param1","Param: param2","Param: param3","Param: param4"
 "benchmark_0","thrpt",80,14,528.8571428571429,278.1419529743147,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_0:secondary1","thrpt",80,14,549.7142857142857,320.2274881224688,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_0:secondary2","thrpt",80,14,615.5,319.2095847883776,"ops/ms","value0","value1","value2","value3","value4"
 "benchmark_1","thrpt",900,1,439.0,NaN,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_1:secondary1","thrpt",900,1,953.0,NaN,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_1:secondary2","thrpt",900,1,367.0,NaN,"ops/ms","value0","value1","value2","value3","value4"
 "benchmark_2","thrpt",466,9,545.0,553.3366994253071,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_2:secondary1","thrpt",466,9,434.44444444444446,465.1825044536822,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_2:secondary2","thrpt",466,9,470.3333333333333,502.27304118350804,"ops/ms","value0","value1","value2","value3","value4"
 "benchmark_3","thrpt",968,14,417.57142857142856,362.81396705929166,"ops/ms","value0","value1","value2","value3","value4"
-"benchmark_4","thrpt",739,1,956.0,NaN,"ops/ms","value0","value1","value2","value3","value4"
\ No newline at end of file
+"benchmark_3:secondary1","thrpt",968,14,672.2142857142857,287.98221103408406,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_3:secondary2","thrpt",968,14,560.1428571428571,289.79996145153024,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_4","thrpt",739,1,956.0,NaN,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_4:secondary1","thrpt",739,1,688.0,NaN,"ops/ms","value0","value1","value2","value3","value4"
+"benchmark_4:secondary2","thrpt",739,1,237.0,NaN,"ops/ms","value0","value1","value2","value3","value4"
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.latex	Fri Nov 07 16:55:12 2014 +0300
@@ -0,0 +1,19 @@
+\begin{tabular}{r|l|l|l|l|l|rl}
+ \multicolumn{1}{c|}{\texttt{Benchmark}} & \texttt{param0} & \texttt{param1} & \texttt{param2} & \texttt{param3} & \texttt{param4} &  \multicolumn{2}{c}{\texttt{Score, ops/ms}} \\
+\hline
+\texttt{benchmark\_0} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{528.857} & \scriptsize $\pm$ \texttt{278.142} \\
+\texttt{benchmark\_0:secondary1} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{549.714} & \scriptsize $\pm$ \texttt{320.227} \\
+\texttt{benchmark\_0:secondary2} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{615.500} & \scriptsize $\pm$ \texttt{319.210} \\
+\texttt{benchmark\_1} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{439.000} & \scriptsize $\pm$ \texttt{  NaN} \\
+\texttt{benchmark\_1:secondary1} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{953.000} & \scriptsize $\pm$ \texttt{  NaN} \\
+\texttt{benchmark\_1:secondary2} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{367.000} & \scriptsize $\pm$ \texttt{  NaN} \\
+\texttt{benchmark\_2} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{545.000} & \scriptsize $\pm$ \texttt{553.337} \\
+\texttt{benchmark\_2:secondary1} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{434.444} & \scriptsize $\pm$ \texttt{465.183} \\
+\texttt{benchmark\_2:secondary2} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{470.333} & \scriptsize $\pm$ \texttt{502.273} \\
+\texttt{benchmark\_3} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{417.571} & \scriptsize $\pm$ \texttt{362.814} \\
+\texttt{benchmark\_3:secondary1} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{672.214} & \scriptsize $\pm$ \texttt{287.982} \\
+\texttt{benchmark\_3:secondary2} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{560.143} & \scriptsize $\pm$ \texttt{289.800} \\
+\texttt{benchmark\_4} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{956.000} & \scriptsize $\pm$ \texttt{  NaN} \\
+\texttt{benchmark\_4:secondary1} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{688.000} & \scriptsize $\pm$ \texttt{  NaN} \\
+\texttt{benchmark\_4:secondary2} & \texttt{value0} & \texttt{value1} & \texttt{value2} & \texttt{value3} & \texttt{value4} & \texttt{237.000} & \scriptsize $\pm$ \texttt{  NaN} \\
+\end{tabular}
\ No newline at end of file
--- a/jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.scsv	Fri Nov 07 15:45:28 2014 +0300
+++ b/jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.scsv	Fri Nov 07 16:55:12 2014 +0300
@@ -1,6 +1,16 @@
 "Benchmark";"Mode";"Threads";"Samples";"Score";"Score Error (99.9%)";"Unit";"Param: param0";"Param: param1";"Param: param2";"Param: param3";"Param: param4"
 "benchmark_0";"thrpt";80;14;528.8571428571429;278.1419529743147;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_0:secondary1";"thrpt";80;14;549.7142857142857;320.2274881224688;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_0:secondary2";"thrpt";80;14;615.5;319.2095847883776;"ops/ms";"value0";"value1";"value2";"value3";"value4"
 "benchmark_1";"thrpt";900;1;439.0;NaN;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_1:secondary1";"thrpt";900;1;953.0;NaN;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_1:secondary2";"thrpt";900;1;367.0;NaN;"ops/ms";"value0";"value1";"value2";"value3";"value4"
 "benchmark_2";"thrpt";466;9;545.0;553.3366994253071;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_2:secondary1";"thrpt";466;9;434.44444444444446;465.1825044536822;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_2:secondary2";"thrpt";466;9;470.3333333333333;502.27304118350804;"ops/ms";"value0";"value1";"value2";"value3";"value4"
 "benchmark_3";"thrpt";968;14;417.57142857142856;362.81396705929166;"ops/ms";"value0";"value1";"value2";"value3";"value4"
-"benchmark_4";"thrpt";739;1;956.0;NaN;"ops/ms";"value0";"value1";"value2";"value3";"value4"
\ No newline at end of file
+"benchmark_3:secondary1";"thrpt";968;14;672.2142857142857;287.98221103408406;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_3:secondary2";"thrpt";968;14;560.1428571428571;289.79996145153024;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_4";"thrpt";739;1;956.0;NaN;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_4:secondary1";"thrpt";739;1;688.0;NaN;"ops/ms";"value0";"value1";"value2";"value3";"value4"
+"benchmark_4:secondary2";"thrpt";739;1;237.0;NaN;"ops/ms";"value0";"value1";"value2";"value3";"value4"
\ No newline at end of file