changeset 276:6a39cdbfd790

Sealing the machine-readable formats.
author shade
date Tue, 26 Nov 2013 21:16:42 +0400
parents b304a8d78c21
children 0697362f0c50
files jmh-core/pom.xml jmh-core/src/main/java/org/openjdk/jmh/output/results/JSONResultFormat.java jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/BenchmarkParams.java jmh-core/src/main/java/org/openjdk/jmh/util/internal/AbstractStatistics.java jmh-core/src/test/java/org/openjdk/jmh/output/results/ResultFormatTest.java jmh-core/src/test/resources/org/openjdk/jmh/output/results/output-golden.csv jmh-core/src/test/resources/org/openjdk/jmh/output/results/output-golden.json
diffstat 7 files changed, 626 insertions(+), 10 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-core/pom.xml	Tue Nov 26 11:44:52 2013 +0400
+++ b/jmh-core/pom.xml	Tue Nov 26 21:16:42 2013 +0400
@@ -77,7 +77,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.0</version>
+                <version>3.1</version>
                 <configuration>
                     <compilerVersion>1.6</compilerVersion>
                     <source>1.6</source>
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/results/JSONResultFormat.java	Tue Nov 26 11:44:52 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/results/JSONResultFormat.java	Tue Nov 26 21:16:42 2013 +0400
@@ -33,7 +33,6 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Map;
 
@@ -72,9 +71,9 @@
                 pw.println("\"measurementIterations\" : " + runResult.getParams().getMeasurement().getCount() + ",");
                 pw.println("\"measurementTime\" : \"" + runResult.getParams().getMeasurement().getTime() + "\",");
                 pw.println("\"primaryMetric\" : {");
-                pw.println("\"score\" : " + runResult.getPrimaryResult().getScore() + ",");
-                pw.println("\"scoreError\" : " + runResult.getPrimaryResult().getStatistics().getMeanErrorAt(0.999) + ",");
-                pw.println("\"scoreConfidence\" : " + Arrays.toString(runResult.getPrimaryResult().getStatistics().getConfidenceIntervalAt(0.999)) + ",");
+                pw.println("\"score\" : " + emit(runResult.getPrimaryResult().getScore()) + ",");
+                pw.println("\"scoreError\" : " + emit(runResult.getPrimaryResult().getStatistics().getMeanErrorAt(0.999)) + ",");
+                pw.println("\"scoreConfidence\" : " + emit(runResult.getPrimaryResult().getStatistics().getConfidenceIntervalAt(0.999)) + ",");
                 pw.println("\"scoreUnit\" : \"" + runResult.getPrimaryResult().getScoreUnit() + "\",");
                 pw.println("\"rawData\" :");
 
@@ -83,7 +82,7 @@
                     for (BenchResult benchResult : runResult.getRawBenchResults()) {
                         Collection<String> scores = new ArrayList<String>();
                         for (Result r : benchResult.getRawPrimaryResults()) {
-                            scores.add(String.valueOf(r.getScore()));
+                            scores.add(emit(r.getScore()));
                         }
                         l1.add(printMultiple(scores, "[", "]"));
                     }
@@ -97,9 +96,9 @@
 
                     StringBuilder sb = new StringBuilder();
                     sb.append("\"").append(secondaryName).append("\" : {");
-                    sb.append("\"score\" : ").append(result.getScore()).append(",");
-                    sb.append("\"scoreError\" : ").append(runResult.getPrimaryResult().getStatistics().getMeanErrorAt(0.999)).append(",");
-                    sb.append("\"scoreConfidence\" : ").append(Arrays.toString(result.getStatistics().getConfidenceIntervalAt(0.999))).append(",");
+                    sb.append("\"score\" : ").append(emit(result.getScore())).append(",");
+                    sb.append("\"scoreError\" : ").append(emit(runResult.getPrimaryResult().getStatistics().getMeanErrorAt(0.999))).append(",");
+                    sb.append("\"scoreConfidence\" : ").append(emit(result.getStatistics().getConfidenceIntervalAt(0.999))).append(",");
                     sb.append("\"scoreUnit\" : \"").append(result.getScoreUnit()).append("\",");
                     sb.append("\"rawData\" :");
 
@@ -107,7 +106,7 @@
                     for (BenchResult benchResult : runResult.getRawBenchResults()) {
                         Collection<String> scores = new ArrayList<String>();
                         for (Result r : benchResult.getRawSecondaryResults().get(secondaryName)) {
-                            scores.add(String.valueOf(r.getScore()));
+                            scores.add(emit(r.getScore()));
                         }
                         l2.add(printMultiple(scores, "[", "]"));
                     }
@@ -128,12 +127,40 @@
         try {
             PrintWriter out = new PrintWriter(output);
             out.println(tidy(sw.toString()));
+            out.flush();
             out.close();
         } catch (IOException e) {
             throw new IllegalStateException(e);
         }
     }
 
+    private String emit(double[] ds) {
+        StringBuilder sb = new StringBuilder();
+
+        boolean isFirst = true;
+        sb.append("[");
+        for (double d : ds) {
+            if (isFirst) {
+                isFirst = false;
+            } else {
+                sb.append(",");
+            }
+            sb.append(emit(d));
+        }
+        sb.append("]");
+        return sb.toString();
+    }
+
+    private String emit(double d) {
+        if (d != d)
+            return "\"NaN\"";
+        if (d == Double.NEGATIVE_INFINITY)
+            return "\"-INF\"";
+        if (d == Double.POSITIVE_INFINITY)
+            return "\"+INF\"";
+        return String.valueOf(d);
+    }
+
     private String tidy(String s) {
         s = s.replaceAll("\n", " ");
         s = s.replaceAll(",", ",\n");
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/BenchmarkParams.java	Tue Nov 26 11:44:52 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/BenchmarkParams.java	Tue Nov 26 21:16:42 2013 +0400
@@ -79,6 +79,19 @@
         return (fork != null) ? fork.value() : -1;
     }
 
+    /**
+     * Test entry method
+     */
+    public BenchmarkParams(boolean synchIterations, int threads, int[] threadGroups, int forks, int warmupForks, int warmupIters, TimeValue warmupTime, int measureIters, TimeValue measureTime) {
+        this.synchIterations = synchIterations;
+        this.threads = threads;
+        this.threadGroups = threadGroups;
+        this.forks = forks;
+        this.warmupForks = warmupForks;
+        this.warmup = new IterationParams(this, warmupIters, warmupTime);
+        this.measurement = new IterationParams(this, measureIters, measureTime);
+    }
+
     public BenchmarkParams(Options options, BenchmarkRecord benchmark, boolean doWarmup, boolean doMeasurement) {
         Class<?> clazz = ClassUtils.loadClass(benchmark.generatedClass());
         Method method = MicroBenchmarkHandlers.findBenchmarkMethod(clazz, benchmark.generatedMethod());
--- a/jmh-core/src/main/java/org/openjdk/jmh/util/internal/AbstractStatistics.java	Tue Nov 26 11:44:52 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/util/internal/AbstractStatistics.java	Tue Nov 26 21:16:42 2013 +0400
@@ -140,6 +140,11 @@
     public double[] getConfidenceIntervalAt(double confidence) {
         double[] interval = new double[2];
 
+        if (getN() <= 2) {
+            interval[0] = interval[1] = Double.NaN;
+            return interval;
+        }
+
         double ip = getStudentT(1 - (1 - confidence)/2, getN() - 1);
         interval[0] = getMean() - ip * (getStandardDeviation() / Math.sqrt(getN()));
         interval[1] = getMean() + ip * (getStandardDeviation() / Math.sqrt(getN()));
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/test/java/org/openjdk/jmh/output/results/ResultFormatTest.java	Tue Nov 26 21:16:42 2013 +0400
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.output.results;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.logic.results.BenchResult;
+import org.openjdk.jmh.logic.results.IterationResult;
+import org.openjdk.jmh.logic.results.ResultRole;
+import org.openjdk.jmh.logic.results.RunResult;
+import org.openjdk.jmh.logic.results.ThroughputResult;
+import org.openjdk.jmh.runner.BenchmarkRecord;
+import org.openjdk.jmh.runner.parameters.BenchmarkParams;
+import org.openjdk.jmh.runner.parameters.TimeValue;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * These tests seal the machine-readable format.
+ * Any change to these tests should be discussed with the maintainers first!
+ */
+public class ResultFormatTest {
+
+    private Map<BenchmarkRecord, RunResult> getStub() {
+        Map<BenchmarkRecord, RunResult> results = new HashMap<BenchmarkRecord, RunResult>();
+
+        Random r = new Random(12345);
+        for (int b = 0; b < r.nextInt(10); b++) {
+            BenchmarkRecord record = new BenchmarkRecord("benchmark_" + b, JSONResultFormat.class.getName() + ".benchmark_" + b, Mode.AverageTime);
+            BenchmarkParams params = new BenchmarkParams(false,
+                    r.nextInt(1000),
+                    new int[]{ r.nextInt(1000) },
+                    r.nextInt(1000),
+                    r.nextInt(1000),
+                    r.nextInt(1000),
+                    TimeValue.seconds(r.nextInt(1000)),
+                    r.nextInt(1000),
+                    TimeValue.seconds(r.nextInt(1000))
+            );
+
+            Collection<BenchResult> benchResults = new ArrayList<BenchResult>();
+            for (int f = 0; f < r.nextInt(10); f++) {
+                Collection<IterationResult> iterResults = new ArrayList<IterationResult>();
+                for (int c = 0; c < r.nextInt(10); c++) {
+                    IterationResult res = new IterationResult(record, params.getMeasurement());
+                    res.addResult(new ThroughputResult(ResultRole.PRIMARY, "test", r.nextInt(1000), 1000 * 1000));
+                    res.addResult(new ThroughputResult(ResultRole.SECONDARY, "secondary1", r.nextInt(1000), 1000 * 1000));
+                    res.addResult(new ThroughputResult(ResultRole.SECONDARY, "secondary2", r.nextInt(1000), 1000 * 1000));
+                    iterResults.add(res);
+                }
+                benchResults.add(new BenchResult(iterResults));
+            }
+            results.put(record, new RunResult(benchResults));
+        }
+        return results;
+    }
+
+    private void compare(String actualFile, String goldenFile) throws IOException {
+        BufferedReader actualReader = new BufferedReader(new FileReader(actualFile));
+        BufferedReader goldenReader = new BufferedReader(new InputStreamReader(ResultFormatTest.class.getResourceAsStream("/org/openjdk/jmh/output/results/" + goldenFile)));
+        while (true) {
+            String goldenLine = goldenReader.readLine();
+            String actualLine = actualReader.readLine();
+            Assert.assertEquals("Mismatch", goldenLine, actualLine);
+            if (goldenLine == null && actualLine == null) break;
+        }
+    }
+
+    @Test
+    public void jsonTest() throws IOException {
+        String actualFile = File.createTempFile("jmh", "test").getAbsolutePath();
+
+        ResultFormatFactory.getInstance(
+                    ResultFormatType.JSON,
+                    actualFile)
+                .writeOut(getStub());
+
+        compare(actualFile, "output-golden.json");
+    }
+
+    @Test
+    public void csvTest() throws IOException {
+        String actualFile = File.createTempFile("jmh", "test").getAbsolutePath();
+
+        ResultFormatFactory.getInstance(
+                    ResultFormatType.CSV,
+                    actualFile)
+                .writeOut(getStub());
+
+        compare(actualFile, "output-golden.csv");
+    }
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/test/resources/org/openjdk/jmh/output/results/output-golden.csv	Tue Nov 26 21:16:42 2013 +0400
@@ -0,0 +1,6 @@
+"Benchmark", "Mode", "Threads", "Iterations", "Iteration time", "Mean", "Mean Error (95%)", "Mean Error (99%)", "Unit"
+"benchmark_3", "avgt", 968, 857, "438 s", 417.57142857142856, 331.1099544524174, "ops/ms"
+"benchmark_4", "avgt", 739, 16, "763 s", 956.0, NaN, "ops/ms"
+"benchmark_0", "avgt", 80, 802, "501 s", 528.8571428571429, 253.83689091986184, "ops/ms"
+"benchmark_1", "avgt", 900, 55, "398 s", 439.0, NaN, "ops/ms"
+"benchmark_2", "avgt", 466, 461, "96 s", 545.0, 493.81293072112726, "ops/ms"
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/test/resources/org/openjdk/jmh/output/results/output-golden.json	Tue Nov 26 21:16:42 2013 +0400
@@ -0,0 +1,440 @@
+[
+    {
+        "benchmark" : "benchmark_3",
+        "mode" : "avgt",
+        "threads" : 968,
+        "forks" : 581,
+        "warmupIterations" : 628,
+        "warmupTime" : "207 s",
+        "measurementIterations" : 857,
+        "measurementTime" : "438 s",
+        "primaryMetric" : {
+            "score" : 417.57141666666666,
+            "scoreError" : 331.1099544524174,
+            "scoreConfidence" : [
+                86.46147411901114,
+                748.6813830238459
+            ],
+            "scoreUnit" : "ops/ms",
+            "rawData" : [
+                [
+                    145.0,
+                    502.0,
+                    686.0,
+                    291.0,
+                    665.0,
+                    120.0,
+                    63.0,
+                    570.0
+                ],
+                [
+                    801.0
+                ],
+                [
+                    731.0,
+                    21.0
+                ],
+                [
+                    147.0,
+                    125.0,
+                    979.0
+                ]
+            ]
+        },
+        "secondaryMetrics" : {
+            "secondary1" : {
+                "score" : 672.2142833333334,
+                "scoreError" : 331.1099544524174,
+                "scoreConfidence" : [
+                    409.39701450647914,
+                    935.0315569220922
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        454.0,
+                        626.0,
+                        584.0,
+                        858.0,
+                        967.0,
+                        814.0,
+                        587.0,
+                        423.0
+                    ],
+                    [
+                        746.0
+                    ],
+                    [
+                        888.0,
+                        284.0
+                    ],
+                    [
+                        965.0,
+                        983.0,
+                        232.0
+                    ]
+                ]
+            },
+            "secondary2" : {
+                "score" : 560.14285,
+                "scoreError" : 331.1099544524174,
+                "scoreConfidence" : [
+                    295.6666772033807,
+                    824.6190370823335
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        114.0,
+                        166.0,
+                        559.0,
+                        560.0,
+                        341.0,
+                        765.0,
+                        353.0,
+                        522.0
+                    ],
+                    [
+                        779.0
+                    ],
+                    [
+                        870.0,
+                        848.0
+                    ],
+                    [
+                        863.0,
+                        392.0,
+                        710.0
+                    ]
+                ]
+            }
+        }
+    }
+    ,
+    {
+        "benchmark" : "benchmark_4",
+        "mode" : "avgt",
+        "threads" : 739,
+        "forks" : 670,
+        "warmupIterations" : 997,
+        "warmupTime" : "651 s",
+        "measurementIterations" : 16,
+        "measurementTime" : "763 s",
+        "primaryMetric" : {
+            "score" : 956.0,
+            "scoreError" : "NaN",
+            "scoreConfidence" : [
+                "NaN",
+                "NaN"
+            ],
+            "scoreUnit" : "ops/ms",
+            "rawData" : [
+                [
+                    956.0
+                ]
+            ]
+        },
+        "secondaryMetrics" : {
+            "secondary1" : {
+                "score" : 688.0,
+                "scoreError" : "NaN",
+                "scoreConfidence" : [
+                    "NaN",
+                    "NaN"
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        688.0
+                    ]
+                ]
+            },
+            "secondary2" : {
+                "score" : 237.0,
+                "scoreError" : "NaN",
+                "scoreConfidence" : [
+                    "NaN",
+                    "NaN"
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        237.0
+                    ]
+                ]
+            }
+        }
+    }
+    ,
+    {
+        "benchmark" : "benchmark_0",
+        "mode" : "avgt",
+        "threads" : 80,
+        "forks" : 828,
+        "warmupIterations" : 84,
+        "warmupTime" : "375 s",
+        "measurementIterations" : 802,
+        "measurementTime" : "501 s",
+        "primaryMetric" : {
+            "score" : 528.8571333333333,
+            "scoreError" : 253.83689091986184,
+            "scoreConfidence" : [
+                275.02025193728105,
+                782.6940337770047
+            ],
+            "scoreUnit" : "ops/ms",
+            "rawData" : [
+                [
+                    942.0,
+                    384.0
+                ],
+                [
+                    351.0,
+                    781.0
+                ],
+                [
+                    397.0,
+                    555.0
+                ],
+                [
+                    582.0,
+                    580.0
+                ],
+                [
+                    644.0
+                ],
+                [
+                    655.0,
+                    87.0,
+                    499.0
+                ],
+                [
+                    121.0,
+                    826.0
+                ]
+            ]
+        },
+        "secondaryMetrics" : {
+            "secondary1" : {
+                "score" : 549.7142833333334,
+                "scoreError" : 253.83689091986184,
+                "scoreConfidence" : [
+                    257.4694477714426,
+                    841.9591236571287
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        390.0,
+                        787.0
+                    ],
+                    [
+                        792.0,
+                        425.0
+                    ],
+                    [
+                        501.0,
+                        952.0
+                    ],
+                    [
+                        99.0,
+                        412.0
+                    ],
+                    [
+                        54.0
+                    ],
+                    [
+                        594.0,
+                        553.0,
+                        370.0
+                    ],
+                    [
+                        918.0,
+                        849.0
+                    ]
+                ]
+            },
+            "secondary2" : {
+                "score" : 615.5,
+                "scoreError" : 253.83689091986184,
+                "scoreConfidence" : [
+                    324.18411727164244,
+                    906.8158827283576
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        806.0,
+                        303.0
+                    ],
+                    [
+                        316.0,
+                        943.0
+                    ],
+                    [
+                        825.0,
+                        831.0
+                    ],
+                    [
+                        758.0,
+                        691.0
+                    ],
+                    [
+                        508.0
+                    ],
+                    [
+                        497.0,
+                        318.0,
+                        773.0
+                    ],
+                    [
+                        56.0,
+                        992.0
+                    ]
+                ]
+            }
+        }
+    }
+    ,
+    {
+        "benchmark" : "benchmark_1",
+        "mode" : "avgt",
+        "threads" : 900,
+        "forks" : 364,
+        "warmupIterations" : 544,
+        "warmupTime" : "409 s",
+        "measurementIterations" : 55,
+        "measurementTime" : "398 s",
+        "primaryMetric" : {
+            "score" : 439.0,
+            "scoreError" : "NaN",
+            "scoreConfidence" : [
+                "NaN",
+                "NaN"
+            ],
+            "scoreUnit" : "ops/ms",
+            "rawData" : [
+                [
+                    439.0
+                ]
+            ]
+        },
+        "secondaryMetrics" : {
+            "secondary1" : {
+                "score" : 953.0,
+                "scoreError" : "NaN",
+                "scoreConfidence" : [
+                    "NaN",
+                    "NaN"
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        953.0
+                    ]
+                ]
+            },
+            "secondary2" : {
+                "score" : 367.0,
+                "scoreError" : "NaN",
+                "scoreConfidence" : [
+                    "NaN",
+                    "NaN"
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        367.0
+                    ]
+                ]
+            }
+        }
+    }
+    ,
+    {
+        "benchmark" : "benchmark_2",
+        "mode" : "avgt",
+        "threads" : 466,
+        "forks" : 677,
+        "warmupIterations" : 384,
+        "warmupTime" : "105 s",
+        "measurementIterations" : 461,
+        "measurementTime" : "96 s",
+        "primaryMetric" : {
+            "score" : 545.0,
+            "scoreError" : 493.81293072112726,
+            "scoreConfidence" : [
+                51.18706927887274,
+                1038.8129307211273
+            ],
+            "scoreUnit" : "ops/ms",
+            "rawData" : [
+                [
+                    749.0,
+                    105.0,
+                    100.0,
+                    863.0
+                ],
+                [
+                    836.0,
+                    497.0,
+                    390.0,
+                    379.0,
+                    986.0
+                ]
+            ]
+        },
+        "secondaryMetrics" : {
+            "secondary1" : {
+                "score" : 434.44443333333334,
+                "scoreError" : 493.81293072112726,
+                "scoreConfidence" : [
+                    19.30274846974106,
+                    849.5861404191478
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        170.0,
+                        532.0,
+                        586.0,
+                        104.0
+                    ],
+                    [
+                        775.0,
+                        88.0,
+                        823.0,
+                        499.0,
+                        333.0
+                    ]
+                ]
+            },
+            "secondary2" : {
+                "score" : 470.3333333333333,
+                "scoreError" : 493.81293072112726,
+                "scoreConfidence" : [
+                    22.091019482393506,
+                    918.5756471842731
+                ],
+                "scoreUnit" : "ops/ms",
+                "rawData" :[
+                    [
+                        574.0,
+                        246.0,
+                        151.0,
+                        665.0
+                    ],
+                    [
+                        173.0,
+                        974.0,
+                        291.0,
+                        339.0,
+                        820.0
+                    ]
+                ]
+            }
+        }
+    }
+]
+
+