changeset 266:51264eec1d55

Use 99.9% as the confidence level universally across the infrastructure. (Less false differences in means).
author shade
date Fri, 22 Nov 2013 18:18:08 +0400
parents cc8b6607f997
children fd83add52cba
files jmh-core-benchmarks/src/main/java/org/openjdk/jmh/MainValidation.java jmh-core/src/main/java/org/openjdk/jmh/logic/results/Result.java jmh-core/src/main/java/org/openjdk/jmh/logic/results/SampleTimeResult.java jmh-core/src/main/java/org/openjdk/jmh/output/format/TextReportFormat.java jmh-core/src/main/java/org/openjdk/jmh/output/results/CSVResultFormat.java jmh-core/src/main/java/org/openjdk/jmh/output/results/JSONResultFormat.java jmh-core/src/main/java/org/openjdk/jmh/util/internal/AbstractStatistics.java jmh-core/src/main/java/org/openjdk/jmh/util/internal/Statistics.java jmh-core/src/test/java/org/openjdk/jmh/util/TestListStatistics.java jmh-core/src/test/java/org/openjdk/jmh/util/TestMultisetStatistics.java
diffstat 10 files changed, 37 insertions(+), 49 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/MainValidation.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/MainValidation.java	Fri Nov 22 18:18:08 2013 +0400
@@ -73,7 +73,7 @@
 
             pw.printf("%50s", "running empty benchmark: ");
             pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.999));
         }
 
         {
@@ -87,7 +87,7 @@
 
             pw.printf("%50s", "Level.Trial, Scope.Benchmark, @Setup: ");
             pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.999));
         }
 
         {
@@ -101,7 +101,7 @@
 
             pw.printf("%50s", "Level.Trial, Scope.Benchmark, @TearDown: ");
             pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.999));
         }
 
         {
@@ -115,7 +115,7 @@
 
             pw.printf("%50s", "Level.Iteration, Scope.Benchmark, @Setup: ");
             pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.999));
         }
 
         {
@@ -129,7 +129,7 @@
 
             pw.printf("%50s", "Level.Iteration, Scope.Benchmark, @TearDown: ");
             pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.999));
         }
 
         {
@@ -143,7 +143,7 @@
 
             pw.printf("%50s", "Level.Invocation, Scope.Benchmark, @Setup: ");
             pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.999));
         }
 
         {
@@ -157,7 +157,7 @@
 
             pw.printf("%50s", "Level.Invocation, Scope.Benchmark, @TearDown: ");
             pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.999));
         }
 
         pw.println();
@@ -183,7 +183,7 @@
 
             pw.printf("%50s", "System.nanoTime latency: ");
             pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.999));
 
             latency = (long) r.getScore();
         }
@@ -200,7 +200,7 @@
 
             pw.printf("%50s", "System.nanoTime granularity: ");
             pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.999));
             granularity = (long) r.getScore();
         }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/logic/results/Result.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/logic/results/Result.java	Fri Nov 22 18:18:08 2013 +0400
@@ -91,16 +91,15 @@
         PrintWriter pw = new PrintWriter(sw);
 
         if (statistics.getN() > 2) {
-            double[] interval95 = statistics.getConfidenceInterval(0.05);
-            double[] interval99 = statistics.getConfidenceInterval(0.01);
-            pw.println(String.format("Result %s: %.3f \u00B1(95%%) %.3f \u00B1(99%%) %.3f %s",
+            double[] interval = statistics.getConfidenceInterval(0.999);
+            pw.println(String.format("Result %s: %.3f \u00B1(99.9%%) %.3f %s",
                     (label == null) ? "" : "\"" + label + "\"",
-                    (interval95[0] + interval95[1]) / 2, (interval95[1] - interval95[0]) / 2, (interval99[1] - interval99[0]) / 2,
+                    statistics.getMean(), (interval[1] - interval[0]) / 2,
                     getScoreUnit()));
-            pw.println(String.format("  Statistics: (min, avg, max) = (%.3f, %.3f, %.3f), stdev = %.3f",
-                    statistics.getMin(), statistics.getMean(), statistics.getMax(), statistics.getStandardDeviation()));
-            pw.println(String.format("  Confidence intervals: 95%% [%.3f, %.3f], 99%% [%.3f, %.3f]",
-                    interval95[0], interval95[1], interval99[0], interval99[1]));
+            pw.println(String.format("  Statistics: (min, avg, max) = (%.3f, %.3f, %.3f), stdev = %.3f%n" +
+                    "  Confidence interval (99.9%%): [%.3f, %.3f]",
+                    statistics.getMin(), statistics.getMean(), statistics.getMax(), statistics.getStandardDeviation(),
+                    interval[0], interval[1]));
         } else {
             pw.println(String.format("Run result: %.2f (<= 2 iterations)", statistics.getMean()));
         }
--- a/jmh-core/src/main/java/org/openjdk/jmh/logic/results/SampleTimeResult.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/logic/results/SampleTimeResult.java	Fri Nov 22 18:18:08 2013 +0400
@@ -117,12 +117,10 @@
         sb.append("    samples = ").append(stats.getN()).append("\n");
 
         if (stats.getN() > 2) {
-            double[] interval95 = stats.getConfidenceInterval(0.05);
-            double[] interval99 = stats.getConfidenceInterval(0.01);
-            sb.append(String.format("        mean = %10.3f \u00B1(95%%) %.3f \u00B1(99%%) %.3f",
+            double[] interval = stats.getConfidenceInterval(0.999);
+            sb.append(String.format("        mean = %10.3f \u00B1(99.9%%) %.3f",
                     convertNs(stats.getMean()),
-                    convertNs((interval95[1] - interval95[0]) / 2),
-                    convertNs((interval99[1] - interval99[0]) / 2)
+                    convertNs((interval[1] - interval[0]) / 2)
             ));
         } else {
             sb.append(String.format("        mean = %10.3f (<= 2 iterations)",
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/format/TextReportFormat.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/format/TextReportFormat.java	Fri Nov 22 18:18:08 2013 +0400
@@ -179,8 +179,6 @@
                 "Benchmark", "Mode", "Thr", "Count", "Sec",
                 "Mean", "Mean error", "Units");
         for (BenchmarkRecord key : runResults.keySet()) {
-            double[] interval = new double[]{Double.NaN, Double.NaN};
-
             RunResult res = runResults.get(key);
 
             int threads = res.getThreads();
@@ -188,31 +186,24 @@
 
             {
                 Statistics stats = res.getPrimaryResult().getStatistics();
-                if (stats.getN() > 2) {
-                    interval = stats.getConfidenceInterval(0.01);
-                }
-
                 out.printf("%-" + nameLen + "s %6s %3d %9d %4d %12.3f %12.3f %8s%n",
                         benchPrefixes.get(key.getUsername()),
                         key.getMode().shortLabel(),
                         threads, stats.getN(),
                         runTime.convertTo(TimeUnit.SECONDS),
-                        stats.getMean(), (interval[1] - interval[0]) / 2,
+                        stats.getMean(), stats.getMeanError(0.999),
                         res.getScoreUnit());
             }
 
             for (String label : res.getSecondaryResults().keySet()) {
                 Statistics stats = res.getSecondaryResults().get(label).getStatistics();
-                if (stats.getN() > 2) {
-                    interval = stats.getConfidenceInterval(0.01);
-                }
 
                 out.printf("%-" + nameLen + "s %6s %3d %9d %4d %12.3f %12.3f %8s%n",
                         benchPrefixes.get(key.getUsername() + ":" + label),
                         key.getMode().shortLabel(),
                         threads, stats.getN(),
                         runTime.convertTo(TimeUnit.SECONDS),
-                        stats.getMean(), (interval[1] - interval[0]) / 2,
+                        stats.getMean(), stats.getMeanError(0.999),
                         res.getScoreUnit());
             }
         }
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/results/CSVResultFormat.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/results/CSVResultFormat.java	Fri Nov 22 18:18:08 2013 +0400
@@ -63,9 +63,7 @@
                 pw.print("\", ");
                 pw.print(runResult.getPrimaryResult().getStatistics().getMean());
                 pw.print(", ");
-                pw.print(runResult.getPrimaryResult().getStatistics().getMeanError(0.95));
-                pw.print(", ");
-                pw.print(runResult.getPrimaryResult().getStatistics().getMeanError(0.99));
+                pw.print(runResult.getPrimaryResult().getStatistics().getMeanError(0.999));
                 pw.print(", \"");
                 pw.print(runResult.getPrimaryResult().getScoreUnit());
                 pw.println("\"");
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/results/JSONResultFormat.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/results/JSONResultFormat.java	Fri Nov 22 18:18:08 2013 +0400
@@ -69,8 +69,9 @@
                 pw.println("  \"primaryMetric\" : {");
                 pw.println("    \"score\" : " + runResult.getPrimaryResult().getScore() + ",");
                 pw.println("    \"scoreStdev\" : " + runResult.getPrimaryResult().getStatistics().getStandardDeviation() + ",");
-                pw.println("    \"scoreConfidence95\" : " + Arrays.toString(runResult.getPrimaryResult().getStatistics().getConfidenceInterval(0.05)) + ",");
-                pw.println("    \"scoreConfidence99\" : " + Arrays.toString(runResult.getPrimaryResult().getStatistics().getConfidenceInterval(0.01)) + ",");
+                pw.println("    \"scoreConfidence95\" : " + Arrays.toString(runResult.getPrimaryResult().getStatistics().getConfidenceInterval(0.95)) + ",");
+                pw.println("    \"scoreConfidence99\" : " + Arrays.toString(runResult.getPrimaryResult().getStatistics().getConfidenceInterval(0.99)) + ",");
+                pw.println("    \"scoreConfidence999\" : " + Arrays.toString(runResult.getPrimaryResult().getStatistics().getConfidenceInterval(0.999)) + ",");
                 pw.println("    \"scoreUnit\" : \"" + runResult.getPrimaryResult().getScoreUnit() + "\",");
                 pw.println("    \"rawData\" : [");
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/util/internal/AbstractStatistics.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/util/internal/AbstractStatistics.java	Fri Nov 22 18:18:08 2013 +0400
@@ -133,28 +133,28 @@
      * Returns the interval c1, c2 of which there's an 1-alpha
      * probability of the mean being within the interval.
      *
-     * @param alpha alpha parameter
+     * @param confidence level
      * @return the confidence interval
      */
     @Override
-    public double[] getConfidenceInterval(double alpha) {
+    public double[] getConfidenceInterval(double confidence) {
         double[] interval = new double[2];
 
-        double ip = getStudentT(1 - alpha / 2.0, getN() - 1);
+        double ip = getStudentT(1 - (1 - confidence)/2, getN() - 1);
         interval[0] = getMean() - ip * (getStandardDeviation() / Math.sqrt(getN()));
         interval[1] = getMean() + ip * (getStandardDeviation() / Math.sqrt(getN()));
 
         return interval;
     }
 
-    protected double getStudentT(double alpha, int n) {
+    protected double getStudentT(double confidence, int n) {
         if (n <= 1) throw new IllegalStateException();
 
         double[] indices = {0.90, 0.95, 0.975, 0.99, 0.995, 0.999};
 
         int index = indices.length - 1;
         for (int i = 0; i < indices.length - 1; i++) {
-            if (indices[i] <= alpha && alpha < indices[i + 1]) {
+            if (indices[i] <= confidence && confidence < indices[i + 1]) {
                 index = i;
                 break;
             }
@@ -168,8 +168,9 @@
     }
 
     @Override
-    public double getMeanError(double alpha) {
-        double ip = getStudentT(1 - alpha / 2.0, getN() - 1);
+    public double getMeanError(double confidence) {
+        if (getN() < 2) return Double.NaN;
+        double ip = getStudentT(1 - (1 - confidence)/2, getN() - 1);
         return ip * (getStandardDeviation() / Math.sqrt(getN()));
     }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/util/internal/Statistics.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/util/internal/Statistics.java	Fri Nov 22 18:18:08 2013 +0400
@@ -28,9 +28,9 @@
 
 public interface Statistics extends Serializable {
 
-    double[] getConfidenceInterval(double alpha);
+    double[] getConfidenceInterval(double confidence);
 
-    double getMeanError(double alpha);
+    double getMeanError(double confidence);
 
     double getStandardDeviation();
 
--- a/jmh-core/src/test/java/org/openjdk/jmh/util/TestListStatistics.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core/src/test/java/org/openjdk/jmh/util/TestListStatistics.java	Fri Nov 22 18:18:08 2013 +0400
@@ -131,7 +131,7 @@
      */
     @Test
     public strictfp void testGetConfidenceInterval() {
-        double[] interval = instance.getConfidenceInterval(0.05);
+        double[] interval = instance.getConfidenceInterval(0.95);
         assertEquals(39.234, interval[0], 0.002);
         assertEquals(62.831, interval[1], 0.002);
     }
--- a/jmh-core/src/test/java/org/openjdk/jmh/util/TestMultisetStatistics.java	Fri Nov 22 15:28:41 2013 +0400
+++ b/jmh-core/src/test/java/org/openjdk/jmh/util/TestMultisetStatistics.java	Fri Nov 22 18:18:08 2013 +0400
@@ -132,7 +132,7 @@
      */
     @Test
     public strictfp void testGetConfidenceInterval() {
-        double[] interval = instance.getConfidenceInterval(0.05);
+        double[] interval = instance.getConfidenceInterval(0.95);
         assertEquals(39.234, interval[0], 0.002);
         assertEquals(62.831, interval[1], 0.002);
     }