changeset 1305:2290f828ce5c

7901580: Print samples histogram alongside with percentiles
author shade
date Thu, 14 Jan 2016 18:23:30 +0300
parents 4efff1f3f56f
children 80b666fe8548
files jmh-core/src/main/java/org/openjdk/jmh/results/Result.java jmh-core/src/main/java/org/openjdk/jmh/results/SampleTimeResult.java jmh-core/src/main/java/org/openjdk/jmh/results/SingleShotResult.java jmh-core/src/main/java/org/openjdk/jmh/util/ListStatistics.java jmh-core/src/main/java/org/openjdk/jmh/util/MultisetStatistics.java jmh-core/src/main/java/org/openjdk/jmh/util/ScoreFormatter.java jmh-core/src/main/java/org/openjdk/jmh/util/SingletonStatistics.java jmh-core/src/main/java/org/openjdk/jmh/util/Statistics.java jmh-core/src/test/java/org/openjdk/jmh/util/TestListStatistics.java jmh-core/src/test/java/org/openjdk/jmh/util/TestMultisetStatistics.java jmh-core/src/test/java/org/openjdk/jmh/util/TestSingletonStatistics.java jmh-core/src/test/java/org/openjdk/jmh/util/Util.java
diffstat 12 files changed, 516 insertions(+), 19 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-core/src/main/java/org/openjdk/jmh/results/Result.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/results/Result.java	Thu Jan 14 18:23:30 2016 +0300
@@ -233,37 +233,82 @@
         return sw.toString();
     }
 
-    protected String percentileExtendedInfo() {
+    protected String distributionExtendedInfo() {
         Statistics stats = getStatistics();
 
         StringBuilder sb = new StringBuilder();
 
         if (stats.getN() > 2) {
-            sb.append("  Samples, N = ").append(stats.getN()).append("\n");
+            sb.append("  N = ").append(stats.getN()).append("\n");
 
             double[] interval = stats.getConfidenceIntervalAt(0.999);
-            sb.append(String.format("        mean = %s \u00B1(99.9%%) %s",
+            sb.append(String.format("  mean = %s \u00B1(99.9%%) %s",
                     ScoreFormatter.format(10, stats.getMean()),
                     ScoreFormatter.formatError((interval[1] - interval[0]) / 2)
             ));
             sb.append(" ").append(getScoreUnit()).append("\n");
 
-            sb.append(String.format("         min = %s %s\n", ScoreFormatter.format(10, stats.getMin()), getScoreUnit()));
-
-            for (double p : new double[]{0.00, 0.50, 0.90, 0.95, 0.99, 0.999, 0.9999, 0.99999, 0.999999}) {
-                sb.append(String.format("  %9s = %s %s\n",
-                        "p(" + String.format("%7.4f", p * 100) + ")",
-                        ScoreFormatter.format(10, stats.getPercentile(p * 100)),
-                        getScoreUnit()
-                        ));
-            }
-
-            sb.append(String.format("         max = %s %s\n",
-                    ScoreFormatter.format(10, stats.getMax()),
-                    getScoreUnit()));
+            printHisto(stats, sb);
+            printPercentiles(stats, sb);
         }
 
         return sb.toString();
     }
 
+    private void printPercentiles(Statistics stats, StringBuilder sb) {
+        sb.append("\n  Percentiles, ").append(getScoreUnit()).append(":\n");
+        for (double p : new double[]{0.00, 0.50, 0.90, 0.95, 0.99, 0.999, 0.9999, 0.99999, 0.999999, 1.0}) {
+            sb.append(String.format("    %11s = %s %s\n",
+                    "p(" + String.format("%.4f", p * 100) + ")",
+                    ScoreFormatter.format(10, stats.getPercentile(p * 100)),
+                    getScoreUnit()
+                    ));
+        }
+    }
+
+    static class LazyProps {
+        private static final int MIN_HISTOGRAM_BINS = Integer.getInteger("jmh.histogramBins", 10);
+    }
+
+    private void printHisto(Statistics stats, StringBuilder sb) {
+        sb.append("\n  Histogram, ").append(getScoreUnit()).append(":\n");
+
+        double min = stats.getMin();
+        double max = stats.getMax();
+
+        double binSize = Math.pow(10, Math.floor(Math.log10(max - min)));
+        min = Math.floor(min / binSize) * binSize;
+        max =  Math.ceil(max / binSize) * binSize;
+        double range = max - min;
+
+        double[] levels;
+        if (range > 0) {
+            while ((range / binSize) < LazyProps.MIN_HISTOGRAM_BINS) {
+                binSize /= 2;
+            }
+
+            int binCount = Math.max(2, (int) Math.ceil(range / binSize));
+
+            levels = new double[binCount];
+            for (int c = 0; c < binCount; c++) {
+                levels[c] = min + (c * binSize);
+            }
+        } else {
+            levels = new double[] {
+                    stats.getMin() - Math.ulp(stats.getMin()),
+                    stats.getMax() + Math.ulp(stats.getMax())
+            };
+        }
+
+        int width = ScoreFormatter.format(1, max).length();
+
+        int[] histo = stats.getHistogram(levels);
+        for (int c = 0; c < levels.length - 1; c++) {
+            sb.append(String.format("    [%" + width + "s, %" + width + "s) = %d %n",
+                    ScoreFormatter.formatExact(width, levels[c]),
+                    ScoreFormatter.formatExact(width, levels[c + 1]),
+                    histo[c]));
+        }
+    }
+
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/results/SampleTimeResult.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/results/SampleTimeResult.java	Thu Jan 14 18:23:30 2016 +0300
@@ -85,7 +85,7 @@
 
     @Override
     public String extendedInfo() {
-        return simpleExtendedInfo() + percentileExtendedInfo();
+        return distributionExtendedInfo();
     }
 
     @Override
--- a/jmh-core/src/main/java/org/openjdk/jmh/results/SingleShotResult.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/results/SingleShotResult.java	Thu Jan 14 18:23:30 2016 +0300
@@ -49,7 +49,7 @@
 
     @Override
     public String extendedInfo() {
-        return simpleExtendedInfo() + percentileExtendedInfo();
+        return distributionExtendedInfo();
     }
 
     @Override
--- a/jmh-core/src/main/java/org/openjdk/jmh/util/ListStatistics.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/util/ListStatistics.java	Thu Jan 14 18:23:30 2016 +0300
@@ -127,6 +127,29 @@
     }
 
     @Override
+    public int[] getHistogram(double[] levels) {
+        if (levels.length < 2) {
+            throw new IllegalArgumentException("Expected more than two levels");
+        }
+
+        double[] vs = Arrays.copyOf(values, count);
+        Arrays.sort(vs);
+
+        int[] result = new int[levels.length - 1];
+
+        int c = 0;
+        values: for (double v : vs) {
+            while (levels[c] > v || v >= levels[c + 1]) {
+                c++;
+                if (c > levels.length - 2) break values;
+            }
+            result[c]++;
+        }
+
+        return result;
+    }
+
+    @Override
     public double getVariance() {
         if (count > 1) {
             double v = 0;
--- a/jmh-core/src/main/java/org/openjdk/jmh/util/MultisetStatistics.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/util/MultisetStatistics.java	Thu Jan 14 18:23:30 2016 +0300
@@ -24,6 +24,10 @@
  */
 package org.openjdk.jmh.util;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
 public class MultisetStatistics extends AbstractStatistics {
     private static final long serialVersionUID = -4401871054963903938L;
 
@@ -110,4 +114,26 @@
         }
     }
 
+    @Override
+    public int[] getHistogram(double[] levels) {
+        if (levels.length < 2) {
+            throw new IllegalArgumentException("Expected more than two levels");
+        }
+
+        List<Double> vs = new ArrayList<Double>(values.keys());
+        Collections.sort(vs);
+
+        int[] result = new int[levels.length - 1];
+
+        int c = 0;
+        values: for (double v : vs) {
+            while (levels[c] > v || v >= levels[c + 1]) {
+                c++;
+                if (c > levels.length - 2) break values;
+            }
+            result[c] += values.count(v);
+        }
+
+        return result;
+    }
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/util/ScoreFormatter.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/util/ScoreFormatter.java	Thu Jan 14 18:23:30 2016 +0300
@@ -52,6 +52,10 @@
         }
     }
 
+    public static String formatExact(int width, double score) {
+        return String.format("%" + width + "." + PRECISION + "f", score);
+    }
+
     public static String formatLatex(double score) {
         if (isApproximate(score)) {
             int power = (int) Math.round(Math.log10(score));
--- a/jmh-core/src/main/java/org/openjdk/jmh/util/SingletonStatistics.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/util/SingletonStatistics.java	Thu Jan 14 18:23:30 2016 +0300
@@ -67,4 +67,15 @@
         return Double.NaN;
     }
 
+    @Override
+    public int[] getHistogram(double[] levels) {
+        int[] result = new int[levels.length - 1];
+        for (int c = 0; c < levels.length - 1; c++) {
+            if (levels[c] <= value && value < levels[c + 1]) {
+                result[c] = 1;
+                break;
+            }
+        }
+        return result;
+    }
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/util/Statistics.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/util/Statistics.java	Thu Jan 14 18:23:30 2016 +0300
@@ -90,4 +90,14 @@
     double getVariance();
 
     double getPercentile(double rank);
+
+    /**
+     * Returns the histogram for this statistics. The histogram bin count would
+     * be equal to number of levels, minus one; so that each i-th bin is the
+     * number of samples in [i-th, (i+1)-th) levels.
+     *
+     * @param levels levels
+     * @return histogram data
+     */
+    int[] getHistogram(double[] levels);
 }
--- a/jmh-core/src/test/java/org/openjdk/jmh/util/TestListStatistics.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/test/java/org/openjdk/jmh/util/TestListStatistics.java	Thu Jan 14 18:23:30 2016 +0300
@@ -268,4 +268,128 @@
         Assert.assertEquals(42.0D, s.getPercentile(100));
     }
 
+    @Test
+    public strictfp void testHistogram_MinMax() {
+        ListStatistics s = new ListStatistics();
+        s.addValue(42.5);
+
+        Util.assertHistogram(s,
+                new double[] {Double.MIN_VALUE, Double.MAX_VALUE},
+                new int[] {1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_42_43() {
+        ListStatistics s = new ListStatistics();
+        s.addValue(42.5);
+
+        Util.assertHistogram(s,
+                new double[] {42, 43},
+                new int[] {1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_0_42() {
+        ListStatistics s = new ListStatistics();
+        s.addValue(42.5);
+
+        Util.assertHistogram(s,
+                new double[] {0, 42},
+                new int[] {0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_43_100() {
+        ListStatistics s = new ListStatistics();
+        s.addValue(42.5);
+
+        Util.assertHistogram(s,
+                new double[] {43, 100},
+                new int[] {0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_leftBound() {
+        ListStatistics s = new ListStatistics();
+        s.addValue(10);
+
+        Util.assertHistogram(s,
+                new double[] {10, 100},
+                new int[] {1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_rightBound() {
+        ListStatistics s = new ListStatistics();
+        s.addValue(10);
+
+        Util.assertHistogram(s,
+                new double[] {0, 10},
+                new int[] {0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_emptyLevels_left() {
+        ListStatistics s = new ListStatistics();
+        s.addValue(9);
+
+        Util.assertHistogram(s,
+                new double[] {0, 2, 4, 8, 10},
+                new int[] {0, 0, 0, 1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_emptyLevels_right() {
+        ListStatistics s = new ListStatistics();
+        s.addValue(1);
+
+        Util.assertHistogram(s,
+                new double[] {0, 2, 4, 8, 10},
+                new int[] {1, 0, 0, 0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_emptyLevels_middle() {
+        ListStatistics s = new ListStatistics();
+        s.addValue(5);
+
+        Util.assertHistogram(s,
+                new double[] {0, 2, 4, 8, 10},
+                new int[] {0, 0, 1, 0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_increasing() {
+        ListStatistics s = new ListStatistics();
+        for (int c = 0; c <= 10; c++) {
+            for (int t = 0; t < c; t++) {
+                s.addValue(c * 10);
+            }
+        }
+
+        Util.assertHistogram(s,
+                new double[] {0, 200},
+                new int[] {55}
+        );
+
+        Util.assertHistogram(s,
+                new double[] {0, 50, 101},
+                new int[] {10, 45}
+        );
+
+        Util.assertHistogram(s,
+                new double[] {0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100},
+                new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+        );
+    }
+
 }
--- a/jmh-core/src/test/java/org/openjdk/jmh/util/TestMultisetStatistics.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/test/java/org/openjdk/jmh/util/TestMultisetStatistics.java	Thu Jan 14 18:23:30 2016 +0300
@@ -263,4 +263,126 @@
     }
 
 
+    @Test
+    public strictfp void testHistogram_MinMax() {
+        MultisetStatistics s = new MultisetStatistics();
+        s.addValue(42.5, 1);
+
+        Util.assertHistogram(s,
+                new double[] {Double.MIN_VALUE, Double.MAX_VALUE},
+                new int[] {1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_42_43() {
+        MultisetStatistics s = new MultisetStatistics();
+        s.addValue(42.5, 1);
+
+        Util.assertHistogram(s,
+                new double[] {42, 43},
+                new int[] {1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_0_42() {
+        MultisetStatistics s = new MultisetStatistics();
+        s.addValue(42.5, 1);
+
+        Util.assertHistogram(s,
+                new double[] {0, 42},
+                new int[] {0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_43_100() {
+        MultisetStatistics s = new MultisetStatistics();
+        s.addValue(42.5, 1);
+
+        Util.assertHistogram(s,
+                new double[] {43, 100},
+                new int[] {0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_leftBound() {
+        MultisetStatistics s = new MultisetStatistics();
+        s.addValue(10, 1);
+
+        Util.assertHistogram(s,
+                new double[] {10, 100},
+                new int[] {1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_rightBound() {
+        MultisetStatistics s = new MultisetStatistics();
+        s.addValue(10, 1);
+
+        Util.assertHistogram(s,
+                new double[] {0, 10},
+                new int[] {0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_emptyLevels_left() {
+        MultisetStatistics s = new MultisetStatistics();
+        s.addValue(9, 1);
+
+        Util.assertHistogram(s,
+                new double[] {0, 2, 4, 8, 10},
+                new int[] {0, 0, 0, 1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_emptyLevels_right() {
+        MultisetStatistics s = new MultisetStatistics();
+        s.addValue(1, 1);
+
+        Util.assertHistogram(s,
+                new double[] {0, 2, 4, 8, 10},
+                new int[] {1, 0, 0, 0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_emptyLevels_middle() {
+        MultisetStatistics s = new MultisetStatistics();
+        s.addValue(5, 1);
+
+        Util.assertHistogram(s,
+                new double[] {0, 2, 4, 8, 10},
+                new int[] {0, 0, 1, 0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_increasing() {
+        MultisetStatistics s = new MultisetStatistics();
+        for (int c = 0; c <= 10; c++) {
+            s.addValue(c * 10, c);
+        }
+
+        Util.assertHistogram(s,
+                new double[] {0, 200},
+                new int[] {55}
+        );
+
+        Util.assertHistogram(s,
+                new double[] {0, 50, 101},
+                new int[] {10, 45}
+        );
+
+        Util.assertHistogram(s,
+                new double[] {0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100},
+                new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+        );
+    }
+
 }
--- a/jmh-core/src/test/java/org/openjdk/jmh/util/TestSingletonStatistics.java	Wed Jan 13 01:49:11 2016 +0300
+++ b/jmh-core/src/test/java/org/openjdk/jmh/util/TestSingletonStatistics.java	Thu Jan 14 18:23:30 2016 +0300
@@ -42,7 +42,7 @@
 
     @BeforeClass
     public static void setUpClass() throws Exception {
-        listStats.addValue(42.73638635);
+        listStats.addValue(VALUE);
     }
 
     /**
@@ -155,4 +155,94 @@
         Assert.assertEquals(42.0D, s.getPercentile(100));
     }
 
+    @Test
+    public strictfp void testHistogram_MinMax() {
+        SingletonStatistics s = new SingletonStatistics(42.5);
+
+        Util.assertHistogram(s,
+                new double[] {Double.MIN_VALUE, Double.MAX_VALUE},
+                new int[] {1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_42_43() {
+        SingletonStatistics s = new SingletonStatistics(42.5);
+
+        Util.assertHistogram(s,
+                new double[] {42, 43},
+                new int[] {1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_0_42() {
+        SingletonStatistics s = new SingletonStatistics(42.5);
+
+        Util.assertHistogram(s,
+                new double[] {0, 42},
+                new int[] {0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_43_100() {
+        SingletonStatistics s = new SingletonStatistics(42.5);
+
+        Util.assertHistogram(s,
+                new double[] {43, 100},
+                new int[] {0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_leftBound() {
+        SingletonStatistics s = new SingletonStatistics(10);
+
+        Util.assertHistogram(s,
+                new double[] {10, 100},
+                new int[] {1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_rightBound() {
+        SingletonStatistics s = new SingletonStatistics(10);
+
+        Util.assertHistogram(s,
+                new double[] {0, 10},
+                new int[] {0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_emptyLevels_left() {
+        SingletonStatistics s = new SingletonStatistics(9);
+
+        Util.assertHistogram(s,
+                new double[] {0, 2, 4, 8, 10},
+                new int[] {0, 0, 0, 1}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_emptyLevels_right() {
+        SingletonStatistics s = new SingletonStatistics(1);
+
+        Util.assertHistogram(s,
+                new double[] {0, 2, 4, 8, 10},
+                new int[] {1, 0, 0, 0}
+        );
+    }
+
+    @Test
+    public strictfp void testHistogram_emptyLevels_middle() {
+        SingletonStatistics s = new SingletonStatistics(5);
+
+        Util.assertHistogram(s,
+                new double[] {0, 2, 4, 8, 10},
+                new int[] {0, 0, 1, 0}
+        );
+    }
+
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/test/java/org/openjdk/jmh/util/Util.java	Thu Jan 14 18:23:30 2016 +0300
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.util;
+
+import junit.framework.Assert;
+
+import java.util.Arrays;
+
+public class Util {
+
+    public static void assertHistogram(Statistics s, double[] vals, int[] expected) {
+        int[] actual = s.getHistogram(vals);
+        Assert.assertEquals(expected.length, actual.length);
+        Assert.assertTrue(
+                "Expected " + Arrays.toString(expected) +
+                        ", but got " + Arrays.toString(actual),
+                Arrays.equals(expected, actual));
+    }
+
+}