changeset 209:1d5924d127bc

Introduce intermediate BenchResult to accommodate the results for the single forked run.
author shade
date Wed, 23 Oct 2013 20:13:56 +0400
parents 5207ecad8a4e
children a38d89ed91ce
files jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java jmh-core/src/main/java/org/openjdk/jmh/link/frames/ResultsFrame.java jmh-core/src/main/java/org/openjdk/jmh/logic/results/BenchResult.java jmh-core/src/main/java/org/openjdk/jmh/logic/results/RunResult.java jmh-core/src/main/java/org/openjdk/jmh/output/format/CsvFormat.java jmh-core/src/main/java/org/openjdk/jmh/output/format/OutputFormat.java jmh-core/src/main/java/org/openjdk/jmh/output/format/SilentFormat.java jmh-core/src/main/java/org/openjdk/jmh/output/format/TextReportFormat.java jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java
diffstat 12 files changed, 230 insertions(+), 212 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java	Wed Oct 23 20:13:56 2013 +0400
@@ -29,7 +29,7 @@
 import org.openjdk.jmh.link.frames.OptionsFrame;
 import org.openjdk.jmh.link.frames.OutputFormatFrame;
 import org.openjdk.jmh.link.frames.ResultsFrame;
-import org.openjdk.jmh.logic.results.RunResult;
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.runner.BenchmarkRecord;
 import org.openjdk.jmh.runner.options.Options;
 
@@ -81,7 +81,7 @@
         clientSocket.close();
     }
 
-    public void pushResults(BenchmarkRecord record, RunResult result) throws IOException {
+    public void pushResults(BenchmarkRecord record, BenchResult result) throws IOException {
         oos.writeObject(new ResultsFrame(record, result));
         oos.flush();
     }
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java	Wed Oct 23 20:13:56 2013 +0400
@@ -29,10 +29,12 @@
 import org.openjdk.jmh.link.frames.OptionsFrame;
 import org.openjdk.jmh.link.frames.OutputFormatFrame;
 import org.openjdk.jmh.link.frames.ResultsFrame;
-import org.openjdk.jmh.logic.results.RunResult;
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.output.format.OutputFormat;
 import org.openjdk.jmh.runner.BenchmarkRecord;
 import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.util.internal.Multimap;
+import org.openjdk.jmh.util.internal.TreeMultimap;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -56,7 +58,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.TreeMap;
 
 /**
  * Accepts the OutputFormat calls from the network and forwards those to given local OutputFormat
@@ -69,7 +70,7 @@
     private final Set<String> forbidden;
     private final Acceptor acceptor;
     private final List<Handler> registeredHandlers;
-    private final Map<BenchmarkRecord, RunResult> results;
+    private final Multimap<BenchmarkRecord, BenchResult> results;
 
     public BinaryLinkServer(Options opts, OutputFormat out) throws IOException {
         this.opts = opts;
@@ -92,7 +93,9 @@
         }
 
         registeredHandlers = Collections.synchronizedList(new ArrayList<Handler>());
-        results = Collections.synchronizedMap(new TreeMap<BenchmarkRecord, RunResult>());
+        synchronized (this) {
+            results = new TreeMultimap<BenchmarkRecord, BenchResult>();
+        }
 
         acceptor = new Acceptor();
         acceptor.start();
@@ -127,8 +130,10 @@
         }
     }
 
-    public Map<BenchmarkRecord, RunResult> getResults() {
-        return results;
+    public Multimap<BenchmarkRecord, BenchResult> getResults() {
+        synchronized (this) {
+            return results;
+        }
     }
 
     private final class Acceptor extends Thread {
@@ -240,8 +245,9 @@
         }
 
         private void handleResults(ResultsFrame obj) {
-            BenchmarkRecord bench = obj.getRecord();
-            results.put(bench, RunResult.merge(results.get(bench), obj.getResult()));
+            synchronized (this) {
+                results.put(obj.getRecord(), obj.getResult());
+            }
         }
 
         private void handleInfra(InfraFrame req) throws IOException {
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/frames/ResultsFrame.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/frames/ResultsFrame.java	Wed Oct 23 20:13:56 2013 +0400
@@ -24,16 +24,16 @@
  */
 package org.openjdk.jmh.link.frames;
 
-import org.openjdk.jmh.logic.results.RunResult;
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.runner.BenchmarkRecord;
 
 import java.io.Serializable;
 
 public class ResultsFrame implements Serializable {
     private final BenchmarkRecord record;
-    private final RunResult result;
+    private final BenchResult result;
 
-    public ResultsFrame(BenchmarkRecord record, RunResult result) {
+    public ResultsFrame(BenchmarkRecord record, BenchResult result) {
         this.record = record;
         this.result = result;
     }
@@ -42,7 +42,7 @@
         return record;
     }
 
-    public RunResult getResult() {
+    public BenchResult getResult() {
         return result;
     }
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/main/java/org/openjdk/jmh/logic/results/BenchResult.java	Wed Oct 23 20:13:56 2013 +0400
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.logic.results;
+
+import org.openjdk.jmh.util.internal.HashMultimap;
+import org.openjdk.jmh.util.internal.Multimap;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Benchmark result.
+ * Contains iteration results.
+ *
+ * @author Aleksey Shipilev
+ */
+public class BenchResult implements Serializable {
+
+    private static final long serialVersionUID = 6467912427356048369L;
+
+    private final Collection<IterationResult> iterationResults;
+
+    public BenchResult(Collection<IterationResult> data) {
+        this.iterationResults = data;
+    }
+
+    public Collection<IterationResult> getRawIterationResults() {
+        return iterationResults;
+    }
+
+    public Result getPrimaryResult() {
+        Result next = iterationResults.iterator().next().getPrimaryResult();
+
+        @SuppressWarnings("unchecked")
+        Aggregator<Result> aggregator = next.getRunAggregator();
+        return aggregator.aggregate(getRawPrimaryResults());
+    }
+
+    public Collection<Result> getRawPrimaryResults() {
+        Collection<Result> rs = new ArrayList<Result>();
+        for (IterationResult k : iterationResults) {
+            rs.add(k.getPrimaryResult());
+        }
+        return rs;
+    }
+
+    public Multimap<String, Result> getRawSecondaryResults() {
+        Multimap<String, Result> rs = new HashMultimap<String, Result>();
+        for (IterationResult k : iterationResults) {
+            for (Map.Entry<String, Result> r : k.getSecondaryResults().entrySet()) {
+                rs.put(r.getKey(), r.getValue());
+            }
+        }
+        return rs;
+    }
+
+    public Map<String, Result> getSecondaryResults() {
+        Multimap<String, Result> rs = getRawSecondaryResults();
+
+        Map<String, Result> answers = new TreeMap<String, Result>();
+        for (String k : rs.keys()) {
+            Collection<Result> results = rs.get(k);
+            Result next = results.iterator().next();
+
+            @SuppressWarnings("unchecked")
+            Aggregator<Result> aggregator = next.getRunAggregator();
+            answers.put(k, aggregator.aggregate(results));
+        }
+
+        return answers;
+    }
+
+    public String getScoreUnit() {
+        return getPrimaryResult().getScoreUnit();
+    }
+
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/logic/results/RunResult.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/logic/results/RunResult.java	Wed Oct 23 20:13:56 2013 +0400
@@ -24,14 +24,13 @@
  */
 package org.openjdk.jmh.logic.results;
 
+import org.openjdk.jmh.runner.parameters.TimeValue;
 import org.openjdk.jmh.util.internal.HashMultimap;
 import org.openjdk.jmh.util.internal.Multimap;
 
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
@@ -46,18 +45,18 @@
 
     private static final long serialVersionUID = 6467912427356048369L;
 
-    private final Collection<IterationResult> iterationResults;
+    private final Collection<BenchResult> benchResults;
 
-    public RunResult(Collection<IterationResult> data) {
-        this.iterationResults = data;
+    public RunResult(Collection<BenchResult> data) {
+        this.benchResults = data;
     }
 
-    public Collection<IterationResult> getRawIterationResults() {
-        return iterationResults;
+    public Collection<BenchResult> getRawBenchResults() {
+        return benchResults;
     }
 
     public Result getPrimaryResult() {
-        Result next = iterationResults.iterator().next().getPrimaryResult();
+        Result next = benchResults.iterator().next().getPrimaryResult();
 
         @SuppressWarnings("unchecked")
         Aggregator<Result> aggregator = next.getRunAggregator();
@@ -66,17 +65,21 @@
 
     public Collection<Result> getRawPrimaryResults() {
         Collection<Result> rs = new ArrayList<Result>();
-        for (IterationResult k : iterationResults) {
-            rs.add(k.getPrimaryResult());
+        for (BenchResult br : benchResults) {
+            for (IterationResult ir : br.getRawIterationResults()) {
+                rs.add(ir.getPrimaryResult());
+            }
         }
         return rs;
     }
 
     public Multimap<String, Result> getRawSecondaryResults() {
         Multimap<String, Result> rs = new HashMultimap<String, Result>();
-        for (IterationResult k : iterationResults) {
-            for (Map.Entry<String, Result> r : k.getSecondaryResults().entrySet()) {
-                rs.put(r.getKey(), r.getValue());
+        for (BenchResult br : benchResults) {
+            for (IterationResult ir : br.getRawIterationResults()) {
+                for (Map.Entry<String, Result> r : ir.getSecondaryResults().entrySet()) {
+                    rs.put(r.getKey(), r.getValue());
+                }
             }
         }
         return rs;
@@ -102,16 +105,12 @@
         return getPrimaryResult().getScoreUnit();
     }
 
-    public static RunResult merge(RunResult... rrs) {
-        return merge(Arrays.asList(rrs));
+    public int getThreads() {
+        return getRawBenchResults().iterator().next().getRawIterationResults().iterator().next().getParams().getThreads();
     }
 
-    public static RunResult merge(Collection<RunResult> rrs) {
-        List<IterationResult> rs = new ArrayList<IterationResult>();
-        for (RunResult rr : rrs) {
-            if (rr == null) continue;
-            rs.addAll(rr.getRawIterationResults());
-        }
-        return new RunResult(rs);
+    public TimeValue getTime() {
+        return getRawBenchResults().iterator().next().getRawIterationResults().iterator().next().getParams().getTime();
     }
+
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/format/CsvFormat.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/format/CsvFormat.java	Wed Oct 23 20:13:56 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.output.format;
 
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.logic.results.IterationResult;
 import org.openjdk.jmh.logic.results.Result;
 import org.openjdk.jmh.logic.results.RunResult;
@@ -32,6 +33,7 @@
 import org.openjdk.jmh.runner.parameters.IterationParams;
 
 import java.io.PrintStream;
+import java.util.Map;
 
 /**
  * CSV implementation of OutputFormat.
@@ -80,8 +82,8 @@
     }
 
     @Override
-    public void endBenchmark(BenchmarkRecord name, RunResult result) {
-        // don't print anything
+    public void endBenchmark(BenchmarkRecord name, BenchResult result) {
+        // do nothing
     }
 
     @Override
@@ -90,8 +92,8 @@
     }
 
     @Override
-    public void endRun() {
-        // don't print anything
+    public void endRun(Map<BenchmarkRecord, RunResult> result) {
+        // do nothing
     }
 
     @Override
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/format/OutputFormat.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/format/OutputFormat.java	Wed Oct 23 20:13:56 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.output.format;
 
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.logic.results.IterationResult;
 import org.openjdk.jmh.logic.results.RunResult;
 import org.openjdk.jmh.runner.BenchmarkRecord;
@@ -31,6 +32,7 @@
 import org.openjdk.jmh.runner.parameters.IterationParams;
 
 import java.io.IOException;
+import java.util.Map;
 
 /**
  * Internal interface for OutputFormat.
@@ -74,7 +76,7 @@
      * @param name       benchmark name
      * @param result statistics of the run
      */
-    public void endBenchmark(BenchmarkRecord name, RunResult result);
+    public void endBenchmark(BenchmarkRecord name, BenchResult result);
 
     /**
      * Format for start-of-benchmark output.
@@ -84,7 +86,7 @@
     /**
      * Format for end-of-benchmark.
      */
-    public void endRun();
+    public void endRun(Map<BenchmarkRecord, RunResult> result);
 
     /* ------------- SPECIAL TRACING METHODS -------------------- */
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/format/SilentFormat.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/format/SilentFormat.java	Wed Oct 23 20:13:56 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.output.format;
 
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.logic.results.IterationResult;
 import org.openjdk.jmh.logic.results.RunResult;
 import org.openjdk.jmh.runner.BenchmarkRecord;
@@ -31,6 +32,7 @@
 import org.openjdk.jmh.runner.parameters.IterationParams;
 
 import java.io.PrintStream;
+import java.util.Map;
 
 /**
  * Silent format, does nothing.
@@ -48,7 +50,7 @@
     }
 
     @Override
-    public void endRun() {
+    public void endRun(Map<BenchmarkRecord, RunResult> results) {
     }
 
     @Override
@@ -57,7 +59,7 @@
     }
 
     @Override
-    public void endBenchmark(BenchmarkRecord name, RunResult result) {
+    public void endBenchmark(BenchmarkRecord name, BenchResult result) {
 
     }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/format/TextReportFormat.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/format/TextReportFormat.java	Wed Oct 23 20:13:56 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.output.format;
 
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.logic.results.IterationResult;
 import org.openjdk.jmh.logic.results.Result;
 import org.openjdk.jmh.logic.results.RunResult;
@@ -31,17 +32,15 @@
 import org.openjdk.jmh.runner.BenchmarkRecord;
 import org.openjdk.jmh.runner.parameters.BenchmarkParams;
 import org.openjdk.jmh.runner.parameters.IterationParams;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 import org.openjdk.jmh.util.ClassUtils;
-import org.openjdk.jmh.util.internal.Multimap;
 import org.openjdk.jmh.util.internal.Statistics;
-import org.openjdk.jmh.util.internal.TreeMultimap;
 
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
 import java.util.Map;
-import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -52,15 +51,8 @@
  */
 public class TextReportFormat extends AbstractOutputFormat {
 
-    private final Map<BenchmarkRecord, IterationParams> benchmarkSettings;
-    private final Multimap<BenchmarkIdentifier, IterationResult> benchmarkResults;
-    private final Multimap<BenchmarkIdentifier, RunResult> benchmarkRunResults;
-
     public TextReportFormat(PrintStream out, boolean verbose) {
         super(out, verbose);
-        benchmarkSettings = new TreeMap<BenchmarkRecord, IterationParams>();
-        benchmarkResults = new TreeMultimap<BenchmarkIdentifier, IterationResult>();
-        benchmarkRunResults = new TreeMultimap<BenchmarkIdentifier, RunResult>();
     }
 
     @Override
@@ -74,8 +66,6 @@
         out.println("# Threads: " + mbParams.getThreads() + " " + getThreadsString(mbParams.getThreads()) + (mbParams.shouldSynchIterations() ? ", will synchronize iterations" : ""));
         out.println("# Benchmark mode: " + name.getMode().longLabel());
         out.println("# Running: " + name.getUsername());
-
-        benchmarkSettings.put(name, mbParams.getIteration());
     }
 
     @Override
@@ -148,16 +138,10 @@
 
         out.println("");
         out.flush();
-        if (type == IterationType.MEASUREMENT) {
-            benchmarkResults.put(new BenchmarkIdentifier(name, params.getThreads()), data);
-        }
     }
 
     @Override
-    public void endBenchmark(BenchmarkRecord name, RunResult result) {
-        IterationParams params = benchmarkSettings.get(name);
-        benchmarkRunResults.put(new BenchmarkIdentifier(name, params.getThreads()), result);
-
+    public void endBenchmark(BenchmarkRecord name, BenchResult result) {
         out.println();
         out.println(result.getPrimaryResult().extendedInfo(null));
         for (Result r : result.getSecondaryResults().values()) {
@@ -172,39 +156,13 @@
     }
 
     @Override
-    public void endRun() {
-        for (BenchmarkIdentifier key1 : benchmarkRunResults.keys()) {
-            Collection<RunResult> forkedResults = benchmarkRunResults.get(key1);
-            if (forkedResults.size() > 1) {
-                out.println("\"" + key1.benchmark.getUsername() + "\", aggregate over forked runs:");
-                out.println();
-
-                RunResult runResult1 = RunResult.merge(forkedResults);
-
-                out.println(runResult1.getPrimaryResult().extendedInfo(null));
-                for (Result r : runResult1.getSecondaryResults().values()) {
-                    out.println(r.extendedInfo(r.getLabel()));
-                }
-            }
-        }
-
-        // generate the full report
-        //
-
-        if (benchmarkResults.isEmpty()) {
-            return;
-        }
-
+    public void endRun(Map<BenchmarkRecord, RunResult> runResults) {
         Collection<String> benchNames = new ArrayList<String>();
-        for (BenchmarkIdentifier key : benchmarkResults.keys()) {
-            Collection<IterationResult> results = benchmarkResults.get(key);
-            if (results != null && !results.isEmpty()) {
-                RunResult runResult = new RunResult(results);
-
-                benchNames.add(key.benchmark.getUsername());
-                for (String label : runResult.getSecondaryResults().keySet()) {
-                    benchNames.add(key.benchmark.getUsername() + ":" + label);
-                }
+        for (BenchmarkRecord key : runResults.keySet()) {
+            RunResult runResult = runResults.get(key);
+            benchNames.add(key.getUsername());
+            for (String label : runResult.getSecondaryResults().keySet()) {
+                benchNames.add(key.getUsername() + ":" + label);
             }
         }
 
@@ -220,111 +178,45 @@
         out.printf("%-" + nameLen + "s %6s %3s %6s %4s %12s %12s %8s%n",
                 "Benchmark", "Mode", "Thr", "Cnt", "Sec",
                 "Mean", "Mean error", "Units");
-        for (BenchmarkIdentifier key : benchmarkResults.keys()) {
-
+        for (BenchmarkRecord key : runResults.keySet()) {
             double[] interval = new double[]{Double.NaN, Double.NaN};
 
-            IterationParams settings = benchmarkSettings.get(key.benchmark);
-            Collection<IterationResult> results = benchmarkResults.get(key);
+            RunResult res = runResults.get(key);
 
-            if (results != null && !results.isEmpty()) {
-                RunResult runResult = new RunResult(results);
+            int threads = res.getThreads();
+            TimeValue runTime = res.getTime();
 
-                {
-                    Statistics stats = runResult.getPrimaryResult().getStatistics();
-                    if (stats.getN() > 2) {
-                        interval = stats.getConfidenceInterval(0.01);
-                    }
-
-                    out.printf("%-" + nameLen + "s %6s %3d %6d %4d %12.3f %12.3f %8s%n",
-                            benchPrefixes.get(key.benchmark.getUsername()),
-                            key.benchmark.getMode().shortLabel(),
-                            key.threads, stats.getN(),
-                            settings.getTime().convertTo(TimeUnit.SECONDS),
-                            stats.getMean(), (interval[1] - interval[0]) / 2,
-                            runResult.getScoreUnit());
+            {
+                Statistics stats = res.getPrimaryResult().getStatistics();
+                if (stats.getN() > 2) {
+                    interval = stats.getConfidenceInterval(0.01);
                 }
 
-                for (String label : runResult.getSecondaryResults().keySet()) {
-                    Statistics stats = runResult.getSecondaryResults().get(label).getStatistics();
-                    if (stats.getN() > 2) {
-                        interval = stats.getConfidenceInterval(0.01);
-                    }
-
-                    out.printf("%-" + nameLen + "s %6s %3d %6d %4d %12.3f %12.3f %8s%n",
-                            benchPrefixes.get(key.benchmark.getUsername() + ":" + label),
-                            key.benchmark.getMode().shortLabel(),
-                            key.threads, stats.getN(),
-                            settings.getTime().convertTo(TimeUnit.SECONDS),
-                            stats.getMean(), (interval[1] - interval[0]) / 2,
-                            runResult.getScoreUnit());
-                }
-            } else {
-                out.printf("%-" + nameLen + "s %6s, %3d %6d %4d %12.3f %12.3f %8s%n",
-                        benchPrefixes.get(key.benchmark.getUsername()),
-                        key.benchmark.getMode().shortLabel(),
-                        key.threads, 0,
-                        settings.getTime().convertTo(TimeUnit.SECONDS),
-                        Double.NaN, Double.NaN,
-                        "N/A");
+                out.printf("%-" + nameLen + "s %6s %3d %6d %4d %12.3f %12.3f %8s%n",
+                        benchPrefixes.get(key.getUsername()),
+                        key.getMode().shortLabel(),
+                        threads, stats.getN(),
+                        runTime.convertTo(TimeUnit.SECONDS),
+                        stats.getMean(), (interval[1] - interval[0]) / 2,
+                        res.getScoreUnit());
             }
 
-        }
-        benchmarkResults.clear();
-        benchmarkSettings.clear();
-    }
+            for (String label : res.getSecondaryResults().keySet()) {
+                Statistics stats = res.getSecondaryResults().get(label).getStatistics();
+                if (stats.getN() > 2) {
+                    interval = stats.getConfidenceInterval(0.01);
+                }
 
-    private static class BenchmarkIdentifier implements Comparable<BenchmarkIdentifier> {
-        final BenchmarkRecord benchmark;
-        final int threads;
-
-        BenchmarkIdentifier(BenchmarkRecord benchmark, int threads) {
-            this.benchmark = benchmark;
-            this.threads = threads;
-        }
-
-        public BenchmarkIdentifier(BenchmarkIdentifier copy) {
-            this.benchmark = copy.benchmark;
-            this.threads = copy.threads;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) {
-                return true;
-            }
-            if (o == null || getClass() != o.getClass()) {
-                return false;
-            }
-
-            BenchmarkIdentifier that = (BenchmarkIdentifier) o;
-
-            if (threads != that.threads) {
-                return false;
-            }
-            if (benchmark != null ? !benchmark.equals(that.benchmark) : that.benchmark != null) {
-                return false;
-            }
-
-            return true;
-        }
-
-        @Override
-        public int hashCode() {
-            int result = benchmark != null ? benchmark.hashCode() : 0;
-            result = 31 * result + threads;
-            return result;
-        }
-
-        @Override
-        public int compareTo(BenchmarkIdentifier o) {
-            int c1 = benchmark.compareTo(o.benchmark);
-            if (c1 == 0) {
-                return ((Integer) threads).compareTo(o.threads);
-            } else {
-                return c1;
+                out.printf("%-" + nameLen + "s %6s %3d %6d %4d %12.3f %12.3f %8s%n",
+                        benchPrefixes.get(key.getUsername() + ":" + label),
+                        key.getMode().shortLabel(),
+                        threads, stats.getN(),
+                        runTime.convertTo(TimeUnit.SECONDS),
+                        stats.getMean(), (interval[1] - interval[0]) / 2,
+                        res.getScoreUnit());
             }
         }
     }
 
+
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Wed Oct 23 20:13:56 2013 +0400
@@ -24,8 +24,8 @@
  */
 package org.openjdk.jmh.runner;
 
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.logic.results.IterationResult;
-import org.openjdk.jmh.logic.results.RunResult;
 import org.openjdk.jmh.output.format.IterationType;
 import org.openjdk.jmh.output.format.OutputFormat;
 import org.openjdk.jmh.runner.options.Options;
@@ -58,7 +58,7 @@
         this.out = handler;
     }
 
-    RunResult runBenchmark(BenchmarkRecord benchmark, boolean doWarmup, boolean doMeasurement) {
+    BenchResult runBenchmark(BenchmarkRecord benchmark, boolean doWarmup, boolean doMeasurement) {
         MicroBenchmarkHandler handler = null;
         try {
             Class<?> clazz = ClassUtils.loadClass(benchmark.generatedClass());
@@ -82,7 +82,7 @@
         return null;
     }
 
-    protected RunResult runBenchmark(BenchmarkParams executionParams, MicroBenchmarkHandler handler) {
+    protected BenchResult runBenchmark(BenchmarkParams executionParams, MicroBenchmarkHandler handler) {
         List<IterationResult> allResults = new ArrayList<IterationResult>();
 
         out.startBenchmark(handler.getBenchmark(), executionParams, this.options.isVerbose());
@@ -127,7 +127,7 @@
 
         // only print end-of-run output if we have actual results
         if (!allResults.isEmpty()) {
-            RunResult result = new RunResult(allResults);
+            BenchResult result = new BenchResult(allResults);
             out.endBenchmark(handler.getBenchmark(), result);
             return result;
         } else {
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Wed Oct 23 20:13:56 2013 +0400
@@ -25,7 +25,7 @@
 package org.openjdk.jmh.runner;
 
 import org.openjdk.jmh.link.BinaryLinkClient;
-import org.openjdk.jmh.logic.results.RunResult;
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.output.OutputFormatFactory;
 import org.openjdk.jmh.runner.options.Options;
 
@@ -51,12 +51,10 @@
             out.println("Benchmarks: ");
             out.println(benchmark.getUsername());
         }
-        out.startRun();
 
-        RunResult result = runBenchmark(benchmark, true, true);
+        BenchResult result = runBenchmark(benchmark, true, true);
         link.pushResults(benchmark, result);
 
-        out.endRun();
         out.flush();
         out.close();
     }
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Wed Oct 23 20:02:07 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Wed Oct 23 20:13:56 2013 +0400
@@ -28,6 +28,7 @@
 import org.openjdk.jmh.annotations.Fork;
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.link.BinaryLinkServer;
+import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.logic.results.RunResult;
 import org.openjdk.jmh.output.OutputFormatFactory;
 import org.openjdk.jmh.output.format.OutputFormat;
@@ -37,6 +38,8 @@
 import org.openjdk.jmh.util.AnnotationUtils;
 import org.openjdk.jmh.util.InputStreamDrainer;
 import org.openjdk.jmh.util.Utils;
+import org.openjdk.jmh.util.internal.Multimap;
+import org.openjdk.jmh.util.internal.TreeMultimap;
 
 import java.io.BufferedOutputStream;
 import java.io.File;
@@ -49,6 +52,7 @@
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -238,15 +242,16 @@
         }
         // run microbenchmarks
         //
-        Map<BenchmarkRecord, RunResult> results = new TreeMap<BenchmarkRecord, RunResult>();
+        Multimap<BenchmarkRecord, BenchResult> results = new TreeMultimap<BenchmarkRecord, BenchResult>();
         for (BenchmarkRecord benchmark : benchmarks) {
             out.println("# Fork: N/A, test runs in same VM");
-            RunResult result = runBenchmark(benchmark, false, true);
-            results.put(benchmark, RunResult.merge(results.get(benchmark), result));
+            BenchResult result = runBenchmark(benchmark, false, true);
+            results.put(benchmark, result);
         }
-        out.endRun();
 
-        return results;
+        Map<BenchmarkRecord, RunResult> runResults = mergeRunResults(results);
+        out.endRun(runResults);
+        return runResults;
     }
 
     private int decideForks(int optionForks, int benchForks) {
@@ -283,24 +288,34 @@
             }
         }
 
-        Map<BenchmarkRecord, RunResult> results = new TreeMap<BenchmarkRecord, RunResult>();
+        Multimap<BenchmarkRecord, BenchResult> results = new TreeMultimap<BenchmarkRecord, BenchResult>();
         for (BenchmarkRecord benchmark : embedded) {
             out.println("# Fork: N/A, test runs in same VM");
-            RunResult r = runBenchmark(benchmark, true, true);
-            results.put(benchmark, RunResult.merge(results.get(benchmark), r));
+            BenchResult r = runBenchmark(benchmark, true, true);
+            results.put(benchmark, r);
         }
 
-        Map<BenchmarkRecord, RunResult> separateResults = runSeparate(forked);
-        for (Map.Entry<BenchmarkRecord, RunResult> e : separateResults.entrySet()) {
-            results.put(e.getKey(), RunResult.merge(results.get(e.getKey()), e.getValue()));
+        Multimap<BenchmarkRecord, BenchResult> separateResults = runSeparate(forked);
+        for (BenchmarkRecord k : separateResults.keys()) {
+            Collection<BenchResult> rs = separateResults.get(k);
+            results.putAll(k, rs);
         }
 
-        out.endRun();
-
-        return results;
+        Map<BenchmarkRecord, RunResult> runResults = mergeRunResults(results);
+        out.endRun(runResults);
+        return runResults;
     }
 
-    private Map<BenchmarkRecord, RunResult> runSeparate(Set<BenchmarkRecord> benchmarksToFork) {
+    private Map<BenchmarkRecord, RunResult> mergeRunResults(Multimap<BenchmarkRecord, BenchResult> results) {
+        Map<BenchmarkRecord, RunResult> result = new TreeMap<BenchmarkRecord, RunResult>();
+        for (BenchmarkRecord key : results.keys()) {
+            Collection<BenchResult> rs = results.get(key);
+            result.put(key, new RunResult(rs));
+        }
+        return result;
+    }
+
+    private Multimap<BenchmarkRecord, BenchResult> runSeparate(Set<BenchmarkRecord> benchmarksToFork) {
         BinaryLinkServer server = null;
         try {
             server = new BinaryLinkServer(options, out);