changeset 1120:18647f128073

7901290: Do not accumulate iteration results in the forked VM
author shade
date Thu, 05 Feb 2015 23:01:02 +0300
parents 87d16fe4bf92
children 903bf758a2dd
files jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/IterationResultAcceptor.java jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java jmh-core/src/main/java/org/openjdk/jmh/runner/link/BinaryLinkClient.java jmh-core/src/main/java/org/openjdk/jmh/runner/link/BinaryLinkServer.java jmh-core/src/main/java/org/openjdk/jmh/runner/link/ResultsFrame.java
diffstat 7 files changed, 170 insertions(+), 101 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Thu Feb 05 21:39:48 2015 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Thu Feb 05 23:01:02 2015 +0300
@@ -69,71 +69,90 @@
         this.out = handler;
     }
 
-    protected Multimap<BenchmarkParams, BenchmarkResult> runBenchmarks(boolean forked, ActionPlan actionPlan) {
+    protected void runBenchmarksForked(ActionPlan actionPlan, IterationResultAcceptor acceptor) {
+        for (Action action : actionPlan.getActions()) {
+            BenchmarkParams params = action.getParams();
+            ActionMode mode = action.getMode();
+
+            doSingle(params, mode, acceptor);
+        }
+    }
+
+    protected Multimap<BenchmarkParams, BenchmarkResult> runBenchmarksEmbedded(ActionPlan actionPlan) {
         Multimap<BenchmarkParams, BenchmarkResult> results = new TreeMultimap<BenchmarkParams, BenchmarkResult>();
 
         for (Action action : actionPlan.getActions()) {
-
             BenchmarkParams params = action.getParams();
             ActionMode mode = action.getMode();
 
-            if (!forked) {
-                String opts = Utils.join(params.getJvmArgs(), " ").trim();
-                String realOpts = Utils.join(ManagementFactory.getRuntimeMXBean().getInputArguments(), " ").trim();
-                if (opts.isEmpty()) {
-                    opts = "<none>";
-                }
-                if (realOpts.isEmpty()) {
-                    realOpts = "<none>";
-                }
-
-                Version.printVersion(out);
-                out.println("# VM invoker: " + params.getJvm());
-                out.println("# VM invoker: " + params.getJvm());
-                out.println("# VM options: " + realOpts + (opts.equals(realOpts) ? "" : " *** WARNING: some JVM options are ignored in non-forked runs ***"));
-
-                out.startBenchmark(params);
-                out.println("");
-                etaBeforeBenchmark();
-                out.println("# Fork: N/A, test runs in the existing VM");
+            String opts = Utils.join(params.getJvmArgs(), " ").trim();
+            String realOpts = Utils.join(ManagementFactory.getRuntimeMXBean().getInputArguments(), " ").trim();
+            if (opts.isEmpty()) {
+                opts = "<none>";
+            }
+            if (realOpts.isEmpty()) {
+                realOpts = "<none>";
             }
 
-            BenchmarkResult r = null;
-            try {
-                switch (mode) {
-                    case WARMUP: {
-                        runBenchmark(params);
-                        out.println("");
-                        break;
-                    }
-                    case WARMUP_MEASUREMENT:
-                    case MEASUREMENT: {
-                        r = runBenchmark(params);
-                        results.put(params, r);
-                        break;
-                    }
-                    default:
-                        throw new IllegalStateException("Unknown mode: " + mode);
+            Version.printVersion(out);
+            out.println("# VM invoker: " + params.getJvm());
+            out.println("# VM invoker: " + params.getJvm());
+            out.println("# VM options: " + realOpts + (opts.equals(realOpts) ? "" : " *** WARNING: some JVM options are ignored in non-forked runs ***"));
 
+            out.startBenchmark(params);
+            out.println("");
+            etaBeforeBenchmark();
+            out.println("# Fork: N/A, test runs in the existing VM");
+
+            final List<IterationResult> res = new ArrayList<IterationResult>();
+
+            IterationResultAcceptor acceptor = new IterationResultAcceptor() {
+                @Override
+                public void accept(IterationResult iterationData) {
+                    res.add(iterationData);
                 }
-            } catch (BenchmarkException be) {
-                out.println("<failure>");
-                out.println("");
-                out.println(Utils.throwableToString(be.getCause()));
-                out.println("");
+            };
 
-                if (options.shouldFailOnError().orElse(Defaults.FAIL_ON_ERROR)) {
-                    throw be;
-                }
+            doSingle(params, mode, acceptor);
+
+            if (!res.isEmpty()) {
+                BenchmarkResult br = new BenchmarkResult(res);
+                results.put(params, br);
+                out.endBenchmark(br);
             }
 
-            if (!forked) {
-                etaAfterBenchmark(params);
-                out.endBenchmark(r);
+            etaAfterBenchmark(params);
+        }
+        return results;
+    }
+
+    private void doSingle(BenchmarkParams params, ActionMode mode, IterationResultAcceptor acceptor) {
+        try {
+            switch (mode) {
+                case WARMUP: {
+                    runBenchmark(params, null);
+                    out.println("");
+                    break;
+                }
+                case WARMUP_MEASUREMENT:
+                case MEASUREMENT: {
+                    runBenchmark(params, acceptor);
+                    break;
+                }
+                default:
+                    throw new IllegalStateException("Unknown mode: " + mode);
+
+            }
+        } catch (BenchmarkException be) {
+            out.println("<failure>");
+            out.println("");
+            out.println(Utils.throwableToString(be.getCause()));
+            out.println("");
+
+            if (options.shouldFailOnError().orElse(Defaults.FAIL_ON_ERROR)) {
+                throw be;
             }
         }
-
-        return results;
     }
 
     protected void etaAfterBenchmark(BenchmarkParams params) {
@@ -206,7 +225,7 @@
         return String.format("%s%02d:%02d:%02d", (days > 0) ? days + " days, " : "", hrs, mins, secs);
     }
 
-    BenchmarkResult runBenchmark(BenchmarkParams benchParams) {
+    void runBenchmark(BenchmarkParams benchParams, IterationResultAcceptor acceptor) {
         BenchmarkHandler handler = null;
         try {
             String target = benchParams.generatedBenchmark();
@@ -216,7 +235,7 @@
 
             handler = BenchmarkHandlers.getInstance(out, clazz, method, benchParams, options);
 
-            return runBenchmark(benchParams, handler);
+            runBenchmark(benchParams, handler, acceptor);
         } catch (BenchmarkException be) {
             throw be;
         } catch (Throwable ex) {
@@ -228,9 +247,7 @@
         }
     }
 
-    protected BenchmarkResult runBenchmark(BenchmarkParams benchParams, BenchmarkHandler handler) {
-        List<IterationResult> allResults = new ArrayList<IterationResult>();
-
+    protected void runBenchmark(BenchmarkParams benchParams, BenchmarkHandler handler, IterationResultAcceptor acceptor) {
         // warmup
         IterationParams wp = benchParams.getWarmup();
         for (int i = 1; i <= wp.getCount(); i++) {
@@ -259,14 +276,10 @@
             boolean isLastIteration = (i == mp.getCount());
             IterationResult iterData = handler.runIteration(benchParams, mp, isLastIteration);
             out.iterationResult(benchParams, mp, i, iterData);
-            allResults.add(iterData);
-        }
 
-        if (!allResults.isEmpty()) {
-            return new BenchmarkResult(allResults);
-        } else {
-            // should be ignored in the caller
-            return null;
+            if (acceptor != null) {
+                acceptor.accept(iterData);
+            }
         }
     }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Thu Feb 05 21:39:48 2015 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Thu Feb 05 23:01:02 2015 +0300
@@ -24,11 +24,9 @@
  */
 package org.openjdk.jmh.runner;
 
-import org.openjdk.jmh.infra.BenchmarkParams;
-import org.openjdk.jmh.results.BenchmarkResult;
+import org.openjdk.jmh.results.IterationResult;
 import org.openjdk.jmh.runner.link.BinaryLinkClient;
 import org.openjdk.jmh.runner.options.Options;
-import org.openjdk.jmh.util.Multimap;
 
 import java.io.IOException;
 
@@ -48,14 +46,40 @@
         ActionPlan actionPlan = link.requestPlan();
 
         try {
-            Multimap<BenchmarkParams,BenchmarkResult> res = runBenchmarks(true, actionPlan);
-            link.pushResults(res);
+            IterationResultAcceptor acceptor = new IterationResultAcceptor() {
+                @Override
+                public void accept(IterationResult iterationData) {
+                    try {
+                        link.pushResults(iterationData);
+                    } catch (IOException e) {
+                        // link had probably failed
+                        throw new SavedIOException(e);
+                    }
+                }
+            };
+
+            runBenchmarksForked(actionPlan, acceptor);
         } catch (BenchmarkException be) {
             link.pushException(be);
+        } catch (SavedIOException ioe) {
+            throw ioe.getCause();
         }
 
         out.flush();
         out.close();
     }
 
+    static class SavedIOException extends RuntimeException {
+        private final IOException e;
+
+        public SavedIOException(IOException e) {
+            super(e);
+            this.e = e;
+        }
+
+        public IOException getCause() {
+            return e;
+        }
+    }
+
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/IterationResultAcceptor.java	Thu Feb 05 23:01:02 2015 +0300
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.runner;
+
+import org.openjdk.jmh.results.IterationResult;
+
+interface IterationResultAcceptor {
+    void accept(IterationResult iterationData);
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Thu Feb 05 21:39:48 2015 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Thu Feb 05 23:01:02 2015 +0300
@@ -32,6 +32,7 @@
 import org.openjdk.jmh.profile.Profiler;
 import org.openjdk.jmh.profile.ProfilerFactory;
 import org.openjdk.jmh.results.BenchmarkResult;
+import org.openjdk.jmh.results.IterationResult;
 import org.openjdk.jmh.results.Result;
 import org.openjdk.jmh.results.RunResult;
 import org.openjdk.jmh.results.format.ResultFormatFactory;
@@ -528,7 +529,7 @@
                 Multimap<BenchmarkParams, BenchmarkResult> res;
                 switch (r.getType()) {
                     case EMBEDDED:
-                        res = runBenchmarks(false, r);
+                        res = runBenchmarksEmbedded(r);
                         break;
                     case FORKED:
                         res = runSeparate(r);
@@ -649,22 +650,24 @@
                     }
                 }
 
-                Multimap<BenchmarkParams, BenchmarkResult> result = doFork(server, commandString, stdOut, stdErr, printOut, printErr);
+                List<IterationResult> result = doFork(server, commandString, stdOut, stdErr, printOut, printErr);
+                if (!result.isEmpty()) {
+                    BenchmarkResult br = new BenchmarkResult(result);
 
-                if (!profilers.isEmpty()) {
-                    out.print("# Processing profiler results: ");
-                    for (ExternalProfiler profiler : profilers) {
-                        out.print(profiler.label() + " ");
-                        for (Result profR : profiler.afterTrial(params, stdOut, stdErr)) {
-                            for (BenchmarkResult r : result.values()) {
-                                r.addBenchmarkResult(profR);
+                    if (!profilers.isEmpty()) {
+                        out.print("# Processing profiler results: ");
+                        for (ExternalProfiler profiler : profilers) {
+                            out.print(profiler.label() + " ");
+                            for (Result profR : profiler.afterTrial(params, stdOut, stdErr)) {
+                                br.addBenchmarkResult(profR);
                             }
                         }
+                        out.println("");
                     }
-                    out.println("");
+
+                    results.put(params, br);
                 }
 
-                results.merge(result);
                 etaAfterBenchmark(params);
                 out.println("");
             }
@@ -689,7 +692,7 @@
         return results;
     }
 
-    private Multimap<BenchmarkParams, BenchmarkResult> doFork(BinaryLinkServer reader, String[] commandString,
+    private List<IterationResult> doFork(BinaryLinkServer reader, String[] commandString,
                                                               File stdOut, File stdErr, boolean printOut, boolean printErr) {
         FileOutputStream fosErr = null;
         FileOutputStream fosOut = null;
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/link/BinaryLinkClient.java	Thu Feb 05 21:39:48 2015 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/link/BinaryLinkClient.java	Thu Feb 05 23:01:02 2015 +0300
@@ -24,14 +24,12 @@
  */
 package org.openjdk.jmh.runner.link;
 
-import org.openjdk.jmh.infra.BenchmarkParams;
-import org.openjdk.jmh.results.BenchmarkResult;
+import org.openjdk.jmh.results.IterationResult;
 import org.openjdk.jmh.runner.ActionPlan;
 import org.openjdk.jmh.runner.BenchmarkException;
 import org.openjdk.jmh.runner.format.OutputFormat;
 import org.openjdk.jmh.runner.options.Options;
 import org.openjdk.jmh.util.FileUtils;
-import org.openjdk.jmh.util.Multimap;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
@@ -161,7 +159,7 @@
         }
     }
 
-    public void pushResults(Multimap<BenchmarkParams, BenchmarkResult> res) throws IOException {
+    public void pushResults(IterationResult res) throws IOException {
         pushFrame(new ResultsFrame(res));
     }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/link/BinaryLinkServer.java	Thu Feb 05 21:39:48 2015 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/link/BinaryLinkServer.java	Thu Feb 05 23:01:02 2015 +0300
@@ -24,20 +24,18 @@
  */
 package org.openjdk.jmh.runner.link;
 
-import org.openjdk.jmh.infra.BenchmarkParams;
-import org.openjdk.jmh.results.BenchmarkResult;
+import org.openjdk.jmh.results.IterationResult;
 import org.openjdk.jmh.runner.ActionPlan;
 import org.openjdk.jmh.runner.BenchmarkException;
 import org.openjdk.jmh.runner.Defaults;
 import org.openjdk.jmh.runner.format.OutputFormat;
 import org.openjdk.jmh.runner.options.Options;
 import org.openjdk.jmh.runner.options.VerboseMode;
-import org.openjdk.jmh.util.HashMultimap;
-import org.openjdk.jmh.util.Multimap;
 import org.openjdk.jmh.util.Utils;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.ObjectInputStream;
@@ -50,8 +48,10 @@
 import java.net.Socket;
 import java.net.SocketException;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
@@ -71,7 +71,7 @@
     private final Set<String> forbidden;
     private final Acceptor acceptor;
     private final AtomicReference<Handler> handler;
-    private final AtomicReference<Multimap<BenchmarkParams, BenchmarkResult>> results;
+    private final AtomicReference<List<IterationResult>> results;
     private final AtomicReference<BenchmarkException> exception;
     private final AtomicReference<ActionPlan> plan;
 
@@ -99,7 +99,7 @@
         acceptor.start();
 
         handler = new AtomicReference<Handler>();
-        results = new AtomicReference<Multimap<BenchmarkParams, BenchmarkResult>>(new HashMultimap<BenchmarkParams, BenchmarkResult>());
+        results = new AtomicReference<List<IterationResult>>(new ArrayList<IterationResult>());
         exception = new AtomicReference<BenchmarkException>();
         plan = new AtomicReference<ActionPlan>();
     }
@@ -137,8 +137,8 @@
         return exception.getAndSet(null);
     }
 
-    public Multimap<BenchmarkParams, BenchmarkResult> getResults() {
-        Multimap<BenchmarkParams, BenchmarkResult> res = results.getAndSet(new HashMultimap<BenchmarkParams, BenchmarkResult>());
+    public List<IterationResult> getResults() {
+        List<IterationResult> res = results.getAndSet(new ArrayList<IterationResult>());
         if (res != null) {
             return res;
         } else {
@@ -284,19 +284,21 @@
                         handleInfra((InfraFrame) obj);
                     }
                     if (obj instanceof ResultsFrame) {
-                        handleResults((ResultsFrame)obj);
+                        handleResults((ResultsFrame) obj);
                     }
                     if (obj instanceof ExceptionFrame) {
-                        handleException((ExceptionFrame)obj);
+                        handleException((ExceptionFrame) obj);
                     }
                     if (obj instanceof OutputFrame) {
-                        handleOutput((OutputFrame)obj);
+                        handleOutput((OutputFrame) obj);
                     }
                     if (obj instanceof FinishingFrame) {
                         // close the streams
                         break;
                     }
                 }
+            } catch (EOFException e) {
+                // ignore
             } catch (Exception e) {
                 out.println("<binary link had failed, forked VM corrupted the stream? Use " + VerboseMode.EXTRA + " verbose to print exception>");
                 if (opts.verbosity().orElse(Defaults.VERBOSITY).equalsOrHigherThan(VerboseMode.EXTRA)) {
@@ -327,7 +329,7 @@
         }
 
         private void handleResults(ResultsFrame obj) {
-            results.get().merge(obj.getRes());
+            results.get().add(obj.getRes());
         }
 
         private void handleInfra(InfraFrame req) throws IOException {
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/link/ResultsFrame.java	Thu Feb 05 21:39:48 2015 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/link/ResultsFrame.java	Thu Feb 05 23:01:02 2015 +0300
@@ -24,22 +24,20 @@
  */
 package org.openjdk.jmh.runner.link;
 
-import org.openjdk.jmh.infra.BenchmarkParams;
-import org.openjdk.jmh.results.BenchmarkResult;
-import org.openjdk.jmh.util.Multimap;
+import org.openjdk.jmh.results.IterationResult;
 
 import java.io.Serializable;
 
 class ResultsFrame implements Serializable {
     private static final long serialVersionUID = -5627086531281515824L;
 
-    private final Multimap<BenchmarkParams, BenchmarkResult> res;
+    private final IterationResult res;
 
-    public ResultsFrame(Multimap<BenchmarkParams, BenchmarkResult> res) {
+    public ResultsFrame(IterationResult res) {
         this.res = res;
     }
 
-    public Multimap<BenchmarkParams, BenchmarkResult> getRes() {
+    public IterationResult getRes() {
         return res;
     }
 }