changeset 376:61a59c1c9453

Advanced recovery for the faulty benchmarks. The results for the correct benchmarks should be always available. Tolerate thrown exceptions, calling System.exit() [forked mode only], and crashed VMs [forked mode only]. More humane stack traces describing the error and omitting the intermediary JMH frames.
author shade
date Tue, 28 Jan 2014 00:59:02 +0400
parents 66fdf41496ea
children 4659c9e09268
files jmh-core-it/src/test/java/org/openjdk/jmh/it/errors/EmbeddedErrorsTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/errors/ForkedErrorsTest.java jmh-core/src/main/java/org/openjdk/jmh/Main.java jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java jmh-core/src/main/java/org/openjdk/jmh/link/frames/ExceptionFrame.java jmh-core/src/main/java/org/openjdk/jmh/output/format/AbstractOutputFormat.java jmh-core/src/main/java/org/openjdk/jmh/output/format/OutputFormat.java jmh-core/src/main/java/org/openjdk/jmh/runner/BaseMicroBenchmarkHandler.java jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkException.java jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/LoopMicroBenchmarkHandler.java jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java jmh-core/src/main/java/org/openjdk/jmh/runner/RunnerException.java
diffstat 15 files changed, 389 insertions(+), 89 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/errors/EmbeddedErrorsTest.java	Tue Jan 28 00:59:02 2014 +0400
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.errors;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.logic.results.RunResult;
+import org.openjdk.jmh.runner.BenchmarkRecord;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.SortedMap;
+import java.util.concurrent.TimeUnit;
+
+@Measurement(iterations = 1, time = 10, timeUnit = TimeUnit.MILLISECONDS)
+@Warmup(iterations = 1, time = 10, timeUnit = TimeUnit.MILLISECONDS)
+public class EmbeddedErrorsTest {
+
+    @GenerateMicroBenchmark
+    public void test00_normal() throws InterruptedException {
+        Thread.sleep(1);
+    }
+
+    @GenerateMicroBenchmark
+    public void test01_exceptional() {
+        throw new IllegalStateException();
+    }
+
+    @GenerateMicroBenchmark
+    public void test02_normal() throws InterruptedException {
+        Thread.sleep(1);
+    }
+
+    // Embedded runs can not possibly survive either System.exit, or VM crash
+
+    @Test
+    public void test_FOE_false() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .forks(0)
+                .shouldFailOnError(false)
+                .build();
+        SortedMap<BenchmarkRecord,RunResult> results = new Runner(opt).run();
+
+        Assert.assertEquals(2, results.size());
+    }
+
+    @Test
+    public void test_FOE_true() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .forks(0)
+                    .shouldFailOnError(true)
+                    .build();
+
+            new Runner(opt).run();
+
+            Assert.fail("Should have thrown the exception");
+        } catch (RunnerException e) {
+            // expected
+        }
+    }
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/errors/ForkedErrorsTest.java	Tue Jan 28 00:59:02 2014 +0400
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.errors;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.logic.results.RunResult;
+import org.openjdk.jmh.runner.BenchmarkRecord;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import sun.misc.Unsafe;
+
+import java.lang.reflect.Field;
+import java.util.SortedMap;
+import java.util.concurrent.TimeUnit;
+
+@Measurement(iterations = 1, time = 10, timeUnit = TimeUnit.MILLISECONDS)
+@Warmup(iterations = 1, time = 10, timeUnit = TimeUnit.MILLISECONDS)
+public class ForkedErrorsTest {
+
+    @GenerateMicroBenchmark
+    public void test00_normal() throws InterruptedException {
+        Thread.sleep(1);
+    }
+
+    @GenerateMicroBenchmark
+    public void test01_exceptional() {
+        throw new IllegalStateException();
+    }
+
+    @GenerateMicroBenchmark
+    public void test02_normal() throws InterruptedException {
+        Thread.sleep(1);
+    }
+
+    @GenerateMicroBenchmark
+    public void test03_exit() throws InterruptedException {
+        System.exit(1);
+    }
+
+    @GenerateMicroBenchmark
+    public void test04_normal() throws InterruptedException {
+        Thread.sleep(1);
+    }
+
+    private static Unsafe getUnsafe() throws NoSuchFieldException, IllegalAccessException {
+        Field f = Unsafe.class.getDeclaredField("theUnsafe");
+        f.setAccessible(true);
+        return (Unsafe) f.get(null);
+    }
+
+    @GenerateMicroBenchmark
+    public void test05_crash() throws InterruptedException, NoSuchFieldException, IllegalAccessException {
+        // SIGSEGV in JVM
+        getUnsafe().getInt(0);
+    }
+
+    @GenerateMicroBenchmark
+    public void test06_normal() throws InterruptedException {
+        Thread.sleep(1);
+    }
+
+    @Test
+    public void test_FOE_false() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .forks(1)
+                .shouldFailOnError(false)
+                .build();
+        SortedMap<BenchmarkRecord,RunResult> results = new Runner(opt).run();
+
+        Assert.assertEquals(4, results.size());
+    }
+
+    @Test
+    public void test_FOE_true() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .forks(1)
+                    .shouldFailOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have thrown the exception");
+        } catch (RunnerException e) {
+            // expected
+        }
+    }
+
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/Main.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/Main.java	Tue Jan 28 00:59:02 2014 +0400
@@ -43,7 +43,7 @@
      *
      * @param argv Command line arguments
      */
-    public static void main(String[] argv) {
+    public static void main(String[] argv) throws RunnerException, IOException {
         try {
             CommandLineOptions cmdOptions = new CommandLineOptions(argv);
 
@@ -70,10 +70,6 @@
             }
 
             runner.run();
-        } catch (RunnerException e) {
-            System.err.println(e.getMessage());
-        } catch (IOException e) {
-            System.err.println(e.getMessage());
         } catch (CommandLineOptionException e) {
             System.err.println("Error parsing command line:");
             System.err.println(" " + e.getMessage());
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java	Tue Jan 28 00:59:02 2014 +0400
@@ -25,6 +25,7 @@
 package org.openjdk.jmh.link;
 
 import org.openjdk.jmh.link.frames.ActionPlanFrame;
+import org.openjdk.jmh.link.frames.ExceptionFrame;
 import org.openjdk.jmh.link.frames.FinishingFrame;
 import org.openjdk.jmh.link.frames.InfraFrame;
 import org.openjdk.jmh.link.frames.OptionsFrame;
@@ -32,6 +33,7 @@
 import org.openjdk.jmh.link.frames.ResultsFrame;
 import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.runner.ActionPlan;
+import org.openjdk.jmh.runner.BenchmarkException;
 import org.openjdk.jmh.runner.BenchmarkRecord;
 import org.openjdk.jmh.runner.options.Options;
 import org.openjdk.jmh.util.internal.Multimap;
@@ -100,4 +102,9 @@
             throw new IllegalStateException("Got the erroneous reply: " + reply);
         }
     }
+
+    public void pushException(BenchmarkException error) throws IOException {
+        oos.writeObject(new ExceptionFrame(error));
+        oos.flush();
+    }
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java	Tue Jan 28 00:59:02 2014 +0400
@@ -25,6 +25,7 @@
 package org.openjdk.jmh.link;
 
 import org.openjdk.jmh.link.frames.ActionPlanFrame;
+import org.openjdk.jmh.link.frames.ExceptionFrame;
 import org.openjdk.jmh.link.frames.FinishingFrame;
 import org.openjdk.jmh.link.frames.InfraFrame;
 import org.openjdk.jmh.link.frames.OptionsFrame;
@@ -33,6 +34,7 @@
 import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.output.format.OutputFormat;
 import org.openjdk.jmh.runner.ActionPlan;
+import org.openjdk.jmh.runner.BenchmarkException;
 import org.openjdk.jmh.runner.BenchmarkRecord;
 import org.openjdk.jmh.runner.options.Options;
 import org.openjdk.jmh.util.internal.HashMultimap;
@@ -72,6 +74,7 @@
     private final Acceptor acceptor;
     private final AtomicReference<Handler> handler;
     private final AtomicReference<Multimap<BenchmarkRecord, BenchResult>> results;
+    private final AtomicReference<BenchmarkException> exception;
     private final AtomicReference<ActionPlan> plan;
 
     public BinaryLinkServer(Options opts, OutputFormat out) throws IOException {
@@ -99,6 +102,7 @@
 
         handler = new AtomicReference<Handler>();
         results = new AtomicReference<Multimap<BenchmarkRecord, BenchResult>>(new HashMultimap<BenchmarkRecord, BenchResult>());
+        exception = new AtomicReference<BenchmarkException>();
         plan = new AtomicReference<ActionPlan>();
     }
 
@@ -131,6 +135,10 @@
         }
     }
 
+    public BenchmarkException getException() {
+        return exception.getAndSet(null);
+    }
+
     public Multimap<BenchmarkRecord, BenchResult> getResults() {
         Multimap<BenchmarkRecord, BenchResult> res = results.getAndSet(new HashMultimap<BenchmarkRecord, BenchResult>());
         if (res != null) {
@@ -259,6 +267,9 @@
                     if (obj instanceof ResultsFrame) {
                         handleResults((ResultsFrame)obj);
                     }
+                    if (obj instanceof ExceptionFrame) {
+                        handleException((ExceptionFrame)obj);
+                    }
                     if (obj instanceof FinishingFrame) {
                         // close the streams
                         break;
@@ -281,6 +292,10 @@
             }
         }
 
+        private void handleException(ExceptionFrame obj) {
+            exception.set(obj.getError());
+        }
+
         private void handleResults(ResultsFrame obj) {
             results.get().merge(obj.getRes());
         }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/frames/ExceptionFrame.java	Tue Jan 28 00:59:02 2014 +0400
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.link.frames;
+
+import org.openjdk.jmh.runner.BenchmarkException;
+
+import java.io.Serializable;
+
+public class ExceptionFrame implements Serializable {
+    private final BenchmarkException error;
+
+    public ExceptionFrame(BenchmarkException error) {
+        this.error = error;
+    }
+
+    public BenchmarkException getError() {
+        return error;
+    }
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/format/AbstractOutputFormat.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/format/AbstractOutputFormat.java	Tue Jan 28 00:59:02 2014 +0400
@@ -28,8 +28,6 @@
 
 import java.io.IOException;
 import java.io.PrintStream;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 public abstract class AbstractOutputFormat implements OutputFormat {
 
@@ -75,9 +73,4 @@
         // do nothing
     }
 
-    @Override
-    public void exception(Throwable ex) {
-        Logger.getAnonymousLogger().log(Level.SEVERE, ex.getMessage(), ex);
-    }
-
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/format/OutputFormat.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/format/OutputFormat.java	Tue Jan 28 00:59:02 2014 +0400
@@ -86,10 +86,6 @@
      */
     public void endRun(Map<BenchmarkRecord, RunResult> result);
 
-    /* ------------- SPECIAL TRACING METHODS -------------------- */
-
-    void exception(Throwable ex);
-
     /* ------------- RAW OUTPUT METHODS ------------------- */
 
     void println(String s);
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseMicroBenchmarkHandler.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseMicroBenchmarkHandler.java	Tue Jan 28 00:59:02 2014 +0400
@@ -215,7 +215,7 @@
             try {
                 iterationResults.addProfileResult(prof.endProfile());
             } catch (Throwable ex) {
-                log(ex);
+                throw new BenchmarkException(ex);
             }
         }
     }
@@ -226,16 +226,11 @@
             try {
                 prof.startProfile();
             } catch (Throwable ex) {
-                log(ex);
+                throw new BenchmarkException(ex);
             }
         }
     }
 
-    protected void log(Throwable ex) {
-        format.exception(ex);
-    }
-
-
     /**
      * {@inheritDoc}
      */
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Tue Jan 28 00:59:02 2014 +0400
@@ -80,20 +80,32 @@
                 out.println("# Fork: N/A, test runs in the existing VM");
             }
 
-            switch (mode) {
-                case WARMUP: {
-                    runBenchmark(benchmark, mode);
-                    out.println("");
-                    break;
+            try {
+                switch (mode) {
+                    case WARMUP: {
+                        runBenchmark(benchmark, mode);
+                        out.println("");
+                        break;
+                    }
+                    case WARMUP_MEASUREMENT:
+                    case MEASUREMENT: {
+                        BenchResult r = runBenchmark(benchmark, mode);
+                        results.put(benchmark, r);
+                        break;
+                    }
+                    default:
+                        throw new IllegalStateException("Unknown mode: " + mode);
+
                 }
-                case WARMUP_MEASUREMENT:
-                case MEASUREMENT: {
-                    BenchResult r = runBenchmark(benchmark, mode);
-                    results.put(benchmark, r);
-                    break;
+            } catch (BenchmarkException be) {
+                out.println("<failure>");
+                out.println("");
+                be.getCause().printStackTrace();
+                out.println("");
+
+                if (options.shouldFailOnError().orElse(Defaults.FAIL_ON_ERROR)) {
+                    throw be;
                 }
-                default:
-                    throw new IllegalStateException("Unknown mode: " + mode);
             }
 
             if (!forked) {
@@ -162,18 +174,15 @@
             handler = MicroBenchmarkHandlers.getInstance(out, benchmark, clazz, method, executionParams, options);
 
             return runBenchmark(executionParams, handler);
+        } catch (BenchmarkException be) {
+            throw be;
         } catch (Throwable ex) {
-            out.exception(ex);
-            if (options.shouldFailOnError().orElse(Defaults.FAIL_ON_ERROR)) {
-                throw new IllegalStateException(ex.getMessage(), ex);
-            }
+            throw new BenchmarkException(ex);
         } finally {
             if (handler != null) {
                 handler.shutdown();
             }
         }
-        // FIXME: Better handling here!
-        return null;
     }
 
     protected BenchResult runBenchmark(BenchmarkParams executionParams, MicroBenchmarkHandler handler) {
@@ -208,24 +217,16 @@
 
             boolean isLastIteration = (i == mp.getCount());
             IterationResult iterData = handler.runIteration(mp, isLastIteration);
-
-            // might get an exception above, in which case the results list will be empty
-            if (iterData.isResultsEmpty()) {
-                out.println("WARNING: No results returned, benchmark payload threw exception?");
-            } else {
-                out.iterationResult(handler.getBenchmark(), mp, i, IterationType.MEASUREMENT, iterData);
-
-                allResults.add(iterData);
-            }
+            out.iterationResult(handler.getBenchmark(), mp, i, IterationType.MEASUREMENT, iterData);
+            allResults.add(iterData);
         }
 
-        // only print end-of-run output if we have actual results
         if (!allResults.isEmpty()) {
             BenchResult result = new BenchResult(allResults);
             out.endBenchmark(handler.getBenchmark(), result);
             return result;
         } else {
-            // FIXME: Better handling here!
+            // should be ignored in the caller
             return null;
         }
     }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkException.java	Tue Jan 28 00:59:02 2014 +0400
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.runner;
+
+public class BenchmarkException extends RuntimeException {
+    public BenchmarkException(Throwable ex) {
+        super(ex);
+    }
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Tue Jan 28 00:59:02 2014 +0400
@@ -49,8 +49,12 @@
     public void run() throws IOException, ClassNotFoundException {
         ActionPlan actionPlan = link.requestPlan();
 
-        Multimap<BenchmarkRecord,BenchResult> res = runBenchmarks(true, actionPlan);
-        link.pushResults(res);
+        try {
+            Multimap<BenchmarkRecord,BenchResult> res = runBenchmarks(true, actionPlan);
+            link.pushResults(res);
+        } catch (BenchmarkException be) {
+            link.pushException(be);
+        }
 
         out.flush();
         out.close();
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/LoopMicroBenchmarkHandler.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/LoopMicroBenchmarkHandler.java	Tue Jan 28 00:59:02 2014 +0400
@@ -119,7 +119,7 @@
         try {
             preSetupBarrier.await();
         } catch (InterruptedException ex) {
-            log(ex);
+            throw new BenchmarkException(ex);
         }
 
         // profilers start when iteration starts
@@ -137,7 +137,7 @@
         try {
             preTearDownBarrier.await();
         } catch (InterruptedException ex) {
-            log(ex);
+            throw new BenchmarkException(ex);
         }
 
         // profilers stop when iteration ends
@@ -152,17 +152,10 @@
                     fr.get(runtime.getTime() * 2, runtime.getTimeUnit());
                     expected--;
                 } catch (InterruptedException ex) {
-                    log(ex);
-                    iterationResults.clearResults();
-                    return iterationResults;
+                    throw new BenchmarkException(ex);
                 } catch (ExecutionException ex) {
                     Throwable cause = ex.getCause().getCause(); // unwrap
-                    log(cause);
-                    iterationResults.clearResults();
-                    if (shouldFailOnError) {
-                        throw new IllegalStateException(cause.getMessage(), cause);
-                    }
-                    return iterationResults;
+                    throw new BenchmarkException(cause);
                 } catch (TimeoutException e) {
                     // do nothing, respin
                 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Tue Jan 28 00:59:02 2014 +0400
@@ -252,7 +252,7 @@
         return result;
     }
 
-    private SortedMap<BenchmarkRecord, RunResult> runBenchmarks(SortedSet<BenchmarkRecord> benchmarks) {
+    private SortedMap<BenchmarkRecord, RunResult> runBenchmarks(SortedSet<BenchmarkRecord> benchmarks) throws RunnerException {
         out.startRun();
 
         Multimap<BenchmarkRecord, BenchResult> results = new TreeMultimap<BenchmarkRecord, BenchResult>();
@@ -260,27 +260,31 @@
 
         beforeBenchmarks(plan);
 
-        for (ActionPlan r : plan) {
-            Multimap<BenchmarkRecord, BenchResult> res;
-            switch (r.getType()) {
-                case EMBEDDED:
-                    res = runBenchmarks(false, r);
-                    break;
-                case FORKED:
-                    res = runSeparate(r);
-                    break;
-                default:
-                    throw new IllegalStateException("Unknown action plan type: " + r.getType());
+        try {
+            for (ActionPlan r : plan) {
+                Multimap<BenchmarkRecord, BenchResult> res;
+                switch (r.getType()) {
+                    case EMBEDDED:
+                        res = runBenchmarks(false, r);
+                        break;
+                    case FORKED:
+                        res = runSeparate(r);
+                        break;
+                    default:
+                        throw new IllegalStateException("Unknown action plan type: " + r.getType());
+                }
+
+                for (BenchmarkRecord br : res.keys()) {
+                    results.putAll(br, res.get(br));
+                }
             }
 
-            for (BenchmarkRecord br : res.keys()) {
-                results.putAll(br, res.get(br));
-            }
+            SortedMap<BenchmarkRecord, RunResult> runResults = mergeRunResults(results);
+            out.endRun(runResults);
+            return runResults;
+        } catch (BenchmarkException be) {
+            throw new RunnerException("Benchmark caught the exception", be.getCause());
         }
-
-        SortedMap<BenchmarkRecord, RunResult> runResults = mergeRunResults(results);
-        out.endRun(runResults);
-        return runResults;
     }
 
     private SortedMap<BenchmarkRecord, RunResult> mergeRunResults(Multimap<BenchmarkRecord, BenchResult> results) {
@@ -378,19 +382,27 @@
             reader.waitFinish();
 
             if (ecode != 0) {
-                out.println("WARNING: Forked process returned code: " + ecode);
+                out.println("<forked VM failed with exit code " + ecode + ">");
+                out.println("");
                 if (options.shouldFailOnError().orElse(Defaults.FAIL_ON_ERROR)) {
-                    throw new IllegalStateException("WARNING: Forked process returned code: " + ecode);
+                    throw new BenchmarkException(
+                        new IllegalStateException("Forked VM failed with exit code " + ecode)
+                    );
                 }
             }
 
         } catch (IOException ex) {
-            out.exception(ex);
+            throw new BenchmarkException(ex);
         } catch (InterruptedException ex) {
-            out.exception(ex);
+            throw new BenchmarkException(ex);
         }
 
-        return reader.getResults();
+        BenchmarkException exception = reader.getException();
+        if (exception == null) {
+            return reader.getResults();
+        } else {
+            throw exception;
+        }
     }
 
     /**
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/RunnerException.java	Mon Jan 27 18:38:20 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/RunnerException.java	Tue Jan 28 00:59:02 2014 +0400
@@ -32,4 +32,8 @@
     public RunnerException() {
         super();
     }
+
+    public RunnerException(String s, Throwable cause) {
+        super(s, cause);
+    }
 }