changeset 1359:950f2655fa2b

7901809: Failing @Setup can be entered multiple times Summary: Implement quick abort when failures are detected in helper methods.
author shade
date Fri, 30 Sep 2016 12:53:54 +0200
parents c9e016659bfe
children 318b0c315634
files jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/BenchmarkInvocationFailureTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/BenchmarkIterationFailureTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/BenchmarkTrialFailureTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/GroupInvocationFailureTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/GroupIterationFailureTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/GroupTrialFailureTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/ThreadInvocationFailureTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/ThreadIterationFailureTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/ThreadTrialFailureTimesTest.java jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java jmh-core/src/main/java/org/openjdk/jmh/generators/core/StateObjectHandler.java jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkException.java jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkHandler.java jmh-core/src/main/java/org/openjdk/jmh/runner/FailureAssistException.java jmh-core/src/main/java/org/openjdk/jmh/runner/InfraControl.java
diffstat 15 files changed, 800 insertions(+), 27 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/BenchmarkInvocationFailureTimesTest.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times.fail;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Benchmark)
+public class BenchmarkInvocationFailureTimesTest {
+
+    private static final AtomicInteger count = new AtomicInteger();
+
+    @Setup(Level.Invocation)
+    public void setup() {
+        count.incrementAndGet();
+        throw new IllegalStateException("Failing");
+    }
+
+    @Benchmark
+    public void test() {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            for (Mode mode : Mode.values()) {
+                if (mode == Mode.All) continue;
+
+                count.set(0);
+
+                Options opt = new OptionsBuilder()
+                        .include(Fixtures.getTestMask(this.getClass()))
+                        .mode(mode)
+                        .threads(Threads.MAX)
+                        .forks(0)
+                        .build();
+                new Runner(opt).run();
+
+                Assert.assertEquals(1, count.get());
+            }
+        }
+    }
+
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/BenchmarkIterationFailureTimesTest.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times.fail;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Benchmark)
+public class BenchmarkIterationFailureTimesTest {
+
+    private static final AtomicInteger count = new AtomicInteger();
+
+    @Setup(Level.Iteration)
+    public void setup() {
+        count.incrementAndGet();
+        throw new IllegalStateException("Failing");
+    }
+
+    @Benchmark
+    public void test() {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            for (Mode mode : Mode.values()) {
+                if (mode == Mode.All) continue;
+
+                count.set(0);
+
+                Options opt = new OptionsBuilder()
+                        .include(Fixtures.getTestMask(this.getClass()))
+                        .mode(mode)
+                        .threads(Threads.MAX)
+                        .forks(0)
+                        .build();
+                new Runner(opt).run();
+
+                Assert.assertEquals(1, count.get());
+            }
+        }
+    }
+
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/BenchmarkTrialFailureTimesTest.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times.fail;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Benchmark)
+public class BenchmarkTrialFailureTimesTest {
+
+    private static final AtomicInteger count = new AtomicInteger();
+
+    @Setup(Level.Trial)
+    public void setup() {
+        count.incrementAndGet();
+        throw new IllegalStateException("Failing");
+    }
+
+    @Benchmark
+    public void test() {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            for (Mode mode : Mode.values()) {
+                if (mode == Mode.All) continue;
+
+                count.set(0);
+
+                Options opt = new OptionsBuilder()
+                        .include(Fixtures.getTestMask(this.getClass()))
+                        .mode(mode)
+                        .threads(Threads.MAX)
+                        .forks(0)
+                        .build();
+                new Runner(opt).run();
+
+                Assert.assertEquals(1, count.get());
+            }
+        }
+    }
+
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/GroupInvocationFailureTimesTest.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times.fail;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Group)
+public class GroupInvocationFailureTimesTest {
+
+    private static final AtomicInteger count = new AtomicInteger();
+
+    @Setup(Level.Invocation)
+    public void setup() {
+        count.incrementAndGet();
+        throw new IllegalStateException("Failing");
+    }
+
+    @Benchmark
+    @Group
+    @GroupThreads(4) // should match the total number of threads
+    public void test() {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            for (Mode mode : Mode.values()) {
+                if (mode == Mode.All) continue;
+
+                count.set(0);
+
+                Options opt = new OptionsBuilder()
+                        .include(Fixtures.getTestMask(this.getClass()))
+                        .mode(mode)
+                        .threads(4)
+                        .forks(0)
+                        .build();
+                new Runner(opt).run();
+
+                Assert.assertEquals(1, count.get());
+            }
+        }
+    }
+
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/GroupIterationFailureTimesTest.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times.fail;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Group)
+public class GroupIterationFailureTimesTest {
+
+    private static final AtomicInteger count = new AtomicInteger();
+
+    @Setup(Level.Iteration)
+    public void setup() {
+        System.err.println("Enter " + Thread.currentThread());
+        count.incrementAndGet();
+        throw new IllegalStateException("Failing");
+    }
+
+    @Benchmark
+    @Group
+    @GroupThreads(4) // should match the total number of threads
+    public void test() {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            for (Mode mode : Mode.values()) {
+                if (mode == Mode.All) continue;
+
+                count.set(0);
+
+                Options opt = new OptionsBuilder()
+                        .include(Fixtures.getTestMask(this.getClass()))
+                        .mode(mode)
+                        .threads(4)
+                        .forks(0)
+                        .build();
+                new Runner(opt).run();
+
+                Assert.assertEquals(1, count.get());
+            }
+        }
+    }
+
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/GroupTrialFailureTimesTest.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times.fail;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Group)
+public class GroupTrialFailureTimesTest {
+
+    private static final AtomicInteger count = new AtomicInteger();
+
+    @Setup(Level.Trial)
+    public void setup() {
+        count.incrementAndGet();
+        throw new IllegalStateException("Failing");
+    }
+
+    @Benchmark
+    @Group
+    public void test() {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            for (Mode mode : Mode.values()) {
+                if (mode == Mode.All) continue;
+
+                count.set(0);
+
+                Options opt = new OptionsBuilder()
+                        .include(Fixtures.getTestMask(this.getClass()))
+                        .mode(mode)
+                        .threads(Threads.MAX)
+                        .forks(0)
+                        .build();
+                new Runner(opt).run();
+
+                Assert.assertEquals(1, count.get());
+            }
+        }
+    }
+
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/ThreadInvocationFailureTimesTest.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times.fail;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Thread)
+public class ThreadInvocationFailureTimesTest {
+
+    private static final AtomicInteger count = new AtomicInteger();
+
+    @Setup(Level.Invocation)
+    public void setup() {
+        count.incrementAndGet();
+        throw new IllegalStateException("Failing");
+    }
+
+    @Benchmark
+    public void test() {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            for (Mode mode : Mode.values()) {
+                if (mode == Mode.All) continue;
+
+                count.set(0);
+
+                Options opt = new OptionsBuilder()
+                        .include(Fixtures.getTestMask(this.getClass()))
+                        .mode(mode)
+                        .threads(1)
+                        .forks(0)
+                        .build();
+                new Runner(opt).run();
+
+                Assert.assertEquals(1, count.get());
+            }
+        }
+    }
+
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/ThreadIterationFailureTimesTest.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times.fail;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Thread)
+public class ThreadIterationFailureTimesTest {
+
+    private static final AtomicInteger count = new AtomicInteger();
+
+    @Setup(Level.Iteration)
+    public void setup() {
+        count.incrementAndGet();
+        throw new IllegalStateException("Failing");
+    }
+
+    @Benchmark
+    public void test() {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            for (Mode mode : Mode.values()) {
+                if (mode == Mode.All) continue;
+
+                count.set(0);
+
+                Options opt = new OptionsBuilder()
+                        .include(Fixtures.getTestMask(this.getClass()))
+                        .mode(mode)
+                        .threads(1)
+                        .forks(0)
+                        .build();
+                new Runner(opt).run();
+
+                Assert.assertEquals(1, count.get());
+            }
+        }
+    }
+
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/fail/ThreadTrialFailureTimesTest.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.times.fail;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Thread)
+public class ThreadTrialFailureTimesTest {
+
+    private static final AtomicInteger count = new AtomicInteger();
+
+    @Setup(Level.Trial)
+    public void setup() {
+        count.incrementAndGet();
+        throw new IllegalStateException("Failing");
+    }
+
+    @Benchmark
+    public void test() {
+        Fixtures.work();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int c = 0; c < Fixtures.repetitionCount(); c++) {
+            for (Mode mode : Mode.values()) {
+                if (mode == Mode.All) continue;
+
+                count.set(0);
+
+                Options opt = new OptionsBuilder()
+                        .include(Fixtures.getTestMask(this.getClass()))
+                        .mode(mode)
+                        .threads(1)
+                        .forks(0)
+                        .build();
+                new Runner(opt).run();
+
+                Assert.assertEquals(1, count.get());
+            }
+        }
+    }
+
+
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java	Fri Sep 30 11:04:24 2016 +0200
+++ b/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java	Fri Sep 30 12:53:54 2016 +0200
@@ -27,10 +27,8 @@
 import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.infra.*;
 import org.openjdk.jmh.results.*;
-import org.openjdk.jmh.runner.BenchmarkList;
-import org.openjdk.jmh.runner.BenchmarkListEntry;
+import org.openjdk.jmh.runner.*;
 import org.openjdk.jmh.runner.Defaults;
-import org.openjdk.jmh.runner.InfraControl;
 import org.openjdk.jmh.util.FileUtils;
 import org.openjdk.jmh.util.HashMultimap;
 import org.openjdk.jmh.util.Multimap;
@@ -534,7 +532,8 @@
                 BenchmarkMode.class, RawResults.class, ResultRole.class,
                 Field.class, BenchmarkParams.class, IterationParams.class,
                 Blackhole.class, Control.class,
-                ScalarResult.class, AggregationPolicy.class
+                ScalarResult.class, AggregationPolicy.class,
+                FailureAssistException.class
         };
 
         for (Class<?> c : imports) {
--- a/jmh-core/src/main/java/org/openjdk/jmh/generators/core/StateObjectHandler.java	Fri Sep 30 11:04:24 2016 +0200
+++ b/jmh-core/src/main/java/org/openjdk/jmh/generators/core/StateObjectHandler.java	Fri Sep 30 12:53:54 2016 +0200
@@ -619,6 +619,7 @@
             if (type == HelperType.SETUP) {
                 result.add("if (" + so.type + ".setup" + helperLevel + "MutexUpdater.compareAndSet(" + so.localIdentifier + ", 0, 1)) {");
                 result.add("    try {");
+                result.add("        if (control.isFailing) throw new FailureAssistException();");
                 result.add("        if (!" + so.localIdentifier + ".ready" + helperLevel + ") {");
                 for (HelperMethodInvocation mi : so.getHelpers()) {
                     if (mi.helperLevel == helperLevel && mi.type == HelperType.SETUP) {
@@ -628,11 +629,15 @@
                 }
                 result.add("            " + so.localIdentifier + ".ready" + helperLevel + " = true;");
                 result.add("        }");
+                result.add("    } catch (Throwable t) {");
+                result.add("        control.isFailing = true;");
+                result.add("        throw t;");
                 result.add("    } finally {");
                 result.add("        " + so.type + ".setup" + helperLevel + "MutexUpdater.set(" + so.localIdentifier + ", 0);");
                 result.add("    }");
                 result.add("} else {");
                 result.add("    while (" + so.type + ".setup" + helperLevel + "MutexUpdater.get(" + so.localIdentifier + ") == 1) {");
+                result.add("        if (control.isFailing) throw new FailureAssistException();");
                 result.add("        if (Thread.interrupted()) throw new InterruptedException();");
                 result.add("    }");
                 result.add("}");
@@ -645,6 +650,7 @@
             if (type == HelperType.TEARDOWN) {
                 result.add("if (" + so.type + ".tear" + helperLevel + "MutexUpdater.compareAndSet(" + so.localIdentifier + ", 0, 1)) {");
                 result.add("    try {");
+                result.add("        if (control.isFailing) throw new FailureAssistException();");
                 result.add("        if (" + so.localIdentifier + ".ready" + helperLevel + ") {");
                 for (HelperMethodInvocation mi : so.getHelpers()) {
                     if (mi.helperLevel == helperLevel && mi.type == HelperType.TEARDOWN) {
@@ -654,6 +660,9 @@
                 }
                 result.add("            " + so.localIdentifier + ".ready" + helperLevel + " = false;");
                 result.add("        }");
+                result.add("    } catch (Throwable t) {");
+                result.add("        control.isFailing = true;");
+                result.add("        throw t;");
                 result.add("    } finally {");
                 result.add("        " + so.type + ".tear" + helperLevel + "MutexUpdater.set(" + so.localIdentifier + ", 0);");
                 result.add("    }");
@@ -673,6 +682,7 @@
                     result.add("        " + so.localIdentifier + "_backoff = Math.max(1024, " + so.localIdentifier + "_backoff * 2);");
                 }
 
+                result.add("        if (control.isFailing) throw new FailureAssistException();");
                 result.add("        if (Thread.interrupted()) throw new InterruptedException();");
                 result.add("    }");
                 result.add("}");
@@ -727,6 +737,8 @@
             result.add("        return val;");
             result.add("    }");
             result.add("    synchronized(this.getClass()) {");
+            result.add("        try {");
+            result.add("        if (control.isFailing) throw new FailureAssistException();");
             result.add("        val = " + so.fieldIdentifier + ";");
             result.add("        if (val != null) {");
             result.add("            return val;");
@@ -750,6 +762,10 @@
             }
             result.add("        val.ready" + Level.Trial + " = true;");
             result.add("        " + so.fieldIdentifier + " = val;");
+            result.add("        } catch (Throwable t) {");
+            result.add("            control.isFailing = true;");
+            result.add("            throw t;");
+            result.add("        }");
             result.add("    }");
             result.add("    return val;");
             result.add("}");
@@ -762,7 +778,7 @@
             result.add(so.type + " " + so.fieldIdentifier + ";");
             result.add("");
             result.add(so.type + " _jmh_tryInit_" + so.fieldIdentifier + "(InfraControl control" + soDependency_TypeArgs(so) + ") throws Throwable {");
-
+            result.add("    if (control.isFailing) throw new FailureAssistException();");
             result.add("    " + so.type + " val = " + so.fieldIdentifier + ";");
             result.add("    if (val == null) {");
             result.add("        val = new " + so.type + "();");
@@ -802,6 +818,8 @@
             result.add("        return val;");
             result.add("    }");
             result.add("    synchronized(this.getClass()) {");
+            result.add("        try {");
+            result.add("        if (control.isFailing) throw new FailureAssistException();");
             result.add("        val = " + so.fieldIdentifier + "_map.get(groupIdx);");
             result.add("        if (val != null) {");
             result.add("            return val;");
@@ -825,6 +843,10 @@
             }
             result.add("        " + "val.ready" + Level.Trial + " = true;");
             result.add("        " + so.fieldIdentifier + "_map.put(groupIdx, val);");
+            result.add("        } catch (Throwable t) {");
+            result.add("            control.isFailing = true;");
+            result.add("            throw t;");
+            result.add("        }");
             result.add("    }");
             result.add("    return val;");
             result.add("}");
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkException.java	Fri Sep 30 11:04:24 2016 +0200
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkException.java	Fri Sep 30 12:53:54 2016 +0200
@@ -33,4 +33,8 @@
     public BenchmarkException(Throwable ex) {
         super(ex);
     }
+
+    public BenchmarkException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkHandler.java	Fri Sep 30 11:04:24 2016 +0200
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkHandler.java	Fri Sep 30 12:53:54 2016 +0200
@@ -400,29 +400,43 @@
         // either normally or abnormally. This means, Future.get() would never block.
         long allOps = 0;
         long measuredOps = 0;
-        IterationResult result;
-        try {
-            for (Future<BenchmarkTaskResult> fr : completed) {
-                try {
-                    BenchmarkTaskResult btr = fr.get();
-                    iterationResults.addAll(btr.getResults());
-                    allOps += btr.getAllOps();
-                    measuredOps += btr.getMeasuredOps();
-                } catch (ExecutionException ex) {
-                    // unwrap: ExecutionException -> Throwable-wrapper -> InvocationTargetException
-                    Throwable cause = ex.getCause().getCause().getCause();
-                    throw new BenchmarkException(cause);
-                } catch (InterruptedException ex) {
-                    throw new BenchmarkException(ex);
+
+        List<Throwable> errors = new ArrayList<Throwable>();
+        for (Future<BenchmarkTaskResult> fr : completed) {
+            try {
+                BenchmarkTaskResult btr = fr.get();
+                iterationResults.addAll(btr.getResults());
+                allOps += btr.getAllOps();
+                measuredOps += btr.getMeasuredOps();
+            } catch (ExecutionException ex) {
+                // unwrap: ExecutionException -> Throwable-wrapper -> InvocationTargetException
+                Throwable cause = ex.getCause().getCause().getCause();
+
+                // record exception, unless it is the assist exception
+                if (!(cause instanceof FailureAssistException)) {
+                    errors.add(cause);
                 }
+            } catch (InterruptedException ex) {
+                // cannot happen here, Future.get() should never block.
+                throw new BenchmarkException(ex);
             }
-        } finally {
-            result = new IterationResult(benchmarkParams, params, new IterationResultMetaData(allOps, measuredOps));
-            result.addResults(iterationResults);
+        }
 
-            // profilers stop when after all threads are confirmed to be
-            // finished to capture the edge behaviors; or, on a failure path
-            stopProfilers(benchmarkParams, params, result);
+        IterationResult result = new IterationResult(benchmarkParams, params, new IterationResultMetaData(allOps, measuredOps));
+        result.addResults(iterationResults);
+
+        // profilers stop when after all threads are confirmed to be
+        // finished to capture the edge behaviors; or, on a failure path
+        stopProfilers(benchmarkParams, params, result);
+
+        if (!errors.isEmpty()) {
+            Throwable first = errors.get(0);
+            if (errors.size() > 1) {
+                // TODO: Once we switch to JDK 7, make use of #addSupressed.
+                throw new BenchmarkException(errors.size() + " pending exceptions, reporting the first", first);
+            } else {
+                throw new BenchmarkException(first);
+            }
         }
 
         return result;
@@ -450,6 +464,10 @@
                 return (BenchmarkTaskResult) method.invoke(td.instance, control, td.params);
             } catch (Throwable e) {
                 // about to fail the iteration;
+
+                // notify other threads we have failed
+                control.isFailing = true;
+
                 // compensate for missed sync-iteration latches, we don't care about that anymore
                 control.preSetupForce();
                 control.preTearDownForce();
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/FailureAssistException.java	Fri Sep 30 12:53:54 2016 +0200
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2016, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.runner;
+
+/**
+ * Thrown by worker threads when they detect other threads have failed.
+ */
+public class FailureAssistException extends RuntimeException {
+
+
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/InfraControl.java	Fri Sep 30 11:04:24 2016 +0200
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/InfraControl.java	Fri Sep 30 12:53:54 2016 +0200
@@ -44,7 +44,7 @@
      * @see org.openjdk.jmh.infra.Blackhole description for the rationale
      */
     static {
-        Utils.check(InfraControl.class, "isDone");
+        Utils.check(InfraControl.class, "isDone", "isFailing");
         Utils.check(InfraControl.class, "volatileSpoiler");
         Utils.check(InfraControl.class, "preSetup", "preTearDown");
         Utils.check(InfraControl.class, "lastIteration");
@@ -133,11 +133,18 @@
 }
 
 abstract class InfraControlL2 extends InfraControlL1 {
-    /* Flag for if we are done or not.
+    /**
+     * Flag that checks for time expiration.
      * This is specifically the public field, so to spare one virtual call.
      */
     public volatile boolean isDone;
 
+    /**
+     * Flag that checks for failure experienced by any measurement thread.
+     * This is specifically the public field, so to spare one virtual call.
+     */
+    public volatile boolean isFailing;
+
     public volatile boolean volatileSpoiler;
 
     public final CountDownLatch preSetup;