changeset 154:b6cfcb9b53a6

MIrror integration tests with API-enabled variants.
author shade
date Wed, 28 Aug 2013 15:45:54 +0400
parents 2621152f8474
children c84f78395242
files jmh-core-it/src/test/java/org/openjdk/jmh/it/IterationCountAnnTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/IterationCountCmdTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/SingleShotTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/WarmupIterationCountAnnTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/WarmupIterationCountCmdTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode0_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode1_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode2_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode3_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode4_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode5_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode6_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/ccontrol/CompilerControlDontInlineTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/ccontrol/CompilerControlExcludeTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/ccontrol/CompilerControlInlineTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkBenchSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkBenchTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkBenchTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkStateSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkStateTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingForkedBenchTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingForkedSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingForkedTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupBenchSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupBenchTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupBenchTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupStateSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupStateTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadBenchSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadBenchTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadBenchTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadStateSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadStateTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkBenchSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkBenchTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkStateSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkStateTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupBenchSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupBenchTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupStateSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupStateTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadBenchSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadBenchTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadStateSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadStateTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkBenchSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkBenchTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkStateSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkStateTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupBenchSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupBenchTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupStateSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupStateTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadBenchSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadBenchTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadStateSetupTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadStateTearDownTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/AnnotatedClassForkedTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/AnnotatedForked1_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/AnnotatedForked2_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmAppendPrependArgs1_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmAppendPrependArgs2_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmArgs1_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmArgs2_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmArgs3_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_D_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_F0_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_F1_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_F_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_D_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_F0_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_F1_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_F_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_D_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_F0_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_F1_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_F_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_D_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_F0_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_F1_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_F_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/BenchmarkBenchOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/BenchmarkStateOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/GroupBenchOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/GroupStateOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/ThreadBenchOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/ThreadStateOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkBenchSetupOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkBenchTearDownOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkStateSetupOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkStateTearDownOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupBenchSetupOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupBenchTearDownOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupStateSetupOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupStateTearDownOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadBenchSetupOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadBenchTearDownOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadStateSetupOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadStateTearDownOrderTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceBenchmarkStateInvocationTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceBenchmarkStateIterationTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceBenchmarkStateRunTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceGroupStateInvocationTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceGroupStateIterationTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceGroupStateRunTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceThreadStateInvocationTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceThreadStateIterationTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceThreadStateRunTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/BenchmarkBenchSharingTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/BenchmarkStateSharingTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupBenchSharingTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupDefaultBenchSharingTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupDefaultStateSharingTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupStateSharingTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/ThreadBenchSharingTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/ThreadStateSharingTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/synciter/SyncIterMeasurementOnlyTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/synciter/SyncIterNotOnlyMeasurementTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/BenchmarkBenchSameThreadTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/BenchmarkStateSameThreadTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/GroupBenchSameThreadTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/GroupStateSameThreadTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/OneThreadCountTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/ThreadBenchSameThreadTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/ThreadStateSameThreadTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/TwoThreadCountTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/ZeroThreadCountTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/BenchmarkBenchHelperTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/BenchmarkStateHelperTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/GroupBenchHelperTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/GroupStateHelperTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/ThreadBenchHelperTimesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/times/ThreadStateHelperTimesTest.java
diffstat 134 files changed, 2437 insertions(+), 173 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/IterationCountAnnTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/IterationCountAnnTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -66,8 +70,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -v");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opts = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .verbose(true)
+                .build();
+        new Runner(opts).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/IterationCountCmdTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/IterationCountCmdTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -64,8 +68,19 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -v -i 1 -wi 0");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .measurementIterations(1)
+                .warmupIterations(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/SingleShotTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/SingleShotTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -30,6 +30,10 @@
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
 import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -59,13 +63,33 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -f");
     }
 
     @Test
-    public void invoke1() {
+    public void invokeCLI_1() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -f 2");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+    }
+
+    @Test
+    public void invokeAPI_1() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(2)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/WarmupIterationCountAnnTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/WarmupIterationCountAnnTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -65,8 +69,17 @@
     }
 
     @Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
     }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/WarmupIterationCountCmdTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/WarmupIterationCountCmdTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -32,6 +32,11 @@
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -61,8 +66,22 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -v -w 1 -r 1 -i 1 -wi 3");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .warmupTime(TimeValue.seconds(1))
+                .measurementTime(TimeValue.seconds(1))
+                .measurementIterations(1)
+                .warmupIterations(3)
+                .build();
+        new Runner(opt).run();
+    }
+
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode0_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode0_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,11 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.it.Fixtures;
 import org.openjdk.jmh.logic.Control;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -96,9 +101,31 @@
     }
 
     @Test
-    public void invoke0() {
+    public void invokeCLI() {
         testSequence.clear();
+
         Main.testMain(Fixtures.getTestMask(this.getClass()) + ".testBig.* -foe -w 1 -r 2 -t 1 -i 1 -wi 2 -f false -si false");
+
+        assertEquals("WWI", getSequence());
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        testSequence.clear();
+
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()) + ".testBig.*")
+                .failOnError(true)
+                .warmupIterations(2)
+                .warmupTime(TimeValue.seconds(1))
+                .measurementIterations(1)
+                .measurementTime(TimeValue.seconds(2))
+                .threads(1)
+                .forks(0)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+
         assertEquals("WWI", getSequence());
     }
 
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode1_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode1_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,11 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.it.Fixtures;
 import org.openjdk.jmh.logic.Control;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -96,9 +101,30 @@
     }
 
     @Test
-    public void invoke1() {
+    public void invokeCLI() {
         testSequence.clear();
         Main.testMain(Fixtures.getTestMask(this.getClass()) + ".testSmall.* -foe -w 1 -r 2 -t 1 -i 1 -wi 2 -f false -si false");
         assertEquals("wwi", getSequence());
     }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        testSequence.clear();
+
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()) + ".testSmall.*")
+                .failOnError(true)
+                .warmupIterations(2)
+                .warmupTime(TimeValue.seconds(1))
+                .measurementIterations(1)
+                .measurementTime(TimeValue.seconds(2))
+                .threads(1)
+                .forks(0)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+
+        assertEquals("wwi", getSequence());
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode2_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode2_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,11 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.it.Fixtures;
 import org.openjdk.jmh.logic.Control;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -96,9 +101,30 @@
     }
 
     @Test
-    public void invoke2() {
+    public void invokeCLI() {
         testSequence.clear();
         Main.testMain(Fixtures.getTestMask(this.getClass()) + ".testBig.* -foe -w 1 -r 2 -t 2 -i 1 -wi 2 -f false -si false");
         assertEquals("WWWWII", getSequence());
     }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        testSequence.clear();
+
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()) + ".testBig.*")
+                .failOnError(true)
+                .warmupIterations(2)
+                .warmupTime(TimeValue.seconds(1))
+                .measurementIterations(1)
+                .measurementTime(TimeValue.seconds(2))
+                .threads(2)
+                .forks(0)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+
+        assertEquals("WWWWII", getSequence());
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode3_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode3_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,11 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.it.Fixtures;
 import org.openjdk.jmh.logic.Control;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -96,9 +101,29 @@
     }
 
     @Test
-    public void invoke3() {
+    public void invokeCLI() {
         testSequence.clear();
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -w 1 -r 2 -t 2 -i 1 -wi 2 -f false -si false");
         assertEquals("WWWWIIwwwwii", getSequence());
     }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        testSequence.clear();
+
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .warmupIterations(2)
+                .warmupTime(TimeValue.seconds(1))
+                .measurementIterations(1)
+                .measurementTime(TimeValue.seconds(2))
+                .threads(2)
+                .forks(0)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+
+        assertEquals("WWWWIIwwwwii", getSequence());
+    }
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode4_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode4_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,11 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.it.Fixtures;
 import org.openjdk.jmh.logic.Control;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -101,4 +106,25 @@
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -w 1 -r 2 -t 1 -i 1 -wi 2 -f false -si false");
         assertEquals("WWIwwi", getSequence());
     }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        testSequence.clear();
+
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .warmupIterations(2)
+                .warmupTime(TimeValue.seconds(1))
+                .measurementIterations(1)
+                .measurementTime(TimeValue.seconds(2))
+                .threads(1)
+                .forks(0)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+
+        assertEquals("WWIwwi", getSequence());
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode5_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode5_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,12 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.it.Fixtures;
 import org.openjdk.jmh.logic.Control;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.WarmupMode;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -96,10 +102,31 @@
     }
 
     @Test
-    public void invoke5() {
+    public void invokeCLI() {
         testSequence.clear();
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -w 1 -r 2 -t 1 -i 1 -wi 2 -wm beforeeach -f false -si false");
         assertEquals("WWIwwi", getSequence());
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        testSequence.clear();
+
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .warmupIterations(2)
+                .warmupTime(TimeValue.seconds(1))
+                .measurementIterations(1)
+                .measurementTime(TimeValue.seconds(2))
+                .threads(1)
+                .forks(0)
+                .syncIterations(false)
+                .warmupMode(WarmupMode.BEFOREEACH)
+                .build();
+        new Runner(opt).run();
+
+        assertEquals("WWIwwi", getSequence());
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode6_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode6_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,12 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.it.Fixtures;
 import org.openjdk.jmh.logic.Control;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.WarmupMode;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -96,9 +102,31 @@
     }
 
     @Test
-    public void invoke6() {
+    public void invokeCLI() {
         testSequence.clear();
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -w 1 -r 2 -t 1 -i 1 -wi 2 -wm beforeany -f false -si false");
         assertEquals("WWwwIi", getSequence());
     }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        testSequence.clear();
+
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .warmupIterations(2)
+                .warmupTime(TimeValue.seconds(1))
+                .measurementIterations(1)
+                .measurementTime(TimeValue.seconds(2))
+                .threads(1)
+                .forks(0)
+                .syncIterations(false)
+                .warmupMode(WarmupMode.BEFOREANY)
+                .build();
+        new Runner(opt).run();
+
+        assertEquals("WWwwIi", getSequence());
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/ccontrol/CompilerControlDontInlineTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/ccontrol/CompilerControlDontInlineTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -31,9 +31,17 @@
 import org.openjdk.jmh.annotations.Measurement;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.WarmupMode;
+import org.openjdk.jmh.runner.parameters.TimeValue;
 
 import java.util.concurrent.TimeUnit;
 
+import static junit.framework.Assert.assertEquals;
+
 public class CompilerControlDontInlineTest {
 
     @GenerateMicroBenchmark
@@ -45,8 +53,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .verbose(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/ccontrol/CompilerControlExcludeTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/ccontrol/CompilerControlExcludeTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -31,6 +31,10 @@
 import org.openjdk.jmh.annotations.Measurement;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -45,8 +49,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .verbose(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/ccontrol/CompilerControlInlineTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/ccontrol/CompilerControlInlineTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -31,6 +31,10 @@
 import org.openjdk.jmh.annotations.Measurement;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -45,8 +49,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .verbose(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkBenchSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkBenchSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -62,10 +66,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -73,4 +77,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkBenchTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkBenchTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -62,10 +66,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -73,4 +77,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkBenchTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkBenchTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -36,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -60,7 +64,7 @@
     public void invoke() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -68,5 +72,19 @@
         Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkStateSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkStateSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -64,10 +68,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -75,4 +79,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkStateTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingBenchmarkStateTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -64,10 +68,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -75,4 +79,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingForkedBenchTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingForkedBenchTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -35,6 +35,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -55,10 +59,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1 -f");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -66,5 +70,20 @@
         Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .forks(1)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingForkedSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingForkedSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -36,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -61,10 +65,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1 -f");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -72,4 +76,20 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .forks(1)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingForkedTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingForkedTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -36,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -61,10 +65,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1 -f");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -72,5 +76,20 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .forks(1)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupBenchSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupBenchSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -65,7 +69,7 @@
     public void invoke() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -73,4 +77,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupBenchTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupBenchTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -62,10 +66,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -73,4 +77,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupBenchTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupBenchTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -36,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -56,10 +60,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -67,5 +71,19 @@
         Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupStateSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupStateSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -64,10 +68,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -75,4 +79,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupStateTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingGroupStateTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -64,10 +68,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -75,4 +79,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadBenchSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadBenchSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -62,10 +66,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -73,4 +77,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadBenchTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadBenchTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -62,10 +66,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -73,4 +77,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadBenchTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadBenchTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -36,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -59,7 +63,7 @@
     public void invoke() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -67,5 +71,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadStateSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadStateSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -67,7 +71,7 @@
     public void invoke() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -75,4 +79,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadStateTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/FailingThreadStateTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -64,10 +68,10 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
-            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -w 1 -r 1");
+            Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
             failed = false;
         } catch (Throwable t) {
             failed = true;
@@ -75,4 +79,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            org.junit.Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkBenchSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkBenchSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -55,7 +60,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -66,4 +71,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkBenchTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkBenchTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -55,7 +60,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -66,4 +71,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkStateSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkStateSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -57,7 +62,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -68,4 +73,20 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkStateTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractBenchmarkStateTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -57,7 +62,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -68,4 +73,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupBenchSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupBenchSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -55,7 +60,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -66,4 +71,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupBenchTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupBenchTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -55,7 +60,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -66,4 +71,20 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupStateSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupStateSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -57,7 +62,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -68,4 +73,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupStateTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractGroupStateTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -57,7 +62,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -68,4 +73,20 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadBenchSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadBenchSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -55,7 +60,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -66,4 +71,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadBenchTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadBenchTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -55,7 +60,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -66,4 +71,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadStateSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadStateSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -57,7 +62,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -68,4 +73,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadStateTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/AbstractThreadStateTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -35,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -57,7 +62,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -68,4 +73,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkBenchSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkBenchSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -52,7 +57,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -63,4 +68,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkBenchTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkBenchTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -52,7 +57,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -63,4 +68,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkStateSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkStateSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -54,7 +59,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -65,4 +70,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkStateTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritBenchmarkStateTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -54,7 +59,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -65,4 +70,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupBenchSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupBenchSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -52,7 +57,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -63,4 +68,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupBenchTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupBenchTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -52,7 +57,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -63,4 +68,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupStateSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupStateSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -54,7 +59,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -65,4 +70,20 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupStateTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritGroupStateTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -54,7 +59,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -65,4 +70,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadBenchSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadBenchSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -52,7 +57,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -63,4 +68,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadBenchTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadBenchTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -52,7 +57,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -63,4 +68,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadStateSetupTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadStateSetupTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -54,7 +59,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -65,4 +70,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadStateTearDownTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fails/inherit/InheritThreadStateTearDownTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -24,6 +24,7 @@
  */
 package org.openjdk.jmh.it.fails.inherit;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -33,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /**
  * Baseline test:
@@ -54,7 +59,7 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         boolean failed;
         try {
             Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
@@ -65,4 +70,19 @@
         junit.framework.Assert.assertTrue("Should have failed", failed);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        try {
+            Options opt = new OptionsBuilder()
+                    .include(Fixtures.getTestMask(this.getClass()))
+                    .failOnError(true)
+                    .build();
+            new Runner(opt).run();
+
+            Assert.fail("Should have failed");
+        } catch (Throwable t) {
+            // expected
+        }
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/AnnotatedClassForkedTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/AnnotatedClassForkedTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -68,8 +72,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/AnnotatedForked1_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/AnnotatedForked1_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -68,13 +72,32 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe");
     }
 
     @Test
-    public void invoke5() {
+    public void invokeF_CLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -f");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
+    @Test
+    public void invokeF_API() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/AnnotatedForked2_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/AnnotatedForked2_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -69,8 +73,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmAppendPrependArgs1_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmAppendPrependArgs1_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -89,8 +93,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmAppendPrependArgs2_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmAppendPrependArgs2_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -114,8 +118,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmArgs1_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmArgs1_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -65,8 +69,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmArgs2_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmArgs2_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -89,8 +93,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmArgs3_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedJvmArgs3_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -67,8 +71,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/ForkedTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -33,6 +33,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -66,18 +70,49 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -f");
     }
 
     @Test
-    public void invoke1() {
+    public void invokeCLI_1() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -f 1");
     }
 
     @Test
-    public void invokeWF() {
+    public void invokeCLI_WF() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + "  -foe -f 1 -wf 2");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+    }
+
+    @Test
+    public void invokeAPI_1() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+    }
+
+    @Test
+    public void invokeAPI_WF() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .warmupForks(2)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_D_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_D_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -33,6 +33,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -55,10 +59,20 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
 
-        // should execute in another VM.
+        Assert.assertEquals(false, sameVM);
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+
         Assert.assertEquals(false, sameVM);
     }
 
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_F0_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_F0_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -33,6 +33,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -55,11 +59,23 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f 0");
 
         // should execute in the same VM.
         Assert.assertEquals(true, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(true, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_F1_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_F1_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -33,6 +33,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -55,11 +59,23 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f 1");
 
         // should execute in another VM.
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_F_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_D_F_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -33,6 +33,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -55,11 +59,23 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f");
 
         // should execute in another VM.
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_D_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_D_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,22 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
 
         // should execute in the same VM.
         Assert.assertEquals(true, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(true, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_F0_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_F0_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,24 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f 0");
 
         // should execute in same VM.
         Assert.assertEquals(true, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(true, sameVM);
+    }
+
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_F1_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_F1_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,23 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f 1");
 
         // should execute in another VM.
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_F_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F0_F_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,23 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f");
 
         // should execute in another VM.
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_D_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_D_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,22 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
 
         // should execute in another VM.
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_F0_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_F0_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,23 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f 0");
 
         // should execute in same VM.
         Assert.assertEquals(true, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(true, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_F1_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_F1_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -64,4 +68,16 @@
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_F_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F1_F_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,23 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f");
 
         // should execute in another VM.
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_D_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_D_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,22 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
 
         // should execute in another VM.
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_F0_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_F0_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,23 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f 0");
 
         // should execute in the same VM.
         Assert.assertEquals(true, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(true, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_F1_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_F1_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,24 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f 1");
 
         // should execute in another VM.
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_F_Test.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/fork/override/ForkOverride_F_F_Test.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -57,11 +61,23 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f");
 
         // should execute in another VM.
         Assert.assertEquals(false, sameVM);
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(1)
+                .build();
+        new Runner(opt).run();
+
+        Assert.assertEquals(false, sameVM);
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/BenchmarkBenchOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/BenchmarkBenchOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -25,6 +25,7 @@
 package org.openjdk.jmh.it.interorder;
 
 import junit.framework.Assert;
+import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
@@ -38,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -107,9 +112,19 @@
         Fixtures.work();
     }
 
-    @org.junit.Test
-    public void invoke() {
+    @Test
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -si false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/BenchmarkStateOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/BenchmarkStateOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -25,6 +25,7 @@
 package org.openjdk.jmh.it.interorder;
 
 import junit.framework.Assert;
+import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
@@ -38,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -109,9 +114,18 @@
         Fixtures.work();
     }
 
-    @org.junit.Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    @Test
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -si false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+    }
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/GroupBenchOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/GroupBenchOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -25,6 +25,7 @@
 package org.openjdk.jmh.it.interorder;
 
 import junit.framework.Assert;
+import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
@@ -38,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -107,9 +112,19 @@
         Fixtures.work();
     }
 
-    @org.junit.Test
-    public void invoke() {
+    @Test
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -si false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/GroupStateOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/GroupStateOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -25,6 +25,7 @@
 package org.openjdk.jmh.it.interorder;
 
 import junit.framework.Assert;
+import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
@@ -38,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -109,9 +114,19 @@
         Fixtures.work();
     }
 
-    @org.junit.Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    @Test
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -si false");
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
     }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/ThreadBenchOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/ThreadBenchOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -25,6 +25,7 @@
 package org.openjdk.jmh.it.interorder;
 
 import junit.framework.Assert;
+import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
@@ -38,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -107,9 +112,19 @@
         Fixtures.work();
     }
 
-    @org.junit.Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    @Test
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -si false");
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
     }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/ThreadStateOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/interorder/ThreadStateOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -25,6 +25,7 @@
 package org.openjdk.jmh.it.interorder;
 
 import junit.framework.Assert;
+import org.junit.Test;
 import org.openjdk.jmh.Main;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
@@ -38,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -109,9 +114,18 @@
         Fixtures.work();
     }
 
-    @org.junit.Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    @Test
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v -si false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+    }
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkBenchSetupOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkBenchSetupOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -141,8 +145,16 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkBenchTearDownOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkBenchTearDownOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -140,8 +144,16 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkStateSetupOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkStateSetupOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -143,8 +147,16 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkStateTearDownOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/BenchmarkStateTearDownOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -142,8 +146,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupBenchSetupOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupBenchSetupOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -141,8 +145,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupBenchTearDownOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupBenchTearDownOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -140,8 +144,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupStateSetupOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupStateSetupOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -143,8 +147,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupStateTearDownOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/GroupStateTearDownOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -142,8 +146,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadBenchSetupOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadBenchSetupOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -141,8 +145,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadBenchTearDownOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadBenchTearDownOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -140,8 +144,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadStateSetupOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadStateSetupOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -143,8 +147,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadStateTearDownOrderTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/intraorder/ThreadStateTearDownOrderTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -37,6 +37,10 @@
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -142,8 +146,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceBenchmarkStateInvocationTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceBenchmarkStateInvocationTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -79,8 +83,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceBenchmarkStateIterationTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceBenchmarkStateIterationTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -78,8 +82,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceBenchmarkStateRunTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceBenchmarkStateRunTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -78,8 +82,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceGroupStateInvocationTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceGroupStateInvocationTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -79,8 +83,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceGroupStateIterationTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceGroupStateIterationTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -78,8 +82,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceGroupStateRunTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceGroupStateRunTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -78,8 +82,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceThreadStateInvocationTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceThreadStateInvocationTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -78,8 +82,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceThreadStateIterationTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceThreadStateIterationTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -78,8 +82,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceThreadStateRunTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/races/RaceThreadStateRunTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -78,8 +82,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -f false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .forks(0)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/BenchmarkBenchSharingTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/BenchmarkBenchSharingTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -68,8 +72,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/BenchmarkStateSharingTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/BenchmarkStateSharingTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -70,8 +74,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupBenchSharingTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupBenchSharingTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -81,8 +85,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupDefaultBenchSharingTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupDefaultBenchSharingTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -68,8 +72,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupDefaultStateSharingTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupDefaultStateSharingTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -70,8 +74,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupStateSharingTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/GroupStateSharingTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -83,8 +87,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/ThreadBenchSharingTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/ThreadBenchSharingTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -68,8 +72,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/ThreadStateSharingTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/sharing/ThreadStateSharingTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -70,8 +74,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/synciter/SyncIterMeasurementOnlyTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/synciter/SyncIterMeasurementOnlyTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -34,6 +34,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -60,8 +64,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -si false");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .syncIterations(false)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/synciter/SyncIterNotOnlyMeasurementTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/synciter/SyncIterNotOnlyMeasurementTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -36,6 +36,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -69,8 +73,18 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -si true");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .syncIterations(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/BenchmarkBenchSameThreadTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/BenchmarkBenchSameThreadTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -113,8 +117,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/BenchmarkStateSameThreadTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/BenchmarkStateSameThreadTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -117,8 +121,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/GroupBenchSameThreadTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/GroupBenchSameThreadTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -113,8 +117,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/GroupStateSameThreadTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/GroupStateSameThreadTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -117,8 +121,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/OneThreadCountTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/OneThreadCountTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -75,13 +79,32 @@
     }
 
     @Test
-    public void invoke1() {
+    public void invokeCLI_1() {
         Main.testMain(Fixtures.getTestMask(this.getClass())+".*test1" + " -foe");
     }
 
     @Test
-    public void invoke2() {
+    public void invokeCLI_2() {
         Main.testMain(Fixtures.getTestMask(this.getClass())+ ".*test2" + " -foe -t 1");
     }
 
+    @Test
+    public void invokeAPI_1() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass())+".*test1")
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
+    @Test
+    public void invokeAPI_2() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass())+".*test2")
+                .failOnError(true)
+                .threads(1)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/ThreadBenchSameThreadTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/ThreadBenchSameThreadTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -130,8 +134,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/ThreadStateSameThreadTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/ThreadStateSameThreadTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 
@@ -134,8 +138,17 @@
     }
 
     @Test
-    public void invoke() {
+    public void invokeCLI() {
         Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
     }
 
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/TwoThreadCountTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/TwoThreadCountTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -75,13 +79,32 @@
     }
 
     @Test
-    public void invoke1() {
-        Main.testMain(Fixtures.getTestMask(this.getClass())+".*test1" + " -foe");
+    public void invokeCLI_1() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + ".*test1" + " -foe");
     }
 
     @Test
-    public void invoke2() {
-        Main.testMain(Fixtures.getTestMask(this.getClass())+ ".*test2" + " -foe -t 2");
+    public void invokeCLI_2() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + ".*test2" + " -foe -t 2");
+    }
+
+    @Test
+    public void invokeAPI_1() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass())+".*test1")
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
+    @Test
+    public void invokeAPI_2() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass())+".*test2")
+                .failOnError(true)
+                .threads(2)
+                .build();
+        new Runner(opt).run();
     }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/ZeroThreadCountTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/threads/ZeroThreadCountTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -38,6 +38,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -75,13 +79,31 @@
     }
 
     @Test
-    public void invoke1() {
+    public void invokeCLI_1() {
         Main.testMain(Fixtures.getTestMask(this.getClass())+".*test1" + " -foe");
     }
 
     @Test
-    public void invoke2() {
+    public void invokeCLI_2() {
         Main.testMain(Fixtures.getTestMask(this.getClass())+ ".*test2" + " -foe -t 0");
     }
 
+    @Test
+    public void invokeAPI_1() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass())+".*test1")
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
+    }
+
+    @Test
+    public void invokeAPI_2() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass())+".*test2")
+                .failOnError(true)
+                .threads(0)
+                .build();
+        new Runner(opt).run();
+    }
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/BenchmarkBenchHelperTimesTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/BenchmarkBenchHelperTimesTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -155,8 +159,16 @@
     }
 
     @Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .build();
+        new Runner(opt).run();
     }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/BenchmarkStateHelperTimesTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/BenchmarkStateHelperTimesTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -159,8 +163,17 @@
     }
 
     @Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
     }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/GroupBenchHelperTimesTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/GroupBenchHelperTimesTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -155,8 +159,17 @@
     }
 
     @Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
     }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/GroupStateHelperTimesTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/GroupStateHelperTimesTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -159,8 +163,17 @@
     }
 
     @Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
     }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/ThreadBenchHelperTimesTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/ThreadBenchHelperTimesTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -151,8 +155,17 @@
     }
 
     @Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
     }
 
 }
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/ThreadStateHelperTimesTest.java	Tue Aug 27 20:29:51 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/times/ThreadStateHelperTimesTest.java	Wed Aug 28 15:45:54 2013 +0400
@@ -39,6 +39,10 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -155,8 +159,17 @@
     }
 
     @Test
-    public void invoke() {
-        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe -v");
+    public void invokeCLI() {
+        Main.testMain(Fixtures.getTestMask(this.getClass()) + " -foe");
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .failOnError(true)
+                .build();
+        new Runner(opt).run();
     }
 
 }