changeset 297:dc7e95227834

Bulk warmup rebuild: missing mode added, able to run in forked VMs now.
author shade
date Wed, 04 Dec 2013 20:00:40 +0400
parents 1098723eff36
children f39c77f5b84a
files jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/NonForkedModesTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode5_Test.java jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode6_Test.java jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java jmh-core/src/main/java/org/openjdk/jmh/link/frames/BenchmarkRecordFrame.java jmh-core/src/main/java/org/openjdk/jmh/link/frames/InfraFrame.java jmh-core/src/main/java/org/openjdk/jmh/link/frames/RecipeFrame.java jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/Recipe.java jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java jmh-core/src/main/java/org/openjdk/jmh/runner/options/WarmupMode.java
diffstat 14 files changed, 442 insertions(+), 177 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/NonForkedModesTest.java	Wed Dec 04 20:00:40 2013 +0400
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.bulkwarmup;
+
+
+import org.junit.Test;
+import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.logic.Control;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.WarmupMode;
+import org.openjdk.jmh.runner.parameters.TimeValue;
+
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import static junit.framework.Assert.assertEquals;
+
+/**
+ * @author Aleksey Shipilev
+ */
+@State(Scope.Thread)
+public class NonForkedModesTest {
+
+    private static Queue<String> testSequence = new ConcurrentLinkedQueue<String>();
+
+    boolean recorded;
+
+    @Setup(Level.Iteration)
+    public void oneShot() {
+        recorded = false;
+    }
+
+    @GenerateMicroBenchmark
+    public void testBig(Control cnt) {
+        if (!recorded) {
+            recorded = true;
+            if (cnt.iterationTime == 100) { // warmup
+                testSequence.add("W");
+            } else if (cnt.iterationTime == 200) {  // iteration
+                testSequence.add("I");
+            }
+        }
+        Fixtures.work();
+    }
+
+    @GenerateMicroBenchmark
+    public void testSmall(Control cnt) {
+        if (!recorded) {
+            recorded = true;
+            if (cnt.iterationTime == 100) { // warmup
+                testSequence.add("w");
+            } else if (cnt.iterationTime == 200) {  // iteration
+                testSequence.add("i");
+            }
+        }
+        Fixtures.work();
+    }
+
+    private static String getSequence() {
+        StringBuilder sb = new StringBuilder();
+        for (String s : testSequence) {
+            sb.append(s);
+        }
+        return sb.toString();
+    }
+
+    @Test
+    public void invokeINDI() throws RunnerException {
+        testWith(WarmupMode.INDI, "WIwi");
+    }
+
+    @Test
+    public void invokeBULK() throws RunnerException {
+        testWith(WarmupMode.BULK, "WwIi");
+    }
+
+    @Test
+    public void invokeBULK_INDI() throws RunnerException {
+        testWith(WarmupMode.BULK_INDI, "WwWIwi");
+    }
+
+    public void testWith(WarmupMode mode, String runSeq) throws RunnerException {
+        testSequence.clear();
+
+        Options opt = new OptionsBuilder()
+                .include(Fixtures.getTestMask(this.getClass()))
+                .shouldFailOnError(true)
+                .warmupIterations(1)
+                .warmupTime(TimeValue.milliseconds(100))
+                .measurementIterations(1)
+                .measurementTime(TimeValue.milliseconds(200))
+                .threads(1)
+                .forks(0)
+                .syncIterations(false)
+                .warmupMode(mode)
+                .build();
+        new Runner(opt).run();
+
+        assertEquals(runSeq, getSequence());
+    }
+
+}
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode5_Test.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode5_Test.java	Wed Dec 04 20:00:40 2013 +0400
@@ -114,7 +114,7 @@
                 .threads(1)
                 .forks(0)
                 .syncIterations(false)
-                .warmupMode(WarmupMode.BEFOREEACH)
+                .warmupMode(WarmupMode.INDI)
                 .build();
         new Runner(opt).run();
 
--- a/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode6_Test.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/bulkwarmup/WarmupMode6_Test.java	Wed Dec 04 20:00:40 2013 +0400
@@ -114,7 +114,7 @@
                 .threads(1)
                 .forks(0)
                 .syncIterations(false)
-                .warmupMode(WarmupMode.BEFOREANY)
+                .warmupMode(WarmupMode.BULK)
                 .build();
         new Runner(opt).run();
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java	Wed Dec 04 20:00:40 2013 +0400
@@ -24,14 +24,15 @@
  */
 package org.openjdk.jmh.link;
 
-import org.openjdk.jmh.link.frames.BenchmarkRecordFrame;
 import org.openjdk.jmh.link.frames.FinishingFrame;
 import org.openjdk.jmh.link.frames.InfraFrame;
 import org.openjdk.jmh.link.frames.OptionsFrame;
 import org.openjdk.jmh.link.frames.OutputFormatFrame;
+import org.openjdk.jmh.link.frames.RecipeFrame;
 import org.openjdk.jmh.link.frames.ResultsFrame;
 import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.runner.BenchmarkRecord;
+import org.openjdk.jmh.runner.Recipe;
 import org.openjdk.jmh.runner.options.Options;
 
 import java.io.IOException;
@@ -87,29 +88,13 @@
         oos.flush();
     }
 
-    public BenchmarkRecord requestNextWarmup() throws IOException, ClassNotFoundException {
-        oos.writeObject(new InfraFrame(InfraFrame.Type.BULK_WARMUP_REQUEST));
+    public Recipe requestRecipe() throws IOException, ClassNotFoundException {
+        oos.writeObject(new InfraFrame(InfraFrame.Type.RECIPE_REQUEST));
         oos.flush();
 
         Object reply = ois.readObject();
-        if (reply == null) {
-            return null;
-        } else if (reply instanceof BenchmarkRecordFrame) {
-            return (((BenchmarkRecordFrame) reply).getBenchmark());
-        } else {
-            throw new IllegalStateException("Got the erroneous reply: " + reply);
-        }
-    }
-
-    public BenchmarkRecord requestNextMeasurement() throws IOException, ClassNotFoundException {
-        oos.writeObject(new InfraFrame(InfraFrame.Type.BENCHMARK_REQUEST));
-        oos.flush();
-
-        Object reply = ois.readObject();
-        if (reply == null) {
-            return null;
-        } else if (reply instanceof BenchmarkRecordFrame) {
-            return (((BenchmarkRecordFrame) reply).getBenchmark());
+        if (reply instanceof RecipeFrame) {
+            return ((RecipeFrame) reply).getRecipe();
         } else {
             throw new IllegalStateException("Got the erroneous reply: " + reply);
         }
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java	Wed Dec 04 20:00:40 2013 +0400
@@ -24,15 +24,15 @@
  */
 package org.openjdk.jmh.link;
 
-import org.openjdk.jmh.link.frames.BenchmarkRecordFrame;
 import org.openjdk.jmh.link.frames.FinishingFrame;
 import org.openjdk.jmh.link.frames.InfraFrame;
 import org.openjdk.jmh.link.frames.OptionsFrame;
 import org.openjdk.jmh.link.frames.OutputFormatFrame;
+import org.openjdk.jmh.link.frames.RecipeFrame;
 import org.openjdk.jmh.link.frames.ResultsFrame;
 import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.output.format.OutputFormat;
-import org.openjdk.jmh.runner.BenchmarkRecord;
+import org.openjdk.jmh.runner.Recipe;
 import org.openjdk.jmh.runner.options.Options;
 
 import java.io.IOException;
@@ -69,7 +69,7 @@
     private final Acceptor acceptor;
     private final AtomicReference<Handler> handler;
     private final AtomicReference<BenchResult> result;
-    private final AtomicReference<BenchmarkRecord> benchmark;
+    private final AtomicReference<Recipe> recipe;
 
     public BinaryLinkServer(Options opts, OutputFormat out) throws IOException {
         this.opts = opts;
@@ -96,7 +96,7 @@
 
         handler = new AtomicReference<Handler>();
         result = new AtomicReference<BenchResult>();
-        benchmark = new AtomicReference<BenchmarkRecord>();
+        recipe = new AtomicReference<Recipe>();
     }
 
     public void terminate() {
@@ -137,10 +137,8 @@
         }
     }
 
-    public void setCurrentBenchmark(BenchmarkRecord benchmark) {
-        if (!this.benchmark.compareAndSet(null, benchmark)) {
-            throw new IllegalStateException("Benchmark is already set");
-        }
+    public void setRecipe(Recipe recipe) {
+        this.recipe.set(recipe);
     }
 
     private final class Acceptor extends Thread {
@@ -265,12 +263,8 @@
                     oos.writeObject(new OptionsFrame(opts));
                     oos.flush();
                     break;
-                case BENCHMARK_REQUEST:
-                    oos.writeObject(new BenchmarkRecordFrame(benchmark.getAndSet(null)));
-                    oos.flush();
-                    break;
-                case BULK_WARMUP_REQUEST:
-                    oos.writeObject(null);
+                case RECIPE_REQUEST:
+                    oos.writeObject(new RecipeFrame(recipe.get()));
                     oos.flush();
                     break;
                 default:
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/frames/BenchmarkRecordFrame.java	Tue Dec 03 22:44:02 2013 +0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package org.openjdk.jmh.link.frames;
-
-import org.openjdk.jmh.runner.BenchmarkRecord;
-
-import java.io.Serializable;
-
-public class BenchmarkRecordFrame implements Serializable {
-    private final BenchmarkRecord benchmark;
-
-    public BenchmarkRecordFrame(BenchmarkRecord benchmark) {
-        this.benchmark = benchmark;
-    }
-
-    public BenchmarkRecord getBenchmark() {
-        return benchmark;
-    }
-}
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/frames/InfraFrame.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/frames/InfraFrame.java	Wed Dec 04 20:00:40 2013 +0400
@@ -40,8 +40,7 @@
 
     public enum Type {
         OPTIONS_REQUEST,
-        BULK_WARMUP_REQUEST,
-        BENCHMARK_REQUEST,
+        RECIPE_REQUEST,
     }
 
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/frames/RecipeFrame.java	Wed Dec 04 20:00:40 2013 +0400
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.link.frames;
+
+import org.openjdk.jmh.runner.Recipe;
+
+import java.io.Serializable;
+
+public class RecipeFrame implements Serializable {
+    private final Recipe recipe;
+
+    public RecipeFrame(Recipe recipe) {
+        this.recipe = recipe;
+    }
+
+    public Recipe getRecipe() {
+        return recipe;
+    }
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Wed Dec 04 20:00:40 2013 +0400
@@ -32,6 +32,8 @@
 import org.openjdk.jmh.runner.parameters.BenchmarkParams;
 import org.openjdk.jmh.runner.parameters.IterationParams;
 import org.openjdk.jmh.util.ClassUtils;
+import org.openjdk.jmh.util.internal.Multimap;
+import org.openjdk.jmh.util.internal.TreeMultimap;
 
 import java.lang.management.GarbageCollectorMXBean;
 import java.lang.management.ManagementFactory;
@@ -58,6 +60,42 @@
         this.out = handler;
     }
 
+    protected Multimap<BenchmarkRecord, BenchResult> runBenchmarks(boolean forked, Recipe recipe) {
+        Multimap<BenchmarkRecord, BenchResult> results = new TreeMultimap<BenchmarkRecord, BenchResult>();
+
+        for (Recipe.Action action : recipe.getActions()) {
+
+            BenchmarkRecord benchmark = action.record;
+            Recipe.Mode mode = action.mode;
+
+            if (!forked) {
+                out.println("# Fork: N/A, test runs in the existing VM");
+            }
+
+            switch (mode) {
+                case WARMUP: {
+                    runBenchmark(benchmark, true, false);
+                    out.println("");
+                    break;
+                }
+                case MEASUREMENT: {
+                    BenchResult r = runBenchmark(benchmark, false, true);
+                    results.put(benchmark, r);
+                    break;
+                }
+                case WARMUP_MEASUREMENT: {
+                    BenchResult r = runBenchmark(benchmark, true, true);
+                    results.put(benchmark, r);
+                    break;
+                }
+                default:
+                    throw new IllegalStateException("Unknown mode: " + mode);
+            }
+        }
+
+        return results;
+    }
+
     BenchResult runBenchmark(BenchmarkRecord benchmark, boolean doWarmup, boolean doMeasurement) {
         MicroBenchmarkHandler handler = null;
         try {
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Wed Dec 04 20:00:40 2013 +0400
@@ -28,6 +28,7 @@
 import org.openjdk.jmh.logic.results.BenchResult;
 import org.openjdk.jmh.output.OutputFormatFactory;
 import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.util.internal.Multimap;
 
 import java.io.IOException;
 
@@ -46,18 +47,13 @@
     }
 
     public void run() throws IOException, ClassNotFoundException {
-        BenchmarkRecord benchmark;
+        Recipe recipe = link.requestRecipe();
 
-        // Bulk warmup benchmarks first
-        while ((benchmark = link.requestNextWarmup()) != null) {
-            BenchResult result = runBenchmark(benchmark, true, false);
-            link.pushResults(benchmark, result);
-        }
-
-        // Measurement then
-        while ((benchmark = link.requestNextMeasurement()) != null) {
-            BenchResult result = runBenchmark(benchmark, true, true);
-            link.pushResults(benchmark, result);
+        Multimap<BenchmarkRecord,BenchResult> res = runBenchmarks(true, recipe);
+        for (BenchmarkRecord br : res.keys()) {
+            for (BenchResult r : res.get(br)) {
+                link.pushResults(br, r);
+            }
         }
 
         out.flush();
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/Recipe.java	Wed Dec 04 20:00:40 2013 +0400
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.runner;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class Recipe implements Serializable {
+
+    private final List<Action> actions;
+
+    public Recipe() {
+        actions = new ArrayList<Action>();
+    }
+
+    void addWarmup(BenchmarkRecord record) {
+        actions.add(new Action(record, Mode.WARMUP));
+    }
+
+    void addWarmup(Collection<BenchmarkRecord> rs) {
+        for (BenchmarkRecord b : rs) {
+            addWarmup(b);
+        }
+    }
+
+    void addMeasurement(BenchmarkRecord record) {
+        actions.add(new Action(record, Mode.MEASUREMENT));
+    }
+
+    void addMeasurement(Collection<BenchmarkRecord> rs) {
+        for (BenchmarkRecord b : rs) {
+            addMeasurement(b);
+        }
+    }
+
+    void addWarmupMeasurement(BenchmarkRecord record) {
+        actions.add(new Action(record, Mode.WARMUP_MEASUREMENT));
+    }
+
+    void addWarmupMeasurement(Collection<BenchmarkRecord> rs) {
+        for (BenchmarkRecord b : rs) {
+            addWarmupMeasurement(b);
+        }
+    }
+
+    public void mixIn(Recipe other) {
+        actions.addAll(other.actions);
+    }
+
+    public List<Action> getActions() {
+        return actions;
+    }
+
+    public List<Action> getMeasurementActions() {
+        List<Action> result = new ArrayList<Action>();
+        for (Action action : actions) {
+            switch (action.mode) {
+                case MEASUREMENT:
+                case WARMUP_MEASUREMENT:
+                    result.add(action);
+                    break;
+
+            }
+        }
+        return result;
+    }
+
+    public static class Action implements Serializable {
+        public final BenchmarkRecord record;
+        public final Mode mode;
+
+        public Action(BenchmarkRecord record, Mode mode) {
+            this.record = record;
+            this.mode = mode;
+        }
+    }
+
+    public static enum Mode {
+        WARMUP,
+        MEASUREMENT,
+        WARMUP_MEASUREMENT,
+    }
+
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Wed Dec 04 20:00:40 2013 +0400
@@ -35,7 +35,6 @@
 import org.openjdk.jmh.output.results.ResultFormat;
 import org.openjdk.jmh.output.results.ResultFormatFactory;
 import org.openjdk.jmh.runner.options.Options;
-import org.openjdk.jmh.runner.options.WarmupMode;
 import org.openjdk.jmh.runner.parameters.BenchmarkParams;
 import org.openjdk.jmh.util.AnnotationUtils;
 import org.openjdk.jmh.util.InputStreamDrainer;
@@ -56,12 +55,12 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -193,13 +192,7 @@
         benchmarks.clear();
         benchmarks.addAll(newBenchmarks);
 
-        Map<BenchmarkRecord, RunResult> results;
-        if ((!options.getWarmupIncludes().isEmpty()) ||
-                (options.getWarmupMode() == WarmupMode.BEFOREANY)) {
-            results = runBulkWarmupBenchmarks(benchmarks);
-        } else {
-            results = runBenchmarks(benchmarks);
-        }
+        Map<BenchmarkRecord, RunResult> results = runBenchmarks(benchmarks);
 
         out.flush();
         out.close();
@@ -210,81 +203,78 @@
         return results;
     }
 
-    /**
-     * Run specified warmup microbenchmarks prior to running any requested mircobenchmarks.
-     * TODO: Currently valid only for non-external JVM runs
-     */
-    private Map<BenchmarkRecord, RunResult> runBulkWarmupBenchmarks(Set<BenchmarkRecord> benchmarks) {
-        out.startRun();
-
-        // list of micros executed before iteration
-        Set<BenchmarkRecord> warmupMicros = new TreeSet<BenchmarkRecord>();
+    private Recipe getEmbeddedRecipe(Set<BenchmarkRecord> benchmarks) {
+        Recipe r = new Recipe();
 
         List<String> warmupMicrosRegexp = options.getWarmupIncludes();
         if (warmupMicrosRegexp != null && !warmupMicrosRegexp.isEmpty()) {
-            warmupMicros.addAll(list.find(out, warmupMicrosRegexp, Collections.<String>emptyList()));
+            r.addWarmup(list.find(out, warmupMicrosRegexp, Collections.<String>emptyList()));
         }
-        if (options.getWarmupMode() == WarmupMode.BEFOREANY) {
-            warmupMicros.addAll(benchmarks);
+        if (options.getWarmupMode().isBulk()) {
+            r.addWarmup(benchmarks);
         }
 
-        if (!warmupMicros.isEmpty()) {
-            // run warmup iterations of the requested benchmarks before running
-            // any measured iterations of any of the requested benchmarks. This
-            // has the effect of getting all the classes loaded getting the JITed
-            // code the the final state, possibly invalidating optimizations that
-            // might not be invalided until later and quite possibly invalidated
-            // during measurement iteration causing a performance shift or simply
-            // increased variance.
-            // currently valid only for non-external JVM runs
-
-            int count = 0;
-            for (BenchmarkRecord benchmark : warmupMicros) {
-                out.println("# Fork: N/A, bulk warmup in progress, " + (++count) + " of " + warmupMicros.size());
-                runBenchmark(benchmark, true, false);
-                out.println("");
-            }
-        }
-        // run microbenchmarks
-        //
-        Multimap<BenchmarkRecord, BenchResult> results = new TreeMultimap<BenchmarkRecord, BenchResult>();
-        for (BenchmarkRecord benchmark : benchmarks) {
-            out.println("# Fork: N/A, test runs in same VM");
-            BenchResult result = runBenchmark(benchmark, false, true);
-            results.put(benchmark, result);
-        }
-
-        Map<BenchmarkRecord, RunResult> runResults = mergeRunResults(results);
-        out.endRun(runResults);
-        return runResults;
-    }
-
-
-    private Map<BenchmarkRecord, RunResult> runBenchmarks(Set<BenchmarkRecord> benchmarks) {
-        Set<BenchmarkRecord> embedded = new TreeSet<BenchmarkRecord>();
-        Set<BenchmarkRecord> forked = new TreeSet<BenchmarkRecord>();
-
-        out.startRun();
-        for (BenchmarkRecord benchmark : benchmarks) {
-            BenchmarkParams params = new BenchmarkParams(options, benchmark, true, true);
-            if (params.getForks() > 0) {
-                forked.add(benchmark);
-            } else {
-                embedded.add(benchmark);
+        for (BenchmarkRecord br : benchmarks) {
+            BenchmarkParams params = new BenchmarkParams(options, br, true, true);
+            if (params.getForks() <= 0) {
+                if (options.getWarmupMode().isIndi()) {
+                    r.addWarmupMeasurement(br);
+                } else {
+                    r.addMeasurement(br);
+                }
             }
         }
 
-        Multimap<BenchmarkRecord, BenchResult> results = new TreeMultimap<BenchmarkRecord, BenchResult>();
-        for (BenchmarkRecord benchmark : embedded) {
-            out.println("# Fork: N/A, test runs in same VM");
-            BenchResult r = runBenchmark(benchmark, true, true);
-            results.put(benchmark, r);
+        return r;
+    }
+
+    private Set<Recipe> getForkedRecipes(Set<BenchmarkRecord> benchmarks) {
+        Recipe base = new Recipe();
+
+        List<String> warmupMicrosRegexp = options.getWarmupIncludes();
+        if (warmupMicrosRegexp != null && !warmupMicrosRegexp.isEmpty()) {
+            base.addWarmup(list.find(out, warmupMicrosRegexp, Collections.<String>emptyList()));
+        }
+        if (options.getWarmupMode().isBulk()) {
+            base.addWarmup(benchmarks);
         }
 
-        Multimap<BenchmarkRecord, BenchResult> separateResults = runSeparate(forked);
-        for (BenchmarkRecord k : separateResults.keys()) {
-            Collection<BenchResult> rs = separateResults.get(k);
-            results.putAll(k, rs);
+        Set<Recipe> result = new HashSet<Recipe>();
+        for (BenchmarkRecord br : benchmarks) {
+            BenchmarkParams params = new BenchmarkParams(options, br, true, true);
+            if (params.getForks() > 0) {
+                Recipe r = new Recipe();
+                r.mixIn(base);
+                if (options.getWarmupMode().isIndi()) {
+                    r.addWarmupMeasurement(br);
+                } else {
+                    r.addMeasurement(br);
+                }
+                result.add(r);
+            }
+        }
+
+        return result;
+    }
+
+    private Map<BenchmarkRecord, RunResult> runBenchmarks(Set<BenchmarkRecord> benchmarks) {
+        out.startRun();
+
+        Multimap<BenchmarkRecord, BenchResult> results = new TreeMultimap<BenchmarkRecord, BenchResult>();
+
+        {
+            Recipe recipe = getEmbeddedRecipe(benchmarks);
+            Multimap<BenchmarkRecord, BenchResult> res = runBenchmarks(false, recipe);
+            for (BenchmarkRecord br : res.keys()) {
+                results.putAll(br, res.get(br));
+            }
+        }
+
+        for (Recipe r : getForkedRecipes(benchmarks)) {
+            Multimap<BenchmarkRecord, BenchResult> res = runSeparate(r);
+            for (BenchmarkRecord br : res.keys()) {
+                results.putAll(br, res.get(br));
+            }
         }
 
         Map<BenchmarkRecord, RunResult> runResults = mergeRunResults(results);
@@ -301,18 +291,21 @@
         return result;
     }
 
-    private Multimap<BenchmarkRecord, BenchResult> runSeparate(Set<BenchmarkRecord> benchmarks) {
+    private Multimap<BenchmarkRecord, BenchResult> runSeparate(Recipe recipe) {
         Multimap<BenchmarkRecord, BenchResult> results = new HashMultimap<BenchmarkRecord, BenchResult>();
 
-        if (benchmarks.isEmpty()) {
-            return results;
+        if (recipe.getMeasurementActions().size() != 1) {
+            throw new IllegalStateException("Expect only single benchmark in the recipe, but was " + recipe.getMeasurementActions().size());
         }
 
         BinaryLinkServer server = null;
         try {
             server = new BinaryLinkServer(options, out);
 
-            for (BenchmarkRecord benchmark : benchmarks) {
+            server.setRecipe(recipe);
+
+            BenchmarkRecord benchmark = recipe.getMeasurementActions().get(0).record;
+
                 // Running microbenchmark in separate JVM requires to read some options from annotations.
                 final Method benchmarkMethod = MicroBenchmarkHandlers.findBenchmarkMethod(benchmark);
                 Fork forkAnnotation = benchmarkMethod.getAnnotation(Fork.class);
@@ -342,7 +335,6 @@
                     out.verbosePrintln("Warmup forking " + warmupForkCount + " times using command: " + Arrays.toString(commandString));
                     for (int i = 0; i < warmupForkCount; i++) {
                         out.println("# Warmup Fork: " + (i + 1) + " of " + forkCount);
-                        server.setCurrentBenchmark(benchmark);
                         doFork(server, commandString);
                     }
                 }
@@ -350,11 +342,10 @@
                 out.verbosePrintln("Forking " + forkCount + " times using command: " + Arrays.toString(commandString));
                 for (int i = 0; i < forkCount; i++) {
                     out.println("# Fork: " + (i + 1) + " of " + forkCount);
-                    server.setCurrentBenchmark(benchmark);
                     BenchResult result = doFork(server, commandString);
                     results.put(benchmark, result);
                 }
-            }
+
         } catch (IOException e) {
             throw new IllegalStateException(e);
         } finally {
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java	Wed Dec 04 20:00:40 2013 +0400
@@ -152,7 +152,7 @@
     @Option(name = "-e", aliases = {"--exclude"}, multiValued = true, metaVar = "REGEXP", usage = "Microbenchmarks to exclude. Regexp filtering out classes or methods which are MicroBenchmarks.")
     protected List<String> excludes = new ArrayList<String>();
 
-    @Option(name = "-wm", aliases = {"--warmupmode"}, usage = "Warmup mode for warming up selected micro benchmarks. Warmup modes are BeforeAny (measurements) or BeforeEach (measurement) (original mode)")
+    @Option(name = "-wm", aliases = {"--warmupmode"}, usage = "Warmup mode for warming up selected micro benchmarks. Warmup modes are: " + WarmupMode.BULK + "," + WarmupMode.BULK_INDI + "," + WarmupMode.INDI + "")
     protected WarmupMode warmupMode = WarmupMode.defaultMode();
 
     @Option(name = "-wmb", aliases = {"--warmupmicrobenchmarks"}, multiValued = true, metaVar = "REGEXP", usage = "Microbenchmarks to run for warmup before running any other benchmarks. These micros may be different from the target micros to warm up the harness or other parts of the JVM prior to running the target micro benchmarks. Regexp filtering out classes or methods which are MicroBenchmarks.")
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/options/WarmupMode.java	Tue Dec 03 22:44:02 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/options/WarmupMode.java	Wed Dec 04 20:00:40 2013 +0400
@@ -30,18 +30,40 @@
 public enum WarmupMode {
 
     /**
-     * Warmup everything before any benchmark starts.
+     * Do the individual warmup for every benchmark
      */
-    BEFOREANY,
+    INDI(false, true),
 
     /**
-     * Warmup each benchmark before benchmark starts.
+     * Do the bulk warmup before any benchmark starts.
      */
-    BEFOREEACH,
+    BULK(true, false),
+
+    /**
+     * Do the bulk warmup before any benchmark starts,
+     * and then also do individual warmups for every
+     * benchmark.
+     */
+    BULK_INDI(true, true),
 
     ;
+    private final boolean bulk;
+    private final boolean indi;
+
+    WarmupMode(boolean bulk, boolean indi) {
+        this.bulk = bulk;
+        this.indi = indi;
+    }
+
+    public boolean isBulk() {
+        return bulk;
+    }
+
+    public boolean isIndi() {
+        return indi;
+    }
 
     static WarmupMode defaultMode() {
-        return BEFOREEACH;
+        return INDI;
     }
 }