changeset 1158:c6413955f9d8

7901345: Refactor batchSize/opsPerInv handling in the generated code and counting the operations
author shade
date Thu, 19 Mar 2015 16:44:12 +0300
parents 964023d59b57
children 02fedaf27367
files jmh-core-it/src/test/java/org/openjdk/jmh/it/batchsize/BatchSizeSanityTest.java jmh-core-it/src/test/java/org/openjdk/jmh/it/batchsize/OpsPerInvSanityTest.java jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java jmh-core/src/main/java/org/openjdk/jmh/results/BenchmarkTaskResult.java jmh-core/src/main/java/org/openjdk/jmh/results/RawResults.java jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkHandler.java
diffstat 6 files changed, 419 insertions(+), 48 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/batchsize/BatchSizeSanityTest.java	Thu Mar 19 16:44:12 2015 +0300
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.batchsize;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.results.RunResult;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Thread)
+public class BatchSizeSanityTest {
+
+    private static final AtomicInteger invCount = new AtomicInteger();
+    private static volatile long startTime;
+    private static volatile long stopTime;
+
+    @Setup(Level.Iteration)
+    public void beforeIter() {
+        startTime = System.nanoTime();
+    }
+
+    @TearDown(Level.Iteration)
+    public void afterIter() {
+        stopTime = System.nanoTime();
+    }
+
+    @Benchmark
+    @Fork(0)
+    @Warmup(iterations = 0)
+    @Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void test() throws InterruptedException {
+        TimeUnit.MILLISECONDS.sleep(1);
+        invCount.incrementAndGet();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int bs : new int[] {1, 10, 100}) {
+            for (Mode m : Mode.values()) {
+                if (m == Mode.All) continue;
+                doWith(m, bs);
+            }
+        }
+    }
+
+    private void doWith(Mode mode, int batchSize) throws RunnerException {
+        invCount.set(0);
+
+        Options opt = new OptionsBuilder()
+            .include(Fixtures.getTestMask(this.getClass()))
+            .shouldFailOnError(true)
+            .measurementBatchSize(batchSize)
+            .mode(mode)
+            .build();
+        RunResult run = new Runner(opt).runSingle();
+
+        final double TOLERANCE = 0.30;
+
+        double expectedScore = 0.0;
+
+        double time = stopTime - startTime;
+        double calls = invCount.get();
+
+        switch (mode) {
+            case Throughput:
+                expectedScore = (calls / batchSize) / time;
+                break;
+            case AverageTime:
+            case SampleTime:
+                expectedScore = time / (calls / batchSize);
+                break;
+            case SingleShotTime:
+                expectedScore = time;
+                break;
+            default:
+                Assert.fail("Unhandled mode: " + mode);
+        }
+
+        double actualScore = run.getPrimaryResult().getScore();
+        Assert.assertTrue(mode + ", " + batchSize + ": " + expectedScore + " vs " + actualScore,
+                Math.abs(1 - actualScore / expectedScore) < TOLERANCE);
+    }
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-it/src/test/java/org/openjdk/jmh/it/batchsize/OpsPerInvSanityTest.java	Thu Mar 19 16:44:12 2015 +0300
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.it.batchsize;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.it.Fixtures;
+import org.openjdk.jmh.results.RunResult;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@State(Scope.Thread)
+public class OpsPerInvSanityTest {
+
+    private static final AtomicInteger invCount = new AtomicInteger();
+    private static volatile long startTime;
+    private static volatile long stopTime;
+
+    @Setup(Level.Iteration)
+    public void beforeIter() {
+        startTime = System.nanoTime();
+    }
+
+    @TearDown(Level.Iteration)
+    public void afterIter() {
+        stopTime = System.nanoTime();
+    }
+
+    @Benchmark
+    @Fork(0)
+    @Warmup(iterations = 0)
+    @Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void test() throws InterruptedException {
+        TimeUnit.MILLISECONDS.sleep(1);
+        invCount.incrementAndGet();
+    }
+
+    @Test
+    public void invokeAPI() throws RunnerException {
+        for (int opsPerInv : new int[] {1, 10, 100}) {
+            for (Mode m : Mode.values()) {
+                if (m == Mode.All) continue;
+                doWith(m, opsPerInv);
+            }
+        }
+    }
+
+    private void doWith(Mode mode, int opsPerInv) throws RunnerException {
+        invCount.set(0);
+
+        Options opt = new OptionsBuilder()
+            .include(Fixtures.getTestMask(this.getClass()))
+            .shouldFailOnError(true)
+            .operationsPerInvocation(opsPerInv)
+            .mode(mode)
+            .build();
+        RunResult run = new Runner(opt).runSingle();
+
+        final double TOLERANCE = 0.30;
+
+        double expectedScore = 0.0;
+
+        double time = stopTime - startTime;
+        double calls = invCount.get();
+
+        switch (mode) {
+            case Throughput:
+                expectedScore = (calls * opsPerInv) / time;
+                break;
+            case AverageTime:
+            case SampleTime:
+                expectedScore = time / (calls * opsPerInv);
+                break;
+            case SingleShotTime:
+                expectedScore = time;
+                break;
+            default:
+                Assert.fail("Unhandled mode: " + mode);
+        }
+
+        double actualScore = run.getPrimaryResult().getScore();
+        Assert.assertTrue(mode + ", " + opsPerInv + ": " + expectedScore + " vs " + actualScore,
+                Math.abs(1 - actualScore / expectedScore) < TOLERANCE);
+    }
+
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java	Thu Mar 19 12:27:40 2015 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java	Thu Mar 19 16:44:12 2015 +0300
@@ -42,6 +42,7 @@
 import org.openjdk.jmh.infra.IterationParams;
 import org.openjdk.jmh.infra.ThreadParams;
 import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
 import org.openjdk.jmh.results.RawResults;
 import org.openjdk.jmh.results.Result;
 import org.openjdk.jmh.results.ResultRole;
@@ -602,6 +603,7 @@
                 Collection.class, ArrayList.class,
                 TimeUnit.class, Generated.class, CompilerControl.class,
                 InfraControl.class, ThreadParams.class,
+                BenchmarkTaskResult.class,
                 Result.class, ThroughputResult.class, AverageTimeResult.class,
                 SampleTimeResult.class, SingleShotResult.class, SampleBuffer.class,
                 Mode.class, Fork.class, Measurement.class, Threads.class, Warmup.class,
@@ -639,7 +641,7 @@
     }
 
     private void generateThroughput(ClassInfo classInfo, PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) {
-        writer.println(ident(1) + "public Collection<ThroughputResult> " + methodGroup.getName() + "_" + benchmarkKind +
+        writer.println(ident(1) + "public HandlerResult " + methodGroup.getName() + "_" + benchmarkKind +
                 "(InfraControl control, ThreadParams threadParams) throws Throwable {");
 
         methodProlog(writer, methodGroup);
@@ -650,6 +652,7 @@
             subGroup++;
 
             writer.println(ident(2) + "if (threadParams.getSubgroupIndex() == " + subGroup + ") {");
+            writer.println(ident(3) + "RawResults res = new RawResults();");
 
             iterationProlog(writer, 3, method, states);
 
@@ -663,6 +666,7 @@
             writer.println(ident(4) + emitCall(method, states) + ';');
             invocationEpilog(writer, 4, method, states, false);
 
+            writer.println(ident(4) + "res.allOps++;");
             writer.println(ident(3) + "}");
             writer.println();
 
@@ -672,14 +676,9 @@
             }
 
             // measurement loop call
-            writer.println(ident(3) + "RawResults res = new RawResults(control.benchmarkParams.getOpsPerInvocation());");
             writer.println(ident(3) + method.getName() + "_" + benchmarkKind.shortLabel() + JMH_STUB_SUFFIX +
                     "(control, res" + prefix(states.getArgList(method)) + ");");
 
-            // pretend we did the batched run; there is no reason to have an additional loop,
-            // when JMH stub already is optimized.
-            writer.println(ident(3) + "res.operations /= control.iterationParams.getBatchSize();");
-
             // control objects get a special treatment
             for (StateObject so : states.getControls()) {
                 writer.println(ident(3) + so.localIdentifier + ".stopMeasurement = true;");
@@ -696,6 +695,7 @@
             writer.println(ident(5) + emitCall(method, states) + ';');
             invocationEpilog(writer, 5, method, states, false);
 
+            writer.println(ident(5) + "res.allOps++;");
             writer.println(ident(4) + "}");
             writer.println(ident(4) + "control.preTearDown();");
             writer.println(ident(3) + "} catch (InterruptedException ie) {");
@@ -705,10 +705,31 @@
             // iteration prolog
             iterationEpilog(writer, 3, method, states);
 
-            writer.println(ident(3) + "Collection<ThroughputResult> results = new ArrayList<ThroughputResult>();");
-            writer.println(ident(3) + "results.add(new ThroughputResult(ResultRole.PRIMARY, \"" + method.getName() + "\", res.getOperations(), res.getTime(), control.benchmarkParams.getTimeUnit()));");
+            writer.println(ident(3) + "res.allOps += res.measuredOps;");
+
+            /*
+               Adjust the operation counts:
+                  1) res.measuredOps counted the individual @Benchmark invocations. Therefore, we need
+                     to adjust for opsPerInv (pretending each @Benchmark invocation counts as $opsPerInv ops);
+                     and we need to adjust down for $batchSize (pretending we had the batched run, and $batchSize
+                     @Benchmark invocations counted as single op);
+                  2) res.allOps counted the individual @Benchmark invocations as well; the same reasoning applies.
+
+               It's prudent to make the multiplication first to get more accuracy.
+             */
+
+            writer.println(ident(3) + "int batchSize = control.iterationParams.getBatchSize();");
+            writer.println(ident(3) + "int opsPerInv = control.benchmarkParams.getOpsPerInvocation();");
+
+            writer.println(ident(3) + "res.allOps *= opsPerInv;");
+            writer.println(ident(3) + "res.allOps /= batchSize;");
+            writer.println(ident(3) + "res.measuredOps *= opsPerInv;");
+            writer.println(ident(3) + "res.measuredOps /= batchSize;");
+
+            writer.println(ident(3) + "HandlerResult results = new HandlerResult(res.allOps, res.measuredOps);");
+            writer.println(ident(3) + "results.add(new ThroughputResult(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), control.benchmarkParams.getTimeUnit()));");
             if (!isSingleMethod) {
-                writer.println(ident(3) + "results.add(new ThroughputResult(ResultRole.SECONDARY, \"" + method.getName() + "\", res.getOperations(), res.getTime(), control.benchmarkParams.getTimeUnit()));");
+                writer.println(ident(3) + "results.add(new ThroughputResult(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), control.benchmarkParams.getTimeUnit()));");
             }
             for (String ops : states.getAuxResultNames(method)) {
                 writer.println(ident(3) + "results.add(new ThroughputResult(ResultRole.SECONDARY, \"" + ops + "\", " + states.getAuxResultAccessor(method, ops) + ", res.getTime(), control.benchmarkParams.getTimeUnit()));");
@@ -741,14 +762,14 @@
             writer.println(ident(2) + "} while(!control.isDone);");
             writer.println(ident(2) + "result.stopTime = System.nanoTime();");
             writer.println(ident(2) + "result.realTime = realTime;");
-            writer.println(ident(2) + "result.operations = operations;");
+            writer.println(ident(2) + "result.measuredOps = operations;");
             writer.println(ident(1) + "}");
             writer.println();
         }
     }
 
     private void generateAverageTime(ClassInfo classInfo, PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) {
-        writer.println(ident(1) + "public Collection<AverageTimeResult> " + methodGroup.getName() + "_" + benchmarkKind +
+        writer.println(ident(1) + "public HandlerResult " + methodGroup.getName() + "_" + benchmarkKind +
                 "(InfraControl control, ThreadParams threadParams) throws Throwable {");
 
         methodProlog(writer, methodGroup);
@@ -759,6 +780,7 @@
             subGroup++;
 
             writer.println(ident(2) + "if (threadParams.getSubgroupIndex() == " + subGroup + ") {");
+            writer.println(ident(3) + "RawResults res = new RawResults();");
 
             iterationProlog(writer, 3, method, states);
 
@@ -772,6 +794,7 @@
             writer.println(ident(4) + emitCall(method, states) + ';');
             invocationEpilog(writer, 4, method, states, false);
 
+            writer.println(ident(4) + "res.allOps++;");
             writer.println(ident(3) + "}");
             writer.println();
 
@@ -781,13 +804,8 @@
             }
 
             // measurement loop call
-            writer.println(ident(3) + "RawResults res = new RawResults(control.benchmarkParams.getOpsPerInvocation());");
             writer.println(ident(3) + method.getName() + "_" + benchmarkKind.shortLabel() + JMH_STUB_SUFFIX + "(control, res" + prefix(states.getArgList(method)) + ");");
 
-            // pretend we did the batched run; there is no reason to have an additional loop,
-            // when JMH stub is already optimized.
-            writer.println(ident(3) + "res.operations /= control.iterationParams.getBatchSize();");
-
             // control objects get a special treatment
             for (StateObject so : states.getControls()) {
                 writer.println(ident(3) + so.localIdentifier + ".stopMeasurement = true;");
@@ -804,6 +822,7 @@
             writer.println(ident(5) + emitCall(method, states) + ';');
             invocationEpilog(writer, 5, method, states, false);
 
+            writer.println(ident(5) + "res.allOps++;");
             writer.println(ident(4) + "}");
             writer.println(ident(4) + "control.preTearDown();");
             writer.println(ident(3) + "} catch (InterruptedException ie) {");
@@ -812,10 +831,31 @@
 
             iterationEpilog(writer, 3, method, states);
 
-            writer.println(ident(3) + "Collection<AverageTimeResult> results = new ArrayList<AverageTimeResult>();");
-            writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + method.getName() + "\", res.getOperations(), res.getTime(), control.benchmarkParams.getTimeUnit()));");
+            writer.println(ident(3) + "res.allOps += res.measuredOps;");
+
+            /*
+               Adjust the operation counts:
+                  1) res.measuredOps counted the individual @Benchmark invocations. Therefore, we need
+                     to adjust for opsPerInv (pretending each @Benchmark invocation counts as $opsPerInv ops);
+                     and we need to adjust down for $batchSize (pretending we had the batched run, and $batchSize
+                     @Benchmark invocations counted as single op)
+                  2) res.measuredOps counted the individual @Benchmark invocations as well; the same reasoning applies.
+
+               It's prudent to make the multiplication first to get more accuracy.
+             */
+
+            writer.println(ident(3) + "int batchSize = control.iterationParams.getBatchSize();");
+            writer.println(ident(3) + "int opsPerInv = control.benchmarkParams.getOpsPerInvocation();");
+
+            writer.println(ident(3) + "res.allOps *= opsPerInv;");
+            writer.println(ident(3) + "res.allOps /= batchSize;");
+            writer.println(ident(3) + "res.measuredOps *= opsPerInv;");
+            writer.println(ident(3) + "res.measuredOps /= batchSize;");
+
+            writer.println(ident(3) + "HandlerResult results = new HandlerResult(res.allOps, res.measuredOps);");
+            writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), control.benchmarkParams.getTimeUnit()));");
             if (!isSingleMethod) {
-                writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.SECONDARY, \"" + method.getName() + "\", res.getOperations(), res.getTime(), control.benchmarkParams.getTimeUnit()));");
+                writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), control.benchmarkParams.getTimeUnit()));");
             }
             for (String ops : states.getAuxResultNames(method)) {
                 writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.SECONDARY, \"" + ops + "\", " + states.getAuxResultAccessor(method, ops) + ", res.getTime(), control.benchmarkParams.getTimeUnit()));");
@@ -848,7 +888,7 @@
             writer.println(ident(2) + "} while(!control.isDone);");
             writer.println(ident(2) + "result.stopTime = System.nanoTime();");
             writer.println(ident(2) + "result.realTime = realTime;");
-            writer.println(ident(2) + "result.operations = operations;");
+            writer.println(ident(2) + "result.measuredOps = operations;");
             writer.println(ident(1) + "}");
             writer.println();
         }
@@ -867,7 +907,7 @@
     }
 
     private void generateSampleTime(ClassInfo classInfo, PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) {
-        writer.println(ident(1) + "public Collection<SampleTimeResult> " + methodGroup.getName() + "_" + benchmarkKind +
+        writer.println(ident(1) + "public HandlerResult " + methodGroup.getName() + "_" + benchmarkKind +
                 "(InfraControl control, ThreadParams threadParams) throws Throwable {");
 
         methodProlog(writer, methodGroup);
@@ -878,6 +918,7 @@
             subGroup++;
 
             writer.println(ident(2) + "if (threadParams.getSubgroupIndex() == " + subGroup + ") {");
+            writer.println(ident(3) + "RawResults res = new RawResults();");
 
             iterationProlog(writer, 3, method, states);
 
@@ -891,6 +932,7 @@
             writer.println(ident(4) + emitCall(method, states) + ';');
             invocationEpilog(writer, 4, method, states, false);
 
+            writer.println(ident(4) + "res.allOps++;");
             writer.println(ident(3) + "}");
             writer.println();
 
@@ -902,8 +944,9 @@
             // measurement loop call
             writer.println(ident(3) + "int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond");
             writer.println(ident(3) + "int batchSize = control.iterationParams.getBatchSize();");
+            writer.println(ident(3) + "int opsPerInv = control.benchmarkParams.getOpsPerInvocation();");
             writer.println(ident(3) + "SampleBuffer buffer = new SampleBuffer();");
-            writer.println(ident(3) + method.getName() + "_" + benchmarkKind.shortLabel() + JMH_STUB_SUFFIX + "(control, buffer, targetSamples, control.benchmarkParams.getOpsPerInvocation(), batchSize" + prefix(states.getArgList(method)) + ");");
+            writer.println(ident(3) + method.getName() + "_" + benchmarkKind.shortLabel() + JMH_STUB_SUFFIX + "(control, res, buffer, targetSamples, opsPerInv, batchSize" + prefix(states.getArgList(method)) + ");");
 
             // control objects get a special treatment
             for (StateObject so : states.getControls()) {
@@ -921,6 +964,7 @@
             writer.println(ident(5) + emitCall(method, states) + ';');
             invocationEpilog(writer, 5, method, states, false);
 
+            writer.println(ident(5) + "res.allOps++;");
             writer.println(ident(4) + "}");
             writer.println(ident(4) + "control.preTearDown();");
             writer.println(ident(3) + "} catch (InterruptedException ie) {");
@@ -929,7 +973,22 @@
 
             iterationEpilog(writer, 3, method, states);
 
-            writer.println(ident(3) + "Collection<SampleTimeResult> results = new ArrayList<SampleTimeResult>();");
+            /*
+               Adjust the operation counts:
+                  1) res.measuredOps counted the batched @Benchmark invocations. Therefore, we need only
+                     to adjust for opsPerInv (pretending each @Benchmark invocation counts as $opsPerInv ops);
+                  2) res.allOps counted the individual @Benchmark invocations; to it needs the adjustment for $batchSize.
+
+               It's prudent to make the multiplication first to get more accuracy.
+             */
+
+            writer.println(ident(3) + "res.allOps += res.measuredOps * batchSize;");
+
+            writer.println(ident(3) + "res.allOps *= opsPerInv;");
+            writer.println(ident(3) + "res.allOps /= batchSize;");
+            writer.println(ident(3) + "res.measuredOps *= opsPerInv;");
+
+            writer.println(ident(3) + "HandlerResult results = new HandlerResult(res.allOps, res.measuredOps);");
             writer.println(ident(3) + "results.add(new SampleTimeResult(ResultRole.PRIMARY, \"" + method.getName() + "\", buffer, control.benchmarkParams.getTimeUnit()));");
             if (!isSingleMethod) {
                 writer.println(ident(3) + "results.add(new SampleTimeResult(ResultRole.SECONDARY, \"" + method.getName() + "\", buffer, control.benchmarkParams.getTimeUnit()));");
@@ -947,8 +1006,9 @@
             String methodName = method.getName() + "_" + benchmarkKind.shortLabel() + JMH_STUB_SUFFIX;
             compilerControl.defaultForceInline(method);
 
-            writer.println(ident(1) + "public" + (methodGroup.isStrictFP() ? " strictfp" : "") + " void " + methodName + "(InfraControl control, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize" + prefix(states.getTypeArgList(method)) + ") throws Throwable {");
+            writer.println(ident(1) + "public" + (methodGroup.isStrictFP() ? " strictfp" : "") + " void " + methodName + "(InfraControl control, RawResults result, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize" + prefix(states.getTypeArgList(method)) + ") throws Throwable {");
             writer.println(ident(2) + "long realTime = 0;");
+            writer.println(ident(2) + "long operations = 0;");
             writer.println(ident(2) + "int rnd = (int)System.nanoTime();");
             writer.println(ident(2) + "int rndMask = startRndMask;");
             writer.println(ident(2) + "long time = 0;");
@@ -979,16 +1039,19 @@
 
             invocationEpilog(writer, 3, method, states, true);
 
+            writer.println(ident(3) + "operations++;");
             writer.println(ident(2) + "} while(!control.isDone);");
             writer.println(ident(2) + "startRndMask = Math.max(startRndMask, rndMask);");
 
+            writer.println(ident(2) + "result.realTime = realTime;");
+            writer.println(ident(2) + "result.measuredOps = operations;");
             writer.println(ident(1) + "}");
             writer.println();
         }
     }
 
     private void generateSingleShotTime(ClassInfo classInfo, PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) {
-        writer.println(ident(1) + "public Collection<SingleShotResult> " + methodGroup.getName() + "_" + benchmarkKind + "(InfraControl control, ThreadParams threadParams) throws Throwable {");
+        writer.println(ident(1) + "public HandlerResult " + methodGroup.getName() + "_" + benchmarkKind + "(InfraControl control, ThreadParams threadParams) throws Throwable {");
 
         methodProlog(writer, methodGroup);
 
@@ -1004,7 +1067,7 @@
             iterationProlog(writer, 3, method, states);
 
             // measurement loop call
-            writer.println(ident(3) + "RawResults res = new RawResults(control.benchmarkParams.getOpsPerInvocation());");
+            writer.println(ident(3) + "RawResults res = new RawResults();");
             writer.println(ident(3) + "int batchSize = control.iterationParams.getBatchSize();");
             writer.println(ident(3) + method.getName() + "_" + benchmarkKind.shortLabel() + JMH_STUB_SUFFIX + "(control, batchSize, res" + prefix(states.getArgList(method)) + ");");
 
@@ -1012,7 +1075,17 @@
 
             iterationEpilog(writer, 3, method, states);
 
-            writer.println(ident(3) + "Collection<SingleShotResult> results = new ArrayList<SingleShotResult>();");
+            /*
+             * Adjust total ops:
+             *   Single shot always does single op.  Therefore, we need to adjust for $opsPerInv (pretending each @Benchmark
+             *   invocation counts as $opsPerInv ops). We *don't need* to adjust down for $batchSize, because we always have
+             *   one "op".
+             */
+
+            writer.println(ident(3) + "int opsPerInv = control.benchmarkParams.getOpsPerInvocation();");
+            writer.println(ident(3) + "long totalOps = opsPerInv;");
+
+            writer.println(ident(3) + "HandlerResult results = new HandlerResult(totalOps, totalOps);");
             writer.println(ident(3) + "results.add(new SingleShotResult(ResultRole.PRIMARY, \"" + method.getName() + "\", res.getTime(), control.benchmarkParams.getTimeUnit()));");
             if (!isSingleMethod) {
                 writer.println(ident(3) + "results.add(new SingleShotResult(ResultRole.SECONDARY, \"" + method.getName() + "\", res.getTime(), control.benchmarkParams.getTimeUnit()));");
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/main/java/org/openjdk/jmh/results/BenchmarkTaskResult.java	Thu Mar 19 16:44:12 2015 +0300
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.results;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+public class BenchmarkTaskResult {
+
+    /**
+     * Total benchmark ops done.
+     */
+    private final long allOperations;
+
+    /**
+     * Measured benchmark ops done: total without sync iterations.
+     */
+    private final long measuredOperations;
+
+    private final Collection<Result> results;
+
+    public BenchmarkTaskResult(long allOperations, long measuredOperations) {
+        this.allOperations = allOperations;
+        this.measuredOperations = measuredOperations;
+        this.results = new ArrayList<Result>();
+    }
+
+    public void add(Result result) {
+        results.add(result);
+    }
+
+    public Collection<Result> getResults() {
+        return results;
+    }
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/results/RawResults.java	Thu Mar 19 12:27:40 2015 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/results/RawResults.java	Thu Mar 19 16:44:12 2015 +0300
@@ -26,22 +26,14 @@
 
 public class RawResults {
 
-    private final int opsPerInv;
-    public long operations;
+    public long allOps;
+    public long measuredOps;
     public long realTime;
     public long startTime;
     public long stopTime;
 
-    public long getOperations() {
-        return opsPerInv * operations;
-    }
-
     public long getTime() {
         return (realTime > 0) ? realTime : (stopTime - startTime);
     }
 
-    public RawResults(int opsPerInv) {
-        this.opsPerInv = opsPerInv;
-    }
-
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkHandler.java	Thu Mar 19 12:27:40 2015 +0300
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkHandler.java	Thu Mar 19 16:44:12 2015 +0300
@@ -30,8 +30,8 @@
 import org.openjdk.jmh.profile.InternalProfiler;
 import org.openjdk.jmh.profile.Profiler;
 import org.openjdk.jmh.profile.ProfilerFactory;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
 import org.openjdk.jmh.results.IterationResult;
-import org.openjdk.jmh.results.Result;
 import org.openjdk.jmh.runner.format.OutputFormat;
 import org.openjdk.jmh.runner.options.Options;
 import org.openjdk.jmh.runner.options.TimeValue;
@@ -41,7 +41,6 @@
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -177,7 +176,7 @@
      * @return
      */
     private static boolean isValidBenchmarkSignature(Method m) {
-        if (m.getReturnType() != Collection.class) {
+        if (m.getReturnType() != BenchmarkTaskResult.class) {
             return false;
         }
         final Class<?>[] parameterTypes = m.getParameterTypes();
@@ -352,7 +351,7 @@
         startProfilers(benchmarkParams, params);
 
         // submit tasks to threadpool
-        Map<BenchmarkTask, Future<Collection<? extends Result>>> results = new HashMap<BenchmarkTask, Future<Collection<? extends Result>>>();
+        Map<BenchmarkTask, Future<BenchmarkTaskResult>> results = new HashMap<BenchmarkTask, Future<BenchmarkTaskResult>>();
         for (BenchmarkTask runner : runners) {
             results.put(runner, executor.submit(runner));
         }
@@ -384,9 +383,9 @@
         try {
             int expected = numThreads;
             while (expected > 0) {
-                for (Map.Entry<BenchmarkTask, Future<Collection<? extends Result>>> re : results.entrySet()) {
+                for (Map.Entry<BenchmarkTask, Future<BenchmarkTaskResult>> re : results.entrySet()) {
                     BenchmarkTask task = re.getKey();
-                    Future<Collection<? extends Result>> fr = re.getValue();
+                    Future<BenchmarkTaskResult> fr = re.getValue();
                     try {
                         long waitFor = Math.max(TimeUnit.MILLISECONDS.toNanos(100), waitDeadline - System.nanoTime());
                         fr.get(waitFor, TimeUnit.NANOSECONDS);
@@ -416,9 +415,9 @@
         // Get the results.
         // Should previous loop allow us to get to this point, we can fully expect
         // all the results ready without the exceptions.
-        for (Future<Collection<? extends Result>> fr : results.values()) {
+        for (Future<BenchmarkTaskResult> fr : results.values()) {
             try {
-                iterationResults.addResults(fr.get());
+                iterationResults.addResults(fr.get().getResults());
             } catch (InterruptedException ex) {
                 throw new IllegalStateException("Impossible to be here");
             } catch (ExecutionException ex) {
@@ -432,7 +431,7 @@
     /**
      * Worker body.
      */
-    class BenchmarkTask implements Callable<Collection<? extends Result>> {
+    class BenchmarkTask implements Callable<BenchmarkTaskResult> {
 
         private volatile Thread runner;
         private final InfraControl control;
@@ -444,13 +443,13 @@
         }
 
         @Override
-        public Collection<? extends Result> call() throws Exception {
+        public BenchmarkTaskResult call() throws Exception {
             try {
                 // bind the executor thread
                 runner = Thread.currentThread();
 
                 // go for the run
-                return (Collection<? extends Result>) method.invoke(instances.get(), control, threadParams);
+                return (BenchmarkTaskResult) method.invoke(instances.get(), control, threadParams);
             } catch (Throwable e) {
                 // about to fail the iteration;
                 // compensate for missed sync-iteration latches, we don't care about that anymore