changeset 398:0801825c8b52

Run @GMB methods in batches to amortize infrastructure costs. Warmup/measurement settings to accept batch sizes. Works (and probably only sensible) with BenchmarkMode(SingleShot). Contributed-by: Sergey Kuksenko <sergey.kuksenko@oracle.com>
author shade
date Fri, 14 Feb 2014 17:12:30 +0400
parents 827d48b75dd5
children 542cef961e16
files jmh-core/src/main/java/org/openjdk/jmh/annotations/Measurement.java jmh-core/src/main/java/org/openjdk/jmh/annotations/Warmup.java jmh-core/src/main/java/org/openjdk/jmh/logic/InfraControl.java jmh-core/src/main/java/org/openjdk/jmh/output/format/TextReportFormat.java jmh-core/src/main/java/org/openjdk/jmh/processor/internal/GenerateMicroBenchmarkProcessor.java jmh-core/src/main/java/org/openjdk/jmh/processor/internal/MethodGroup.java jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkRecord.java jmh-core/src/main/java/org/openjdk/jmh/runner/LoopMicroBenchmarkHandler.java jmh-core/src/main/java/org/openjdk/jmh/runner/options/ChainedOptionsBuilder.java jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java jmh-core/src/main/java/org/openjdk/jmh/runner/options/Options.java jmh-core/src/main/java/org/openjdk/jmh/runner/options/OptionsBuilder.java jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/BenchmarkParams.java jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/Defaults.java jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/IterationParams.java jmh-core/src/test/java/org/openjdk/jmh/logic/results/TestAggregateResult.java jmh-core/src/test/java/org/openjdk/jmh/output/results/ResultFormatTest.java jmh-core/src/test/java/org/openjdk/jmh/runner/options/TestOptions.java jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_26_BatchSize.java
diffstat 19 files changed, 373 insertions(+), 37 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-core/src/main/java/org/openjdk/jmh/annotations/Measurement.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/annotations/Measurement.java	Fri Feb 14 17:12:30 2014 +0400
@@ -45,6 +45,7 @@
 
     public static final int BLANK_ITERATIONS = -1;
     public static final long BLANK_TIME = -1L;
+    public static final int BLANK_BATCHSIZE = -1;
 
     /** Amount of iterations */
     int iterations() default BLANK_ITERATIONS;
@@ -55,5 +56,8 @@
     /** time unit of the time value */
     TimeUnit timeUnit() default TimeUnit.SECONDS;
 
+    /** Batch size: number of benchmark method calls per operation (some benchmark modes can ignore this setting) */
+    int batchSize() default BLANK_BATCHSIZE;
+
 }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/annotations/Warmup.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/annotations/Warmup.java	Fri Feb 14 17:12:30 2014 +0400
@@ -45,6 +45,7 @@
 
     public static final int BLANK_ITERATIONS = -1;
     public static final long BLANK_TIME = -1L;
+    public static final int BLANK_BATCHSIZE = -1;
 
     /** Amount of iterations */
     int iterations() default BLANK_ITERATIONS;
@@ -52,7 +53,11 @@
     /** time of each iteration */
     long time() default BLANK_TIME;
 
+    /** time unit of the time value */
     TimeUnit timeUnit() default TimeUnit.SECONDS;
 
+    /** Batch size: number of benchmark method calls per operation (some benchmark modes can ignore this setting) */
+    int batchSize() default BLANK_BATCHSIZE;
+
 }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/logic/InfraControl.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/logic/InfraControl.java	Fri Feb 14 17:12:30 2014 +0400
@@ -80,8 +80,8 @@
         }
     }
 
-    public InfraControl(int threads, boolean syncIterations, TimeValue loopTime, CountDownLatch preSetup, CountDownLatch preTearDown, boolean lastIteration, TimeUnit timeUnit) {
-        super(threads, syncIterations, loopTime, preSetup, preTearDown, lastIteration, timeUnit);
+    public InfraControl(int threads, boolean syncIterations, TimeValue loopTime, CountDownLatch preSetup, CountDownLatch preTearDown, boolean lastIteration, TimeUnit timeUnit, int batchSize) {
+        super(threads, syncIterations, loopTime, preSetup, preTearDown, lastIteration, timeUnit, batchSize);
     }
 
     /**
@@ -179,7 +179,9 @@
     public final AtomicInteger warmupVisited, warmdownVisited;
     public volatile boolean warmupShouldWait, warmdownShouldWait;
 
-    public InfraControlL2(int threads, boolean syncIterations, TimeValue loopTime, CountDownLatch preSetup, CountDownLatch preTearDown, boolean lastIteration, TimeUnit timeUnit) {
+    public final int batchSize;
+
+    public InfraControlL2(int threads, boolean syncIterations, TimeValue loopTime, CountDownLatch preSetup, CountDownLatch preTearDown, boolean lastIteration, TimeUnit timeUnit, int batchSize) {
         this.threads = threads;
         this.syncIterations = syncIterations;
         this.warmupVisited = new AtomicInteger();
@@ -192,6 +194,7 @@
         this.duration = loopTime.convertTo(TimeUnit.NANOSECONDS);
         this.lastIteration = lastIteration;
         this.timeUnit = timeUnit;
+        this.batchSize = batchSize;
     }
 
     public void announceWarmupReady() {
@@ -238,16 +241,16 @@
     private boolean q161, q162, q163, q164, q165, q166, q167, q168;
     private boolean q171, q172, q173, q174, q175, q176, q177, q178;
 
-    public InfraControlL3(int threads, boolean syncIterations, TimeValue loopTime, CountDownLatch preSetup, CountDownLatch preTearDown, boolean lastIteration, TimeUnit timeUnit) {
-        super(threads, syncIterations, loopTime, preSetup, preTearDown, lastIteration, timeUnit);
+    public InfraControlL3(int threads, boolean syncIterations, TimeValue loopTime, CountDownLatch preSetup, CountDownLatch preTearDown, boolean lastIteration, TimeUnit timeUnit, int batchSize) {
+        super(threads, syncIterations, loopTime, preSetup, preTearDown, lastIteration, timeUnit, batchSize);
     }
 }
 
 abstract class InfraControlL4 extends InfraControlL3 {
     public int markerEnd;
 
-    public InfraControlL4(int threads, boolean syncIterations, TimeValue loopTime, CountDownLatch preSetup, CountDownLatch preTearDown, boolean lastIteration, TimeUnit timeUnit) {
-        super(threads, syncIterations, loopTime, preSetup, preTearDown, lastIteration, timeUnit);
+    public InfraControlL4(int threads, boolean syncIterations, TimeValue loopTime, CountDownLatch preSetup, CountDownLatch preTearDown, boolean lastIteration, TimeUnit timeUnit, int batchSize) {
+        super(threads, syncIterations, loopTime, preSetup, preTearDown, lastIteration, timeUnit, batchSize);
     }
 }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/output/format/TextReportFormat.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/output/format/TextReportFormat.java	Fri Feb 14 17:12:30 2014 +0400
@@ -55,8 +55,12 @@
 
     @Override
     public void startBenchmark(BenchmarkRecord name, BenchmarkParams mbParams) {
-        out.println("# Warmup: " + mbParams.getWarmup().getCount() + " iterations, " + mbParams.getWarmup().getTime() + " each");
-        out.println("# Measurement: " + mbParams.getMeasurement().getCount() + " iterations, " + mbParams.getMeasurement().getTime() + " each");
+        out.println("# Warmup: " + mbParams.getWarmup().getCount() + " iterations, " +
+                mbParams.getWarmup().getTime() + " each" +
+                (mbParams.getWarmup().getBatchSize() <= 1 ? "" : ", " + mbParams.getWarmup().getBatchSize() + " calls per batch"));
+        out.println("# Measurement: " + mbParams.getMeasurement().getCount() + " iterations, " +
+                mbParams.getMeasurement().getTime() + " each"+
+                (mbParams.getMeasurement().getBatchSize() <= 1 ? "" : ", " + mbParams.getMeasurement().getBatchSize() + " calls per batch"));
         out.println("# Threads: " + mbParams.getThreads() + " " + getThreadsString(mbParams.getThreads()) + (mbParams.shouldSynchIterations() ? ", will synchronize iterations" : ""));
         out.println("# Benchmark mode: " + name.getMode().longLabel());
         out.println("# Benchmark: " + name.getUsername());
--- a/jmh-core/src/main/java/org/openjdk/jmh/processor/internal/GenerateMicroBenchmarkProcessor.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/processor/internal/GenerateMicroBenchmarkProcessor.java	Fri Feb 14 17:12:30 2014 +0400
@@ -159,8 +159,10 @@
                                         group.getTotalThreadCount(),
                                         group.getWarmupIterations(),
                                         group.getWarmupTime(),
+                                        group.getWarmupBatchSize(),
                                         group.getMeasurementIterations(),
                                         group.getMeasurementTime(),
+                                        group.getMeasurementBatchSize(),
                                         group.getForks(),
                                         group.getWarmupForks(),
                                         group.getJVMArgs(),
@@ -955,7 +957,10 @@
             invocationProlog(writer, 3, method, states, false);
 
             writer.println(ident(3) + "long time1 = System.nanoTime();");
-            writer.println(ident(3) + emitCall(method, states) + ';');
+            writer.println(ident(3) + "int batchSize = control.batchSize;");
+            writer.println(ident(3) + "for (int b = 0; b < batchSize; b++) {");
+            writer.println(ident(4) + emitCall(method, states) + ';');
+            writer.println(ident(3) + "}");
             writer.println(ident(3) + "long time2 = System.nanoTime();");
 
             invocationEpilog(writer, 3, method, states, false);
--- a/jmh-core/src/main/java/org/openjdk/jmh/processor/internal/MethodGroup.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/processor/internal/MethodGroup.java	Fri Feb 14 17:12:30 2014 +0400
@@ -158,6 +158,14 @@
         return Optional.none();
     }
 
+    public Optional<Integer> getWarmupBatchSize() {
+        Warmup ann = getFinal(Warmup.class);
+        if (ann != null && ann.batchSize() != Warmup.BLANK_BATCHSIZE) {
+            return Optional.of(ann.batchSize());
+        }
+        return Optional.none();
+    }
+
     public Optional<Integer> getMeasurementIterations() {
         Measurement ann = getFinal(Measurement.class);
         if (ann != null && ann.iterations() != Measurement.BLANK_ITERATIONS) {
@@ -174,6 +182,14 @@
         return Optional.none();
     }
 
+    public Optional<Integer> getMeasurementBatchSize() {
+        Measurement ann = getFinal(Measurement.class);
+        if (ann != null && ann.batchSize() != Measurement.BLANK_BATCHSIZE) {
+            return Optional.of(ann.batchSize());
+        }
+        return Optional.none();
+    }
+
     public Optional<Integer> getForks() {
         Fork ann = getFinal(Fork.class);
         if (ann != null && ann.value() != Fork.BLANK_FORKS) {
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkRecord.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BenchmarkRecord.java	Fri Feb 14 17:12:30 2014 +0400
@@ -45,8 +45,10 @@
     private final Optional<Integer> threads;
     private final Optional<Integer> warmupIterations;
     private final Optional<TimeValue> warmupTime;
+    private final Optional<Integer> warmupBatchSize;
     private final Optional<Integer> measurementIterations;
     private final Optional<TimeValue> measurementTime;
+    private final Optional<Integer> measurementBatchSize;
     private final Optional<Integer> forks;
     private final Optional<Integer> warmupForks;
     private final Optional<Collection<String>> jvmArgs;
@@ -54,8 +56,8 @@
     private final Optional<Collection<String>> jvmArgsAppend;
 
     public BenchmarkRecord(String userName, String generatedName, Mode mode, int[] threadGroups, Optional<Integer> threads,
-                           Optional<Integer> warmupIterations, Optional<TimeValue> warmupTime,
-                           Optional<Integer> measurementIterations, Optional<TimeValue> measurementTime,
+                           Optional<Integer> warmupIterations, Optional<TimeValue> warmupTime, Optional<Integer> warmupBatchSize,
+                           Optional<Integer> measurementIterations, Optional<TimeValue> measurementTime, Optional<Integer> measurementBatchSize,
                            Optional<Integer> forks, Optional<Integer> warmupForks, Optional<Collection<String>> jvmArgs, Optional<Collection<String>> jvmArgsPrepend, Optional<Collection<String>> jvmArgsAppend) {
         this.userName = userName;
         this.generatedName = generatedName;
@@ -64,8 +66,10 @@
         this.threads = threads;
         this.warmupIterations = warmupIterations;
         this.warmupTime = warmupTime;
+        this.warmupBatchSize = warmupBatchSize;
         this.measurementIterations = measurementIterations;
         this.measurementTime = measurementTime;
+        this.measurementBatchSize = measurementBatchSize;
         this.forks = forks;
         this.warmupForks = warmupForks;
         this.jvmArgs = jvmArgs;
@@ -76,7 +80,7 @@
     public BenchmarkRecord(String line) {
         String[] args = line.split(BR_SEPARATOR);
 
-        if (args.length != 14) {
+        if (args.length != 16) {
             throw new IllegalStateException("Mismatched format for the line: " + line);
         }
 
@@ -87,25 +91,28 @@
         this.threads = Optional.of(args[4], Optional.INTEGER_UNMARSHALLER);
         this.warmupIterations = Optional.of(args[5], Optional.INTEGER_UNMARSHALLER);
         this.warmupTime = Optional.of(args[6], Optional.TIME_VALUE_UNMARSHALLER);
-        this.measurementIterations = Optional.of(args[7], Optional.INTEGER_UNMARSHALLER);
-        this.measurementTime = Optional.of(args[8], Optional.TIME_VALUE_UNMARSHALLER);
-        this.forks = Optional.of(args[9], Optional.INTEGER_UNMARSHALLER);
-        this.warmupForks = Optional.of(args[10], Optional.INTEGER_UNMARSHALLER);
-        this.jvmArgs = Optional.of(args[11], Optional.STRING_COLLECTION_UNMARSHALLER);
-        this.jvmArgsPrepend = Optional.of(args[12], Optional.STRING_COLLECTION_UNMARSHALLER);
-        this.jvmArgsAppend = Optional.of(args[13], Optional.STRING_COLLECTION_UNMARSHALLER);
+        this.warmupBatchSize = Optional.of(args[7], Optional.INTEGER_UNMARSHALLER);
+        this.measurementIterations = Optional.of(args[8], Optional.INTEGER_UNMARSHALLER);
+        this.measurementTime = Optional.of(args[9], Optional.TIME_VALUE_UNMARSHALLER);
+        this.measurementBatchSize = Optional.of(args[10], Optional.INTEGER_UNMARSHALLER);
+        this.forks = Optional.of(args[11], Optional.INTEGER_UNMARSHALLER);
+        this.warmupForks = Optional.of(args[12], Optional.INTEGER_UNMARSHALLER);
+        this.jvmArgs = Optional.of(args[13], Optional.STRING_COLLECTION_UNMARSHALLER);
+        this.jvmArgsPrepend = Optional.of(args[14], Optional.STRING_COLLECTION_UNMARSHALLER);
+        this.jvmArgsAppend = Optional.of(args[15], Optional.STRING_COLLECTION_UNMARSHALLER);
     }
 
     public BenchmarkRecord(String userName, String generatedName, Mode mode) {
         this(userName, generatedName, mode, new int[]{}, Optional.<Integer>none(),
-                Optional.<Integer>none(), Optional.<TimeValue>none(), Optional.<Integer>none(), Optional.<TimeValue>none(),
+                Optional.<Integer>none(), Optional.<TimeValue>none(), Optional.<Integer>none(), Optional.<Integer>none(), Optional.<TimeValue>none(), Optional.<Integer>none(),
                 Optional.<Integer>none(), Optional.<Integer>none(), Optional.<Collection<String>>none(), Optional.<Collection<String>>none(), Optional.<Collection<String>>none());
     }
 
     public String toLine() {
         return userName + BR_SEPARATOR + generatedName + BR_SEPARATOR + mode + BR_SEPARATOR + convert(threadGroups) + BR_SEPARATOR +
-                threads + BR_SEPARATOR + warmupIterations + BR_SEPARATOR + warmupTime + BR_SEPARATOR + measurementIterations + BR_SEPARATOR +
-                measurementTime + BR_SEPARATOR + forks + BR_SEPARATOR + warmupForks + BR_SEPARATOR +
+                threads + BR_SEPARATOR + warmupIterations + BR_SEPARATOR + warmupTime + BR_SEPARATOR + warmupBatchSize + BR_SEPARATOR +
+                measurementIterations + BR_SEPARATOR + measurementTime + BR_SEPARATOR + measurementBatchSize + BR_SEPARATOR +
+                forks + BR_SEPARATOR + warmupForks + BR_SEPARATOR +
                 jvmArgs.toString(Optional.STRING_COLLECTION_MARSHALLER) + BR_SEPARATOR +
                 jvmArgsPrepend.toString(Optional.STRING_COLLECTION_MARSHALLER) + BR_SEPARATOR +
                 jvmArgsAppend.toString(Optional.STRING_COLLECTION_MARSHALLER);
@@ -113,7 +120,8 @@
 
     public BenchmarkRecord cloneWith(Mode mode) {
         return new BenchmarkRecord(userName, generatedName, mode, threadGroups, threads,
-                warmupIterations, warmupTime, measurementIterations, measurementTime,
+                warmupIterations, warmupTime, warmupBatchSize,
+                measurementIterations, measurementTime, measurementBatchSize,
                 forks, warmupForks, jvmArgs, jvmArgsPrepend, jvmArgsAppend);
     }
 
@@ -213,6 +221,10 @@
         return warmupIterations;
     }
 
+    public Optional<Integer> getWarmupBatchSize() {
+        return warmupBatchSize;
+    }
+
     public Optional<TimeValue> getMeasurementTime() {
         return measurementTime;
     }
@@ -221,6 +233,10 @@
         return measurementIterations;
     }
 
+    public Optional<Integer> getMeasurementBatchSize() {
+        return measurementBatchSize;
+    }
+
     public Optional<Integer> getForks() {
         return forks;
     }
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/LoopMicroBenchmarkHandler.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/LoopMicroBenchmarkHandler.java	Fri Feb 14 17:12:30 2014 +0400
@@ -85,7 +85,7 @@
         // result object to accumulate the results in
         IterationResult iterationResults = new IterationResult(microbenchmark, params);
 
-        InfraControl control = new InfraControl(numThreads, shouldSynchIterations, runtime, preSetupBarrier, preTearDownBarrier, last, timeUnit);
+        InfraControl control = new InfraControl(numThreads, shouldSynchIterations, runtime, preSetupBarrier, preTearDownBarrier, last, timeUnit, params.getBatchSize());
 
         // preparing the worker runnables
         BenchmarkTask[] runners = new BenchmarkTask[numThreads];
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/options/ChainedOptionsBuilder.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/options/ChainedOptionsBuilder.java	Fri Feb 14 17:12:30 2014 +0400
@@ -144,6 +144,13 @@
     ChainedOptionsBuilder warmupIterations(int value);
 
     /**
+     * How large warmup batchSize should be?
+     * @param value batch size
+     * @return builder
+     */
+    ChainedOptionsBuilder warmupBatchSize(int value);
+
+    /**
      * How long each warmup iteration should take?
      * @param value time
      * @return builder
@@ -172,6 +179,13 @@
     ChainedOptionsBuilder measurementIterations(int count);
 
     /**
+     * How large measurement batchSize should be?
+     * @param value batch size
+     * @return builder
+     */
+    ChainedOptionsBuilder measurementBatchSize(int value);
+
+    /**
      * How long each measurement iteration should take?
      * @param value time
      * @return builder
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java	Fri Feb 14 17:12:30 2014 +0400
@@ -58,8 +58,10 @@
 
     private final Optional<Integer> iterations;
     private final Optional<TimeValue> runTime;
+    private final Optional<Integer> batchSize;
     private final Optional<Integer> warmupIterations;
     private final Optional<TimeValue> warmupTime;
+    private final Optional<Integer> warmupBatchSize;
     private final List<Mode> benchMode = new ArrayList<Mode>();
     private final Optional<Integer> threads;
     private final List<Integer> threadGroups = new ArrayList<Integer>();
@@ -94,12 +96,20 @@
         OptionSpec<Integer> optMeasureCount = parser.accepts("i", "Number of measurement iterations to do.")
                 .withRequiredArg().ofType(Integer.class).describedAs("int");
 
+        OptionSpec<Integer> optMeasureBatchSize = parser.accepts("bs", "Batch size: number of benchmark method calls per operation. " +
+                "(some benchmark modes can ignore this setting)")
+                .withRequiredArg().ofType(Integer.class).describedAs("int");
+
         OptionSpec<String> optMeasureTime = parser.accepts("r", "Time to spend at each measurement iteration.")
                 .withRequiredArg().ofType(String.class).describedAs("time");
 
         OptionSpec<Integer> optWarmupCount = parser.accepts("wi", "Number of warmup iterations to do.")
                 .withRequiredArg().ofType(Integer.class).describedAs("int");
 
+        OptionSpec<Integer> optWarmupBatchSize = parser.accepts("wbs", "Warmup batch size: number of benchmark method calls per operation. " +
+                "(some benchmark modes can ignore this setting)")
+                .withRequiredArg().ofType(Integer.class).describedAs("int");
+
         OptionSpec<String> optWarmupTime = parser.accepts("w", "Time to spend at each warmup iteration.")
                 .withRequiredArg().ofType(String.class).describedAs("time");
 
@@ -237,6 +247,8 @@
 
             iterations = Optional.eitherOf(optMeasureCount.value(set));
 
+            batchSize = Optional.eitherOf(optMeasureBatchSize.value(set));
+
             if (set.has(optMeasureTime)) {
                 String value = optMeasureTime.value(set);
                 try {
@@ -250,6 +262,8 @@
 
             warmupIterations = Optional.eitherOf(optWarmupCount.value(set));
 
+            warmupBatchSize = Optional.eitherOf(optWarmupBatchSize.value(set));
+
             if (set.has(optWarmupTime)) {
                 String value = optWarmupTime.value(set);
                 try {
@@ -568,6 +582,16 @@
      * @return the value
      */
     @Override
+    public Optional<Integer> getMeasurementBatchSize() {
+        return batchSize;
+    }
+
+    /**
+     * Getter
+     *
+     * @return the value
+     */
+    @Override
     public Optional<TimeValue> getMeasurementTime() {
         return runTime;
     }
@@ -598,6 +622,16 @@
      * @return the value
      */
     @Override
+    public Optional<Integer> getWarmupBatchSize() {
+        return warmupBatchSize;
+    }
+
+    /**
+     * Getter
+     *
+     * @return the value
+     */
+    @Override
     public Optional<Integer> getThreads() {
         return threads;
     }
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/options/Options.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/options/Options.java	Fri Feb 14 17:12:30 2014 +0400
@@ -122,6 +122,12 @@
     Optional<TimeValue> getWarmupTime();
 
     /**
+     * Number of batch size for warmup
+     * @return number of batch size for warmup
+     */
+    Optional<Integer> getWarmupBatchSize();
+
+    /**
      * Warmup mode.
      * @return warmup mode
      */
@@ -146,6 +152,12 @@
     Optional<TimeValue> getMeasurementTime();
 
     /**
+     * Number of batch size for measurement
+     * @return number of batch size for measurement
+     */
+    Optional<Integer> getMeasurementBatchSize();
+
+    /**
      * Benchmarks modes to execute.
      * @return modes to execute the benchmarks in; empty to use the default modes
      */
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/options/OptionsBuilder.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/options/OptionsBuilder.java	Fri Feb 14 17:12:30 2014 +0400
@@ -315,6 +315,25 @@
 
     // ---------------------------------------------------------------------------
 
+    private Optional<Integer> warmupBatchSize = Optional.none();
+
+    @Override
+    public ChainedOptionsBuilder warmupBatchSize(int value) {
+        this.warmupBatchSize = Optional.of(value);
+        return this;
+    }
+
+    @Override
+    public Optional<Integer> getWarmupBatchSize() {
+        if (otherOptions != null) {
+            return warmupBatchSize.orAnother(otherOptions.getWarmupBatchSize());
+        } else {
+            return warmupBatchSize;
+        }
+    }
+
+    // ---------------------------------------------------------------------------
+
     private Optional<TimeValue> warmupTime = Optional.none();
 
     @Override
@@ -411,6 +430,26 @@
 
     // ---------------------------------------------------------------------------
 
+    private Optional<Integer> measurementBatchSize = Optional.none();
+
+    @Override
+    public ChainedOptionsBuilder measurementBatchSize(int value) {
+        this.measurementBatchSize = Optional.of(value);
+        return this;
+    }
+
+    @Override
+    public Optional<Integer> getMeasurementBatchSize() {
+        if (otherOptions != null) {
+            return measurementBatchSize.orAnother(otherOptions.getMeasurementBatchSize());
+        } else {
+            return measurementBatchSize;
+        }
+    }
+
+
+    // ---------------------------------------------------------------------------
+
     private EnumSet<Mode> benchModes = EnumSet.noneOf(Mode.class);
 
     @Override
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/BenchmarkParams.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/BenchmarkParams.java	Fri Feb 14 17:12:30 2014 +0400
@@ -48,14 +48,16 @@
     /**
      * Test entry method
      */
-    public BenchmarkParams(boolean synchIterations, int threads, int[] threadGroups, int forks, int warmupForks, int warmupIters, TimeValue warmupTime, int measureIters, TimeValue measureTime) {
+    public BenchmarkParams(boolean synchIterations, int threads, int[] threadGroups, int forks, int warmupForks,
+                           int warmupIters, TimeValue warmupTime, int warmupBatchSize,
+                           int measureIters, TimeValue measureTime, int measureBatchSize) {
         this.synchIterations = synchIterations;
         this.threads = threads;
         this.threadGroups = threadGroups;
         this.forks = forks;
         this.warmupForks = warmupForks;
-        this.warmup = new IterationParams(this, warmupIters, warmupTime);
-        this.measurement = new IterationParams(this, measureIters, measureTime);
+        this.warmup = new IterationParams(this, warmupIters, warmupTime, warmupBatchSize);
+        this.measurement = new IterationParams(this, measureIters, measureTime, measureBatchSize);
     }
 
     public BenchmarkParams(Options options, BenchmarkRecord benchmark, ActionMode mode) {
@@ -76,11 +78,11 @@
 
         this.measurement = mode.doMeasurement() ?
                 getMeasurement(options, benchmark) :
-                new IterationParams(this, 0, TimeValue.NONE);
+                new IterationParams(this, 0, TimeValue.NONE, 1);
 
         this.warmup = mode.doWarmup() ?
                 getWarmup(options, benchmark) :
-                new IterationParams(this, 0, TimeValue.NONE);
+                new IterationParams(this, 0, TimeValue.NONE, 1);
 
         this.forks = options.getForkCount().orElse(
                 benchmark.getForks().orElse(
@@ -101,7 +103,13 @@
                 options.getWarmupTime().orElse(
                         benchmark.getWarmupTime().orElse(
                             (benchmark.getMode() == Mode.SingleShotTime) ? TimeValue.NONE : Defaults.WARMUP_TIME
-                ))
+                )),
+                (benchmark.getMode() != Mode.SingleShotTime) ? 1 :
+                        options.getWarmupBatchSize().orElse(
+                                benchmark.getWarmupBatchSize().orElse(
+                                        Defaults.WARMUP_BATCHSIZE
+                                )
+                        )
         );
     }
 
@@ -111,11 +119,17 @@
                 options.getMeasurementIterations().orElse(
                         benchmark.getMeasurementIterations().orElse(
                                 (benchmark.getMode() == Mode.SingleShotTime) ? Defaults.SINGLESHOT_MEASUREMENT_ITERATIONS : Defaults.MEASUREMENT_ITERATIONS
-                        )),
+                )),
                 options.getMeasurementTime().orElse(
                         benchmark.getMeasurementTime().orElse(
                                 (benchmark.getMode() == Mode.SingleShotTime) ? TimeValue.NONE : Defaults.ITERATION_TIME
-                        ))
+                )),
+                (benchmark.getMode() != Mode.SingleShotTime) ? 1 :
+                        options.getMeasurementBatchSize().orElse(
+                                benchmark.getMeasurementBatchSize().orElse(
+                                        Defaults.MEASUREMENT_BATCHSIZE
+                                )
+                        )
         );
     }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/Defaults.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/Defaults.java	Fri Feb 14 17:12:30 2014 +0400
@@ -46,9 +46,11 @@
 
     public static final int MEASUREMENT_ITERATIONS = 20;
     public static final int SINGLESHOT_MEASUREMENT_ITERATIONS = 1;
+    public static final int MEASUREMENT_BATCHSIZE = 1;
 
     public static final int WARMUP_ITERATIONS = 20;
     public static final int SINGLESHOT_WARMUP_ITERATIONS = 0;
+    public static final int WARMUP_BATCHSIZE = 1;
 
     public static final TimeValue WARMUP_TIME = new TimeValue(1, TimeUnit.SECONDS);
     public static final TimeValue ITERATION_TIME = new TimeValue(1, TimeUnit.SECONDS);
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/IterationParams.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/parameters/IterationParams.java	Fri Feb 14 17:12:30 2014 +0400
@@ -46,11 +46,16 @@
      */
     private final TimeValue timeValue;
 
+    /**
+     * batch size (method invocations inside the single op)
+     */
+    private final int batchSize;
 
-    public IterationParams(BenchmarkParams params, int count, TimeValue time) {
+    public IterationParams(BenchmarkParams params, int count, TimeValue time, int batchSize) {
         this.count = count;
         this.timeValue = time;
         this.benchmarkParams = params;
+        this.batchSize = batchSize;
     }
 
     public int getCount() {
@@ -61,6 +66,10 @@
         return timeValue;
     }
 
+    public int getBatchSize() {
+        return batchSize;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -69,6 +78,7 @@
         IterationParams that = (IterationParams) o;
 
         if (count != that.count) return false;
+        if (batchSize != that.batchSize) return false;
         if (timeValue != null ? !timeValue.equals(that.timeValue) : that.timeValue != null) return false;
 
         return true;
@@ -77,13 +87,14 @@
     @Override
     public int hashCode() {
         int result = count;
+        result = 31 * result + batchSize;
         result = 31 * result + (timeValue != null ? timeValue.hashCode() : 0);
         return result;
     }
 
     @Override
     public String toString() {
-        return "IterationParams("+ getCount()+", "+ getTime()+")";
+        return "IterationParams("+ getCount()+", "+ getTime()+", "+ getBatchSize()+")";
     }
 
     public BenchmarkParams getBenchmarkParams() {
--- a/jmh-core/src/test/java/org/openjdk/jmh/logic/results/TestAggregateResult.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/test/java/org/openjdk/jmh/logic/results/TestAggregateResult.java	Fri Feb 14 17:12:30 2014 +0400
@@ -46,7 +46,7 @@
 
     @BeforeClass
     public static void setupClass() {
-        result = new IterationResult(new BenchmarkRecord("blah", "blah", Mode.AverageTime), new IterationParams(null, 1, TimeValue.days(1)));
+        result = new IterationResult(new BenchmarkRecord("blah", "blah", Mode.AverageTime), new IterationParams(null, 1, TimeValue.days(1), 1));
         for (double d : values) {
             result.addResult(new ThroughputResult(ResultRole.PRIMARY, "test1", (long) d, 10 * 1000 * 1000));
         }
--- a/jmh-core/src/test/java/org/openjdk/jmh/output/results/ResultFormatTest.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/test/java/org/openjdk/jmh/output/results/ResultFormatTest.java	Fri Feb 14 17:12:30 2014 +0400
@@ -66,8 +66,10 @@
                     r.nextInt(1000),
                     r.nextInt(1000),
                     TimeValue.seconds(r.nextInt(1000)),
+                    1,
                     r.nextInt(1000),
-                    TimeValue.seconds(r.nextInt(1000))
+                    TimeValue.seconds(r.nextInt(1000)),
+                    1
             );
 
             Collection<BenchResult> benchResults = new ArrayList<BenchResult>();
--- a/jmh-core/src/test/java/org/openjdk/jmh/runner/options/TestOptions.java	Fri Feb 14 15:49:52 2014 +0400
+++ b/jmh-core/src/test/java/org/openjdk/jmh/runner/options/TestOptions.java	Fri Feb 14 17:12:30 2014 +0400
@@ -455,4 +455,28 @@
         Assert.assertEquals(EMPTY_BUILDER.getJvmArgs(), EMPTY_CMDLINE.getJvmArgs());
     }
 
+    @Test
+    public void testBatchSize() throws Exception {
+        CommandLineOptions cmdLine = new CommandLineOptions("-bs", "42");
+        Options builder = new OptionsBuilder().measurementBatchSize(42).build();
+        Assert.assertEquals(builder.getMeasurementBatchSize(), cmdLine.getMeasurementBatchSize());
+    }
+
+    @Test
+    public void testBatchSize_Default() throws Exception {
+        Assert.assertEquals(EMPTY_BUILDER.getMeasurementBatchSize(), EMPTY_CMDLINE.getMeasurementBatchSize());
+    }
+
+    @Test
+    public void testWarmupBatchSize() throws Exception {
+        CommandLineOptions cmdLine = new CommandLineOptions("-wbs", "43");
+        Options builder = new OptionsBuilder().warmupBatchSize(43).build();
+        Assert.assertEquals(builder.getWarmupBatchSize(), cmdLine.getWarmupBatchSize());
+    }
+
+    @Test
+    public void testWarmupBatchSize_Default() throws Exception {
+        Assert.assertEquals(EMPTY_BUILDER.getWarmupBatchSize(), EMPTY_CMDLINE.getWarmupBatchSize());
+    }
+
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_26_BatchSize.java	Fri Feb 14 17:12:30 2014 +0400
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.samples;
+
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.LinkedList;
+import java.util.List;
+
+@State(Scope.Thread)
+public class JMHSample_26_BatchSize {
+
+    /*
+     * Sometimes you need to evaluate operation which doesn't have
+     * the steady state. The cost of a benchmarked operation may
+     * significantly vary from invocation to invocation.
+     *
+     * In this case, using the timed measurements is not a good idea,
+     * and the only acceptable benchmark mode is a single shot. On the
+     * other hand, the operation may be too small for reliable single
+     * shot measurement.
+     *
+     * We can use "batch size" parameter to describe the number of
+     * benchmark calls to do per one invocation without looping the method
+     * manually and protect from problems described in JMHSample_11_Loops.
+     */
+
+    /*
+     * Suppose we want to measure insertion in the middle of the list.
+     */
+
+    List<String> list = new LinkedList<String>();
+
+    @GenerateMicroBenchmark
+    @Warmup(iterations = 5, time = 1)
+    @Measurement(iterations = 5, time = 1)
+    @BenchmarkMode(Mode.AverageTime)
+    public List<String> measureWrong_1() {
+        list.add(list.size() / 2, "something");
+        return list;
+    }
+
+    @GenerateMicroBenchmark
+    @Warmup(iterations = 5, time = 5)
+    @Measurement(iterations = 5, time = 5)
+    @BenchmarkMode(Mode.AverageTime)
+    public List<String> measureWrong_5() {
+        list.add(list.size() / 2, "something");
+        return list;
+    }
+
+    /*
+     * This is what you do with JMH.
+     */
+    @GenerateMicroBenchmark
+    @Warmup(iterations = 5, batchSize = 5000)
+    @Measurement(iterations = 5, batchSize = 5000)
+    @BenchmarkMode(Mode.SingleShotTime)
+    public List<String> measureRight() {
+        list.add(list.size() / 2, "something");
+        return list;
+    }
+
+    @Setup(Level.Iteration)
+    public void setup(){
+        list.clear();
+    }
+
+    /*
+     * ============================== HOW TO RUN THIS TEST: ====================================
+     *
+     * You can see completely different results for measureWrong_1 and measureWrong_5; this
+     * is because the workload has no steady state. The result of the workload is dependent
+     * on the measurement time. measureRight does not have this drawback, because it measures
+     * the N invocations of the test method and measures it's time.
+     *
+     * We measure batch of 5000 invocations and consider the batch as the single operation.
+     *
+     * You can run this test:
+     *
+     * a) Via the command line:
+     *    $ mvn clean install
+     *    $ java -jar target/microbenchmarks.jar ".*JMHSample_26.*" -f 1
+     *
+     * b) Via the Java API:
+     */
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(".*" + JMHSample_26_BatchSize.class.getSimpleName() + ".*")
+                .forks(1)
+                .build();
+
+        new Runner(opt).run();
+    }
+
+}