changeset 180:192f2396e6db

Validation workload extended.
author shade
date Tue, 10 Sep 2013 21:45:22 +0400
parents 3894e4876293
children d88ac2fba077
files jmh-core-benchmarks/pom.xml jmh-core-benchmarks/src/main/java/org/openjdk/jmh/MainValidation.java jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/EmptyBench.java jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LevelInvocationBench.java jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LevelIterationBench.java jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LevelTrialBench.java jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/MainValidation.java
diffstat 7 files changed, 280 insertions(+), 108 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-core-benchmarks/pom.xml	Tue Sep 10 16:07:07 2013 +0400
+++ b/jmh-core-benchmarks/pom.xml	Tue Sep 10 21:45:22 2013 +0400
@@ -107,7 +107,7 @@
                             <transformers>
                                 <transformer
                                         implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>org.openjdk.jmh.benchmarks.MainValidation</mainClass>
+                                    <mainClass>org.openjdk.jmh.MainValidation</mainClass>
                                 </transformer>
                             </transformers>
                         </configuration>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/MainValidation.java	Tue Sep 10 21:45:22 2013 +0400
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh;
+
+import org.openjdk.jmh.benchmarks.EmptyBench;
+import org.openjdk.jmh.benchmarks.LevelInvocationBench;
+import org.openjdk.jmh.benchmarks.LevelIterationBench;
+import org.openjdk.jmh.benchmarks.LevelTrialBench;
+import org.openjdk.jmh.benchmarks.NanoTimerBench;
+import org.openjdk.jmh.logic.results.Result;
+import org.openjdk.jmh.logic.results.RunResult;
+import org.openjdk.jmh.output.OutputFormatType;
+import org.openjdk.jmh.runner.BenchmarkRecord;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.Map;
+
+public class MainValidation {
+
+    public static void main(String[] args) throws RunnerException {
+
+        PrintWriter pw = new PrintWriter(System.out, true);
+
+        pw.println("JMH Infrastructure Validation Tests");
+        pw.println("-------------------------------------------------------------------------");
+        pw.println();
+
+        timingMeasurements(pw);
+        basic(pw);
+    }
+
+    private static void basic(PrintWriter pw) throws RunnerException {
+        pw.println("Basic benchmark tests:");
+        pw.println("  These tests show the basic overheads of calling the benchmark method.");
+        pw.println("  Additionally, the overheads of calling empty Fixture methods are measured.");
+        pw.println("  ");
+
+        {
+            Options opts = new OptionsBuilder()
+                    .include(EmptyBench.class.getCanonicalName() + ".*")
+                    .outputFormat(OutputFormatType.Silent)
+                    .build();
+
+            RunResult result = new Runner(opts).runSingle();
+            Result r = result.getPrimaryResult();
+
+            pw.printf("%50s", "running empty benchmark: ");
+            pw.flush();
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+        }
+
+        {
+            Options opts = new OptionsBuilder()
+                    .include(LevelTrialBench.class.getCanonicalName() + ".*benchmark_setup")
+                    .outputFormat(OutputFormatType.Silent)
+                    .build();
+
+            RunResult result = new Runner(opts).runSingle();
+            Result r = result.getPrimaryResult();
+
+            pw.printf("%50s", "Level.Trial, Scope.Benchmark, @Setup: ");
+            pw.flush();
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+        }
+
+        {
+            Options opts = new OptionsBuilder()
+                    .include(LevelTrialBench.class.getCanonicalName() + ".*benchmark_teardown")
+                    .outputFormat(OutputFormatType.Silent)
+                    .build();
+
+            RunResult result = new Runner(opts).runSingle();
+            Result r = result.getPrimaryResult();
+
+            pw.printf("%50s", "Level.Trial, Scope.Benchmark, @TearDown: ");
+            pw.flush();
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+        }
+
+        {
+            Options opts = new OptionsBuilder()
+                    .include(LevelIterationBench.class.getCanonicalName() + ".*benchmark_setup")
+                    .outputFormat(OutputFormatType.Silent)
+                    .build();
+
+            RunResult result = new Runner(opts).runSingle();
+            Result r = result.getPrimaryResult();
+
+            pw.printf("%50s", "Level.Iteration, Scope.Benchmark, @Setup: ");
+            pw.flush();
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+        }
+
+        {
+            Options opts = new OptionsBuilder()
+                    .include(LevelIterationBench.class.getCanonicalName() + ".*benchmark_teardown")
+                    .outputFormat(OutputFormatType.Silent)
+                    .build();
+
+            RunResult result = new Runner(opts).runSingle();
+            Result r = result.getPrimaryResult();
+
+            pw.printf("%50s", "Level.Iteration, Scope.Benchmark, @TearDown: ");
+            pw.flush();
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+        }
+
+        {
+            Options opts = new OptionsBuilder()
+                    .include(LevelInvocationBench.class.getCanonicalName() + ".*benchmark_setup")
+                    .outputFormat(OutputFormatType.Silent)
+                    .build();
+
+            RunResult result = new Runner(opts).runSingle();
+            Result r = result.getPrimaryResult();
+
+            pw.printf("%50s", "Level.Invocation, Scope.Benchmark, @Setup: ");
+            pw.flush();
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+        }
+
+        {
+            Options opts = new OptionsBuilder()
+                    .include(LevelInvocationBench.class.getCanonicalName() + ".*benchmark_teardown")
+                    .outputFormat(OutputFormatType.Silent)
+                    .build();
+
+            RunResult result = new Runner(opts).runSingle();
+            Result r = result.getPrimaryResult();
+
+            pw.printf("%50s", "Level.Invocation, Scope.Benchmark, @TearDown: ");
+            pw.flush();
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+        }
+
+        pw.println();
+    }
+
+    private static void timingMeasurements(PrintWriter pw) throws RunnerException {
+        pw.println("Timing measurements test");
+        pw.println("  This test shows the minimal individual timing possible to measure.");
+        pw.println("  This normally affects only SampleTime and SingleShot benchmark modes.");
+        pw.println("  Throughput/AverageTime tests can do better since they do only a few");
+        pw.println("  timestamps before and after the complete iteration.");
+        pw.println("  ");
+
+        long latency;
+        {
+            Options opts = new OptionsBuilder()
+                    .include(NanoTimerBench.class.getCanonicalName() + ".*latency")
+                    .outputFormat(OutputFormatType.Silent)
+                    .build();
+
+            RunResult result = new Runner(opts).runSingle();
+            Result r = result.getPrimaryResult();
+
+            pw.printf("%50s", "System.nanoTime latency: ");
+            pw.flush();
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+
+            latency = (long) r.getScore();
+        }
+
+        long granularity;
+        {
+            Options opts = new OptionsBuilder()
+                    .include(NanoTimerBench.class.getCanonicalName() + ".*granularity")
+                    .outputFormat(OutputFormatType.Silent)
+                    .build();
+
+            RunResult result = new Runner(opts).runSingle();
+            Result r = result.getPrimaryResult();
+
+            pw.printf("%50s", "System.nanoTime granularity: ");
+            pw.flush();
+            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
+            granularity = (long) r.getScore();
+        }
+
+        pw.println();
+        pw.printf("   Verdict: ");
+        if (latency < 100 && granularity < 100) {
+            pw.println("OK!");
+        } else {
+            pw.println("WARNING, latency and/or granularity is suspiciously high.");
+        }
+        pw.println();
+    }
+
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/EmptyBench.java	Tue Sep 10 21:45:22 2013 +0400
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.benchmarks;
+
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+public class EmptyBench {
+
+    @GenerateMicroBenchmark
+    public void empty() {
+
+    }
+
+}
--- a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LevelInvocationBench.java	Tue Sep 10 16:07:07 2013 +0400
+++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LevelInvocationBench.java	Tue Sep 10 21:45:22 2013 +0400
@@ -27,17 +27,21 @@
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
 import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
 
 import java.util.concurrent.TimeUnit;
 
-@BenchmarkMode(Mode.All)
+@BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
 public class LevelInvocationBench {
 
     @State(Scope.Benchmark)
--- a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LevelIterationBench.java	Tue Sep 10 16:07:07 2013 +0400
+++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LevelIterationBench.java	Tue Sep 10 21:45:22 2013 +0400
@@ -27,17 +27,21 @@
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
 import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
 
 import java.util.concurrent.TimeUnit;
 
-@BenchmarkMode(Mode.All)
+@BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
 public class LevelIterationBench {
 
     @State(Scope.Benchmark)
--- a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LevelTrialBench.java	Tue Sep 10 16:07:07 2013 +0400
+++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LevelTrialBench.java	Tue Sep 10 21:45:22 2013 +0400
@@ -27,17 +27,21 @@
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
 import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
 
 import java.util.concurrent.TimeUnit;
 
-@BenchmarkMode(Mode.All)
+@BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
 public class LevelTrialBench {
 
     @State(Scope.Benchmark)
--- a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/MainValidation.java	Tue Sep 10 16:07:07 2013 +0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,104 +0,0 @@
-/*
- * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package org.openjdk.jmh.benchmarks;
-
-import org.openjdk.jmh.logic.results.Result;
-import org.openjdk.jmh.logic.results.RunResult;
-import org.openjdk.jmh.output.OutputFormatType;
-import org.openjdk.jmh.runner.BenchmarkRecord;
-import org.openjdk.jmh.runner.Runner;
-import org.openjdk.jmh.runner.RunnerException;
-import org.openjdk.jmh.runner.options.Options;
-import org.openjdk.jmh.runner.options.OptionsBuilder;
-
-import java.io.PrintWriter;
-import java.util.Arrays;
-import java.util.Map;
-
-public class MainValidation {
-
-    public static void main(String[] args) throws RunnerException {
-
-        PrintWriter pw = new PrintWriter(System.out, true);
-
-        pw.println("JMH Infrastructure Validation Tests");
-        pw.println("-------------------------------------------------------------------------");
-        pw.println();
-
-        timingMeasurements(pw);
-
-    }
-
-    private static void timingMeasurements(PrintWriter pw) throws RunnerException {
-        pw.println("Timing measurements test");
-        pw.println("  This test shows the minimal individual timing possible to measure.");
-        pw.println("  This normally affects only SampleTime and SingleShot benchmark modes.");
-        pw.println("  Throughput/AverageTime tests can do better since they do only a few");
-        pw.println("  timestamps before and after the complete iteration.");
-        pw.println("  ");
-
-        long latency;
-        {
-            Options opts = new OptionsBuilder()
-                    .include(NanoTimerBench.class.getCanonicalName() + ".*latency")
-                    .outputFormat(OutputFormatType.Silent)
-                    .build();
-
-            RunResult result = new Runner(opts).runSingle();
-            Result r = result.getPrimaryResult();
-
-            pw.printf("%40s", "System.nanoTime latency: ");
-            pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
-
-            latency = (long) r.getScore();
-        }
-
-        long granularity;
-        {
-            Options opts = new OptionsBuilder()
-                    .include(NanoTimerBench.class.getCanonicalName() + ".*granularity")
-                    .outputFormat(OutputFormatType.Silent)
-                    .build();
-
-            RunResult result = new Runner(opts).runSingle();
-            Result r = result.getPrimaryResult();
-
-            pw.printf("%40s", "System.nanoTime granularity: ");
-            pw.flush();
-            pw.printf("%.2f +- %.2f ns\n", r.getScore(), r.getStatistics().getMeanError(0.95));
-            granularity = (long) r.getScore();
-        }
-
-        pw.println();
-        pw.printf("   Verdict: ");
-        if (latency < 100 && granularity < 100) {
-            pw.println("OK!");
-        } else {
-            pw.println("WARNING, latency and/or granularity is suspiciously high.");
-        }
-    }
-
-}