changeset 157:8ee765583a16

First experimental slack on API returning results.
author shade
date Wed, 28 Aug 2013 18:43:34 +0400
parents a9eec858c97b
children ee135e6ab5c0
files jmh-api-samples/src/main/java/org/openjdk/jmh/MyAPIBench.java jmh-api-samples/src/main/java/org/openjdk/jmh/SimpleTest.java jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java jmh-core/src/main/java/org/openjdk/jmh/link/frames/ResultsFrame.java jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java
diffstat 8 files changed, 165 insertions(+), 19 deletions(-) [+]
line wrap: on
line diff
--- a/jmh-api-samples/src/main/java/org/openjdk/jmh/MyAPIBench.java	Wed Aug 28 16:03:53 2013 +0400
+++ b/jmh-api-samples/src/main/java/org/openjdk/jmh/MyAPIBench.java	Wed Aug 28 18:43:34 2013 +0400
@@ -26,11 +26,13 @@
 
 import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
 
+import java.util.concurrent.TimeUnit;
+
 public class MyAPIBench {
 
     @GenerateMicroBenchmark
-    public void myTest() {
-
+    public void myTest() throws InterruptedException {
+        TimeUnit.MILLISECONDS.sleep(1);
     }
 
 }
--- a/jmh-api-samples/src/main/java/org/openjdk/jmh/SimpleTest.java	Wed Aug 28 16:03:53 2013 +0400
+++ b/jmh-api-samples/src/main/java/org/openjdk/jmh/SimpleTest.java	Wed Aug 28 18:43:34 2013 +0400
@@ -24,16 +24,53 @@
  */
 package org.openjdk.jmh;
 
+import org.openjdk.jmh.logic.results.Result;
+import org.openjdk.jmh.logic.results.internal.RunResult;
+import org.openjdk.jmh.output.OutputFormatType;
+import org.openjdk.jmh.runner.BenchmarkRecord;
 import org.openjdk.jmh.runner.Runner;
 import org.openjdk.jmh.runner.RunnerException;
 import org.openjdk.jmh.runner.options.OptionsBuilder;
 import org.openjdk.jmh.runner.options.Options;
 import org.openjdk.jmh.runner.parameters.TimeValue;
 
+import java.util.Map;
+
 public class SimpleTest {
 
+    /*
+     * This sample uses VERY EXPERIMENTAL API; use with care!
+     */
+
     public static void main(String[] args) throws RunnerException {
-        Options opts = new OptionsBuilder().include(".*").warmupTime(TimeValue.seconds(1)).jvmArgs("-server").build();
-        new Runner(opts).run();
+        Options opts = new OptionsBuilder()
+                .include(".*")
+                .warmupTime(TimeValue.milliseconds(100))
+                .measurementTime(TimeValue.milliseconds(100))
+                .jvmArgs("-server")
+                .outputFormat(OutputFormatType.TextReport)
+                .build();
+
+        Map<BenchmarkRecord, RunResult> results = new Runner(opts).run();
+        RunResult runResult = extractSingleResult(results);
+        Result result = extractSingleMetric(runResult);
+
+        System.out.println("API replied benchmark score: " + result.getScore() + " " + result.getScoreUnit());
+    }
+
+    private static Result extractSingleMetric(RunResult result) {
+        Map<String,Result> results = result.getResults();
+        if (results.size() != 1) {
+            throw new IllegalArgumentException("More than one metric: " + results);
+        }
+        return results.values().iterator().next();
+    }
+
+    public static RunResult extractSingleResult(Map<BenchmarkRecord, RunResult> results) {
+        if (results.size() != 1) {
+            throw new IllegalArgumentException("More than one result: " + results);
+        }
+
+        return results.values().iterator().next();
     }
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java	Wed Aug 28 16:03:53 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkClient.java	Wed Aug 28 18:43:34 2013 +0400
@@ -28,6 +28,9 @@
 import org.openjdk.jmh.link.frames.InfraFrame;
 import org.openjdk.jmh.link.frames.OptionsFrame;
 import org.openjdk.jmh.link.frames.OutputFormatFrame;
+import org.openjdk.jmh.link.frames.ResultsFrame;
+import org.openjdk.jmh.logic.results.internal.RunResult;
+import org.openjdk.jmh.runner.BenchmarkRecord;
 import org.openjdk.jmh.runner.options.Options;
 
 import java.io.IOException;
@@ -77,4 +80,9 @@
         oos.close();
         clientSocket.close();
     }
+
+    public void pushResults(BenchmarkRecord record, RunResult result) throws IOException {
+        oos.writeObject(new ResultsFrame(record, result));
+        oos.flush();
+    }
 }
--- a/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java	Wed Aug 28 16:03:53 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/BinaryLinkServer.java	Wed Aug 28 18:43:34 2013 +0400
@@ -28,7 +28,11 @@
 import org.openjdk.jmh.link.frames.InfraFrame;
 import org.openjdk.jmh.link.frames.OptionsFrame;
 import org.openjdk.jmh.link.frames.OutputFormatFrame;
+import org.openjdk.jmh.link.frames.ResultsFrame;
+import org.openjdk.jmh.logic.results.Result;
+import org.openjdk.jmh.logic.results.internal.RunResult;
 import org.openjdk.jmh.output.format.OutputFormat;
+import org.openjdk.jmh.runner.BenchmarkRecord;
 import org.openjdk.jmh.runner.options.Options;
 
 import java.io.IOException;
@@ -53,6 +57,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
 
 /**
  * Accepts the OutputFormat calls from the network and forwards those to given local OutputFormat
@@ -65,6 +70,7 @@
     private final Set<String> forbidden;
     private final Acceptor acceptor;
     private final List<Handler> registeredHandlers;
+    private final Map<BenchmarkRecord, RunResult> results;
 
     public BinaryLinkServer(Options opts, OutputFormat out) throws IOException {
         this.opts = opts;
@@ -87,6 +93,7 @@
         }
 
         registeredHandlers = Collections.synchronizedList(new ArrayList<Handler>());
+        results = Collections.synchronizedMap(new TreeMap<BenchmarkRecord, RunResult>());
 
         acceptor = new Acceptor();
         acceptor.start();
@@ -121,6 +128,10 @@
         }
     }
 
+    public Map<BenchmarkRecord, RunResult> getResults() {
+        return results;
+    }
+
     private final class Acceptor extends Thread {
 
         private final ServerSocket server;
@@ -204,6 +215,9 @@
                     if (obj instanceof InfraFrame) {
                         handleInfra((InfraFrame) obj);
                     }
+                    if (obj instanceof ResultsFrame) {
+                        handleResults((ResultsFrame)obj);
+                    }
                     if (obj instanceof FinishingFrame) {
                         // close the streams
                         break;
@@ -226,6 +240,10 @@
             }
         }
 
+        private void handleResults(ResultsFrame obj) {
+            results.put(obj.getRecord(), obj.getResult());
+        }
+
         private void handleInfra(InfraFrame req) throws IOException {
             switch (req.getType()) {
                 case OPTIONS_REQUEST:
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jmh-core/src/main/java/org/openjdk/jmh/link/frames/ResultsFrame.java	Wed Aug 28 18:43:34 2013 +0400
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.link.frames;
+
+import org.openjdk.jmh.logic.results.internal.RunResult;
+import org.openjdk.jmh.runner.BenchmarkRecord;
+
+import java.io.Serializable;
+
+public class ResultsFrame implements Serializable {
+    private final BenchmarkRecord record;
+    private final RunResult result;
+
+    public ResultsFrame(BenchmarkRecord record, RunResult result) {
+        this.record = record;
+        this.result = result;
+    }
+
+    public BenchmarkRecord getRecord() {
+        return record;
+    }
+
+    public RunResult getResult() {
+        return result;
+    }
+}
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Wed Aug 28 16:03:53 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java	Wed Aug 28 18:43:34 2013 +0400
@@ -60,7 +60,7 @@
         this.out = handler;
     }
 
-    void runBenchmark(BenchmarkRecord benchmark, boolean doWarmup, boolean doMeasurement) {
+    RunResult runBenchmark(BenchmarkRecord benchmark, boolean doWarmup, boolean doMeasurement) {
         MicroBenchmarkHandler handler = null;
         try {
             Class<?> clazz = ClassUtils.loadClass(benchmark.generatedClass());
@@ -69,7 +69,7 @@
             MicroBenchmarkParameters executionParams = MicroBenchmarkParametersFactory.makeParams(options, benchmark, method, doWarmup, doMeasurement);
             handler = MicroBenchmarkHandlers.getInstance(out, benchmark, clazz, method, executionParams, options);
 
-            runBenchmark(executionParams, handler);
+            return runBenchmark(executionParams, handler);
         } catch (Throwable ex) {
             out.exception(ex);
             if (this.options.shouldFailOnError()) {
@@ -80,9 +80,11 @@
                 handler.shutdown();
             }
         }
+        // FIXME: Better handling here!
+        return null;
     }
 
-    protected void runBenchmark(MicroBenchmarkParameters executionParams, MicroBenchmarkHandler handler) {
+    protected RunResult runBenchmark(MicroBenchmarkParameters executionParams, MicroBenchmarkHandler handler) {
         List<IterationData> allResults = new ArrayList<IterationData>();
 
         out.startBenchmark(handler.getBenchmark(), executionParams, this.options.isVerbose());
@@ -133,6 +135,10 @@
         if (!allResults.isEmpty()) {
             RunResult result = aggregateIterationData(allResults);
             out.endBenchmark(handler.getBenchmark(), result);
+            return result;
+        } else {
+            // FIXME: Better handling here!
+            return null;
         }
     }
 
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Wed Aug 28 16:03:53 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/ForkedRunner.java	Wed Aug 28 18:43:34 2013 +0400
@@ -25,9 +25,12 @@
 package org.openjdk.jmh.runner;
 
 import org.openjdk.jmh.link.BinaryLinkClient;
+import org.openjdk.jmh.logic.results.internal.RunResult;
 import org.openjdk.jmh.output.OutputFormatFactory;
 import org.openjdk.jmh.runner.options.Options;
 
+import java.io.IOException;
+
 /**
  * Runner frontend class. Responsible for running micro benchmarks in forked JVM.
  *
@@ -35,18 +38,24 @@
  */
 public class ForkedRunner extends BaseRunner {
 
+    private final BinaryLinkClient link;
+
     public ForkedRunner(Options options, BinaryLinkClient link) {
         super(options, OutputFormatFactory.createBinaryHook(link));
+        this.link = link;
     }
 
-    public void run(BenchmarkRecord benchmark) {
+    public void run(BenchmarkRecord benchmark) throws IOException {
         // expect the tuple from the parent process
         if (options.isVerbose()) {
             out.println("Benchmarks: ");
             out.println(benchmark.getUsername());
         }
         out.startRun();
-        runBenchmark(benchmark, true, true);
+
+        RunResult result = runBenchmark(benchmark, true, true);
+        link.pushResults(benchmark, result);
+
         out.endRun();
         out.flush();
         out.close();
--- a/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Wed Aug 28 16:03:53 2013 +0400
+++ b/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java	Wed Aug 28 18:43:34 2013 +0400
@@ -27,6 +27,7 @@
 import org.openjdk.jmh.ForkedMain;
 import org.openjdk.jmh.annotations.Fork;
 import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.logic.results.internal.RunResult;
 import org.openjdk.jmh.output.OutputFormatFactory;
 import org.openjdk.jmh.output.format.OutputFormat;
 import org.openjdk.jmh.link.BinaryLinkServer;
@@ -50,8 +51,10 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -101,14 +104,14 @@
     }
 
     /** Main entry point */
-    public void run() throws RunnerException {
+    public Map<BenchmarkRecord, RunResult> run() throws RunnerException {
         Set<BenchmarkRecord> benchmarks = list.find(out, options.getRegexps(), options.getExcludes());
 
         if (benchmarks.isEmpty()) {
             out.println("No matching benchmarks. Miss-spelled regexp? Use -v for verbose output.");
             out.flush();
             out.close();
-            return;
+            return null;
         }
 
         // list microbenchmarks if -v
@@ -147,22 +150,25 @@
         benchmarks.clear();
         benchmarks.addAll(newBenchmarks);
 
+        Map<BenchmarkRecord, RunResult> results;
         if ((!options.getWarmupMicros().isEmpty()) ||
                 (options.getWarmupMode() == WarmupMode.BEFOREANY)) {
-            runBulkWarmupBenchmarks(benchmarks);
+            results = runBulkWarmupBenchmarks(benchmarks);
         } else {
-            runBenchmarks(benchmarks);
+            results = runBenchmarks(benchmarks);
         }
 
         out.flush();
         out.close();
+
+        return results;
     }
 
     /**
      * Run specified warmup microbenchmarks prior to running any requested mircobenchmarks.
      * TODO: Currently valid only for non-external JVM runs
      */
-    private void runBulkWarmupBenchmarks(Set<BenchmarkRecord> benchmarks) {
+    private Map<BenchmarkRecord, RunResult> runBulkWarmupBenchmarks(Set<BenchmarkRecord> benchmarks) {
         out.startRun();
 
         // list of micros executed before iteration
@@ -192,10 +198,14 @@
         }
         // run microbenchmarks
         //
+        Map<BenchmarkRecord, RunResult> results = new TreeMap<BenchmarkRecord, RunResult>();
         for (BenchmarkRecord benchmark : benchmarks) {
-            runBenchmark(benchmark, false, true);
+            RunResult result = runBenchmark(benchmark, false, true);
+            results.put(benchmark, result);
         }
         out.endRun();
+
+        return results;
     }
 
     private int decideForks(int optionForks, int benchForks) {
@@ -218,7 +228,7 @@
         }
     }
 
-    private void runBenchmarks(Set<BenchmarkRecord> benchmarks) {
+    private Map<BenchmarkRecord, RunResult> runBenchmarks(Set<BenchmarkRecord> benchmarks) {
         Set<BenchmarkRecord> embedded = new TreeSet<BenchmarkRecord>();
         Set<BenchmarkRecord> forked = new TreeSet<BenchmarkRecord>();
 
@@ -232,21 +242,29 @@
             }
         }
 
+        Map<BenchmarkRecord, RunResult> results = new TreeMap<BenchmarkRecord, RunResult>();
         for (BenchmarkRecord benchmark : embedded) {
-            runBenchmark(benchmark, true, true);
+            RunResult r = runBenchmark(benchmark, true, true);
+            results.put(benchmark, r);
         }
 
-        runSeparate(forked);
+
+        Map<BenchmarkRecord, RunResult> separateResults = runSeparate(forked);
+        results.putAll(separateResults);
+
         out.endRun();
+
+        return results;
     }
 
-    private void runSeparate(Set<BenchmarkRecord> benchmarksToFork) {
+    private Map<BenchmarkRecord, RunResult> runSeparate(Set<BenchmarkRecord> benchmarksToFork) {
         BinaryLinkServer server = null;
         try {
             server = new BinaryLinkServer(options, out);
             for (BenchmarkRecord benchmark : benchmarksToFork) {
                 runSeparateMicroBenchmark(server, benchmark, server.getHost(), server.getPort());
             }
+            return server.getResults();
         } catch (IOException e) {
             throw new IllegalStateException(e);
         } finally {