changeset 327:3b07f039cca7

Runners: better reporting at the end of the run.
author shade
date Thu, 20 Oct 2016 10:08:42 +0200
parents d48af41e33bf
children 01fb61958140
files jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ExceptionReportPrinter.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java
diffstat 7 files changed, 242 insertions(+), 105 deletions(-) [+]
line wrap: on
line diff
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java	Fri Oct 07 11:26:33 2016 +0200
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java	Thu Oct 20 10:08:42 2016 +0200
@@ -30,6 +30,7 @@
 import org.openjdk.jcstress.infra.collectors.*;
 import org.openjdk.jcstress.infra.grading.ConsoleReportPrinter;
 import org.openjdk.jcstress.infra.grading.ExceptionReportPrinter;
+import org.openjdk.jcstress.infra.grading.TextReportPrinter;
 import org.openjdk.jcstress.infra.grading.HTMLReportPrinter;
 import org.openjdk.jcstress.infra.runners.Runner;
 import org.openjdk.jcstress.infra.runners.TestConfig;
@@ -129,29 +130,27 @@
 
         diskCollector.close();
 
+        out.println();
+        out.println();
+        out.println("RUN COMPLETE.");
+        out.println();
+
         parseResults();
     }
 
     public void parseResults() throws Exception {
-        out.println();
-        out.println("Reading the results back... ");
-
         InProcessCollector collector = new InProcessCollector();
         new DiskReadCollector(opts.getResultFile(), collector).dump();
 
-        out.println("Generating the report... ");
-
-        HTMLReportPrinter p = new HTMLReportPrinter(opts, collector);
-        p.parse();
+        new TextReportPrinter(opts, collector).work();
+        new HTMLReportPrinter(opts, collector).work();
 
         out.println();
-        out.println();
-        out.println("Look at " + opts.getResultDest() + "index.html for the complete run results.");
+        out.println("HTML report was generated. Look at " + opts.getResultDest() + "index.html for the complete run results.");
         out.println();
 
         out.println("Will throw any pending exceptions at this point.");
-        ExceptionReportPrinter e = new ExceptionReportPrinter(collector);
-        e.parse();
+        new ExceptionReportPrinter(collector).work();
 
         out.println("Done.");
     }
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java	Fri Oct 07 11:26:33 2016 +0200
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java	Thu Oct 20 10:08:42 2016 +0200
@@ -25,6 +25,7 @@
 package org.openjdk.jcstress.infra.collectors;
 
 import org.openjdk.jcstress.infra.Status;
+import org.openjdk.jcstress.infra.grading.TestGrading;
 import org.openjdk.jcstress.infra.runners.TestConfig;
 import org.openjdk.jcstress.util.Environment;
 import org.openjdk.jcstress.util.HashMultiset;
@@ -102,4 +103,8 @@
     public TestConfig getConfig() {
         return config;
     }
+
+    public TestGrading grading() {
+        return TestGrading.grade(this);
+    }
 }
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java	Fri Oct 07 11:26:33 2016 +0200
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java	Thu Oct 20 10:08:42 2016 +0200
@@ -88,37 +88,26 @@
         observedIterations++;
         observedCount += r.getTotalCount();
 
-        printResult(r, verbose);
+        printResult(r);
     }
 
-    private void printResult(TestResult r, boolean isVerbose) {
-        TestGrading grading = TestGrading.grade(r);
+    private void printResult(TestResult r) {
+        TestGrading grading = r.grading();
 
         switch (r.status()) {
             case TIMEOUT_ERROR:
-                printLine("TIMEOUT", r);
-                hardErrors++;
-                return;
             case CHECK_TEST_ERROR:
             case TEST_ERROR:
-                printLine("ERROR", r);
+            case VM_ERROR:
                 hardErrors++;
-                return;
-            case VM_ERROR:
-                printLine("VM ERROR", r);
-                hardErrors++;
-                return;
+                break;
             case API_MISMATCH:
-                printLine("SKIPPED", r);
                 softErrors++;
-                return;
+                break;
             case NORMAL:
                 if (grading.isPassed) {
-                    printLine("OK", r);
                     passed++;
                 } else {
-                    output.println();
-                    printLine("FAILED", r);
                     failed++;
                 }
                 break;
@@ -126,46 +115,10 @@
                 throw new IllegalStateException("Illegal status: " + r.status());
         }
 
+        printLine(r);
+
         if (!grading.isPassed || grading.hasInteresting || verbose) {
-            output.format("    (fork: #%d, iteration #%d, JVM args: %s)%n",
-                    r.getConfig().forkId + 1,
-                    r.getIteration() + 1,
-                    r.getConfig().jvmArgs
-            );
-
-            int idLen = "Observed state".length();
-            int occLen = "Occurrences".length();
-            int expectLen = "Expectation".length();
-            int descLen = 60;
-
-            for (String s : r.getStateKeys()) {
-                idLen = Math.max(idLen, s.length());
-                occLen = Math.max(occLen, String.format("%,d", r.getCount(s)).length());
-                expectLen = Math.max(expectLen, Expect.UNKNOWN.toString().length());
-            }
-
-            TestInfo test = TestList.getInfo(r.getName());
-            for (StateCase c : test.cases()) {
-                idLen = Math.max(idLen, c.matchPattern().length());
-                expectLen = Math.max(expectLen, c.expect().toString().length());
-            }
-            expectLen = Math.max(expectLen, test.unmatched().expect().toString().length());
-
-            idLen += 2;
-            occLen += 2;
-            expectLen += 2;
-
-            output.printf("%" + idLen + "s %" + occLen +"s %" + expectLen + "s  %-" + descLen + "s%n", "Observed state", "Occurrences", "Expectation", "Interpretation");
-
-            for (GradingResult gradeRes : grading.gradingResults) {
-                output.printf("%" + idLen + "s %," + occLen + "d %" + expectLen + "s  %-" + descLen + "s%n",
-                        StringUtils.cutoff(gradeRes.id, idLen),
-                        gradeRes.count,
-                        gradeRes.expect,
-                        StringUtils.cutoff(gradeRes.description, descLen));
-            }
-
-            output.println();
+            ReportUtils.printDetails(output, r, true);
         }
 
         if (!r.getAuxData().isEmpty()) {
@@ -179,7 +132,8 @@
         printProgress();
     }
 
-    private void printLine(String label, TestResult r) {
+    private void printLine(TestResult r) {
+        String label = ReportUtils.statusToLabel(r);
         output.printf("\r%" + progressLen + "s\r", "");
         output.printf("%10s %s\n", "[" + label + "]", StringUtils.chunkName(r.getName()));
     }
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ExceptionReportPrinter.java	Fri Oct 07 11:26:33 2016 +0200
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ExceptionReportPrinter.java	Thu Oct 20 10:08:42 2016 +0200
@@ -52,10 +52,10 @@
 
     public ExceptionReportPrinter(InProcessCollector collector) throws JAXBException, FileNotFoundException {
         this.collector = collector;
-        failures = new ArrayList<>();
+        this.failures = new ArrayList<>();
     }
 
-    public void parse() throws FileNotFoundException, JAXBException {
+    public void work() throws FileNotFoundException, JAXBException {
         List<TestResult> results = ReportUtils.mergedByConfig(collector.getTestResults());
 
         for (TestResult k : results) {
@@ -87,7 +87,7 @@
                 failures.add(label + " had failed with the VM error.");
                 break;
             case NORMAL:
-                TestGrading grading = TestGrading.grade(result);
+                TestGrading grading = result.grading();
                 if (!grading.failureMessages.isEmpty()) {
                     for (String msg : grading.failureMessages) {
                         failures.add(label + ": " + msg);
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java	Fri Oct 07 11:26:33 2016 +0200
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java	Thu Oct 20 10:08:42 2016 +0200
@@ -53,18 +53,13 @@
     private final InProcessCollector collector;
     private int cellStyle = 1;
 
-    private final ConsoleReportPrinter printer;
-    private final boolean verbose;
-
     public HTMLReportPrinter(Options opts, InProcessCollector collector) throws FileNotFoundException {
         this.collector = collector;
-        this.printer = new ConsoleReportPrinter(opts, new PrintWriter(System.out, true), 0, 0);
         this.resultDir = opts.getResultDest();
-        this.verbose = opts.isVerbose();
         new File(resultDir).mkdirs();
     }
 
-    public void parse() throws FileNotFoundException {
+    public void work() throws FileNotFoundException {
         List<TestResult> byName = ReportUtils.mergedByName(collector.getTestResults());
 
         PrintWriter output = new PrintWriter(resultDir + "/index.html");
@@ -81,7 +76,7 @@
             int sanityFailedCount = 0;
             for (TestResult result : byName) {
                 if (result.status() == Status.NORMAL) {
-                    if (TestGrading.grade(result).isPassed) {
+                    if (result.grading().isPassed) {
                         passedCount++;
                     } else {
                         failedCount++;
@@ -167,48 +162,33 @@
         printXTests(byName, output,
                 "FAILED tests",
                 "Strong asserts were violated. Correct implementations should have no assert failures here.",
-                (s) -> s == Status.NORMAL,
-                (g) -> !g.isPassed);
+                r -> r.status() == Status.NORMAL && !r.grading().isPassed);
 
         printXTests(byName, output,
                 "ERROR tests",
                 "Tests break for some reason, other than failing the assert. Correct implementations should have none.",
-                (s) -> s != Status.NORMAL && s != Status.API_MISMATCH,
-                (g) -> true);
+                r -> r.status() != Status.NORMAL && r.status() != Status.API_MISMATCH);
 
         printXTests(byName, output,
                 "SPEC tests",
                 "Formally acceptable, but surprising results are observed. Implementations going beyond the minimal requirements should have none.",
-                (s) -> s == Status.NORMAL,
-                (g) -> g.hasSpec);
+                r -> r.status() == Status.NORMAL && r.grading().hasSpec);
 
         printXTests(byName, output,
                 "INTERESTING tests",
                 "Some interesting behaviors observed. This is for the plain curiosity.",
-                (s) -> s == Status.NORMAL,
-                (g) -> g.hasInteresting);
+                r -> r.status() == Status.NORMAL && r.grading().hasInteresting);
 
         printXTests(byName, output,
                 "All tests",
                 "",
-                (s) -> true,
-                (g) -> true);
+                r -> true);
 
         printFooter(output);
 
         output.close();
 
         emitTestReports(ReportUtils.byName(collector.getTestResults()));
-
-        if (verbose) {
-            List<TestResult> byConfig = ReportUtils.mergedByConfig(collector.getTestResults());
-            for (TestResult result : byConfig) {
-                TestGrading grading = TestGrading.grade(result);
-                if (!grading.isPassed || grading.hasInteresting) {
-                    printer.add(result);
-                }
-            }
-        }
     }
 
     private void printFooter(PrintWriter output) {
@@ -247,8 +227,7 @@
                              PrintWriter output,
                              String header,
                              String subheader,
-                             Predicate<Status> filterStatus,
-                             Predicate<TestGrading> filterGrading) {
+                             Predicate<TestResult> filterResults) {
         output.println("<hr>");
         output.println("<h3>" + header + "</h3>");
         output.println("<p>" + subheader + "</p>");
@@ -256,8 +235,7 @@
 
         boolean hadAnyTests = false;
         for (TestResult result : byName) {
-            TestGrading grading = TestGrading.grade(result);
-            if (filterStatus.test(result.status()) && filterGrading.test(grading)) {
+            if (filterResults.test(result)) {
                 if (result.status() == Status.NORMAL) {
                     emitTest(output, result);
                 } else {
@@ -282,7 +260,7 @@
         output.println("<td>&nbsp;&nbsp;&nbsp;<a href=\"" + result.getName() + ".html\">" + StringUtils.chunkName(result.getName()) + "</a></td>");
         output.printf("<td>%s</td>", getRoughCount(result));
 
-        TestGrading grading = TestGrading.grade(result);
+        TestGrading grading = result.grading();
         if (grading.isPassed) {
             output.println("<td class=\"passed\">PASSED</td>");
         } else {
@@ -384,8 +362,7 @@
             output.println("<th>Interpretation</th>");
             output.println("</tr>");
 
-            TestGrading grading = TestGrading.grade(r);
-            for (GradingResult c : grading.gradingResults) {
+            for (GradingResult c : r.grading().gradingResults) {
                 output.println("<tr bgColor=" + selectHTMLColor(c.expect, c.count == 0) + ">");
                 output.println("<td>" + c.id + "</td>");
                 output.println("<td align=center>" + c.count + "</td>");
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java	Fri Oct 07 11:26:33 2016 +0200
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java	Thu Oct 20 10:08:42 2016 +0200
@@ -24,11 +24,16 @@
  */
 package org.openjdk.jcstress.infra.grading;
 
+import org.openjdk.jcstress.annotations.Expect;
+import org.openjdk.jcstress.infra.StateCase;
 import org.openjdk.jcstress.infra.Status;
+import org.openjdk.jcstress.infra.TestInfo;
 import org.openjdk.jcstress.infra.collectors.TestResult;
 import org.openjdk.jcstress.infra.runners.TestConfig;
+import org.openjdk.jcstress.infra.runners.TestList;
 import org.openjdk.jcstress.util.*;
 
+import java.io.PrintWriter;
 import java.util.*;
 
 public class ReportUtils {
@@ -103,5 +108,73 @@
         return root;
     }
 
+    public static void printDetails(PrintWriter pw, TestResult r, boolean inProgress) {
+        if (inProgress) {
+            pw.format("    (fork: #%d, iteration #%d, JVM args: %s)%n",
+                    r.getConfig().forkId + 1,
+                    r.getIteration() + 1,
+                    r.getConfig().jvmArgs
+            );
+        } else {
+            pw.format("    (JVM args: %s)%n",
+                    r.getConfig().jvmArgs
+            );
+        }
 
+        int idLen = "Observed state".length();
+        int occLen = "Occurrences".length();
+        int expectLen = "Expectation".length();
+        int descLen = 60;
+
+        for (String s : r.getStateKeys()) {
+            idLen = Math.max(idLen, s.length());
+            occLen = Math.max(occLen, String.format("%,d", r.getCount(s)).length());
+            expectLen = Math.max(expectLen, Expect.UNKNOWN.toString().length());
+        }
+
+        TestInfo test = TestList.getInfo(r.getName());
+        for (StateCase c : test.cases()) {
+            idLen = Math.max(idLen, c.matchPattern().length());
+            expectLen = Math.max(expectLen, c.expect().toString().length());
+        }
+        expectLen = Math.max(expectLen, test.unmatched().expect().toString().length());
+
+        idLen += 2;
+        occLen += 2;
+        expectLen += 2;
+
+        pw.printf("%" + idLen + "s %" + occLen +"s %" + expectLen + "s  %-" + descLen + "s%n", "Observed state", "Occurrences", "Expectation", "Interpretation");
+
+        for (GradingResult gradeRes : r.grading().gradingResults) {
+            pw.printf("%" + idLen + "s %," + occLen + "d %" + expectLen + "s  %-" + descLen + "s%n",
+                    StringUtils.cutoff(gradeRes.id, idLen),
+                    gradeRes.count,
+                    gradeRes.expect,
+                    StringUtils.cutoff(gradeRes.description, descLen));
+        }
+
+        pw.println();
+    }
+
+    public static String statusToLabel(TestResult result) {
+        switch (result.status()) {
+            case TIMEOUT_ERROR:
+                return "TIMEOUT";
+            case CHECK_TEST_ERROR:
+            case TEST_ERROR:
+                return "ERROR";
+            case VM_ERROR:
+                return "VM ERROR";
+            case API_MISMATCH:
+                return "SKIPPED";
+            case NORMAL:
+                if (result.grading().isPassed) {
+                    return "OK";
+                } else {
+                    return "FAILED";
+                }
+            default:
+                throw new IllegalStateException("Illegal status: " + result.status());
+        }
+    }
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java	Thu Oct 20 10:08:42 2016 +0200
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jcstress.infra.grading;
+
+
+import org.openjdk.jcstress.Options;
+import org.openjdk.jcstress.infra.Status;
+import org.openjdk.jcstress.infra.collectors.InProcessCollector;
+import org.openjdk.jcstress.infra.collectors.TestResult;
+import org.openjdk.jcstress.util.StringUtils;
+
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Predicate;
+
+/**
+ * Prints HTML reports.
+ *
+ * @author Aleksey Shipilev (aleksey.shipilev@oracle.com)
+ */
+public class TextReportPrinter {
+
+    private final InProcessCollector collector;
+    private final boolean verbose;
+    private final PrintWriter pw;
+    private final Set<TestResult> emittedTests;
+
+    public TextReportPrinter(Options opts, InProcessCollector collector) throws FileNotFoundException {
+        this.collector = collector;
+        this.pw = new PrintWriter(System.out, true);
+        this.verbose = opts.isVerbose();
+        this.emittedTests = new HashSet<>();
+    }
+
+    public void work() throws FileNotFoundException {
+        emittedTests.clear();
+
+        List<TestResult> byConfig = ReportUtils.mergedByConfig(collector.getTestResults());
+
+        pw.println("RUN RESULTS:");
+        pw.println("------------------------------------------------------------------------------------------------------------------------");
+        pw.println();
+
+        printXTests(byConfig,
+                "INTERESTING tests",
+                "Some interesting behaviors observed. This is for the plain curiosity.",
+                r -> r.status() == Status.NORMAL && r.grading().hasInteresting);
+
+        printXTests(byConfig,
+                "SPEC tests",
+                "Formally acceptable, but surprising results are observed. Implementations going beyond the minimal requirements should have none.",
+                r -> r.status() == Status.NORMAL && r.grading().hasSpec);
+
+        printXTests(byConfig,
+                "FAILED tests",
+                "Strong asserts were violated. Correct implementations should have no assert failures here.",
+                r -> r.status() == Status.NORMAL && !r.grading().isPassed);
+
+        printXTests(byConfig,
+                "ERROR tests",
+                "Tests break for some reason, other than failing the assert. Correct implementations should have none.",
+                r -> r.status() != Status.NORMAL && r.status() != Status.API_MISMATCH);
+
+        if (verbose) {
+            printXTests(byConfig,
+                    "All remaining tests",
+                    "Tests that do not fall into any of the previous categories.",
+                    r -> !emittedTests.contains(r));
+        }
+
+        pw.println("------------------------------------------------------------------------------------------------------------------------");
+    }
+
+    private void printXTests(List<TestResult> byName,
+                             String header,
+                             String subHeader,
+                             Predicate<TestResult> predicate) {
+        boolean hadHeader = false;
+
+        for (TestResult result : byName) {
+
+            if (predicate.test(result)) {
+                if (!hadHeader) {
+                    pw.println("*** " + header);
+                    pw.println("  " + subHeader);
+                    pw.println();
+                    hadHeader = true;
+                }
+                emitTest(result);
+            }
+        }
+
+        if (hadHeader) {
+            pw.println();
+        }
+    }
+
+    public void emitTest(TestResult result) {
+        emittedTests.add(result);
+        pw.printf("%10s %s\n", "[" + ReportUtils.statusToLabel(result) + "]", StringUtils.chunkName(result.getName()));
+        ReportUtils.printDetails(pw, result, false);
+    }
+
+}