changeset 286:0709374cd9bd

Runners: move all test result grading to TestGrading.
author shade
date Mon, 06 Jun 2016 23:34:34 +0300
parents 9c8a067c5565
children 039eee0573fc
files jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ExceptionReportPrinter.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/GradingResult.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TestGrading.java jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestList.java
diffstat 7 files changed, 170 insertions(+), 213 deletions(-) [+]
line wrap: on
line diff
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java	Thu Jun 02 17:00:07 2016 +0300
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java	Mon Jun 06 23:34:34 2016 +0300
@@ -92,6 +92,8 @@
     }
 
     private void printResult(TestResult r, boolean isVerbose) {
+        TestGrading grading = TestGrading.grade(r);
+
         switch (r.status()) {
             case TIMEOUT_ERROR:
                 printLine("TIMEOUT", r);
@@ -121,22 +123,14 @@
                 softErrors++;
                 return;
             case NORMAL:
-                TestInfo test = TestList.getInfo(r.getName());
-                if (test == null) {
+                if (grading.isPassed) {
+                    printLine("OK", r);
+                    passed++;
+                } else {
                     output.println();
-                    printLine("UNKNOWN", r);
+                    printLine("FAILED", r);
                     isVerbose = true;
-                } else {
-                    TestGrading grading = new TestGrading(r, test);
-                    if (grading.isPassed) {
-                        printLine("OK", r);
-                        passed++;
-                    } else {
-                        output.println();
-                        printLine("FAILED", r);
-                        isVerbose = true;
-                        failed++;
-                    }
+                    failed++;
                 }
                 break;
             default:
@@ -162,18 +156,6 @@
             }
 
             TestInfo test = TestList.getInfo(r.getName());
-            if (test == null) {
-                output.printf("%" + idLen + "s %" + occLen +"s %" + expectLen + "s  %-" + descLen + "s%n", "Observed state", "Occurrences", "Expectation", "Interpretation");
-                for (String s : r.getStateKeys()) {
-                    output.printf("%" + idLen + "s %," + occLen + "d %" + expectLen + "s  %-" + descLen + "s%n",
-                            StringUtils.cutoff(s, idLen),
-                            r.getCount(s),
-                            Expect.UNKNOWN,
-                            "N/A");
-                }
-                return;
-            }
-
             for (StateCase c : test.cases()) {
                 idLen = Math.max(idLen, c.matchPattern().length());
                 expectLen = Math.max(expectLen, c.expect().toString().length());
@@ -186,40 +168,12 @@
 
             output.printf("%" + idLen + "s %" + occLen +"s %" + expectLen + "s  %-" + descLen + "s%n", "Observed state", "Occurrences", "Expectation", "Interpretation");
 
-            List<String> unmatchedStates = new ArrayList<>();
-            unmatchedStates.addAll(r.getStateKeys());
-
-            for (StateCase c : test.cases()) {
-                boolean matched = false;
-
-                for (String s : r.getStateKeys()) {
-                    if (c.matches(s)) {
-                        // match!
-                        output.printf("%" + idLen + "s %," + occLen + "d %" + expectLen + "s  %-" + descLen + "s%n",
-                                StringUtils.cutoff(s, idLen),
-                                r.getCount(s),
-                                c.expect(),
-                                StringUtils.cutoff(c.description(), descLen));
-                        matched = true;
-                        unmatchedStates.remove(s);
-                    }
-                }
-
-                if (!matched) {
-                    output.printf("%" + idLen + "s %," + occLen + "d %" + expectLen + "s  %-" + descLen + "s%n",
-                                StringUtils.cutoff(c.matchPattern(), idLen),
-                                0,
-                                c.expect(),
-                                StringUtils.cutoff(c.description(), descLen));
-                }
-            }
-
-            for (String s : unmatchedStates) {
+            for (GradingResult gradeRes : grading.gradingResults) {
                 output.printf("%" + idLen + "s %," + occLen + "d %" + expectLen + "s  %-" + descLen + "s%n",
-                        StringUtils.cutoff(s, idLen),
-                        r.getCount(s),
-                        test.unmatched().expect(),
-                        StringUtils.cutoff(test.unmatched().description(), descLen));
+                        StringUtils.cutoff(gradeRes.id, idLen),
+                        gradeRes.count,
+                        gradeRes.expect,
+                        StringUtils.cutoff(gradeRes.description, descLen));
             }
 
             output.println();
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ExceptionReportPrinter.java	Thu Jun 02 17:00:07 2016 +0300
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ExceptionReportPrinter.java	Mon Jun 06 23:34:34 2016 +0300
@@ -56,20 +56,10 @@
     }
 
     public void parse() throws FileNotFoundException, JAXBException {
-        Map<TestConfig, TestResult> results = ReportUtils.mergedByConfig(collector.getTestResults());
+        List<TestResult> results = ReportUtils.mergedByConfig(collector.getTestResults());
 
-        // build prefixes
-        Multimap<String, TestConfig> packages = new TreeMultimap<>();
-        for (TestConfig k : results.keySet()) {
-            String name = k.name;
-            String pack = name.substring(0, name.lastIndexOf("."));
-            packages.put(pack, k);
-        }
-
-        for (TestConfig k : results.keySet()) {
-            TestInfo test = TestList.getInfo(k.name);
-            TestResult result = results.get(k);
-            emitTest(result, test);
+        for (TestResult k : results) {
+            emitTest(k);
         }
 
         if (!failures.isEmpty()) {
@@ -81,7 +71,7 @@
         }
     }
 
-    public void emitTest(TestResult result, TestInfo description) throws FileNotFoundException, JAXBException {
+    public void emitTest(TestResult result) throws FileNotFoundException, JAXBException {
         switch (result.status()) {
             case CHECK_TEST_ERROR:
                 failures.add(result.getName() + " had failed with the pre-test error.");
@@ -96,15 +86,11 @@
                 failures.add(result.getName() + " had failed with the VM error.");
                 break;
             case NORMAL:
-                if (description != null) {
-                    TestGrading grading = new TestGrading(result, description);
-                    if (!grading.failureMessages.isEmpty()) {
-                        for (String msg : grading.failureMessages) {
-                            failures.add(result.getName() + ": " + msg);
-                        }
+                TestGrading grading = TestGrading.grade(result);
+                if (!grading.failureMessages.isEmpty()) {
+                    for (String msg : grading.failureMessages) {
+                        failures.add(result.getName() + ": " + msg);
                     }
-                } else {
-                    failures.add("TEST BUG: " + result.getName() + " description is not found.");
                 }
                 break;
             case API_MISMATCH:
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/GradingResult.java	Mon Jun 06 23:34:34 2016 +0300
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jcstress.infra.grading;
+
+import org.openjdk.jcstress.annotations.Expect;
+
+public class GradingResult {
+
+    public final String id;
+    public final Expect expect;
+    public final long count;
+    public final String description;
+
+    public GradingResult(String id, Expect expect, long count, String description) {
+        this.id = id;
+        this.expect = expect;
+        this.count = count;
+        this.description = description;
+    }
+
+}
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java	Thu Jun 02 17:00:07 2016 +0300
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java	Mon Jun 06 23:34:34 2016 +0300
@@ -27,12 +27,10 @@
 
 import org.openjdk.jcstress.Options;
 import org.openjdk.jcstress.annotations.Expect;
-import org.openjdk.jcstress.infra.StateCase;
 import org.openjdk.jcstress.infra.Status;
 import org.openjdk.jcstress.infra.TestInfo;
 import org.openjdk.jcstress.infra.collectors.InProcessCollector;
 import org.openjdk.jcstress.infra.collectors.TestResult;
-import org.openjdk.jcstress.infra.runners.TestConfig;
 import org.openjdk.jcstress.infra.runners.TestList;
 import org.openjdk.jcstress.util.*;
 
@@ -67,7 +65,7 @@
     }
 
     public void parse() throws FileNotFoundException {
-        Map<String, TestResult> byName = ReportUtils.mergedByName(collector.getTestResults());
+        List<TestResult> byName = ReportUtils.mergedByName(collector.getTestResults());
 
         PrintWriter output = new PrintWriter(resultDir + "/index.html");
 
@@ -81,12 +79,9 @@
             int passedCount = 0;
             int failedCount = 0;
             int sanityFailedCount = 0;
-            for (String name : byName.keySet()) {
-                TestResult result = byName.get(name);
-                TestInfo test = TestList.getInfo(name);
-
+            for (TestResult result : byName) {
                 if (result.status() == Status.NORMAL) {
-                    if (new TestGrading(result, test).isPassed) {
+                    if (TestGrading.grade(result).isPassed) {
                         passedCount++;
                     } else {
                         failedCount++;
@@ -132,8 +127,7 @@
 
         {
             SortedMap<String, String> env = new TreeMap<>();
-            for (String k : byName.keySet()) {
-                TestResult result = byName.get(k);
+            for (TestResult result : byName) {
                 if (result != null) {
                     for (Map.Entry<String, String> kv : result.getEnv().entries().entrySet()) {
                         String key = kv.getKey();
@@ -207,9 +201,8 @@
         emitTestReports(ReportUtils.byName(collector.getTestResults()));
 
         if (verbose) {
-            Map<TestConfig, TestResult> byConfig = ReportUtils.mergedByConfig(collector.getTestResults());
-            for (TestConfig k : byConfig.keySet()) {
-                TestResult result = byConfig.get(k);
+            List<TestResult> byConfig = ReportUtils.mergedByConfig(collector.getTestResults());
+            for (TestResult result : byConfig) {
                 printer.add(result);
             }
         }
@@ -247,7 +240,7 @@
                 "<body>");
     }
 
-    private void printXTests(Map<String, TestResult> byName,
+    private void printXTests(List<TestResult> byName,
                              PrintWriter output,
                              String header,
                              String subheader,
@@ -259,15 +252,13 @@
         output.println("<table cellspacing=0 cellpadding=3 width=\"100%\">");
 
         boolean hadAnyTests = false;
-        for (String name : byName.keySet()) {
-            TestInfo test = TestList.getInfo(name);
-            TestResult result = byName.get(name);
-            TestGrading grading = new TestGrading(result, test);
+        for (TestResult result : byName) {
+            TestGrading grading = TestGrading.grade(result);
             if (filterStatus.test(result.status()) && filterGrading.test(grading)) {
                 if (result.status() == Status.NORMAL) {
-                    emitTest(output, result, test);
+                    emitTest(output, result);
                 } else {
-                    emitTestFailure(output, result, test);
+                    emitTestFailure(output, result);
                 }
                 hadAnyTests = true;
             }
@@ -282,77 +273,64 @@
         output.println("<br>");
     }
 
-    public void emitTest(PrintWriter output, TestResult result, TestInfo description) {
+    public void emitTest(PrintWriter output, TestResult result) {
         cellStyle = 3 - cellStyle;
         output.println("<tr class=\"cell" + cellStyle + "\">");
         output.println("<td>&nbsp;&nbsp;&nbsp;<a href=\"" + result.getName() + ".html\">" + StringUtils.chunkName(result.getName()) + "</a></td>");
         output.printf("<td>%s</td>", getRoughCount(result));
-        if (description != null) {
-            TestGrading grading = new TestGrading(result, description);
-            if (grading.isPassed) {
-                output.println("<td class=\"passed\">PASSED</td>");
-            } else {
-                output.println("<td class=\"failed\">FAILED</td>");
-            }
 
-            if (grading.hasInteresting) {
-                output.println("<td class=\"interesting\">INTERESTING</td>");
-            } else {
-                output.println("<td class=\"interesting\"></td>");
-            }
-            if (grading.hasSpec) {
-                output.println("<td class=\"spec\">SPEC</td>");
-            } else {
-                output.println("<td class=\"spec\"></td>");
-            }
-            output.println("<td class=\"passed\"></td>");
+        TestGrading grading = TestGrading.grade(result);
+        if (grading.isPassed) {
+            output.println("<td class=\"passed\">PASSED</td>");
         } else {
-            output.println("<td class=\"failed\">MISSING DESCRIPTION</td>");
-            output.println("<td class=\"failed\"></td>");
-            output.println("<td class=\"failed\"></td>");
-            output.println("<td class=\"failed\"></td>");
+            output.println("<td class=\"failed\">FAILED</td>");
         }
+
+        if (grading.hasInteresting) {
+            output.println("<td class=\"interesting\">INTERESTING</td>");
+        } else {
+            output.println("<td class=\"interesting\"></td>");
+        }
+        if (grading.hasSpec) {
+            output.println("<td class=\"spec\">SPEC</td>");
+        } else {
+            output.println("<td class=\"spec\"></td>");
+        }
+        output.println("<td class=\"passed\"></td>");
         output.println("</tr>");
     }
 
-    public void emitTestFailure(PrintWriter output, TestResult result, TestInfo description) {
+    public void emitTestFailure(PrintWriter output, TestResult result) {
         cellStyle = 3 - cellStyle;
         output.println("<tr class=\"cell" + cellStyle + "\">");
         output.println("<td>&nbsp;&nbsp;&nbsp;<a href=\"" + result.getName() + ".html\">" + StringUtils.chunkName(result.getName()) + "</a></td>");
         output.println("<td></td>");
-        if (description != null) {
-            switch (result.status()) {
-                case API_MISMATCH:
-                    output.println("<td class=\"interesting\">API MISMATCH</td>");
-                    output.println("<td class=\"interesting\"></td>");
-                    output.println("<td class=\"interesting\"></td>");
-                    output.println("<td class=\"interesting\">Sanity check failed, API mismatch?</td>");
-                    break;
-                case TEST_ERROR:
-                case CHECK_TEST_ERROR:
-                    output.println("<td class=\"failed\">ERROR</td>");
-                    output.println("<td class=\"failed\"></td>");
-                    output.println("<td class=\"failed\"></td>");
-                    output.println("<td class=\"failed\">Error while running the test</td>");
-                    break;
-                case TIMEOUT_ERROR:
-                    output.println("<td class=\"failed\">ERROR</td>");
-                    output.println("<td class=\"failed\"></td>");
-                    output.println("<td class=\"failed\"></td>");
-                    output.println("<td class=\"failed\">Timeout while running the test</td>");
-                    break;
-                case VM_ERROR:
-                    output.println("<td class=\"failed\">VM ERROR</td>");
-                    output.println("<td class=\"failed\"></td>");
-                    output.println("<td class=\"failed\"></td>");
-                    output.println("<td class=\"failed\">Error running the VM</td>");
-                    break;
-            }
-        } else {
-            output.println("<td class=\"failed\">MISSING DESCRIPTION</td>");
-            output.println("<td class=\"failed\"></td>");
-            output.println("<td class=\"failed\"></td>");
-            output.println("<td class=\"failed\"></td>");
+        switch (result.status()) {
+            case API_MISMATCH:
+                output.println("<td class=\"interesting\">API MISMATCH</td>");
+                output.println("<td class=\"interesting\"></td>");
+                output.println("<td class=\"interesting\"></td>");
+                output.println("<td class=\"interesting\">Sanity check failed, API mismatch?</td>");
+                break;
+            case TEST_ERROR:
+            case CHECK_TEST_ERROR:
+                output.println("<td class=\"failed\">ERROR</td>");
+                output.println("<td class=\"failed\"></td>");
+                output.println("<td class=\"failed\"></td>");
+                output.println("<td class=\"failed\">Error while running the test</td>");
+                break;
+            case TIMEOUT_ERROR:
+                output.println("<td class=\"failed\">ERROR</td>");
+                output.println("<td class=\"failed\"></td>");
+                output.println("<td class=\"failed\"></td>");
+                output.println("<td class=\"failed\">Timeout while running the test</td>");
+                break;
+            case VM_ERROR:
+                output.println("<td class=\"failed\">VM ERROR</td>");
+                output.println("<td class=\"failed\"></td>");
+                output.println("<td class=\"failed\"></td>");
+                output.println("<td class=\"failed\">Error running the VM</td>");
+                break;
         }
         output.println("</tr>");
     }
@@ -403,42 +381,13 @@
             output.println("<th>Interpretation</th>");
             output.println("</tr>");
 
-            List<String> unmatchedStates = new ArrayList<>();
-            unmatchedStates.addAll(r.getStateKeys());
-            for (StateCase c : test.cases()) {
-
-                boolean matched = false;
-
-                for (String s : r.getStateKeys()) {
-                    if (c.matches(s)) {
-                        // match!
-                        output.println("<tr bgColor=" + selectHTMLColor(c.expect(), r.getCount(s) == 0) + ">");
-                        output.println("<td>" + s + "</td>");
-                        output.println("<td align=center>" + r.getCount(s) + "</td>");
-                        output.println("<td align=center>" + c.expect() + "</td>");
-                        output.println("<td>" + c.description() + "</td>");
-                        output.println("</tr>");
-                        matched = true;
-                        unmatchedStates.remove(s);
-                    }
-                }
-
-                if (!matched) {
-                    output.println("<tr bgColor=" + selectHTMLColor(c.expect(), true) + ">");
-                    output.println("<td>" + c.matchPattern() + "</td>");
-                    output.println("<td align=center>" + 0 + "</td>");
-                    output.println("<td align=center>" + c.expect() + "</td>");
-                    output.println("<td>" + c.description() + "</td>");
-                    output.println("</tr>");
-                }
-            }
-
-            for (String s : unmatchedStates) {
-                output.println("<tr bgColor=" + selectHTMLColor(test.unmatched().expect(), r.getCount(s) == 0) + ">");
-                output.println("<td>" + s + "</td>");
-                output.println("<td align=center>" + r.getCount(s) + "</td>");
-                output.println("<td align=center>" + test.unmatched().expect() + "</td>");
-                output.println("<td>" + test.unmatched().expect() + "</td>");
+            TestGrading grading = TestGrading.grade(r);
+            for (GradingResult c : grading.gradingResults) {
+                output.println("<tr bgColor=" + selectHTMLColor(c.expect, c.count == 0) + ">");
+                output.println("<td>" + c.id + "</td>");
+                output.println("<td align=center>" + c.count + "</td>");
+                output.println("<td align=center>" + c.expect + "</td>");
+                output.println("<td>" + c.description + "</td>");
                 output.println("</tr>");
             }
 
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java	Thu Jun 02 17:00:07 2016 +0300
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java	Mon Jun 06 23:34:34 2016 +0300
@@ -33,33 +33,33 @@
 
 public class ReportUtils {
 
-    public static Map<TestConfig, TestResult> mergedByConfig(Collection<TestResult> src) {
+    public static List<TestResult> mergedByConfig(Collection<TestResult> src) {
         Multimap<TestConfig, TestResult> multiResults = new HashMultimap<>();
         for (TestResult r : src) {
             multiResults.put(r.getConfig(), r);
         }
 
-        Map<TestConfig, TestResult> results = new HashMap<>();
+        List<TestResult> results = new ArrayList<>();
         for (TestConfig config : multiResults.keys()) {
             Collection<TestResult> mergeable = multiResults.get(config);
             TestResult root = merged(config, mergeable);
-            results.put(config, root);
+            results.add(root);
         }
 
         return results;
     }
 
-    public static Map<String, TestResult> mergedByName(Collection<TestResult> src) {
+    public static List<TestResult> mergedByName(Collection<TestResult> src) {
         Multimap<String, TestResult> multiResults = new HashMultimap<>();
         for (TestResult r : src) {
             multiResults.put(r.getConfig().name, r);
         }
 
-        Map<String, TestResult> results = new HashMap<>();
+        List<TestResult> results = new ArrayList<>();
         for (String name : multiResults.keys()) {
             Collection<TestResult> mergeable = multiResults.get(name);
             TestResult root = merged(mergeable.iterator().next().getConfig(), mergeable);
-            results.put(name, root);
+            results.add(root);
         }
 
         return results;
@@ -67,9 +67,8 @@
 
     public static Multimap<String, TestResult> byName(Collection<TestResult> src) {
         Multimap<String, TestResult> result = new HashMultimap<>();
-        Map<TestConfig, TestResult> m = mergedByConfig(src);
-        for (TestConfig key : m.keySet()) {
-            result.put(key.name, m.get(key));
+        for (TestResult r : mergedByConfig(src)) {
+            result.put(r.getName(), r);
         }
         return result;
     }
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TestGrading.java	Thu Jun 02 17:00:07 2016 +0300
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TestGrading.java	Mon Jun 06 23:34:34 2016 +0300
@@ -28,6 +28,7 @@
 import org.openjdk.jcstress.infra.StateCase;
 import org.openjdk.jcstress.infra.TestInfo;
 import org.openjdk.jcstress.infra.collectors.TestResult;
+import org.openjdk.jcstress.infra.runners.TestList;
 import org.openjdk.jcstress.util.NonNullArrayList;
 
 import java.util.ArrayList;
@@ -40,25 +41,25 @@
     public boolean isPassed;
     public boolean hasInteresting;
     public boolean hasSpec;
+    public final List<GradingResult> gradingResults;
     public final List<String> failureMessages;
 
-    public TestGrading(TestResult r, TestInfo test) {
+    public static TestGrading grade(TestResult r) {
+        return new TestGrading(r);
+    }
+
+    private TestGrading(TestResult r) {
+        TestInfo test = TestList.getInfo(r.getName());
+        gradingResults = new ArrayList<>();
         failureMessages = new NonNullArrayList<>();
 
-        if (test == null) {
-            isPassed = false;
-            hasInteresting = false;
-            hasSpec = false;
-            failureMessages.add("No test.");
-            return;
-        }
-
         isPassed = true;
         hasInteresting = false;
         hasSpec = false;
 
         List<String> unmatchedStates = new ArrayList<>();
         unmatchedStates.addAll(r.getStateKeys());
+
         for (StateCase c : test.cases()) {
             boolean matched = false;
 
@@ -72,6 +73,13 @@
                     failureMessages.add(failureMessage(s, ex, count));
                     matched = true;
                     unmatchedStates.remove(s);
+
+                    gradingResults.add(new GradingResult(
+                            c.matchPattern(),
+                            c.expect(),
+                            r.getCount(s),
+                            c.description()
+                    ));
                 }
             }
 
@@ -80,6 +88,13 @@
                 hasInteresting |= hasInteresting(ex, 0);
                 hasSpec |= hasSpec(ex, 0);
                 failureMessages.add(failureMessage("N/A", ex, 0));
+
+                gradingResults.add(new GradingResult(
+                        c.matchPattern(),
+                        c.expect(),
+                        0,
+                        c.description()
+                ));
             }
         }
 
@@ -90,6 +105,13 @@
             hasInteresting |= hasInteresting(expect, count);
             hasSpec |= hasSpec(expect, count);
             failureMessages.add(failureMessage(s, expect, count));
+
+            gradingResults.add(new GradingResult(
+                    s,
+                    test.unmatched().expect(),
+                    r.getCount(s),
+                    test.unmatched().description()
+            ));
         }
     }
 
--- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestList.java	Thu Jun 02 17:00:07 2016 +0300
+++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestList.java	Mon Jun 06 23:34:34 2016 +0300
@@ -107,6 +107,10 @@
     }
 
     public static TestInfo getInfo(String name) {
-        return getTests().get(name);
+        TestInfo info = getTests().get(name);
+        if (info == null) {
+            throw new IllegalArgumentException("Unable to find info for test: " + name);
+        }
+        return info;
     }
 }