changeset 54:ba3be59cfe06

Distinguish between pre-test error, test error, VM error.
author shade
date Thu, 03 Oct 2013 14:41:42 +0400
parents b63e0544fd19
children c4ebc9b5ea9d
files harness/src/main/java/org/openjdk/jcstress/JCStress.java harness/src/main/java/org/openjdk/jcstress/infra/Status.java harness/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java harness/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java harness/src/main/java/org/openjdk/jcstress/infra/grading/ExceptionReportPrinter.java harness/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor1_Runner.java harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor2_Arbiter1_Runner.java harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor2_Runner.java harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor3_Runner.java harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor4_Runner.java
diffstat 11 files changed, 107 insertions(+), 93 deletions(-) [+]
line wrap: on
line diff
--- a/harness/src/main/java/org/openjdk/jcstress/JCStress.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/JCStress.java	Thu Oct 03 14:41:42 2013 +0400
@@ -193,7 +193,7 @@
 
             if (ecode != 0) {
                 // Test had failed, record this.
-                collector.add(new TestResult(test.getName(), Status.ERROR));
+                collector.add(new TestResult(test.getName(), Status.VM_ERROR));
             }
 
             errDrainer.join();
--- a/harness/src/main/java/org/openjdk/jcstress/infra/Status.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/Status.java	Thu Oct 03 14:41:42 2013 +0400
@@ -30,29 +30,21 @@
  * @author Aleksey Shipilev (aleksey.shipilev@oracle.com)
  */
 public enum Status {
-    NORMAL {
-        @Override
-        public Status combine(Status other) {
-            return other;
-        }
-    },
-    API_MISMATCH {
-        @Override
-        public Status combine(Status other) {
-            if (other == ERROR) {
-                return other;
-            } else {
-                return this;
-            }
-        }
-    },
-    ERROR {
-        @Override
-        public Status combine(Status other) {
-            return this;
-        }
-    },;
 
-    public abstract Status combine(Status other);
+    NORMAL(0),
+    API_MISMATCH(1),
+    CHECK_TEST_ERROR(2),
+    TEST_ERROR(3),
+    VM_ERROR(4);
+
+    private int severity;
+
+    Status(int severity) {
+        this.severity = severity;
+    }
+
+    public Status combine(Status other) {
+        return severity >= other.severity ? this : other;
+    }
 
 }
--- a/harness/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/collectors/TestResult.java	Thu Oct 03 14:41:42 2013 +0400
@@ -75,18 +75,6 @@
         return states.values();
     }
 
-    public boolean isNormal() {
-        return status == Status.NORMAL;
-    }
-
-    public boolean isError() {
-        return status == Status.ERROR;
-    }
-
-    public boolean isMismatch() {
-        return status == Status.API_MISMATCH;
-    }
-
     public Environment getEnv() {
         return env;
     }
--- a/harness/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java	Thu Oct 03 14:41:42 2013 +0400
@@ -98,9 +98,11 @@
 
         observedResults.incrementAndGet();
 
+        int totalCount = 0;
         for (State s : r.getStates()) {
-            observedCount.addAndGet(s.getCount());
+            totalCount += s.getCount();
         }
+        observedCount.addAndGet(totalCount);
 
         if (verbose) {
             output.println();
@@ -111,40 +113,48 @@
     }
 
     private void parseSummary(PrintWriter output, TestResult r) {
-        if (!r.isNormal()) {
-            if (r.isError()) {
+        switch (r.status()) {
+            case TEST_ERROR:
                 output.println();
                 printLine(output, "ERROR", r);
                 parseVerbose(output, r);
                 output.println();
                 return;
-            } else {
+            case CHECK_TEST_ERROR:
+                printLine(output, "CHECK ERROR", r);
+                return;
+            case VM_ERROR:
+                printLine(output, "VM ERROR", r);
+                return;
+            case API_MISMATCH:
                 printLine(output, "SKIPPED", r);
                 return;
-            }
-        }
-
-        Test test = testDescriptions.get(r.getName());
-        if (test == null) {
-            output.println();
-            printLine(output, "ERROR", r);
-            parseVerbose(output, r);
-            output.println();
-        } else {
-            TestGrading grading = new TestGrading(r, test);
-            if (grading.isPassed) {
-                printLine(output, "OK", r);
-            } else {
-                output.println();
-                printLine(output, "FAILED", r);
-                parseVerbose(output, r);
-                output.println();
-            }
+            case NORMAL:
+                Test test = testDescriptions.get(r.getName());
+                if (test == null) {
+                    output.println();
+                    printLine(output, "UNKNOWN", r);
+                    parseVerbose(output, r);
+                    output.println();
+                } else {
+                    TestGrading grading = new TestGrading(r, test);
+                    if (grading.isPassed) {
+                        printLine(output, "OK", r);
+                    } else {
+                        output.println();
+                        printLine(output, "FAILED", r);
+                        parseVerbose(output, r);
+                        output.println();
+                    }
+                }
+                break;
+            default:
+                throw new IllegalStateException("Illegal status: " + r.status());
         }
     }
 
     private PrintWriter printLine(PrintWriter output, String label, TestResult r) {
-        return output.printf(" (ETA: %10s) (R: %s) (T:%4d/%d) (F:%2d/%d) (I:%2d/%d) %9s %s\n",
+        return output.printf(" (ETA: %10s) (R: %s) (T:%4d/%d) (F:%2d/%d) (I:%2d/%d) %10s %s\n",
                 computeETA(),
                 computeSpeed(),
                 testsProgress.size(), expectedTests, testsProgress.get(r.getName()).getVMindex(r.getVmID()), expectedForks, testsProgress.get(r.getName()).getIteration(r.getVmID()), expectedIterations,
--- a/harness/src/main/java/org/openjdk/jcstress/infra/grading/ExceptionReportPrinter.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/grading/ExceptionReportPrinter.java	Thu Oct 03 14:41:42 2013 +0400
@@ -121,21 +121,33 @@
     }
 
     public void emitTest(TestResult result, Test description) throws FileNotFoundException, JAXBException {
-        if (result.isError()) {
-            failures.add(result.getName() + " is in error.");
-        }
-
-        if (result.isNormal()) {
-            if (description != null) {
-                TestGrading grading = new TestGrading(result, description);
-                if (!grading.failureMessages.isEmpty()) {
-                    for (String msg : grading.failureMessages) {
-                        failures.add(result.getName() + ": " + msg);
+        switch (result.status()) {
+            case CHECK_TEST_ERROR:
+                failures.add(result.getName() + " had failed the pre-test.");
+                break;
+            case TEST_ERROR:
+                failures.add(result.getName() + " had failed the test.");
+                break;
+            case VM_ERROR:
+                failures.add(result.getName() + " had failed with the VM error.");
+                break;
+            case NORMAL:
+                if (description != null) {
+                    TestGrading grading = new TestGrading(result, description);
+                    if (!grading.failureMessages.isEmpty()) {
+                        for (String msg : grading.failureMessages) {
+                            failures.add(result.getName() + ": " + msg);
+                        }
                     }
+                } else {
+                    failures.add("TEST BUG: " + result.getName() + " description is not found.");
                 }
-            } else {
-                failures.add("TEST BUG: " + result.getName() + " description is not found.");
-            }
+                break;
+            case API_MISMATCH:
+                // silently ignore
+                break;
+            default:
+                throw new IllegalStateException("Unhandled status: " + result.status());
         }
     }
 
--- a/harness/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java	Thu Oct 03 14:41:42 2013 +0400
@@ -153,14 +153,14 @@
                 for (String testName : testNames) {
                     Test test = testDescriptions.get(testName);
                     TestResult result = results.get(testName);
-                    if (result.isNormal()) {
+                    if (result.status() == Status.NORMAL) {
                         if (new TestGrading(result, test).isPassed) {
                             passedCount++;
                         } else {
                             failedCount++;
                         }
                     } else {
-                        if (result.isMismatch()) {
+                        if (result.status() == Status.API_MISMATCH) {
                             sanityFailedCount++;
                         } else {
                             failedCount++;
@@ -255,7 +255,7 @@
             for (String testName : testNames) {
                 Test test = testDescriptions.get(testName);
                 TestResult result = results.get(testName);
-                if (result.isNormal()) {
+                if (result.status() == Status.NORMAL) {
                     emitTest(output, result, test);
                 } else {
                     emitTestFailure(output, result, test);
@@ -319,13 +319,20 @@
         output.println("<td>&nbsp;&nbsp;&nbsp;<a href=\"" + result.getName() + ".html\">" + cutKlass(result.getName()) + "</a></td>");
         output.println("<td></td>");
         if (description != null) {
-            if (result.isMismatch()) {
-                output.println("<td class=\"special\">SKIPPED</td>");
-                output.println("<td class=\"special\">Sanity check failed, API mismatch?</td>");
-            }
-            if (result.isError()) {
-                output.println("<td class=\"failed\">ERROR</td>");
-                output.println("<td class=\"failed\">Error running the test</td>");
+            switch (result.status()) {
+                case API_MISMATCH:
+                    output.println("<td class=\"special\">SKIPPED</td>");
+                    output.println("<td class=\"special\">Sanity check failed, API mismatch?</td>");
+                    break;
+                case CHECK_TEST_ERROR:
+                case TEST_ERROR:
+                    output.println("<td class=\"failed\">ERROR</td>");
+                    output.println("<td class=\"failed\">Error running the test</td>");
+                    break;
+                case VM_ERROR:
+                    output.println("<td class=\"failed\">VM ERROR</td>");
+                    output.println("<td class=\"failed\">Error running the VM</td>");
+                    break;
             }
         } else {
             output.println("<td class=\"failed\">MISSING DESCRIPTION</td>");
--- a/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor1_Runner.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor1_Runner.java	Thu Oct 03 14:41:42 2013 +0400
@@ -110,7 +110,8 @@
         } catch (NoSuchMethodError e) {
             return Status.API_MISMATCH;
         } catch (Throwable e) {
-            return Status.ERROR;
+            e.printStackTrace();
+            return Status.CHECK_TEST_ERROR;
         }
     }
 
@@ -159,7 +160,7 @@
         controlHolder.isStopped = true;
 
         if (!waitFor(tasks)) {
-            dumpFailure(test, Status.ERROR);
+            dumpFailure(test, Status.TEST_ERROR);
         }
 
         return counter;
--- a/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor2_Arbiter1_Runner.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor2_Arbiter1_Runner.java	Thu Oct 03 14:41:42 2013 +0400
@@ -104,7 +104,8 @@
         } catch (NoSuchMethodError e) {
             return Status.API_MISMATCH;
         } catch (Throwable e) {
-            return Status.ERROR;
+            e.printStackTrace();
+            return Status.CHECK_TEST_ERROR;
         }
     }
 
@@ -173,7 +174,7 @@
         controlHolder.isStopped = true;
 
         if (!waitFor(tasks)) {
-            dumpFailure(test, Status.ERROR);
+            dumpFailure(test, Status.TEST_ERROR);
         }
 
         return counter;
--- a/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor2_Runner.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor2_Runner.java	Thu Oct 03 14:41:42 2013 +0400
@@ -111,7 +111,8 @@
         } catch (NoSuchMethodError e) {
             return Status.API_MISMATCH;
         } catch (Throwable e) {
-            return Status.ERROR;
+            e.printStackTrace();
+            return Status.CHECK_TEST_ERROR;
         }
     }
 
@@ -169,7 +170,7 @@
         controlHolder.isStopped = true;
 
         if (!waitFor(tasks)) {
-            dumpFailure(test, Status.ERROR);
+            dumpFailure(test, Status.TEST_ERROR);
         }
 
         return counter;
--- a/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor3_Runner.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor3_Runner.java	Thu Oct 03 14:41:42 2013 +0400
@@ -113,7 +113,8 @@
         } catch (NoSuchMethodError e) {
             return Status.API_MISMATCH;
         } catch (Throwable e) {
-            return Status.ERROR;
+            e.printStackTrace();
+            return Status.CHECK_TEST_ERROR;
         }
     }
 
@@ -180,7 +181,7 @@
         controlHolder.isStopped = true;
 
         if (!waitFor(tasks)) {
-            dumpFailure(test, Status.ERROR);
+            dumpFailure(test, Status.TEST_ERROR);
         }
 
         return counter;
--- a/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor4_Runner.java	Thu Oct 03 13:12:25 2013 +0400
+++ b/harness/src/main/java/org/openjdk/jcstress/infra/runners/Actor4_Runner.java	Thu Oct 03 14:41:42 2013 +0400
@@ -115,7 +115,8 @@
         } catch (NoSuchMethodError e) {
             return Status.API_MISMATCH;
         } catch (Throwable e) {
-            return Status.ERROR;
+            e.printStackTrace();
+            return Status.CHECK_TEST_ERROR;
         }
     }
 
@@ -191,7 +192,7 @@
         controlHolder.isStopped = true;
 
         if (!waitFor(tasks)) {
-            dumpFailure(test, Status.ERROR);
+            dumpFailure(test, Status.TEST_ERROR);
         }
 
         return counter;