changeset 252:18ce1cd3026c

8014225: Rerun only failed 262 tests Reviewed-by: jlaskey, lagergren
author attila
date Wed, 08 May 2013 16:48:33 +0200
parents d28180d97c61
children 9073bcc4307b
files make/project.properties test/src/jdk/nashorn/internal/test/framework/AbstractScriptRunnable.java test/src/jdk/nashorn/internal/test/framework/ParallelTestRunner.java test/src/jdk/nashorn/internal/test/framework/TestConfig.java test/src/jdk/nashorn/internal/test/framework/TestFinder.java
diffstat 5 files changed, 69 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- a/make/project.properties	Wed May 08 15:51:36 2013 +0200
+++ b/make/project.properties	Wed May 08 16:48:33 2013 +0200
@@ -194,6 +194,8 @@
 test262-test-sys-prop.test.js.exclude.dir=\
     ${test262.suite.dir}/intl402/
 
+test262-test-sys-prop.test.failed.list.file=${build.dir}/test/failedTests
+
 # test262 test frameworks
 test262-test-sys-prop.test.js.framework=\
     -timezone=PST \
--- a/test/src/jdk/nashorn/internal/test/framework/AbstractScriptRunnable.java	Wed May 08 15:51:36 2013 +0200
+++ b/test/src/jdk/nashorn/internal/test/framework/AbstractScriptRunnable.java	Wed May 08 16:48:33 2013 +0200
@@ -117,7 +117,7 @@
     // run this test - compile or compile-and-run depending on option passed
     public void runTest() throws IOException {
         log(toString());
-
+        Thread.currentThread().setName(testFile.getPath());
         if (shouldRun) {
             // Analysis of failing tests list -
             // if test is in failing list it must fail
--- a/test/src/jdk/nashorn/internal/test/framework/ParallelTestRunner.java	Wed May 08 15:51:36 2013 +0200
+++ b/test/src/jdk/nashorn/internal/test/framework/ParallelTestRunner.java	Wed May 08 16:48:33 2013 +0200
@@ -25,6 +25,7 @@
 
 package jdk.nashorn.internal.test.framework;
 
+import static jdk.nashorn.internal.test.framework.TestConfig.TEST_FAILED_LIST_FILE;
 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE;
 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE;
 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_LIST;
@@ -37,6 +38,7 @@
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FileReader;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
@@ -324,7 +326,8 @@
         });
     }
 
-    public void run() {
+    @SuppressWarnings("resource")
+    public boolean run() throws IOException {
         final int testCount = tests.size();
         int passCount = 0;
         int doneCount = 0;
@@ -371,23 +374,36 @@
         });
 
         boolean hasFailed = false;
-        for (final ScriptRunnable.Result result : results) {
-            if (!result.passed()) {
-                if (hasFailed == false) {
-                    hasFailed = true;
-                    System.out.println();
-                    System.out.println("FAILED TESTS");
-                }
+        final String failedList = System.getProperty(TEST_FAILED_LIST_FILE);
+        final boolean hasFailedList = failedList != null;
+        final boolean hadPreviouslyFailingTests = hasFailedList && new File(failedList).length() > 0;
+        final FileWriter failedFileWriter = hasFailedList ? new FileWriter(failedList) : null;
+        try {
+            final PrintWriter failedListWriter = failedFileWriter == null ? null : new PrintWriter(failedFileWriter);
+            for (final ScriptRunnable.Result result : results) {
+                if (!result.passed()) {
+                    if (hasFailed == false) {
+                        hasFailed = true;
+                        System.out.println();
+                        System.out.println("FAILED TESTS");
+                    }
 
-                System.out.println(result.getTest());
-                if (result.exception != null) {
-                    final String exceptionString = result.exception instanceof TestFailedError ? result.exception.getMessage() : result.exception.toString();
-                    System.out.print(exceptionString.endsWith("\n") ? exceptionString : exceptionString + "\n");
-                    System.out.print(result.out != null ? result.out : "");
+                    System.out.println(result.getTest());
+                    if(failedFileWriter != null) {
+                        failedListWriter.println(result.getTest().testFile.getPath());
+                    }
+                    if (result.exception != null) {
+                        final String exceptionString = result.exception instanceof TestFailedError ? result.exception.getMessage() : result.exception.toString();
+                        System.out.print(exceptionString.endsWith("\n") ? exceptionString : exceptionString + "\n");
+                        System.out.print(result.out != null ? result.out : "");
+                    }
                 }
             }
+        } finally {
+            if(failedFileWriter != null) {
+                failedFileWriter.close();
+            }
         }
-
         final double timeElapsed = (System.nanoTime() - startTime) / 1e9; // [s]
         System.out.printf("Tests run: %d/%d tests, passed: %d (%.2f%%), failed: %d. Time elapsed: %.0fmin %.0fs.\n", doneCount, testCount, passCount, 100d * passCount / doneCount, doneCount - passCount, timeElapsed / 60, timeElapsed % 60);
         System.out.flush();
@@ -397,12 +413,25 @@
         if (hasFailed) {
             throw new AssertionError("TEST FAILED");
         }
+
+        if(hasFailedList) {
+            new File(failedList).delete();
+        }
+
+        if(hadPreviouslyFailingTests) {
+            System.out.println();
+            System.out.println("Good job on getting all your previously failing tests pass!");
+            System.out.println("NOW re-running all tests to make sure you haven't caused any NEW test failures.");
+            System.out.println();
+        }
+
+        return hadPreviouslyFailingTests;
     }
 
     public static void main(final String[] args) throws Exception {
         parseArgs(args);
 
-        new ParallelTestRunner().run();
+        while(new ParallelTestRunner().run());
     }
 
     private static void parseArgs(final String[] args) {
--- a/test/src/jdk/nashorn/internal/test/framework/TestConfig.java	Wed May 08 15:51:36 2013 +0200
+++ b/test/src/jdk/nashorn/internal/test/framework/TestConfig.java	Wed May 08 16:48:33 2013 +0200
@@ -72,4 +72,7 @@
 
     // shared context mode or not
     static final String TEST_JS_SHARED_CONTEXT              = "test.js.shared.context";
+
+    // file for storing last run's failed tests
+    static final String TEST_FAILED_LIST_FILE = "test.failed.list.file";
 }
--- a/test/src/jdk/nashorn/internal/test/framework/TestFinder.java	Wed May 08 15:51:36 2013 +0200
+++ b/test/src/jdk/nashorn/internal/test/framework/TestFinder.java	Wed May 08 16:48:33 2013 +0200
@@ -31,6 +31,7 @@
 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_EXPECT_RUN_FAIL;
 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_IGNORE_STD_ERROR;
 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_RUN;
+import static jdk.nashorn.internal.test.framework.TestConfig.TEST_FAILED_LIST_FILE;
 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE;
 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE;
 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_DIR;
@@ -41,7 +42,9 @@
 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ROOTS;
 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_UNCHECKED_DIR;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileReader;
 import java.io.IOException;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
@@ -86,6 +89,22 @@
     static <T> void findAllTests(final List<T> tests, final Set<String> orphans, final TestFactory<T> testFactory) throws Exception {
         final String framework = System.getProperty(TEST_JS_FRAMEWORK);
         final String testList = System.getProperty(TEST_JS_LIST);
+        final String failedTestFileName = System.getProperty(TEST_FAILED_LIST_FILE);
+        if(failedTestFileName != null) {
+            File failedTestFile = new File(failedTestFileName);
+            if(failedTestFile.exists() && failedTestFile.length() > 0L) {
+                try(final BufferedReader r = new BufferedReader(new FileReader(failedTestFile))) {
+                    for(;;) {
+                        final String testFileName = r.readLine();
+                        if(testFileName == null) {
+                            break;
+                        }
+                        handleOneTest(framework, new File(testFileName).toPath(), tests, orphans, testFactory);
+                    }
+                }
+                return;
+            }
+        }
         if (testList == null || testList.length() == 0) {
             // Run the tests under the test roots dir, selected by the
             // TEST_JS_INCLUDES patterns