changeset 294:1f57afd14cc1

8014219: Make the run-octane harness more deterministic by not measuring elapsed time every iteration. Also got rid of most of the run logic in base.js and call benchmarks directly for the same purpose Reviewed-by: jlaskey, attila
author lagergren
date Mon, 27 May 2013 13:11:13 +0200
parents 751cfefff5eb
children 910fd2849c4c
files make/build-benchmark.xml src/jdk/nashorn/internal/runtime/AccessorProperty.java src/jdk/nashorn/internal/runtime/Property.java src/jdk/nashorn/internal/runtime/UserAccessorProperty.java test/script/basic/compile-octane.js.EXPECTED test/script/basic/run-octane.js
diffstat 6 files changed, 174 insertions(+), 196 deletions(-) [+]
line wrap: on
line diff
--- a/make/build-benchmark.xml	Fri May 24 23:27:52 2013 +0530
+++ b/make/build-benchmark.xml	Mon May 27 13:11:13 2013 +0200
@@ -32,250 +32,263 @@
     <pathconvert pathsep=" " property="octane-tests" refid="octane-set"/>
   </target>
 
+
   <!-- box2d -->
   <target name="octane-box2d" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/box2d.js"/>
+      <param name="octane-tests" value="box2d"/>
     </antcall>
   </target>
 
   <target name="octane-box2d-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/box2d.js"/>
+      <param name="octane-tests" value="box2d"/>
     </antcall>
   </target>
 
   <target name="octane-box2d-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/box2d.js"/>
+      <param name="octane-tests" value="box2d"/>
     </antcall>
   </target>
 
+
   <!-- code-load -->  
   <target name="octane-code-load" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/code-load.js"/>
+      <param name="octane-tests" value="code-load"/>
     </antcall>
   </target>
 
   <target name="octane-code-load-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/code-load.js"/>
+      <param name="octane-tests" value="code-load"/>
     </antcall>
   </target>
 
   <target name="octane-code-load-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/code-load.js"/>
+      <param name="octane-tests" value="code-load"/>
     </antcall>
   </target>
 
+
   <!-- crypto -->
   <target name="octane-crypto" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/crypto.js"/>
+      <param name="octane-tests" value="crypto"/>
     </antcall>
   </target>
 
   <target name="octane-crypto-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/crypto.js"/>
+      <param name="octane-tests" value="crypto"/>
     </antcall>
   </target>
 
   <target name="octane-crypto-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/crypto.js"/>
+      <param name="octane-tests" value="crypto"/>
     </antcall>
   </target>
 
+
   <!-- deltablue -->
   <target name="octane-deltablue" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/deltablue.js"/>
+      <param name="octane-tests" value="deltablue"/>
     </antcall>
   </target>
 
   <target name="octane-deltablue-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/deltablue.js"/>
+      <param name="octane-tests" value="deltablue"/>
     </antcall>
   </target>
 
   <target name="octane-deltablue-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/deltablue.js"/>
+      <param name="octane-tests" value="deltablue"/>
     </antcall>
   </target>
 
+
   <!-- earley-boyer -->
   <target name="octane-earley-boyer" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/earley-boyer.js"/>
+      <param name="octane-tests" value="earley-boyer"/>
     </antcall>
   </target>
 
   <target name="octane-earley-boyer-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/earley-boyer.js"/>
+      <param name="octane-tests" value="earley-boyer"/>
     </antcall>
   </target>
 
   <target name="octane-earley-boyer-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/earley-boyer.js"/>
+      <param name="octane-tests" value="earley-boyer"/>
     </antcall>
   </target>
 
+
   <!-- gbemu -->  
   <target name="octane-gbemu" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/gbemu.js"/>
+      <param name="octane-tests" value="gbemu"/>
     </antcall>
   </target>
 
   <target name="octane-gbemu-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/gbemu.js"/>
+      <param name="octane-tests" value="gbemu"/>
     </antcall>
   </target>
 
   <target name="octane-gbemu-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/gbemu.js"/>
+      <param name="octane-tests" value="gbemu"/>
     </antcall>
   </target>
 
+
   <!-- mandreel -->  
   <target name="octane-mandreel" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/mandreel.js"/>
+      <param name="octane-tests" value="mandreel"/>
     </antcall>
   </target>
 
   <target name="octane-mandreel-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/mandreel.js"/>
+      <param name="octane-tests" value="mandreel"/>
     </antcall>
   </target>
 
   <target name="octane-mandreel-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/mandreel.js"/>
+      <param name="octane-tests" value="mandreel"/>
     </antcall>
   </target>
 
+
   <!-- navier-stokes -->
   <target name="octane-navier-stokes" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/navier-stokes.js"/>
+      <param name="octane-tests" value="navier-stokes"/>
     </antcall>
   </target>
 
   <target name="octane-navier-stokes-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/navier-stokes.js"/>
+      <param name="octane-tests" value="navier-stokes"/>
     </antcall>
   </target>
 
   <target name="octane-navier-stokes-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/navier-stokes.js"/>
+      <param name="octane-tests" value="navier-stokes"/>
     </antcall>
   </target>
 
+
   <!-- pdfjs -->  
   <target name="octane-pdfjs" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/pdfjs.js"/>
+      <param name="octane-tests" value="pdfjs"/>
     </antcall>
   </target>
 
   <target name="octane-pdfjs-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/pdfjs.js"/>
+      <param name="octane-tests" value="pdfjs"/>
     </antcall>
   </target>
 
   <target name="octane-pdfjs-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/pdfjs.js"/>
+      <param name="octane-tests" value="pdfjs"/>
     </antcall>
   </target>
 
+
   <!-- raytrace -->
   <target name="octane-raytrace" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/raytrace.js"/>
+      <param name="octane-tests" value="raytrace"/>
     </antcall>
   </target>
 
   <target name="octane-raytrace-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/raytrace.js"/>
+      <param name="octane-tests" value="raytrace"/>
     </antcall>
   </target>
 
   <target name="octane-raytrace-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/raytrace.js"/>
+      <param name="octane-tests" value="raytrace"/>
     </antcall>
   </target>
 
+
   <!-- regexp -->
   <target name="octane-regexp" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/regexp.js"/>
+      <param name="octane-tests" value="regexp"/>
     </antcall>
   </target>
 
   <target name="octane-regexp-octane-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/regexp.js"/>
+      <param name="octane-tests" value="regexp"/>
     </antcall>
   </target>
 
   <target name="octane-regexp-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/regexp.js"/>
+      <param name="octane-tests" value="regexp"/>
     </antcall>
   </target>
 
+
   <!-- richards -->
   <target name="octane-richards" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/richards.js"/>
+      <param name="octane-tests" value="richards"/>
     </antcall>
   </target>
 
   <target name="octane-richards-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/richards.js"/>
+      <param name="octane-tests" value="richards"/>
     </antcall>
   </target>
 
   <target name="octane-richards-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/richards.js"/>
+      <param name="octane-tests" value="richards"/>
     </antcall>
   </target>
 
+
   <!-- splay -->
   <target name="octane-splay" depends="jar">
     <antcall target="run-octane">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/splay.js"/>
+      <param name="octane-tests" value="splay"/>
     </antcall>
   </target>
 
   <target name="octane-splay-v8" depends="jar">
     <antcall target="run-octane-v8">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/splay.js"/>
+      <param name="octane-tests" value="splay"/>
     </antcall>
   </target>
 
   <target name="octane-splay-rhino" depends="jar">
     <antcall target="run-octane-rhino">
-      <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/splay.js"/>
+      <param name="octane-tests" value="splay"/>
     </antcall>
   </target>
 
--- a/src/jdk/nashorn/internal/runtime/AccessorProperty.java	Fri May 24 23:27:52 2013 +0530
+++ b/src/jdk/nashorn/internal/runtime/AccessorProperty.java	Mon May 27 13:11:13 2013 +0200
@@ -306,14 +306,14 @@
     protected Object getObjectValue(final ScriptObject self, final ScriptObject owner) {
         if (isSpill()) {
             return self.spill[getSlot()];
-        } else {
-            try {
-                return getGetter(Object.class).invokeExact((Object)self);
-            } catch (final Error|RuntimeException e) {
-                throw e;
-            } catch (final Throwable e) {
-                throw new RuntimeException(e);
-            }
+        }
+
+        try {
+            return getGetter(Object.class).invokeExact((Object)self);
+        } catch (final Error|RuntimeException e) {
+            throw e;
+        } catch (final Throwable e) {
+            throw new RuntimeException(e);
         }
     }
 
--- a/src/jdk/nashorn/internal/runtime/Property.java	Fri May 24 23:27:52 2013 +0530
+++ b/src/jdk/nashorn/internal/runtime/Property.java	Mon May 27 13:11:13 2013 +0200
@@ -52,6 +52,7 @@
      * we can use leave flag byte initialized with (the default) zero value.
      */
 
+    /** Mask for property being both writable, enumerable and configurable */
     public static final int WRITABLE_ENUMERABLE_CONFIGURABLE = 0b0000_0000_0000;
 
     /** ECMA 8.6.1 - Is this property not writable? */
@@ -360,6 +361,7 @@
      * @param self the this object
      * @param owner the owner object
      * @param value the new property value
+     * @param strict is this a strict setter?
      */
     protected abstract void setObjectValue(ScriptObject self, ScriptObject owner, Object value, boolean strict);
 
--- a/src/jdk/nashorn/internal/runtime/UserAccessorProperty.java	Fri May 24 23:27:52 2013 +0530
+++ b/src/jdk/nashorn/internal/runtime/UserAccessorProperty.java	Mon May 27 13:11:13 2013 +0200
@@ -193,7 +193,6 @@
     // getter/setter may be inherited. If so, proto is bound during lookup. In either
     // inherited or self case, slot is also bound during lookup. Actual ScriptFunction
     // to be called is retrieved everytime and applied.
-    @SuppressWarnings("unused")
     static Object userAccessorGetter(final ScriptObject proto, final int slot, final Object self) {
         final ScriptObject container = (proto != null) ? proto : (ScriptObject)self;
         final Object       func      = container.getSpill(slot);
@@ -211,7 +210,6 @@
         return UNDEFINED;
     }
 
-    @SuppressWarnings("unused")
     static void userAccessorSetter(final ScriptObject proto, final int slot, final String name, final Object self, final Object value) {
         final ScriptObject container = (proto != null) ? proto : (ScriptObject)self;
         final Object       func      = container.getSpill(slot);
--- a/test/script/basic/compile-octane.js.EXPECTED	Fri May 24 23:27:52 2013 +0530
+++ b/test/script/basic/compile-octane.js.EXPECTED	Mon May 27 13:11:13 2013 +0200
@@ -1,39 +1,13 @@
-Compiling... box2d.js
-Compiled OK: box2d.js
-
-Compiling... code-load.js
-Compiled OK: code-load.js
-
-Compiling... crypto.js
-Compiled OK: crypto.js
-
-Compiling... deltablue.js
-Compiled OK: deltablue.js
-
-Compiling... earley-boyer.js
-Compiled OK: earley-boyer.js
-
-Compiling... gbemu.js
-Compiled OK: gbemu.js
-
-Compiling... mandreel.js
-Compiled OK: mandreel.js
-
-Compiling... navier-stokes.js
-Compiled OK: navier-stokes.js
-
-Compiling... pdfjs.js
-Compiled OK: pdfjs.js
-
-Compiling... raytrace.js
-Compiled OK: raytrace.js
-
-Compiling... regexp.js
-Compiled OK: regexp.js
-
-Compiling... richards.js
-Compiled OK: richards.js
-
-Compiling... splay.js
-Compiled OK: splay.js
-
+Compiled OK: box2d
+Compiled OK: code-load
+Compiled OK: crypto
+Compiled OK: deltablue
+Compiled OK: earley-boyer
+Compiled OK: gbemu
+Compiled OK: mandreel
+Compiled OK: navier-stokes
+Compiled OK: pdfjs
+Compiled OK: raytrace
+Compiled OK: regexp
+Compiled OK: richards
+Compiled OK: splay
--- a/test/script/basic/run-octane.js	Fri May 24 23:27:52 2013 +0530
+++ b/test/script/basic/run-octane.js	Mon May 27 13:11:13 2013 +0200
@@ -26,36 +26,20 @@
  */
 
 var tests = [
-    "box2d.js",
-    "code-load.js",
-    "crypto.js", 
-    "deltablue.js", 
-    "earley-boyer.js", 
-    "gbemu.js",
-    "mandreel.js",
-    "navier-stokes.js", 
-    "pdfjs.js",
-    "raytrace.js",
-    "regexp.js", 
-    "richards.js", 
-    "splay.js" 
+    {file:"box2d",suite:"Box2DBenchmark"},
+    {file:"code-load",suite:"CodeLoad"},
+    {file:"crypto",suite:"Crypto"},
+    {file:"deltablue",suite:"DeltaBlue"},
+    {file:"earley-boyer", suite:"EarleyBoyer"},
+    {file:"gbemu", suite:"GameboyBenchmark"},
+    {file:"mandreel", suite:"MandreelBenchmark"},
+    {file:"navier-stokes", suite:"NavierStokes"},
+    {file:"pdfjs", suite:"PdfJS"},
+    {file:"raytrace", suite:"RayTrace"},
+    {file:"regexp", suite:"RegExpSuite"},
+    {file:"richards", suite:"Richards"},
+    {file:"splay", suite:"Splay"}
 ];
-
-// hack, teardown breaks things defined in the global space, making it impossible
-// to do multiple consecutive benchmark runs with the same harness. I think it's a bug
-// that the setup and teardown aren't each others constructor and destructor but rather
-// that the benchmarks rely on partial global state. For shame, Octane! 
-var ignoreTeardown = [
-    { name: "box2d.js" },
-    { name: "gbemu.js" },
-];
-
-
-//TODO mandreel can be compiled as a test, but not run multiple times unless modified to not have global state
-var compileOnly = {
-    "mandreel.js" : true
-};
-
 var dir = (typeof(__DIR__) == 'undefined') ? "test/script/basic/" : __DIR__;
 
 // TODO: why is this path hard coded when it's defined in project properties?
@@ -71,110 +55,96 @@
 }
 
 function should_compile_only(name) {
-    return (typeof compile_only !== 'undefined') || compileOnly[name] === true;
+    return (typeof compile_only !== 'undefined')
 }
 
 function run_one_benchmark(arg, iters) {
-
     var file_name;
-    var file = arg.split('/');
-    if (file.length == 1) {
-        file = arg.split('\\');
-    }    
-
-    //trim off trailing path separators
-    while (file[file.length - 1].indexOf(".js") == -1) {
-	file.pop();
-    }
-    file_name = file[file.length - 1];
-
+    var file = (arg.file + ".js").split('/');
+    
+    file_name = path + file[file.length - 1];
+    
     var compile_and_return = should_compile_only(file_name);
     if (compile_and_return) {
 	if (typeof compile_only === 'undefined') { //for a run, skip compile onlies, don't even compile them
 	    return;
 	}
-	print("Compiling... " + file_name);
     }
-
-    load(path + 'base.js');
-    load(arg);
+    
+    print_verbose("Loading... " + file_name);
+    load(file_name);
     
     if (compile_and_return) {
-	print("Compiled OK: " + file_name);
-	print("");
+	print_always("Compiled OK: " + arg.file);
 	return;
     }
     
     var success = true;
-    var hiscore = 0;
-    var loscore = 10e8;
     var current_name;
     
-    function PrintResult(name, result) {
-	current_name = name;
-    }
-        
-    function PrintError(name, error) {
-	current_name = name;
-	PrintResult(name, error);
-	success = false;
-    }
-        
-    function PrintScore(score) {
-	if (success) {
-	    if (+score >= hiscore) {
-		hiscore = +score;
-	    }
-	    if (+score <= loscore) {
-		loscore = +score;
-	    }
-	}
-
-	if (verbose) {
-	    print("Score: " + score);
-	}
-    }
-    
     if (iters == undefined) {
 	iters = numberOfIterations;
     } else {
 	numberOfIterations = iters;
     }
+    
+    var benchmarks = eval(arg.suite + ".benchmarks");
+    for (var x = 0; x < benchmarks.length ; x++) { 
+	benchmarks[x].Setup();
+    }
+    print_verbose("Running '" + arg.file + "' for " + iters + " iterations of no less than " + min_time + " seconds (" + runtime + ")");
+    
+    var scores = [];
+    
+    var min_time_ms = min_time * 1000;
+    var len = benchmarks.length;
+    
+    for (var it = 0; it < iters + 1; it++) {
+	//every iteration must take a minimum of 10 secs
+	var ops = 0;
+	var elapsed = 0;
+	var start = new Date;
+	do {
+	    for (var i = 0; i < len; i++) {
+		benchmarks[i].run();
+	    }	    
+	    ops += len;
+	    elapsed = new Date - start;
+	} while (elapsed < min_time * 1000);
+	
+	var score = ops / elapsed * 1000 * 60;
+	scores.push(score);
+	var name = it == 0 ? "warmup" : "iteration " + it;   
+	print_verbose("[" + arg.file + "] " + name + " finished " + score.toFixed(0) + " ops/minute");
+    }
+    for (var x = 0; x < benchmarks.length ; x++) { 
+	benchmarks[x].TearDown();
+    }
 
-    print(runtime + ": running " + file_name + "...");
+    var min_score  = 1e9;
+    var max_score  = 0;
+    var mean_score = 0;
+    for (var x = 1; x < iters + 1 ; x++) {
+	mean_score += scores[x];
+	min_score = Math.min(min_score, scores[x]);
+	max_score = Math.max(max_score, scores[x]);
+    }
+    mean_score /= iters;    
+    var res = "[" + arg.file + "] " + mean_score.toFixed(0);
+    if (verbose) {
+	res += " ops/minute (" + min_score.toFixed(0) + "-" + max_score.toFixed(0) + "), warmup=" + scores[0].toFixed(0);
+    }
+    print_always(res);
+}
 
-    for (var i = 0; i < numberOfIterations; i++) {
-	var callbacks =
-	    { NotifyResult: PrintResult,
-	      NotifyError: PrintError,
-	      NotifyScore: PrintScore };	
+function print_always(x) {
+    print(x);
+}
 
-	for (j in ignoreTeardown) {
-	    var ignore = ignoreTeardown[j];
-	    if (endsWith(arg, ignore.name)) {
-		var teardownOverride = ignore.teardown;
-		if (!teardownOverride) {
-		    teardownOverride = function() {};
-		}
-
-		for (k in BenchmarkSuite.suites) {
-		    var benchmarks = BenchmarkSuite.suites[k].benchmarks;
-		    for (l in benchmarks) {
-			benchmarks[l].TearDown = teardownOverride;
-		    }
-                }
-		break;
-	    }
-	}
-	
-	BenchmarkSuite.RunSuites(callbacks);
+function print_verbose(x) {
+    if (verbose) {
+	print(x);
     }
-    
-    var start = "Score: ";
-    if (runtime != "") {
-	start = runtime + ": ";
-    } 
-    print(start + current_name + ' (version ' + BenchmarkSuite.version + '): ' + loscore + '-' + hiscore);
 }
 
 function run_suite(tests, iters) {
@@ -211,6 +181,7 @@
 
 var tests_found = [];
 var iters = undefined;
+var min_time = 5;
 
 for (var i = 0; i < args.length; i++) { 
     arg = args[i];
@@ -220,21 +191,41 @@
 	runtime = args[++i];
     } else if (arg == "--verbose") {
 	verbose = true;
+    } else if (arg == "--min-time") {
+	min_time = +args[++i];
     } else if (arg == "") {
 	continue; //skip
     } else {
-	tests_found.push(arg);
+	var found = false;
+	for (j in tests) {
+	    if (tests[j].file === arg) {
+		tests_found.push(tests[j]);
+		found = true;
+		break;
+	    }
+	}
+	if (!found) {
+	    var str = "unknown test name: '" + arg + "' -- valid names are: ";
+	    for (j in tests) {
+		if (j != 0) {
+		    str += ", ";
+		}
+		str += "'" + tests[j].file + "'";
+	    }
+	    throw str;
+	}
     }
 }
 
 if (tests_found.length == 0) {    
     for (i in tests) {
-	tests_found.push(path + tests[i]);
+	tests_found.push(tests[i]);
     }
 } 
 
 tests_found.sort();
 
+load(path + 'base.js');
 run_suite(tests_found, iters);