runtests framework & config for batch test runs on Imperial infrastructure

.gitlab-ci.yml

@@ -2,8 +2,65 @@
 stages:
 - Git Sync
 - build
-- test
-- deploy
+- integration
+
+cache:
+  key: "branch-$CI_BUILD_REF_NAME"
+  paths:
+  - .git/modules
+  - tools/runtests/env
+
+variables:
+  OPAMYES: "true"
+  OPAMROOT: "/tmp/opam-$CI_BUILD_ID"
+
+mljsref:
+  stage: build
+  before_script:
+  - opam init -n
+  - opam update -u
+  - make init
+  - eval `opam config env`
+  script:
+  - make mljsref
+  after_script:
+  - rm -Rf $OPAMROOT
+  except:
+  - triggers
+  artifacts:
+    paths:
+    - jsref/
+    expire_in: 1 day
+
+# FIXME: Packaged JS_Parser means that we now need to distribute opam instance to condor runs
+condor:
+  before_script:
+  - export CI_BUILD_REF_SUBJ="$(git show --pretty=format:%s -s)"
+  - export TEST262_VERSION="$(git rev-parse --verify HEAD:test/data/test262)"
+  - git submodule update --init --force tools/runtests test/data/test262
+  - source tools/runtests/scripts/set-voldir.sh jsexplain
+  - "# OPAM initialisation"
+  - export OPAMROOT=$VOLDIR/.opam
+  - opam init -n
+  - opam update -u
+  - make init
+  - eval `opam config env`
+  - export OCAMLFIND_CONF=$OPAMROOT/$(opam switch show)/lib/findlib.conf
+  - "# Runtests initialisation"
+  - source tools/runtests/scripts/init-env.sh
+  - cp -R tools/runtests test/data/test262 test/data/test_prelude.js jsref/main.byte $VOLDIR
+  - cd $VOLDIR
+  script:
+  - 'runtests/runtests.py --interp mljsref --executor condor --batch_size 2 --condor_log --db postgres --db_pg_schema jsil --interp_version $CI_BUILD_REF --title "mljsref $CI_BUILD_REF_NAME autobuild: $CI_BUILD_REF_SUBJ" --tests_version $TEST262_VERSION --timeout 300 test262/test/suite'
+  - runtests/scripts/condor-wait.sh
+  after_script:
+  - "#tools/runtests/scripts/clean-voldir.sh jsexplain"
+  stage: integration
+  tags:
+  - condor
+  except:
+  - triggers
+
 git-sync:
   before_script:
   - eval `ssh-agent`

.gitmodules

 [submodule "test/test262"]
-	path = test/test262
+	path = test/data/test262
 	url = https://github.com/tc39/test262.git
 	branch = es5-tests
+[submodule "tools/runtests"]
+	path = tools/runtests
+	url = git@github.com:edgemaster/runtests.git

Makefile

@@ -7,18 +7,17 @@ init:
 	opam pin -yn add jsjsref .
 	opam pin -yn add JS_Parser "https://github.com/resource-reasoning/JS_Parser.git#v0.1.0"
 	opam install -y jsjsref --deps-only
+	@echo
+	@echo 'You now need to execute: eval `opam config env`'
 
 # Build Stages
 generator:
 	$(MAKE) -C generator
 
-generator-stdlib:
-	$(MAKE) -C generator stdlib
-
 jsjsref: generator
 	$(MAKE) -C jsref jsjsref
 
-mljsref: generator-stdlib
+mljsref: generator # (requires the ppx)
 	$(MAKE) -C jsref mljsref
 
 # Test Stages

test/data/test_prelude.js

+// Martin's original error function, which sets the secret state. Updated to keep more than one error message.
+// We use a try for the first time we set __$ERROR__: as it is not set
+// initially, trying to access it will result in an exception.
+function $ERROR (str) {
+    try {
+        __$ERROR__ = __$ERROR__ + " | " + str
+    }
+    catch(ex) {
+        __$ERROR__ = str
+    }
+}
+
+// Some tests use this instead
+var $FAIL = $ERROR
+
+// Gareth's addition, which brings it up to Test262 conformance. Hopefully.
+var t262 = {$ERROR:$ERROR , TestFailureError: function(err){return {__$ERROR__:err}}}
+var Test262Error = function (){}
+// TypeError = function(err){return {__$ERROR__:err}}
+function runTestCase(f) {
+    if (!f()) {
+        $ERROR("runTestCase returned false")
+    }
+}
+
+// Used in some places to fetch the global object
+function fnGlobalObject() {
+    return this
+}
+
+// Used to test if argument is a function
+// taken from test262/harness/fnExists.js
+function fnExists(/*arguments*/) {
+    for (var i = 0; i < arguments.length; i++) {
+        if (typeof (arguments[i]) !== "function") return false;
+    }
+    return true;
+}
+
+// This could be used to print ...
+function $PRINT(s){ }
+

test/helpers/test262.js

@@ -33,7 +33,7 @@ function testNegativity(str) {
 before(function(done) {
   this.timeout(0); // Otherwise it fails on slow filesystems
 
-  var test262path = fs.readlinkSync(__dirname + '/../test262');
+  var test262path = fs.readlinkSync(__dirname + '/../data/test262');
   var tests = [];
 
   walk(test262path)

runtests @ 530c1615

+Subproject commit 530c16152ec9423fd75df1a76537d02c722e0972