Use travis ci to run new or modified test files on engines

.travis.yml

-language: python
-install:
-  - python -m pip install --requirement tools/generation/requirements.txt
-  - python -m pip install --requirement tools/lint/requirements.txt
-script:
-  - ./tools/scripts/ci_build.sh
-  - ./tools/generation/test/run.py
-  - ./tools/lint/test/run.py
-  - ./tools/scripts/ci_lint.sh
-after_success:
-  - ./tools/scripts/deploy.sh
+matrix:
+  include:
+  - language: python
+    name: Project lint, generation tests and build
+    python:
+      - "2.7"
+    install:
+      - python -m pip install --requirement tools/generation/requirements.txt
+      - python -m pip install --requirement tools/lint/requirements.txt
+    script:
+      - ./tools/scripts/ci_build.sh
+      - ./tools/generation/test/run.py
+      - ./tools/lint/test/run.py
+      - ./tools/scripts/ci_lint.sh
+    after_success:
+      - ./tools/scripts/deploy.sh
+  - language: node_js
+    name: "New or modified tests on SpiderMonkey"
+    if: type = pull_request
+    node_js: "node"
+    script: ./tools/scripts/ci_test.sh
+    env: T262ENGINE=spidermonkey hostType=jsshell
+  - language: node_js
+    name: "New or modified tests on V8"
+    if: type = pull_request
+    node_js: "node"
+    script: ./tools/scripts/ci_test.sh
+    env: T262ENGINE=v8 hostType=d8
+  - language: node_js
+    name: "New or modified tests on V8 with --harmony"
+    if: type = pull_request
+    node_js: "node"
+    script: ./tools/scripts/ci_test.sh
+    env: T262ENGINE=v8 hostType=d8 hostArgs='--harmony'
+  - language: node_js
+    name: "New or modified tests on ChakraCore"
+    if: type = pull_request
+    node_js: "node"
+    script: ./tools/scripts/ci_test.sh
+    env: T262ENGINE=chakra hostType=ch
+  - language: node_js
+    name: "New or modified tests on JavaScriptCore"
+    if: type = pull_request
+    node_js: "node"
+    script: ./tools/scripts/ci_test.sh
+    env: T262ENGINE=javascriptcore hostType=jsc
+  - language: node_js
+    name: "New or modified tests on Node"
+    if: type = pull_request
+    node_js: "node"
+    script: ./tools/scripts/ci_test.sh
+    env: T262ENGINE=node hostType=node
 notifications:
   email:
     on_failure: always

package.json

@@ -11,5 +11,13 @@
     "url": "https://github.com/tc39/test262/issues"
   },
   "private": true,
-  "homepage": "https://github.com/tc39/test262#readme"
+  "homepage": "https://github.com/tc39/test262#readme",
+  "devDependencies": {
+    "jsvu": "^1.3.1",
+    "test262-harness": "^4.7.0"
+  },
+  "scripts": {
+    "ci-jsvu": "jsvu --os=linux64 --engines=$T262ENGINE",
+    "ci-test-pr": "test262-harness --hostType=$HOSTTYPE --hostPath=~/.jsvu/$T262ENGINE --hostArgs=\"$HOSTARGS\" $(git diff --name-only --diff-filter=AMR master test/)"
+  }
 }

tools/scripts/ci_test.sh

+#!/bin/bash
+
+if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
+  paths=$(git diff --diff-filter ACMR --name-only $TRAVIS_BRANCH -- test/)
+
+  if [ "$paths" == "" ]; then
+    echo No test files added or modified. Exiting.
+    exit 0
+  fi
+
+  echo New or modified test files:
+  echo "$paths"
+  echo ""
+
+  if [ "$T262ENGINE" != "node" ]; then
+    echo "installing engine with jsvu"
+    echo "jsvu --os=linux64 --engines=$T262ENGINE"
+    jsvu --os=linux64 --engines=$T262ENGINE
+    hostPath=$HOME/.jsvu/$T262ENGINE
+  else
+    hostPath=node
+  fi
+
+  
+
+  echo ""
+  echo "Running the tests with test262-harness"
+  echo "test262-harness -t 1 --hostType=$hostType --hostPath=$hostPath --hostArgs=\"$hostArgs\" -- $paths"
+  test262-harness -t 1 --hostType=$hostType --hostPath=$hostPath --hostArgs="$hostArgs" -- $paths
+fi