Introduce automated validation for test format (#994)

This script is intended to identify common test file formatting errors
prior to their acceptance into the project. It is designed to support
future extensions for additional validation rules.

.travis.yml

 language: python
 install: pip install --requirement tools/generation/requirements.txt
 script:
-  - echo The test generation tool should be working.
+  - ./tools/scripts/ci_build.sh
   - ./tools/generation/test/run.py
-  - sh ./tools/scripts/ci.sh
+  - ./tools/lint/test/run.py
+  - ./tools/scripts/ci_lint.sh
 after_success:
-  - sh ./tools/scripts/deploy.sh
+  - ./tools/scripts/deploy.sh

CONTRIBUTING.md

@@ -249,6 +249,19 @@ p.then(function () {
 
 As above, exceptions that are thrown from a `then` clause are passed to a later `$DONE` function and reported asynchronously.
 
+## Linting
+
+Some of the expectations documented here are enforced via a "linting" script. This script is used to validate patches automatically at submission time, but it may also be invoked locally via the following command:
+
+    python tools/lint/lint.py --whitelist lint.whitelist [paths to tests]
+
+...where `[paths to tests]` is a list of one or more paths to test files or directories containing test files.
+
+In some cases, it may be necessary for a test to intentionally violate the rules enforced by the linting tool. Such violations can be allowed by including the path of the test(s) in the `lint.whitelist` file. Each path must appear on a dedicated line in that file, and a space-separated list of rules to ignore must follow each path. Lines beginning with the pound sign (`#`) will be ignored. For example:
+
+    # This file documents authorship information and is not itself a test
+    test/built-ins/Simd/AUTHORS FRONTMATTER LICENSE
+
 ## Procedurally-generated tests
 
 Some language features are expressed through a number of distinct syntactic forms. Test262 maintains these tests as a set of "test cases" and "test templates" in order to ensure equivalent coverage across all forms. The sub-directories within the `src/` directory describe the various language features that benefit from this approach.

lint.whitelist

+# This file documents authorship information and is not itself a test
+test/built-ins/Simd/AUTHORS FRONTMATTER LICENSE

test/built-ins/Object/assign/Override.js

@@ -8,13 +8,13 @@ es6id:  19.1.2.1.5.c
 
 //"a" will be an property of the final object and the value should be 1
 var target = {a:1};
-/*---
+/*
 "1a2c3" have own enumerable properties, so it Should be wrapped to objects;
 {b:6} is an object,should be assigned to final object.
 undefined and null should be ignored;
 125 is a number,it cannot has own enumerable properties;
 {a:"c"},{a:5} will override property a, the value should be 5.
----*/
+*/
 var result = Object.assign(target,"1a2c3",{a:"c"},undefined,{b:6},null,125,{a:5});
 
 assert.sameValue(Object.keys(result).length, 7 , "The length should be 7 in the final object.");

tools/lint/lib/check.py

+class Check(object):
+    '''Base class for  defining linting checks.'''
+    ID = None
+
+    def run(self, name, meta, source):
+        return True

tools/lint/lib/checks/frontmatter.py

+from ..check import Check
+
+_REQUIRED_FIELDS = set(['description'])
+_OPTIONAL_FIELDS = set([
+    'author', 'es5id', 'es6id', 'esid', 'features', 'flags', 'includes',
+    'info', 'negative', 'timeout'
+])
+_VALID_FIELDS = _REQUIRED_FIELDS | _OPTIONAL_FIELDS
+
+class CheckFrontmatter(Check):
+    '''Ensure tests have the expected YAML-formatted metadata.'''
+    ID = 'FRONTMATTER'
+
+    def run(self, name, meta, source):
+        if name.endswith('_FIXTURE.js'):
+            if meta is not None:
+                return '"Fixture" files cannot specify metadata'
+            return
+
+        if meta is None:
+            return 'No valid YAML-formatted frontmatter'
+
+        fields = set(meta.keys())
+
+        missing = _REQUIRED_FIELDS - fields
+        if len(missing) > 0:
+            return 'Required fields missing: %s' % ', '.join(list(missing))
+
+        unrecognized = fields - _VALID_FIELDS
+        if len(unrecognized) > 0:
+            return 'Unrecognized fields: %s' % ', '.join(list(unrecognized))
+
+        if 'negative' in meta:
+            negative = meta['negative']
+            if not isinstance(negative, dict):
+                return '"negative" must be a dictionary with fields "type" and "phase"'
+
+            if not 'type' in negative:
+                return '"negative" must specify a "type" field'
+
+            if not 'phase' in negative:
+                return '"negative" must specify a "phase" field'

tools/lint/lib/checks/license.py

+import re
+
+from ..check import Check
+
+_MIN_YEAR = 2009
+_MAX_YEAR = 2030
+
+_LICENSE_PATTERN = re.compile(
+    r'\/\/ Copyright( \([cC]\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
+    r'(' +
+        r'\/\/ (' +
+            r'This code is governed by the( BSD)? license found in the LICENSE file\.' +
+            r'|' +
+            r'See LICENSE for details' +
+        r')' +
+        r'|' +
+        r'\/\/ Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
+        r'\/\/ found in the LICENSE file\.' +
+        r'|' +
+        r'\/\/ See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
+    r')', re.IGNORECASE)
+
+class CheckLicense(Check):
+    '''Ensure tests declare valid license information.'''
+    ID = 'LICENSE'
+
+    def run(self, name, meta, source):
+        if meta and 'flags' in meta and 'generated' in meta['flags']:
+            return
+
+        match = _LICENSE_PATTERN.search(source)
+
+        if not match:
+            return 'No license information found.'
+
+        year_str = match.group(2)
+        try:
+            year = int(year_str)
+
+            if year < _MIN_YEAR or year > _MAX_YEAR:
+                raise ValueError()
+        except ValueError:
+            return 'Invalid year: %s' % year_str

tools/lint/lib/collect_files.py

+import os
+
+def collect_files(path):
+    '''Given a path to a file, yield that path. Given a path to a directory,
+    yield the path of all files within that directory recursively, omitting any
+    that begin with a period (.) character.'''
+
+    if os.path.isfile(path):
+        yield path
+        return
+
+    if not os.path.isdir(path):
+        raise ValueError('Not found: "%s"' % path)
+
+    for root, dirs, file_names in os.walk(path):
+        for file_name in file_names:
+            if file_name.startswith('.'):
+                continue
+
+            yield os.path.join(root, file_name)

tools/lint/lib/eprint.py

+from __future__ import print_function
+import sys
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)

tools/lint/lib/frontmatter.py

+import re
+import yaml
+
+def parse(src):
+    '''Parse the YAML-formatted metadata found in a given string of source
+    code. Tolerate missing or invalid metadata; those conditions are handled by
+    a dedicated "Check" instance.'''
+
+    match = re.search(r'/\*---(.*)---\*/', src, re.DOTALL)
+    if not match:
+        return None
+
+    try:
+        return yaml.load(match.group(1))
+    except (yaml.scanner.ScannerError, yaml.parser.ParserError):
+        return None

tools/lint/lib/whitelist.py

+def parse(handle):
+    '''Parse the contents of the provided file descriptor as a linting
+    whitelist file. Return a dictionary whose keys are test file names and
+    whose values are Python sets of "Check" ID strings.'''
+
+    whitelist = dict()
+
+    for line in handle:
+        if line.startswith('#'):
+            continue
+
+        parts = line.split()
+        file_name = parts[0]
+        check_names = set(parts[1:])
+
+        assert file_name not in whitelist, (
+            'Whitelist should have a single entry for each file')
+
+        assert len(check_names) > 0, (
+            'Each whitelist entry should specify at least on check')
+
+        whitelist[file_name] = check_names
+
+    return whitelist

tools/lint/lint.py

+#!/usr/bin/env python
+# Copyright (C) 2017 Mike Pennisi. All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import argparse
+import sys
+
+from lib.collect_files import collect_files
+from lib.checks.frontmatter import CheckFrontmatter
+from lib.checks.license import CheckLicense
+from lib.eprint import eprint
+import lib.frontmatter
+import lib.whitelist
+
+parser = argparse.ArgumentParser(description='Test262 linting tool')
+parser.add_argument('--whitelist',
+        type=argparse.FileType('r'),
+        help='file containing expected linting errors')
+parser.add_argument('path',
+        nargs='+',
+        help='file name or directory of files to lint')
+
+checks = [CheckFrontmatter(), CheckLicense()]
+
+def lint(file_names):
+    errors = dict()
+
+    for file_name in file_names:
+        with open(file_name, 'r') as f:
+            content = f.read()
+        meta = lib.frontmatter.parse(content)
+        for check in checks:
+            error = check.run(file_name, meta, content)
+
+            if error is not None:
+                if file_name not in errors:
+                    errors[file_name] = dict()
+                errors[file_name][check.ID] = error
+
+    return errors
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    if args.whitelist:
+        whitelist = lib.whitelist.parse(args.whitelist)
+    else:
+        whitelist = dict()
+
+    files = [path for _path in args.path for path in collect_files(_path)]
+    file_count = len(files)
+    print 'Linting %s file%s.' % (file_count, 's' if file_count != 1 else '')
+
+    all_errors = lint(files)
+    unexpected_errors = dict(all_errors)
+
+    for file_name, failures in all_errors.iteritems():
+        if file_name not in whitelist:
+            continue
+        if set(failures.keys()) == whitelist[file_name]:
+            del unexpected_errors[file_name]
+
+    error_count = len(unexpected_errors)
+    s = 's' if error_count != 1 else ''
+
+    print 'Linting complete. %s error%s found.' % (error_count, s)
+
+    if error_count == 0:
+        sys.exit(0)
+
+    for file_name, failures in unexpected_errors.iteritems():
+        for ID, message in failures.iteritems():
+            eprint('%s: %s - %s' % (file_name, ID, message))
+
+    sys.exit(1)

tools/lint/requirements.txt

+PyYAML==3.11

tools/lint/test/fixtures/frontmatter_invalid_yaml.js

+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+es6id: 12.14.1
+description: Applied to a "covered" YieldExpression
+info: This is some information
+features: [generators
+---*/
+
+function* g() {
+  yield 23;
+}

tools/lint/test/fixtures/frontmatter_missing_desc.js

+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+es6id: 12.14.1
+info: This is some information
+---*/
+
+function* g() {
+  yield 23;
+}

tools/lint/test/fixtures/frontmatter_module_FIXTURE.js

+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+
+function* g() {
+  yield 23;
+}

tools/lint/test/fixtures/frontmatter_module_with_meta_FIXTURE.js

+FRONTMATTER
+^ expected errors | v input
+// Copyright (C) 2017 Mike Pennisi. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-assignment-operators-static-semantics-early-errors
+es6id: 12.14.1
+description: Applied to a "covered" YieldExpression
+info: This is some information
+features: [generators]
+---*/
+
+function* g() {
+  yield 23;
+}