diff --git a/tools/generation/generator.py b/tools/generation/generator.py new file mode 100755 index 0000000000000000000000000000000000000000..754f00fcf5cd8c0dd40977d976f3ae425bf4004f --- /dev/null +++ b/tools/generation/generator.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# Copyright (C) 2016 the V8 project authors. All rights reserved. +# This code is governed by the BSD license found in the LICENSE file. + +from __future__ import print_function +import argparse +import os, sys + +from lib.expander import Expander +from lib.test import Test + +def print_error(*values): + print('ERROR:', *values, file=sys.stderr) + +def find_cases(location): + # When a file is specified, return the file name and its containing + # directory + if os.path.isfile(location): + return location, [os.path.dirname(location)] + + # When a directory is specified, if that directory contains a sub-directory + # names "default" interpret it as a "case directory" + if (os.path.isdir(os.path.join(location, 'default'))): + return None, [location] + else: + return None, map( + lambda x: os.path.join(args.cases, x), os.listdir(args.cases)) + +def clean(args): + for (subdir, _, fileNames) in os.walk(args.directory): + for fileName in map(lambda x: os.path.join(subdir, x), fileNames): + test = Test(fileName) + test.load() + if test.is_generated(): + print('Deleting file "' + fileName + '"...') + os.remove(fileName) + +def create(args): + caseFile, caseDirs = find_cases(args.cases) + + for caseDir in caseDirs: + exp = Expander(caseDir) + for test in exp.expand('utf-8', caseFile): + if args.out: + try: + test.load(args.out) + + if args.no_clobber: + print_error( + 'Refusing to overwrite file: ' + test.file_name) + exit(1) + + if not test.is_generated(): + print_error( + 'Refusing to overwrite non-generated file: ' + + test.file_name) + exit(1) + except IOError: + pass + + test.write(args.out, parents=args.parents) + else: + print(test.to_string()) + +parser = argparse.ArgumentParser(description='Test262 test generator tool') +subparsers = parser.add_subparsers() + +create_parser = subparsers.add_parser('create', + help='''Generate test material''') +create_parser.add_argument('-o', '--out', help='''The directory to write the + compiled tests. If unspecified, tests will be written to standard out.''') +create_parser.add_argument('-p', '--parents', action='store_true', + help='''Create non-existent directories as necessary.''') +create_parser.add_argument('-n', '--no-clobber', action='store_true', + help='''Do not produce test if a corresponding file exists within this + directory.''') +create_parser.add_argument('cases', + help='''Test cases to generate. May be a file or a directory.''') +create_parser.set_defaults(func=create) + +clean_parser = subparsers.add_parser('clean', + help='''Remove previously-generated files''') +clean_parser.add_argument('directory', + help='''Remove any generated tests from this directory''') +clean_parser.set_defaults(func=clean) + +args = parser.parse_args() +args.func(args) diff --git a/tools/generation/lib/__init__.py b/tools/generation/lib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2ae28399f5fda2dfb1b04405f4a3b4895f5fac1e --- /dev/null +++ b/tools/generation/lib/__init__.py @@ -0,0 +1 @@ +pass diff --git a/tools/generation/lib/case.py b/tools/generation/lib/case.py new file mode 100644 index 0000000000000000000000000000000000000000..f43d4f664ba24c0965aa8f8d14f5e6308aeb7468 --- /dev/null +++ b/tools/generation/lib/case.py @@ -0,0 +1,44 @@ +# Copyright (C) 2016 the V8 project authors. All rights reserved. +# This code is governed by the BSD license found in the LICENSE file. + +import re + +from util.find_comments import find_comments +from util.parse_yaml import parse_yaml + +regionStartPattern = re.compile(r'-\s+(\S+)') + +class Case: + def __init__(self, file_name): + self.attribs = dict(meta=None, regions=dict()) + + with open(file_name) as handle: + self.attribs = self._parse(handle.read()) + + def _parse(self, source): + case = dict(meta=None, regions=dict()) + region_name = None + region_start = 0 + lines = source.split('\n') + + for comment in find_comments(source): + meta = parse_yaml(comment['source']) + if meta: + case['meta'] = meta + continue + + match = regionStartPattern.match(comment['source']) + if match: + if region_name: + case['regions'][region_name] = \ + '\n'.join(lines[region_start:comment['lineno'] - 1]) + + region_name = match.group(1) + region_start = comment['lineno'] + continue + + if region_name: + case['regions'][region_name] = \ + '\n'.join(lines[region_start:-1]) + + return case diff --git a/tools/generation/lib/expander.py b/tools/generation/lib/expander.py new file mode 100644 index 0000000000000000000000000000000000000000..5703a698b6d5851b14b7307a84f9e8d30246191b --- /dev/null +++ b/tools/generation/lib/expander.py @@ -0,0 +1,58 @@ +# Copyright (C) 2016 the V8 project authors. All rights reserved. +# This code is governed by the BSD license found in the LICENSE file. + +import re, os + +from case import Case +from template import Template + +caseFilenamePattern = re.compile(r'^[^\.].*\.case$') +templateFilenamePattern = re.compile(r'^[^\.].*\.template$') + +class Expander: + def __init__(self, case_dir): + self.templates = dict() + self.case_dir = case_dir + + def _load_templates(self, template_class): + directory = os.path.join(self.case_dir, template_class) + file_names = map( + lambda x: os.path.join(directory, x), + filter(self.is_template_file, os.listdir(directory)) + ) + + self.templates[template_class] = [Template(x) for x in file_names] + + def _get_templates(self, template_class): + if not template_class in self.templates: + self._load_templates(template_class) + + return self.templates[template_class] + + def is_template_file(self, filename): + return re.match(templateFilenamePattern, filename) + + def list_cases(self): + for name in os.listdir(self.case_dir): + full = os.path.join(self.case_dir, name) + if os.path.isfile(full) and caseFilenamePattern.match(name): + yield full + + def expand(self, encoding, case_file = None): + if case_file: + case_files = [case_file] + else: + case_files = self.list_cases() + + for case_file in case_files: + for test in self.expand_case(case_file, encoding): + yield test + + def expand_case(self, file_name, encoding): + case = Case(file_name) + + template_class = case.attribs['meta']['template'] + templates = self.templates.get(template_class) + + for template in self._get_templates(template_class): + yield template.expand(file_name, os.path.basename(file_name[:-5]), case.attribs, encoding) diff --git a/tools/generation/lib/template.py b/tools/generation/lib/template.py new file mode 100644 index 0000000000000000000000000000000000000000..fa891f00bef53eaa663578f936fe4f4df174ac66 --- /dev/null +++ b/tools/generation/lib/template.py @@ -0,0 +1,160 @@ +# Copyright (C) 2016 the V8 project authors. All rights reserved. +# This code is governed by the BSD license found in the LICENSE file. + +import os, re +import codecs, yaml + +from util.find_comments import find_comments +from util.parse_yaml import parse_yaml +from test import Test + +indentPattern = re.compile(r'^(\s*)') +interpolatePattern = re.compile(r'\{\s*(\S+)\s*\}') + +def indent(text, prefix = ' '): + '''Prefix a block of text (as defined by the "line break" control + character) with some character sequence.''' + + if isinstance(text, list): + lines = text + else: + lines = text.split('\n') + + return prefix + ('\n' + prefix).join(lines) + +class Template: + def __init__(self, filename): + self.filename = filename + + with open(filename) as template_file: + self.source = template_file.read() + + self.attribs = dict() + self.regions = [] + + self._parse() + + def _remove_comment(self, comment): + '''Create a region that is not intended to be referenced by any case, + ensuring that the comment is not emitted in the rendered file.''' + name = '__remove_comment_' + str(comment['firstchar']) + '__' + + # When a removed comment ends the line, the following newline character + # should also be removed from the generated file. + lastchar = comment['lastchar'] + if self.source[lastchar] == '\n': + comment['lastchar'] = comment['lastchar'] + 1 + + self.regions.insert(0, dict(name=name, **comment)) + + def _parse(self): + for comment in find_comments(self.source): + meta = parse_yaml(comment['source']) + + # Do not emit the template's frontmatter in generated files + # (file-specific frontmatter is generated as part of the rendering + # process) + if meta: + self.attribs['meta'] = meta + self._remove_comment(comment) + continue + + # Do not emit license information in generated files (recognized as + # comments preceeding the YAML frontmatter) + if not self.attribs.get('meta'): + self._remove_comment(comment) + continue + + match = interpolatePattern.match(comment['source']) + + if match == None: + continue + + self.regions.insert(0, dict(name=match.group(1), **comment)) + + def expand_regions(self, source, context): + lines = source.split('\n') + + for region in self.regions: + whitespace = indentPattern.match(lines[region['lineno']]).group(1) + value = context['regions'].get(region['name'], '') + source = source[:region['firstchar']] + \ + indent(value, whitespace).lstrip() + \ + source[region['lastchar']:] + + setup = context['regions'].get('setup') + + if setup: + source = setup + '\n' + source + + teardown = context['regions'].get('teardown') + + if teardown: + source += '\n' + teardown + '\n' + + return source + + def _frontmatter(self, case_filename, case_values): + description = case_values['meta']['desc'].strip() + \ + ' (' + self.attribs['meta']['name'].strip() + ')' + lines = [] + + lines += [ + '// This file was procedurally generated from the following sources:', + '// - ' + case_filename, + '// - ' + self.filename, + '/*---', + 'description: ' + description, + ] + + esid = self.attribs['meta'].get('esid') + if esid: + lines.append('esid: ' + esid) + + es6id = self.attribs['meta'].get('es6id') + if es6id: + lines.append('es6id: ' + es6id) + + features = [] + features += case_values['meta'].get('features', []) + features += self.attribs['meta'].get('features', []) + if len(features): + lines += ['features: ' + yaml.dump(features)] + + flags = ['generated'] + flags += case_values['meta'].get('flags', []) + flags += self.attribs['meta'].get('flags', []) + lines += ['flags: ' + yaml.dump(flags).strip()] + + includes = [] + includes += case_values['meta'].get('includes', []) + includes += self.attribs['meta'].get('includes', []) + if len(includes): + lines += ['includes: ' + yaml.dump(includes).strip()] + + if case_values['meta'].get('negative'): + lines += ['negative: ' + case_values['meta'].get('negative')] + + info = [] + + if 'info' in self.attribs['meta']: + info.append(indent(self.attribs['meta']['info'])) + if 'info' in case_values['meta']: + if len(info): + info.append('') + info.append(indent(case_values['meta']['info'])) + + if len(info): + lines.append('info: >') + lines += info + + lines.append('---*/') + + return '\n'.join(lines) + + def expand(self, case_filename, case_name, case_values, encoding): + frontmatter = self._frontmatter(case_filename, case_values) + body = self.expand_regions(self.source, case_values) + + return Test(self.attribs['meta']['path'] + case_name + '.js', + source=codecs.encode(frontmatter + '\n' + body, encoding)) diff --git a/tools/generation/lib/test.py b/tools/generation/lib/test.py new file mode 100644 index 0000000000000000000000000000000000000000..1dd6d45f7354001c0e1b0c5ef73c4dec1736f6d4 --- /dev/null +++ b/tools/generation/lib/test.py @@ -0,0 +1,63 @@ +# Copyright (C) 2016 the V8 project authors. All rights reserved. +# This code is governed by the BSD license found in the LICENSE file. + +import os, re + +from util.find_comments import find_comments +from util.parse_yaml import parse_yaml + +class Test: + """Representation of a generated test. Specifies a file location which may + or may not exist.""" + def __init__(self, file_name, source=None): + self.file_name = file_name + self.source = source + self.attribs = dict(meta=None) + + if self.source: + self._parse() + + def load(self, prefix = None): + location = os.path.join(prefix or '', self.file_name) + with open(location) as handle: + self.source = handle.read() + self._parse() + + def _parse(self): + for comment in find_comments(self.source): + meta = parse_yaml(comment['source']) + if meta: + self.attribs['meta'] = meta + break + + def is_generated(self): + if not self.attribs['meta']: + return False + flags = self.attribs['meta'].get('flags') + + if not flags: + return False + + return 'generated' in flags + + def to_string(self): + return '\n'.join([ + '/**', + ' * ----------------------------------------------------------------', + ' * ' + self.file_name, + ' * ----------------------------------------------------------------', + ' */', + self.source, + '\n']) + + def write(self, prefix, parents=False): + location = os.path.join(prefix, self.file_name) + path = os.path.dirname(location) + if not os.path.exists(path): + if parents: + os.makedirs(path) + else: + raise Exception('Directory does not exist: ' + path) + + with open(location, 'w') as handle: + handle.write(self.source) diff --git a/tools/generation/lib/util/__init__.py b/tools/generation/lib/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2ae28399f5fda2dfb1b04405f4a3b4895f5fac1e --- /dev/null +++ b/tools/generation/lib/util/__init__.py @@ -0,0 +1 @@ +pass diff --git a/tools/generation/lib/util/find_comments.py b/tools/generation/lib/util/find_comments.py new file mode 100644 index 0000000000000000000000000000000000000000..81ad0ab55206fb6a858156bc36b641c32cd8cc35 --- /dev/null +++ b/tools/generation/lib/util/find_comments.py @@ -0,0 +1,68 @@ +# Copyright (C) 2016 the V8 project authors. All rights reserved. +# This code is governed by the BSD license found in the LICENSE file. + +def find_comments(source): + '''Parse input string describing JavaScript source and yield dictionaries + describing the JavaScript comments in the order they appear in the source. + + Each dictionary defines the following attributes: + + - source: the source text of the comment + - firstchar: the zero-indexed position of the token that begins the comment + - lastchar: the zero-indexed position of the token that closes the comment + - lineno: the zero-indexed offset of the line on which the comment appears + ''' + in_string = False + in_s_comment = False + in_m_comment = False + follows_escape = False + comment = '' + lineno = 0 + + for idx in xrange(len(source)): + if source[idx] == '\n': + lineno += 1 + + # Within comments and strings, any odd number of back-slashes begins an + # escape sequence. + if source[idx - 1] == '\\': + follows_escape = not follows_escape + else: + follows_escape = False + + if in_s_comment: + if source[idx] == '\n': + in_s_comment = False + yield dict( + source=comment[1:], + firstchar=idx - len(comment) - 1, + lastchar=idx, + lineno=lineno) + continue + elif in_m_comment: + if source[idx - 1] == '*' and source[idx] == '/': + in_m_comment = False + yield dict( + source=comment[1:-1], + firstchar=idx - len(comment) - 1, + lastchar=idx + 1, + lineno=lineno) + continue + elif in_string: + if source[idx] == in_string and not follows_escape: + in_string = False + elif source[idx] == '\n' and in_string != '`' and not follows_escape: + in_string = False + continue + + if in_m_comment or in_s_comment: + comment += source[idx] + continue + + in_m_comment = source[idx] == '/' and source[idx + 1] == '*' + in_s_comment = source[idx] == '/' and source[idx + 1] == '/' + + if in_m_comment or in_s_comment: + comment = '' + elif source[idx] == '\'' or source[idx] == '"' or source[idx] == '`': + in_string = source[idx] diff --git a/tools/generation/lib/util/parse_yaml.py b/tools/generation/lib/util/parse_yaml.py new file mode 100644 index 0000000000000000000000000000000000000000..3ad356db8c5caf32e12d7bc55c75d3ed7fa17429 --- /dev/null +++ b/tools/generation/lib/util/parse_yaml.py @@ -0,0 +1,17 @@ +# Copyright (C) 2016 the V8 project authors. All rights reserved. +# This code is governed by the BSD license found in the LICENSE file. + +import yaml, re + +yamlPattern = re.compile(r'\---\n([\s]*)((?:\s|\S)*)[\n\s*]---', + flags=re.DOTALL|re.MULTILINE) + +def parse_yaml(string): + match = yamlPattern.match(string) + if not match: + return False + + unindented = re.sub('^' + match.group(1), '', + match.group(2), flags=re.MULTILINE) + + return yaml.safe_load(unindented) diff --git a/tools/generation/requirements.txt b/tools/generation/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..efb082d8dec90c386fc31b924ee44eebb0a845ac --- /dev/null +++ b/tools/generation/requirements.txt @@ -0,0 +1 @@ +PyYAML==3.11 diff --git a/tools/generation/test/expected/normal/nested/path2-normal.js b/tools/generation/test/expected/normal/nested/path2-normal.js new file mode 100644 index 0000000000000000000000000000000000000000..d9da70500a9c2f385515839272d312b072ea7898 --- /dev/null +++ b/tools/generation/test/expected/normal/nested/path2-normal.js @@ -0,0 +1,19 @@ +// This file was procedurally generated from the following sources: +// - tools/generation/test/fixtures/normal.case +// - tools/generation/test/fixtures/normal/normal2.template +/*--- +description: foobar (Second template name) +esid: sec-a-generic-id +flags: [generated, a, b] +includes: [foo.js, bar.js] +info: > + template info + + case info +---*/ + +before-Third valueSecond value-after + +/* Improperly-terminated comments should not break the tokenizer * + +'This is "teardown" code.'; diff --git a/tools/generation/test/expected/normal/no-info-normal.js b/tools/generation/test/expected/normal/no-info-normal.js new file mode 100644 index 0000000000000000000000000000000000000000..903028036b6e74f0a06a6f370c8a2335f80b9beb --- /dev/null +++ b/tools/generation/test/expected/normal/no-info-normal.js @@ -0,0 +1,15 @@ +// This file was procedurally generated from the following sources: +// - tools/generation/test/fixtures/normal.case +// - tools/generation/test/fixtures/normal/no-info.template +/*--- +description: foobar (First template name) +es6id: 1.2.3 +flags: [generated, a, b] +includes: [foo.js] +info: > + case info +---*/ + +First value + +'This is "teardown" code.'; diff --git a/tools/generation/test/expected/normal/path1-normal.js b/tools/generation/test/expected/normal/path1-normal.js new file mode 100644 index 0000000000000000000000000000000000000000..3bc10d0e9f283f5b882fc99207b812c3e75399a6 --- /dev/null +++ b/tools/generation/test/expected/normal/path1-normal.js @@ -0,0 +1,33 @@ +// This file was procedurally generated from the following sources: +// - tools/generation/test/fixtures/normal.case +// - tools/generation/test/fixtures/normal/normal.template +/*--- +description: foobar (First template name) +es6id: 1.2.3 +flags: [generated, a, b, c, d] +includes: [foo.js] +info: > + template info + + case info +---*/ + +before-First value-between-Third value-after + +before*Second value*between*First value*after + +before/* " */Third valueafter + +The following should not be expanded: + +/* */*{ first }*/ +/* +*/*{ first }*/ +//*{ first }*/ +// /*{ first }*/ +"/*{ first }*/" +'/*{ first }*/' +` +/*{ first }*/` + +'This is "teardown" code.'; diff --git a/tools/generation/test/fixtures/normal.case b/tools/generation/test/fixtures/normal.case new file mode 100644 index 0000000000000000000000000000000000000000..276b58ccddc87eab2b6e4c211a670f09fae54ee5 --- /dev/null +++ b/tools/generation/test/fixtures/normal.case @@ -0,0 +1,27 @@ +// Copyright (C) 2016 the V8 project authors. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +template: normal +desc: foobar +info: case info +flags: [a, b] +includes: [foo.js] +---*/ + +Because this test appears before any "region" delimiters, it should not appear +in the generated files. + +// - first +this is not a valid region delimiter + +/* *//- first +this is also not a valid region delimiter + +//- first +First value +//- second +Second value +//- third +Third value +//- teardown +'This is "teardown" code.'; diff --git a/tools/generation/test/fixtures/normal/no-info.template b/tools/generation/test/fixtures/normal/no-info.template new file mode 100644 index 0000000000000000000000000000000000000000..0f1b340ea36f42c5ea3f6abc0757c43ae8e58f85 --- /dev/null +++ b/tools/generation/test/fixtures/normal/no-info.template @@ -0,0 +1,9 @@ +// Copyright (C) 2016 the V8 project authors. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +name: First template name +path: normal/no-info- +es6id: 1.2.3 +---*/ + +/*{ first }*/ diff --git a/tools/generation/test/fixtures/normal/normal.template b/tools/generation/test/fixtures/normal/normal.template new file mode 100644 index 0000000000000000000000000000000000000000..ac0bea1e159ce65e91b93b86d2921aed63effb4b --- /dev/null +++ b/tools/generation/test/fixtures/normal/normal.template @@ -0,0 +1,27 @@ +// Copyright (C) 2016 the V8 project authors. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +name: First template name +path: normal/path1- +es6id: 1.2.3 +info: template info +flags: [c, d] +---*/ + +before-/*{ first }*/-between-/*{ third }*/-after + +before*/*{ second }*/*between*/*{ first }*/*after + +before/* " *//*{ third }*/after + +The following should not be expanded: + +/* */*{ first }*/ +/* +*/*{ first }*/ +//*{ first }*/ +// /*{ first }*/ +"/*{ first }*/" +'/*{ first }*/' +` +/*{ first }*/` diff --git a/tools/generation/test/fixtures/normal/normal2.template b/tools/generation/test/fixtures/normal/normal2.template new file mode 100644 index 0000000000000000000000000000000000000000..f7ae0b6830b8cc65b9767b93cfe7103214a8b16a --- /dev/null +++ b/tools/generation/test/fixtures/normal/normal2.template @@ -0,0 +1,13 @@ +// Copyright (C) 2016 the V8 project authors. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +name: Second template name +path: normal/nested/path2- +esid: sec-a-generic-id +includes: [bar.js] +info: template info +---*/ + +before-/*{ third }*//*{ second }*/-after + +/* Improperly-terminated comments should not break the tokenizer * diff --git a/tools/generation/test/run.py b/tools/generation/test/run.py new file mode 100755 index 0000000000000000000000000000000000000000..297815d679413afb9d4d600d8267f770f19e7507 --- /dev/null +++ b/tools/generation/test/run.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# Copyright (C) 2016 the V8 project authors. All rights reserved. +# This code is governed by the BSD license found in the LICENSE file. + +import shutil, subprocess, sys, os, unittest + +testDir = os.path.dirname(os.path.relpath(__file__)) +OUT_DIR = os.path.join(testDir, 'out') +EXPECTED_DIR = os.path.join(testDir, 'expected') +ex = os.path.join(testDir, '..', 'generator.py') + +class TestGeneration(unittest.TestCase): + maxDiff = None + + def fixture(self, name): + relpath = os.path.relpath(os.path.join(testDir, 'fixtures', name)) + sp = subprocess.Popen( + [ex, 'create', '-o', OUT_DIR, '-p', relpath], + stdout=subprocess.PIPE) + stdout, stderr = sp.communicate() + return dict(stdout=stdout, stderr=stderr, returncode=sp.returncode) + + def getFiles(self, path): + names = [] + for root, _, fileNames in os.walk(path): + for fileName in filter(lambda x: x[0] != '.', fileNames): + names.append(os.path.join(root, fileName)) + names.sort() + return names + + def compareTrees(self, targetName): + expectedPath = os.path.join(EXPECTED_DIR, targetName) + actualPath = os.path.join(OUT_DIR, targetName) + + expectedFiles = self.getFiles(expectedPath) + actualFiles = self.getFiles(actualPath) + + self.assertListEqual( + map(lambda x: os.path.relpath(x, expectedPath), expectedFiles), + map(lambda x: os.path.relpath(x, actualPath), actualFiles)) + + for expectedFile, actualFile in zip(expectedFiles, actualFiles): + with open(expectedFile) as expectedHandle: + with open(actualFile) as actualHandle: + self.assertMultiLineEqual( + expectedHandle.read(), + actualHandle.read()) + + def tearDown(self): + shutil.rmtree(OUT_DIR, ignore_errors=True) + + def test_normal(self): + result = self.fixture('normal.case') + self.assertEqual(result['returncode'], 0) + self.compareTrees('normal') + +if __name__ == '__main__': + unittest.main()