diff --git a/tools/generation/lib/case.py b/tools/generation/lib/case.py index f43d4f664ba24c0965aa8f8d14f5e6308aeb7468..5ed1c63de00faabf8af6cc5921411cb02429a7e0 100644 --- a/tools/generation/lib/case.py +++ b/tools/generation/lib/case.py @@ -1,6 +1,7 @@ # Copyright (C) 2016 the V8 project authors. All rights reserved. # This code is governed by the BSD license found in the LICENSE file. +import codecs import re from util.find_comments import find_comments @@ -9,10 +10,10 @@ from util.parse_yaml import parse_yaml regionStartPattern = re.compile(r'-\s+(\S+)') class Case: - def __init__(self, file_name): + def __init__(self, file_name, encoding): self.attribs = dict(meta=None, regions=dict()) - with open(file_name) as handle: + with codecs.open(file_name, 'r', encoding) as handle: self.attribs = self._parse(handle.read()) def _parse(self, source): diff --git a/tools/generation/lib/expander.py b/tools/generation/lib/expander.py index 5703a698b6d5851b14b7307a84f9e8d30246191b..c0b7a571361430035a4c12bed2d2026394a601d6 100644 --- a/tools/generation/lib/expander.py +++ b/tools/generation/lib/expander.py @@ -14,18 +14,20 @@ class Expander: self.templates = dict() self.case_dir = case_dir - def _load_templates(self, template_class): + def _load_templates(self, template_class, encoding): directory = os.path.join(self.case_dir, template_class) file_names = map( lambda x: os.path.join(directory, x), filter(self.is_template_file, os.listdir(directory)) ) - self.templates[template_class] = [Template(x) for x in file_names] + self.templates[template_class] = [ + Template(x, encoding) for x in file_names + ] - def _get_templates(self, template_class): + def _get_templates(self, template_class, encoding): if not template_class in self.templates: - self._load_templates(template_class) + self._load_templates(template_class, encoding) return self.templates[template_class] @@ -49,10 +51,10 @@ class Expander: yield test def expand_case(self, file_name, encoding): - case = Case(file_name) + case = Case(file_name, encoding) template_class = case.attribs['meta']['template'] templates = self.templates.get(template_class) - for template in self._get_templates(template_class): + for template in self._get_templates(template_class, encoding): yield template.expand(file_name, os.path.basename(file_name[:-5]), case.attribs, encoding) diff --git a/tools/generation/lib/template.py b/tools/generation/lib/template.py index 14cc3caebece6574a9d7d81f313e871ba605d869..bc66b7e8cb783e5a68aa329fddc58540425b6737 100644 --- a/tools/generation/lib/template.py +++ b/tools/generation/lib/template.py @@ -50,10 +50,10 @@ def indent(text, prefix = ' ', js_value = False): return '\n'.join(indented) class Template: - def __init__(self, filename): + def __init__(self, filename, encoding): self.filename = filename - with open(filename) as template_file: + with codecs.open(filename, 'r', encoding) as template_file: self.source = template_file.read() self.attribs = dict() @@ -203,5 +203,6 @@ class Template: frontmatter = self._frontmatter(case_filename, case_values) body = self.expand_regions(self.source, case_values) + assert encoding == 'utf-8' return Test(self.attribs['meta']['path'] + case_name + '.js', source=codecs.encode(frontmatter + '\n' + body, encoding)) diff --git a/tools/generation/test/expected/normal/nested/path2-normal.js b/tools/generation/test/expected/normal/nested/path2-normal.js index fc85583e7508e78c6ccb2afb79352765df95ad7f..36bfd2509e1e96e614384f512c3d7dd46189da6c 100644 --- a/tools/generation/test/expected/normal/nested/path2-normal.js +++ b/tools/generation/test/expected/normal/nested/path2-normal.js @@ -12,7 +12,7 @@ info: | case info ---*/ -before-Third valueSecond value-after +before-Third value (Special characters like `≠` should be tolerated.)Second value-after /* Improperly-terminated comments should not break the tokenizer * diff --git a/tools/generation/test/expected/normal/path1-normal.js b/tools/generation/test/expected/normal/path1-normal.js index e80120c38bd0744728b7cf47a372876472e12591..4d1481fb59bbf14f50263939260eb71c70abec0e 100644 --- a/tools/generation/test/expected/normal/path1-normal.js +++ b/tools/generation/test/expected/normal/path1-normal.js @@ -12,11 +12,13 @@ info: | case info ---*/ -before-First value-between-Third value-after +before-First value-between-Third value (Special characters like `≠` should be tolerated.)-after before*Second value*between*First value*after -before/* " */Third valueafter +before/* " */Third value (Special characters like `≠` should be tolerated.)after + +// Special characters like `≠` should be tolerated. The following should not be expanded: diff --git a/tools/generation/test/fixtures/normal.case b/tools/generation/test/fixtures/normal.case index c97a6e7b7fc54cc2f069bcad94ceb31c69ef5093..2b9a71936de12f06602c627dc705d065a6b5c203 100644 --- a/tools/generation/test/fixtures/normal.case +++ b/tools/generation/test/fixtures/normal.case @@ -22,7 +22,7 @@ First value //- second Second value //- third -Third value +Third value (Special characters like `≠` should be tolerated.) //- fourth Quote characters: " ' ` //- teardown diff --git a/tools/generation/test/fixtures/normal/normal.template b/tools/generation/test/fixtures/normal/normal.template index 32bc9cff19f8c9f69fcc27427c32b9add851573e..29ff4d5915ff5ebe4c65c838bc5f1c040550b3eb 100644 --- a/tools/generation/test/fixtures/normal/normal.template +++ b/tools/generation/test/fixtures/normal/normal.template @@ -14,6 +14,8 @@ before*/*{ second }*/*between*/*{ first }*/*after before/* " *//*{ third }*/after +// Special characters like `≠` should be tolerated. + The following should not be expanded: /* */*{ first }*/