add some unit tests

add test to repro #114
fix issue #114
factor out PercentFormat function
new fn write; start factoring summary output
fix tests to match
consolidate "Failed Tests" message
add test for successful run
consolidate all_succeeded message
consolidate summary line
consolidate ntests
failing negative tests
squeeze whitespace

tools/packaging/test/README.md

@@ -6,6 +6,6 @@ This directory holds tests for the python code, not tests of EMCAScript
 
 ````
 $ cd tools/packaging/test
-$ python test*.py
+$ for x in test*.py; do python $x; done
 ````
 

tools/packaging/test/test_test262.py

+#!/usr/bin/env python
+
+# Copyright 2014 by Sam Mikes.  All rights reserved.
+# This code is governed by the BSD license found in the LICENSE file.
+
+import unittest
+
+import sys
+import os
+import cStringIO
+from functools import wraps
+
+sys.path.insert(0, "..")
+
+import test262
+
+class TestTest262(unittest.TestCase):
+
+    def test_that_tests_run(self):
+        self.assertEqual(1 + 2, 3)
+
+class MockTest(object):
+
+    def __init__(self, name, negative):
+        self.name = name
+        self.negative = negative if negative else False
+        self.strict_mode = False
+
+    def GetName(self):
+        return self.name
+
+    def IsNegative(self):
+        return self.negative
+
+    def GetMode(self):
+        if self.strict_mode:
+            return "strict mode"
+
+        return "non-strict mode"
+
+class MockResult(object):
+
+    def __init__(self, case):
+        self.case = case
+
+    
+
+class TestTestSuite(unittest.TestCase):
+
+    def test_that_tests_run(self):
+        self.assertEqual(1 + 2, 3)
+
+    def test_create_test_suite(self):
+        test_suite = test262.TestSuite(".",
+                                       False,
+                                       False,
+                                       False,
+                                       None)
+        self.assertNotEqual(test_suite, None)
+
+    def test_summary(self):
+        test_suite = test262.TestSuite(".",
+                                       False,
+                                       False,
+                                       False,
+                                       None)
+
+        progress = test262.ProgressIndicator(100)
+        progress.succeeded = 98
+        progress.failed = 2
+
+        result = mute(True)(test_suite.PrintSummary)(progress, None)
+        self.assertEqual("""
+=== Summary ===
+ - Ran 100 tests
+ - Passed 98 tests (98.0%)
+ - Failed 2 tests (2.0%)
+""", result)
+
+    def test_summary_logfile(self):
+        test_suite = test262.TestSuite(".",
+                                       False,
+                                       False,
+                                       False,
+                                       None)
+
+        progress = test262.ProgressIndicator(100)
+        progress.succeeded = 98
+        progress.failed = 2
+
+        fake_log = cStringIO.StringIO()
+        test_suite.logf = fake_log
+
+        result = mute(True)(test_suite.PrintSummary)(progress, True)
+
+        expected_out = """
+=== Summary ===
+ - Ran 100 tests
+ - Passed 98 tests (98.0%)
+ - Failed 2 tests (2.0%)
+"""
+
+        expected_log = """=== Summary ===
+ - Ran 100 tests
+ - Passed 98 tests (98.0%)
+ - Failed 2 tests (2.0%)
+"""
+        self.assertEqual(expected_out, result)
+        self.assertEqual(expected_log, fake_log.getvalue())
+
+
+    def test_summary_withfails(self):
+        test_suite = test262.TestSuite(".",
+                                       False,
+                                       False,
+                                       False,
+                                       None)
+
+        progress = test262.ProgressIndicator(100)
+        progress.succeeded = 98
+        progress.failed = 2
+        progress.failed_tests = [
+            MockResult(MockTest("foo", False)),
+            MockResult(MockTest("bar", True))
+        ]
+
+        result = mute(True)(test_suite.PrintSummary)(progress, None)
+        self.assertEqual("""
+=== Summary ===
+ - Ran 100 tests
+ - Passed 98 tests (98.0%)
+ - Failed 2 tests (2.0%)
+
+Failed Tests
+  foo in non-strict mode
+
+Expected to fail but passed ---
+  bar in non-strict mode
+""", result)
+
+
+    def test_summary_withfails_andlog(self):
+        test_suite = test262.TestSuite(".",
+                                       False,
+                                       False,
+                                       False,
+                                       None)
+
+        progress = test262.ProgressIndicator(100)
+        progress.succeeded = 98
+        progress.failed = 2
+        progress.failed_tests = [
+            MockResult(MockTest("foo", False)),
+            MockResult(MockTest("bar", True))
+        ]
+
+        fake_log = cStringIO.StringIO()
+        test_suite.logf = fake_log
+
+        expected_out = """
+=== Summary ===
+ - Ran 100 tests
+ - Passed 98 tests (98.0%)
+ - Failed 2 tests (2.0%)
+
+Failed Tests
+  foo in non-strict mode
+
+Expected to fail but passed ---
+  bar in non-strict mode
+"""
+        expected_log = """=== Summary ===
+ - Ran 100 tests
+ - Passed 98 tests (98.0%)
+ - Failed 2 tests (2.0%)
+Failed Tests
+  foo in non-strict mode
+Expected to fail but passed ---
+  bar in non-strict mode
+"""
+
+        result = mute(True)(test_suite.PrintSummary)(progress, True)
+        self.assertEqual(expected_out, result)
+        self.assertEqual(expected_log, fake_log.getvalue())
+
+
+    def test_summary_success_logfile(self):
+        test_suite = test262.TestSuite(".",
+                                       False,
+                                       False,
+                                       False,
+                                       None)
+
+        progress = test262.ProgressIndicator(100)
+        progress.succeeded = 100
+        progress.failed = 0
+
+        fake_log = cStringIO.StringIO()
+        test_suite.logf = fake_log
+
+        result = mute(True)(test_suite.PrintSummary)(progress, True)
+
+        expected_out = """
+=== Summary ===
+ - Ran 100 tests
+ - All tests succeeded
+"""
+
+        expected_log = """=== Summary ===
+ - Ran 100 tests
+ - All tests succeeded
+"""
+        self.assertEqual(expected_out, result)
+        self.assertEqual(expected_log, fake_log.getvalue())
+
+
+    def test_percent_format(self):
+        self.assertEqual(test262.PercentFormat(1, 100), "1 test (1.0%)")
+        self.assertEqual(test262.PercentFormat(0, 100), "0 tests (0.0%)")
+        self.assertEqual(test262.PercentFormat(99, 100), "99 tests (99.0%)")
+
+
+# module level utility functions
+# copied from https://stackoverflow.com/questions/2828953/silence-the-stdout-of-a-function-in-python-without-trashing-sys-stdout-and-resto
+
+
+def mute(returns_output=False):
+    """
+        Decorate a function that prints to stdout, intercepting the output.
+        If "returns_output" is True, the function will return a generator
+        yielding the printed lines instead of the return values.
+
+        The decorator litterally hijack sys.stdout during each function
+        execution for ALL THE THREADS, so be careful with what you apply it to
+        and in which context.
+
+        >>> def numbers():
+            print "42"
+            print "1984"
+        ...
+        >>> numbers()
+        42
+        1984
+        >>> mute()(numbers)()
+        >>> list(mute(True)(numbers)())
+        ['42', '1984']
+
+    """
+
+    def decorator(func):
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+
+            saved_stdout = sys.stdout
+            sys.stdout = cStringIO.StringIO()
+
+            try:
+                out = func(*args, **kwargs)
+                if returns_output:
+                    out = sys.stdout.getvalue()
+            finally:
+                sys.stdout = saved_stdout
+
+            return out
+
+        return wrapper
+
+    return decorator
+
+
+if __name__ == '__main__':
+    unittest.main()
+

tools/packaging/test262.py

@@ -382,6 +382,11 @@ def MakePlural(n):
   else:
     return (n, "s")
 
+def PercentFormat(partial, total):
+  return "%i test%s (%.1f%%)" % (MakePlural(partial) +
+                                 ((100.0 * partial)/total,))
+
+
 class TestSuite(object):
 
   def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle):
@@ -464,49 +469,37 @@ class TestSuite(object):
     logging.info("Done listing tests")
     return cases
 
+
   def PrintSummary(self, progress, logfile):
+
+    def write(s):
+      if logfile:
+        self.logf.write(s + "\n")
+      print s
+
     print
-    if logfile:
-       self.logf.write("=== Summary === \n")
-    print "=== Summary ==="
+    write("=== Summary ===");
     count = progress.count
     succeeded = progress.succeeded
     failed = progress.failed
-    if logfile:
-      self.logf.write(" - Ran %i test%s \n" % MakePlural(count))
-    print " - Ran %i test%s" % MakePlural(count)
+    write(" - Ran %i test%s" % MakePlural(count))
     if progress.failed == 0:
-      if logfile:
-        self.logf.write(" - All tests succeeded \n")
-      print " - All tests succeeded"
-   
+      write(" - All tests succeeded")
     else:
-      percent = ((100.0 * succeeded) / count,)
-      if logfile:
-        self.logf.write(" - Passed %i test%s (%.1f%%)\n" % (MakePlural(succeeded) + percent))
-      print " - Passed %i test%s (%.1f%%)" % (MakePlural(succeeded) + percent)
-      percent = ((100.0 * failed) / count,)
-      if logfile:
-        self.logf.write(" - Failed %i test%s (%.1f%%) \n" % (MakePlural(failed) + percent))
-      print " - Failed %i test%s (%.1f%%)" % (MakePlural(failed) + percent)
+      write(" - Passed " + PercentFormat(succeeded, count))
+      write(" - Failed " + PercentFormat(failed, count))
       positive = [c for c in progress.failed_tests if not c.case.IsNegative()]
       negative = [c for c in progress.failed_tests if c.case.IsNegative()]
       if len(positive) > 0:
         print
-        if logfile:
-          self.logf.write("Failed Tests \n") 
-        print "Failed tests"
+        write("Failed Tests")
         for result in positive:
-          if logfile:
-            self.logf.write("  %s in %s \n" % (result.case.GetName(), result.case.GetMode()))
-          print "  %s in %s" % (result.case.GetName(), result.case.GetMode())
+          write("  %s in %s" % (result.case.GetName(), result.case.GetMode()))
       if len(negative) > 0:
         print
-        print "Expected to fail but passed ---"
+        write("Expected to fail but passed ---")
         for result in negative:
-          if logfile:
-            self.logfile.append(" %s in %s \n" % (result.case.GetName(), result.case.GetMode()))
-          print " %s in %s" % (result.case.GetName(), result.case.GetMode())
+          write("  %s in %s" % (result.case.GetName(), result.case.GetMode()))
 
   def PrintFailureOutput(self, progress, logfile):
     for result in progress.failed_tests: