Handle utf8 better

James Graham · James Graham · commit 6d29b334a292 · 2010-07-15T13:38:48.000+02:00
diff --git a/html5lib/tests/support.py b/html5lib/tests/support.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import codecs
 import glob
 
 base_path = os.path.split(__file__)[0]
@@ -74,7 +75,7 @@ def __getitem__(self, key):
 
 class TestData(object):
     def __init__(self, filename, newTestHeading="data"):
-        self.f = open(filename)
+        self.f = codecs.open(filename, encoding="utf8")
         self.newTestHeading = newTestHeading
     
     def __iter__(self):
diff --git a/html5lib/tests/test_parser.py b/html5lib/tests/test_parser.py
@@ -84,22 +84,20 @@ def runParserTest(self, innerHTML, input, expected, errors, treeClass,
             p = html5parser.HTMLParser(tree = treeClass,
                                        namespaceHTMLElements=namespaceHTMLElements)
         except constants.DataLossWarning:
-            return 
+            return
 
-        errors = [item.decode("utf-8") for item in errors]
-        
         try:
             if innerHTML:
-                document = p.parseFragment(StringIO.StringIO(input), innerHTML)
+                document = p.parseFragment(input, innerHTML)
             else:
                 try:
-                    document = p.parse(StringIO.StringIO(input))
+                    document = p.parse(input)
                 except constants.DataLossWarning:
                     return 
         except:
-            errorMsg = "\n".join(["\n\nInput:", input, "\nExpected:", expected,
-                                  "\nTraceback:", traceback.format_exc()])
-            self.assertTrue(False, errorMsg)
+            errorMsg = u"\n".join([u"\n\nInput:", input, u"\nExpected:", expected,
+                                   u"\nTraceback:", traceback.format_exc()])
+            self.assertTrue(False, errorMsg.encode("utf8"))
         
         output = convertTreeDump(p.tree.testSerializer(document))
         output = attrlist.sub(sortattrs, output)
@@ -109,9 +107,9 @@ def runParserTest(self, innerHTML, input, expected, errors, treeClass,
         if namespaceHTMLElements:
             expected = namespaceExpected(r"\1<html \2>", expected)
         
-        errorMsg = "\n".join(["\n\nInput:", input, "\nExpected:", expected,
-                              "\nReceived:", output])
-        self.assertEquals(expected, output, errorMsg.encode("utf-8"))
+        errorMsg = u"\n".join([u"\n\nInput:", input, u"\nExpected:", expected,
+                               u"\nReceived:", output])
+        self.assertEquals(expected, output, errorMsg.encode("utf8"))
         errStr = [u"Line: %i Col: %i %s"%(line, col, 
                                           constants.E[errorcode] % datavars if isinstance(datavars, dict) else (datavars,)) for
                   ((line,col), errorcode, datavars) in p.errors]
@@ -152,6 +150,8 @@ def testFunc(self, innerHTML=innerHTML, input=input,
                     testFunc.__name__ = "test_%s_%d_%s_%s" % (testName,index+1,treeName, namespaceHTMLElements and "namespaced" or "no_html_namespace")
                     setattr(TestCase, testFunc.__name__,
                          testFunc)
+                    break
+        break
 
     return unittest.TestLoader().loadTestsFromTestCase(TestCase)
 
