Placate pyflakes and fix tests after fixing issues.

This is mostly just removing dead variables, however, there are a few
substantial changes in here:

 - Move to using try/except ImportError/else in tests where we are
   checking some module existing, as this was hiding genuine bugs
   that manifested themselves as ImportError (the ElementTree treewalker was
   throwing ImportError when being imported).

 - Fixes the ImportError the ElementTree treewalker was throwing (this
   was, too, reported as a bug by pyflakes, thereby showing its
   value).

 - Fixes the assertion then shown in the ElementTree treewalker (this
   is simply a case of it having been missed in the move to Python 3,
   due to the tests not being run).

 - Fixes the ElementTree treewalker with xml.etree.cElementTree under
   2.6, where the ElementTree Comment factory isn't the tag attribute
   on Comment objects (this is effectively the treewalker equivalent
   of 3e50aad).

 - The parser defined the scriptDataDoubleEscapedDashState twice,
   therefore everything that should've been run in this state was in
   fact run in the scriptDataDoubleEscapedDashDashState.

This also adds flake8 to Travis, albeit running it without any PEP 8
errors showing.

Author: gsnedders

.travis.yml

@@ -10,6 +10,14 @@ env:
   - USE_OPTIONAL=true
   - USE_OPTIONAL=false
 
+matrix:
+  exclude:
+    - python: "2.7"
+      env: USE_OPTIONAL=false
+  include:
+    - python: "2.7"
+      env: USE_OPTIONAL=false FLAKE=true
+
 before_install:
   - git submodule update --init --recursive
 
@@ -19,9 +27,11 @@ install:
   - if [[ $TRAVIS_PYTHON_VERSION != 3.* && $USE_OPTIONAL == "true" ]]; then pip install -r requirements-optional-2.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3.* && $USE_OPTIONAL == "true" ]]; then pip install -r requirements-optional-3.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION != "pypy" && $USE_OPTIONAL == "true" ]]; then pip install -r requirements-optional-cpython.txt --use-mirrors; fi
+  - if [[ $FLAKE == "true" ]]; then pip install --use-mirrors flake8; fi
 
 script:
   - nosetests
+  - if [[ $FLAKE == "true" ]]; then flake8 --exclude=E,W html5lib; fi
 
 after_script:
   - python debug-info.py

html5lib/__init__.py

@@ -18,4 +18,6 @@
 from .treewalkers import getTreeWalker
 from .serializer import serialize
 
+__all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder",
+           "getTreeWalker", "serialize"]
 __version__ = "1.0b1"

html5lib/html5parser.py

@@ -1,7 +1,6 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import with_metaclass
 
-import sys
 import types
 
 from . import inputstream
@@ -14,10 +13,10 @@
 from . import utils
 from . import constants
 from .constants import spaceCharacters, asciiUpper2Lower
-from .constants import formattingElements, specialElements
-from .constants import headingElements, tableInsertModeElements
-from .constants import cdataElements, rcdataElements, voidElements
-from .constants import tokenTypes, ReparseException, namespaces, spaceCharacters
+from .constants import specialElements
+from .constants import headingElements
+from .constants import cdataElements, rcdataElements
+from .constants import tokenTypes, ReparseException, namespaces
 from .constants import htmlIntegrationPointElements, mathmlTextIntegrationPointElements
 
 def parse(doc, treebuilder="simpletree", encoding=None,
@@ -88,7 +87,7 @@ def _parse(self, stream, innerHTML=False, container="div",
             try:
                 self.mainLoop()
                 break
-            except ReparseException as e:
+            except ReparseException:
                 self.reset()
 
     def reset(self):
@@ -405,7 +404,7 @@ def parseRCDataRawtext(self, token, contentType):
         """
         assert contentType in ("RAWTEXT", "RCDATA")
 
-        element = self.tree.insertElement(token)
+        self.tree.insertElement(token)
 
         if contentType == "RAWTEXT":
             self.tokenizer.state = self.tokenizer.rawtextState
@@ -1402,7 +1401,6 @@ def endTagFormatting(self, token):
             """The much-feared adoption agency algorithm"""
             # http://www.whatwg.org/specs/web-apps/current-work/#adoptionAgency
             # XXX Better parseError messages appreciated.
-            name = token["name"]
 
             # Step 1
             outerLoopCounter = 0
@@ -1620,7 +1618,7 @@ def endTagScript(self, token):
             #document.write works
 
         def endTagOther(self, token):
-            node = self.tree.openElements.pop()
+            self.tree.openElements.pop()
             self.parser.phase = self.parser.originalPhase
 
     class InTablePhase(Phase):

html5lib/inputstream.py

@@ -3,7 +3,6 @@
 
 import codecs
 import re
-import types
 import sys
 
 from .constants import EOF, spaceCharacters, asciiLetters, asciiUppercase
@@ -107,8 +106,7 @@ def _readFromBuffer(self, bytes):
                 bytesToRead = len(bufferedData) - bufferOffset
                 self.position = [bufferIndex, len(bufferedData)]
                 bufferIndex += 1
-            data = rv.append(bufferedData[bufferOffset:
-                                          bufferOffset + bytesToRead])
+            rv.append(bufferedData[bufferOffset:bufferOffset + bytesToRead])
             remainingBytes -= bytesToRead
 
             bufferOffset = 0
@@ -290,7 +288,6 @@ def characterErrorsUCS2(self, data):
         #Someone picked the wrong compile option
         #You lose
         skip = False
-        import sys
         for match in invalid_unicode_re.finditer(data):
             if skip:
                 continue

html5lib/serializer/htmlserializer.py

@@ -24,8 +24,6 @@
 else:
     unicode_encode_errors = "htmlentityreplace"
 
-    from html5lib.constants import entities
-
     encode_entity_map = {}
     is_ucs4 = len("\U0010FFFF") == 1
     for k, v in list(entities.items()):
@@ -228,7 +226,6 @@ def serialize(self, treewalker, encoding=None):
                     in_cdata = True
                 elif in_cdata:
                     self.serializeError(_("Unexpected child element of a CDATA element"))
-                attributes = []
                 for (attr_namespace,attr_name),attr_value in sorted(token["data"].items()):
                     #TODO: Add namespace support here
                     k = attr_name

html5lib/tests/__init__.py

@@ -1,12 +1 @@
 from __future__ import absolute_import, division, unicode_literals
-
-import sys
-import os
-
-parent_path = os.path.abspath(os.path.join(os.path.split(__file__)[0], ".."))
-
-if not parent_path in sys.path:
-    sys.path.insert(0, parent_path)
-del parent_path
-
-from . import support

html5lib/tests/support.py

@@ -12,8 +12,7 @@
                                                 os.path.pardir,
                                                 os.path.pardir)))
 
-import html5lib
-from html5lib import html5parser, treebuilders
+from html5lib import treebuilders
 del base_path
 
 #Build a dict of avaliable trees
@@ -43,10 +42,11 @@
         pass
 
 try:
-    import lxml.etree as lxml
-    treeTypes['lxml'] = treebuilders.getTreeBuilder("lxml")
+    import lxml.etree as lxml # flake8: noqa
 except ImportError:
     pass
+else:
+    treeTypes['lxml'] = treebuilders.getTreeBuilder("lxml")
 
 def get_data_files(subdirectory, files='*.dat'):
     return glob.glob(os.path.join(test_dir,subdirectory,files))

html5lib/tests/test_encoding.py

@@ -1,6 +1,5 @@
 from __future__ import absolute_import, division, unicode_literals
 
-import re
 import os
 import unittest
 
@@ -27,7 +26,7 @@ def test_codec_name_d(self):
 
 def runParserEncodingTest(data, encoding):
     p = HTMLParser()
-    t = p.parse(data, useChardet=False)
+    p.parse(data, useChardet=False)
     encoding = encoding.lower().decode("ascii")
 
     assert encoding == p.tokenizer.stream.charEncoding[0], errorMessage(data, encoding, p.tokenizer.stream.charEncoding[0])
@@ -44,18 +43,17 @@ def runPreScanEncodingTest(data, encoding):
 
 def test_encoding():
     for filename in get_data_files("encoding"):
-        test_name = os.path.basename(filename).replace('.dat',''). \
-            replace('-','')
         tests = TestData(filename, b"data", encoding=None)
         for idx, test in enumerate(tests):
             yield (runParserEncodingTest, test[b'data'], test[b'encoding'])
             yield (runPreScanEncodingTest, test[b'data'], test[b'encoding'])
 
 try:
-    import chardet
+    import chardet # flake8: noqa
+except ImportError:
+    print("chardet not found, skipping chardet tests")
+else:
     def test_chardet():
         with open(os.path.join(test_dir, "encoding" , "chardet", "test_big5.txt"), "rb") as fp:
             encoding = inputstream.HTMLInputStream(fp.read()).charEncoding
             assert encoding[0].lower() == "big5"
-except ImportError:
-    print("chardet not found, skipping chardet tests")

html5lib/tests/test_parser.py

@@ -3,16 +3,14 @@
 import os
 import sys
 import traceback
-import io
 import warnings
 import re
 
 warnings.simplefilter("error")
 
 from .support import get_data_files
 from .support import TestData, convert, convertExpected, treeTypes
-import html5lib
-from html5lib import html5parser, treebuilders, constants
+from html5lib import html5parser, constants
 
 #Run the parse error checks
 checkParseErrors = False

html5lib/tests/test_parser2.py

@@ -2,7 +2,7 @@
 
 import io
 
-from . import support
+from . import support # flake8: noqa
 from html5lib import html5parser
 from html5lib.constants import namespaces
 from html5lib.treebuilders import dom