Recognize short comments during encoding prescan

Author: muneikh

AUTHORS.rst

@@ -63,4 +63,4 @@ Patches and suggestions
 - Ville Skyttä
 - Hugo van Kemenade
 - Mark Vasilkov
-
+- Muneeb Sheikh

CHANGES.rst

@@ -18,6 +18,9 @@ Features:
 
 Bug fixes:
 
+* Detect declared encodings after short comments whose terminators overlap
+  the comment opener. (#427)
+
 * The sanitizer now permits ``<summary>`` tags. It used to allow ``<details>``
   already. (#423)
 

html5lib/_inputstream.py

@@ -712,7 +712,8 @@ def getEncoding(self):
         return self.encoding
 
     def handleComment(self):
-        """Skip over comments"""
+        """Skip over comments, including terminators overlapping the opener."""
+        self.data.position -= 2
         return self.data.jumpTo(b"-->")
 
     def handleMeta(self):

html5lib/tests/test_encoding.py

@@ -115,3 +115,19 @@ def test_chardet():
             encoding = _inputstream.HTMLInputStream(fp.read()).charEncoding
             assert encoding[0].name == "big5"
 # pylint:enable=wrong-import-position
+
+
+@pytest.mark.parametrize("comment", [b"<!-->", b"<!--->", b"<!---->", b"<!--normal-->"])
+def test_prescan_short_comments(comment):
+    data = comment + b"<meta charset=iso-8859-2>"
+    encoding = _inputstream.EncodingParser(data).getEncoding()
+    assert encoding is not None
+    assert encoding.name == "iso-8859-2"
+    stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False)
+    assert stream.charEncoding[0].name == "iso-8859-2"
+
+
+def test_prescan_ignores_meta_inside_comment():
+    data = b"<!--<meta charset=utf-8>--><meta charset=iso-8859-2>"
+    encoding = _inputstream.EncodingParser(data).getEncoding()
+    assert encoding.name == "iso-8859-2"