diff --git a/History.md b/History.md index 95af4940e..e469cbdfb 100644 --- a/History.md +++ b/History.md @@ -1,5 +1,9 @@ # Liquid Change Log +## Unreleased + +* Raise `Liquid::SyntaxError` instead of `ArgumentError` when a template with invalid bytes starts with a tag, a variable, or is a `{% liquid %}` body [Josh Faigan] + ## 5.14.0 * Avoid materializing integer ranges in `for` and `tablerow` loops, and account for each visited range item [Ian Ker-Seymer] diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb index 8b331d93c..b3f065832 100644 --- a/lib/liquid/tokenizer.rb +++ b/lib/liquid/tokenizer.rb @@ -24,6 +24,8 @@ def initialize( @line_number = line_number || (line_numbers ? 1 : nil) @for_liquid_tag = for_liquid_tag @source = source.to_s.to_str + raise SyntaxError, "Invalid byte sequence in #{@source.encoding}" unless @source.valid_encoding? + @offset = 0 @tokens = [] @@ -103,12 +105,6 @@ def next_text_token pos = @ss.pos -= 2 @source.byteslice(start, pos - start) - rescue ::ArgumentError => e - if e.message == "invalid byte sequence in #{@ss.string.encoding}" - raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}" - else - raise - end end def next_variable_token diff --git a/test/unit/tokenizer_unit_test.rb b/test/unit/tokenizer_unit_test.rb index 76d379d36..09b7305ca 100644 --- a/test/unit/tokenizer_unit_test.rb +++ b/test/unit/tokenizer_unit_test.rb @@ -48,6 +48,52 @@ def test_unmatching_start_and_end assert_equal(["{%%}", "}"], tokenize('{%%}}')) end + def test_invalid_utf8_after_leading_tag_raises_syntax_error + error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{% if true %}\xFF") } + assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message) + end + + def test_invalid_utf8_after_variable_then_tag_raises_syntax_error + error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{{ x }}{% if true %}\xFF") } + assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message) + end + + def test_invalid_utf8_in_unterminated_variable_with_tag_raises_syntax_error + error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{{ x {% if true %}\xFF") } + assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message) + end + + def test_invalid_utf8_deep_in_source_raises_syntax_error + source = "{% if true %}hi{% endif %}" + ("x" * 5000) + "\xFF" + error = assert_raises(Liquid::SyntaxError) { new_tokenizer(source) } + assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message) + end + + def test_invalid_utf8_with_line_numbers_raises_syntax_error + error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{% if true %}\xFF", start_line_number: 1) } + assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message) + end + + def test_invalid_utf8_in_liquid_tag_body_raises_syntax_error + error = assert_raises(Liquid::SyntaxError) do + Liquid::ParseContext.new.new_tokenizer("echo \xFF\nx", for_liquid_tag: true, start_line_number: 1) + end + assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message) + end + + def test_valid_multibyte_utf8_after_leading_tag + assert_equal(["{% if true %}", "é", "{% endif %}"], tokenize("{% if true %}é{% endif %}")) + end + + def test_binary_source_with_high_bytes_tokenizes + assert_equal(["{% if true %}".b, "\xFF".b], tokenize("{% if true %}\xFF".b)) + end + + def test_invalid_utf8_in_variable_only_source_raises_syntax_error + error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{{ \"\xFF\" }}") } + assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message) + end + private def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)