Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions History.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Liquid Change Log

## Unreleased

* Raise `Liquid::SyntaxError` instead of `ArgumentError` when a template with invalid bytes starts with a tag, a variable, or is a `{% liquid %}` body [Josh Faigan]

## 5.14.0

* Avoid materializing integer ranges in `for` and `tablerow` loops, and account for each visited range item [Ian Ker-Seymer]
Expand Down
8 changes: 2 additions & 6 deletions lib/liquid/tokenizer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ def initialize(
@line_number = line_number || (line_numbers ? 1 : nil)
@for_liquid_tag = for_liquid_tag
@source = source.to_s.to_str
raise SyntaxError, "Invalid byte sequence in #{@source.encoding}" unless @source.valid_encoding?

@offset = 0
@tokens = []

Expand Down Expand Up @@ -103,12 +105,6 @@ def next_text_token

pos = @ss.pos -= 2
@source.byteslice(start, pos - start)
rescue ::ArgumentError => e
if e.message == "invalid byte sequence in #{@ss.string.encoding}"
raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}"
else
raise
end
end

def next_variable_token
Expand Down
46 changes: 46 additions & 0 deletions test/unit/tokenizer_unit_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,52 @@ def test_unmatching_start_and_end
assert_equal(["{%%}", "}"], tokenize('{%%}}'))
end

def test_invalid_utf8_after_leading_tag_raises_syntax_error
error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{% if true %}\xFF") }
assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message)
end

def test_invalid_utf8_after_variable_then_tag_raises_syntax_error
error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{{ x }}{% if true %}\xFF") }
assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message)
end

def test_invalid_utf8_in_unterminated_variable_with_tag_raises_syntax_error
error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{{ x {% if true %}\xFF") }
assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message)
end

def test_invalid_utf8_deep_in_source_raises_syntax_error
source = "{% if true %}hi{% endif %}" + ("x" * 5000) + "\xFF"
error = assert_raises(Liquid::SyntaxError) { new_tokenizer(source) }
assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message)
end

def test_invalid_utf8_with_line_numbers_raises_syntax_error
error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{% if true %}\xFF", start_line_number: 1) }
assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message)
end

def test_invalid_utf8_in_liquid_tag_body_raises_syntax_error
error = assert_raises(Liquid::SyntaxError) do
Liquid::ParseContext.new.new_tokenizer("echo \xFF\nx", for_liquid_tag: true, start_line_number: 1)
end
assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message)
end

def test_valid_multibyte_utf8_after_leading_tag
assert_equal(["{% if true %}", "é", "{% endif %}"], tokenize("{% if true %}é{% endif %}"))
end

def test_binary_source_with_high_bytes_tokenizes
assert_equal(["{% if true %}".b, "\xFF".b], tokenize("{% if true %}\xFF".b))
end

def test_invalid_utf8_in_variable_only_source_raises_syntax_error
error = assert_raises(Liquid::SyntaxError) { new_tokenizer("{{ \"\xFF\" }}") }
assert_equal("Liquid syntax error: Invalid byte sequence in UTF-8", error.message)
end

private

def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)
Expand Down
Loading