From 96bc2b50a692679ca4ff8c14aa280254ad38d6c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 28 Sep 2026 14:11:05 +0200 Subject: [PATCH] Accept transfer codings with parameters in Transfer-Encoding A Transfer-Encoding value with a parameter, such as "custom; level=1, chunked", failed with {:invalid_token_list, value} because the value was parsed as a plain list of tokens. RFC 9110 10.1.4 allows parameters on a transfer coding, as a token or a quoted string. Transfer-Encoding values are now parsed with their parameters and only the coding names are kept, so the final coding of a response still decides between chunked framing and reading until close. The chunked coding defines no parameters and RFC 9112 7.1 says their presence should be treated as an error, so chunked with parameters is still rejected. The same parser checks the Transfer-Encoding headers passed to request/5 with a streamed body and no Content-Length, so a value such as "gzip;q=1" is accepted there. The fields are combined with ", " into one value before parsing (RFC 9110 5.3). If chunked or identity is among the codings the headers are sent unchanged, as before. Otherwise chunked is appended to the last field, so it ends up as the final coding once. --- lib/mint/http1.ex | 32 +++++++++--- lib/mint/http1/parse.ex | 76 ++++++++++++++++++++++++++- test/mint/http1/conn_test.exs | 96 ++++++++++++++++++++++++++++++++++ test/mint/http1/parse_test.exs | 23 ++++++++ 4 files changed, 218 insertions(+), 9 deletions(-) diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index 2e5a2325..6fed3027 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -1358,17 +1358,14 @@ defmodule Mint.HTTP1 do Headers.has?(headers, "content-length") -> {:ok, headers, :identity} - found = Headers.find(headers, "transfer-encoding") -> - {raw_name, value} = found + Headers.has?(headers, "transfer-encoding") -> + values = for {_name, "transfer-encoding", value} <- headers, do: value - with {:ok, tokens} <- Parse.transfer_encoding_header(value) do - if "chunked" in tokens or "identity" in tokens do + with {:ok, codings} <- transfer_codings(values) do + if "chunked" in codings or "identity" in codings do {:ok, headers, :identity} else - headers = - Headers.replace(headers, raw_name, "transfer-encoding", value <> ",chunked") - - {:ok, headers, :chunked} + {:ok, append_chunked_coding(headers), :chunked} end end @@ -1393,6 +1390,25 @@ defmodule Mint.HTTP1 do :identity} end + # Adds chunked as the final transfer coding, after the codings of the last + # Transfer-Encoding field. + defp append_chunked_coding(headers) do + headers = Enum.reverse(headers) + {name, "transfer-encoding", value} = List.keyfind(headers, "transfer-encoding", 1) + + headers + |> Headers.replace(name, "transfer-encoding", value <> ",chunked") + |> Enum.reverse() + end + + # RFC 9110 5.3: the field lines are combined with ", " into one field value, + # so a quoted string can span two of them. + defp transfer_codings(values) do + values + |> Enum.join(", ") + |> Parse.transfer_encoding_header() + end + defp wrap_error(reason) do %HTTPError{reason: reason, module: __MODULE__} end diff --git a/lib/mint/http1/parse.ex b/lib/mint/http1/parse.ex index 09ec7b18..27fc590c 100644 --- a/lib/mint/http1/parse.ex +++ b/lib/mint/http1/parse.ex @@ -135,10 +135,84 @@ defmodule Mint.HTTP1.Parse do split_into_downcase_tokens(string) end + # RFC 9110 10.1.4: transfer-coding = token *( OWS ";" OWS transfer-parameter ) + # and transfer-parameter = token BWS "=" BWS ( token / quoted-string ). Only the + # coding names are returned. RFC 9112 7.1: chunked defines no parameters and + # their presence should be treated as an error. def transfer_encoding_header(string) do - split_into_downcase_tokens(string) + case transfer_coding_list(string, []) do + {:ok, []} -> {:error, :empty_token_list} + {:ok, codings} -> {:ok, codings} + :error -> {:error, {:invalid_token_list, string}} + end + end + + defp transfer_coding_list(<<>>, acc), do: {:ok, :lists.reverse(acc)} + + defp transfer_coding_list(<>, acc) + when is_whitespace(char) or is_comma(char), + do: transfer_coding_list(rest, acc) + + defp transfer_coding_list(<> = string, acc) when is_tchar(char) do + {coding, rest} = take_token_downcase(string, <<>>) + transfer_parameters(trim_leading_whitespace(rest), [coding | acc]) end + defp transfer_coding_list(_string, _acc), do: :error + + # Parses the parameters of the last coding in acc, then the list separator. + defp transfer_parameters(<<>>, acc), do: {:ok, :lists.reverse(acc)} + + defp transfer_parameters(<>, acc) when is_comma(char), + do: transfer_coding_list(rest, acc) + + defp transfer_parameters(<>, [coding | _] = acc) when coding != "chunked" do + with {:ok, rest} <- transfer_parameter(trim_leading_whitespace(rest)), + do: transfer_parameters(trim_leading_whitespace(rest), acc) + end + + defp transfer_parameters(_string, _acc), do: :error + + defp transfer_parameter(<> = string) when is_tchar(char) do + rest = skip_token(string) + + case trim_leading_whitespace(rest) do + <> -> transfer_parameter_value(trim_leading_whitespace(rest)) + _other -> :error + end + end + + defp transfer_parameter(_string), do: :error + + defp transfer_parameter_value(<>), do: quoted_string(rest) + + defp transfer_parameter_value(<> = string) when is_tchar(char) do + {:ok, skip_token(string)} + end + + defp transfer_parameter_value(_string), do: :error + + # RFC 9110 5.6.4: qdtext and quoted-pair, after the opening quote. + defp quoted_string(<>), do: {:ok, rest} + + defp quoted_string(<>) + when char == 9 or char in 32..126 or char in 128..255, + do: quoted_string(rest) + + defp quoted_string(<>) + when char in [9, 32, 33] or char in 35..91 or char in 93..126 or char in 128..255, + do: quoted_string(rest) + + defp quoted_string(_string), do: :error + + defp skip_token(<>) when is_tchar(char), do: skip_token(rest) + defp skip_token(rest), do: rest + + defp take_token_downcase(<>, acc) when is_tchar(char), + do: take_token_downcase(rest, <>) + + defp take_token_downcase(rest, acc), do: {acc, rest} + defp split_into_downcase_tokens(string) do case token_list_downcase(string) do {:ok, []} -> {:error, :empty_token_list} diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index a3620eb4..cb1fe45a 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -1207,6 +1207,43 @@ defmodule Mint.HTTP1Test do assert done == {:done, ref} end + test "chunked framing applies after a transfer coding with parameters", %{conn: conn} do + {:ok, conn, ref1} = HTTP1.request(conn, "GET", "/", [], nil) + {:ok, conn, ref2} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: custom; level=1 ;name=\"a, b\", chunked\r\n\r\n" <> + "5\r\nhello\r\n0\r\n\r\n" <> + "HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok" + + assert {:ok, conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [ + {:status, ^ref1, 200}, + {:headers, ^ref1, _}, + {:data, ^ref1, "hello"}, + {:done, ^ref1}, + {:status, ^ref2, 200}, + {:headers, ^ref2, _}, + {:data, ^ref2, "ok"}, + {:done, ^ref2} + ] = responses + + assert HTTP1.open?(conn) + end + + test "the chunked transfer coding with parameters is an error", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked; level=1\r\n\r\n5\r\nhello\r\n0\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: {:invalid_token_list, "chunked; level=1"}}, + [{:status, ^ref, 200}]} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + + refute HTTP1.open?(conn) + end + test "close/1", %{conn: conn} do assert HTTP1.open?(conn) assert {:ok, conn} = HTTP1.close(conn) @@ -1863,6 +1900,65 @@ defmodule Mint.HTTP1Test do assert HTTP1.open?(conn) end + test "chunked is appended to a transfer coding with parameters", + %{conn: conn, server_socket: server_socket, port: port} do + {:ok, conn, ref} = + HTTP1.request(conn, "GET", "/", [{"transfer-encoding", "gzip;q=1"}], :stream) + + assert receive_request_string(server_socket) == + request_string(""" + GET / HTTP/1.1 + host: localhost:#{port} + user-agent: #{mint_user_agent()} + transfer-encoding: gzip;q=1,chunked + + \ + """) + + {:ok, conn} = HTTP1.stream_request_body(conn, ref, "hello") + assert receive_request_string(server_socket) == "5\r\nhello\r\n" + end + + test "chunked is appended to the last of several transfer-encoding headers", + %{conn: conn, server_socket: server_socket, port: port} do + headers = [{"transfer-encoding", "custom;p=1"}, {"transfer-encoding", "gzip"}] + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", headers, :stream) + + assert receive_request_string(server_socket) == + request_string(""" + GET / HTTP/1.1 + host: localhost:#{port} + user-agent: #{mint_user_agent()} + transfer-encoding: custom;p=1 + transfer-encoding: gzip,chunked + + \ + """) + + {:ok, conn} = HTTP1.stream_request_body(conn, ref, "hello") + assert receive_request_string(server_socket) == "5\r\nhello\r\n" + end + + test "transfer-encoding headers are combined before they're parsed", + %{conn: conn, server_socket: server_socket, port: port} do + headers = [{"transfer-encoding", ~s(custom;p="a)}, {"transfer-encoding", ~s(b", chunked)}] + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", headers, :stream) + + assert receive_request_string(server_socket) == + request_string(""" + GET / HTTP/1.1 + host: localhost:#{port} + user-agent: #{mint_user_agent()} + transfer-encoding: custom;p="a + transfer-encoding: b", chunked + + \ + """) + + {:ok, conn} = HTTP1.stream_request_body(conn, ref, "hello") + assert receive_request_string(server_socket) == "hello" + end + test "transfer-encoding is not set to chunked if already set to identity", %{conn: conn, server_socket: server_socket, port: port} do {:ok, conn, ref} = diff --git a/test/mint/http1/parse_test.exs b/test/mint/http1/parse_test.exs index 8cbe0dbd..6986e915 100644 --- a/test/mint/http1/parse_test.exs +++ b/test/mint/http1/parse_test.exs @@ -164,6 +164,29 @@ defmodule Mint.HTTP1.ParseTest do assert transfer_encoding_header("") == {:error, :empty_token_list} end + test "transfer_encoding_header/1 with transfer coding parameters" do + assert transfer_encoding_header("Custom;Level=1") == {:ok, ["custom"]} + + assert transfer_encoding_header("custom ; a=1 ;b = x, chunked") == + {:ok, ["custom", "chunked"]} + + assert transfer_encoding_header(~s(custom; a="x, \\"y\\"; z", gzip)) == + {:ok, ["custom", "gzip"]} + + for value <- [ + "custom;", + "custom; a", + "custom; a=", + "custom; =1", + "custom; a=1 b", + ~s(custom; a="x), + "chunked; a=1", + "Chunked;a=1, gzip" + ] do + assert transfer_encoding_header(value) == {:error, {:invalid_token_list, value}} + end + end + describe "token_list_downcase/1" do property "returns an empty list if there's no token" do no_tokens_generator = string([?\s, ?\t, ?,])