Accept transfer codings with parameters in Transfer-Encoding

lib/mint/http1.ex

@@ -1358,17 +1358,14 @@ defmodule Mint.HTTP1 do
       Headers.has?(headers, "content-length") ->
         {:ok, headers, :identity}
 
-      found = Headers.find(headers, "transfer-encoding") ->
-        {raw_name, value} = found
+      Headers.has?(headers, "transfer-encoding") ->
+        values = for {_name, "transfer-encoding", value} <- headers, do: value
 
-        with {:ok, tokens} <- Parse.transfer_encoding_header(value) do
-          if "chunked" in tokens or "identity" in tokens do
+        with {:ok, codings} <- transfer_codings(values) do
+          if "chunked" in codings or "identity" in codings do
             {:ok, headers, :identity}
           else
-            headers =
-              Headers.replace(headers, raw_name, "transfer-encoding", value <> ",chunked")
-
-            {:ok, headers, :chunked}
+            {:ok, append_chunked_coding(headers), :chunked}
           end
         end
 
@@ -1393,6 +1390,25 @@ defmodule Mint.HTTP1 do
      :identity}
   end
 
+  # Adds chunked as the final transfer coding, after the codings of the last
+  # Transfer-Encoding field.
+  defp append_chunked_coding(headers) do
+    headers = Enum.reverse(headers)
+    {name, "transfer-encoding", value} = List.keyfind(headers, "transfer-encoding", 1)
+
+    headers
+    |> Headers.replace(name, "transfer-encoding", value <> ",chunked")
+    |> Enum.reverse()
+  end
+
+  # RFC 9110 5.3: the field lines are combined with ", " into one field value,
+  # so a quoted string can span two of them.
+  defp transfer_codings(values) do
+    values
+    |> Enum.join(", ")
+    |> Parse.transfer_encoding_header()
+  end
+
   defp wrap_error(reason) do
     %HTTPError{reason: reason, module: __MODULE__}
   end

lib/mint/http1/parse.ex

@@ -135,10 +135,84 @@ defmodule Mint.HTTP1.Parse do
     split_into_downcase_tokens(string)
   end
 
+  # RFC 9110 10.1.4: transfer-coding = token *( OWS ";" OWS transfer-parameter )
+  # and transfer-parameter = token BWS "=" BWS ( token / quoted-string ). Only the
+  # coding names are returned. RFC 9112 7.1: chunked defines no parameters and
+  # their presence should be treated as an error.
   def transfer_encoding_header(string) do
-    split_into_downcase_tokens(string)
+    case transfer_coding_list(string, []) do
+      {:ok, []} -> {:error, :empty_token_list}
+      {:ok, codings} -> {:ok, codings}
+      :error -> {:error, {:invalid_token_list, string}}
+    end
+  end
+
+  defp transfer_coding_list(<<>>, acc), do: {:ok, :lists.reverse(acc)}
+
+  defp transfer_coding_list(<<char, rest::binary>>, acc)
+       when is_whitespace(char) or is_comma(char),
+       do: transfer_coding_list(rest, acc)
+
+  defp transfer_coding_list(<<char, _::binary>> = string, acc) when is_tchar(char) do
+    {coding, rest} = take_token_downcase(string, <<>>)
+    transfer_parameters(trim_leading_whitespace(rest), [coding | acc])
   end
 
+  defp transfer_coding_list(_string, _acc), do: :error
+
+  # Parses the parameters of the last coding in acc, then the list separator.
+  defp transfer_parameters(<<>>, acc), do: {:ok, :lists.reverse(acc)}
+
+  defp transfer_parameters(<<char, rest::binary>>, acc) when is_comma(char),
+    do: transfer_coding_list(rest, acc)
+
+  defp transfer_parameters(<<?;, rest::binary>>, [coding | _] = acc) when coding != "chunked" do
+    with {:ok, rest} <- transfer_parameter(trim_leading_whitespace(rest)),
+         do: transfer_parameters(trim_leading_whitespace(rest), acc)
+  end
+
+  defp transfer_parameters(_string, _acc), do: :error
+
+  defp transfer_parameter(<<char, _::binary>> = string) when is_tchar(char) do
+    rest = skip_token(string)
+
+    case trim_leading_whitespace(rest) do
+      <<?=, rest::binary>> -> transfer_parameter_value(trim_leading_whitespace(rest))
+      _other -> :error
+    end
+  end
+
+  defp transfer_parameter(_string), do: :error
+
+  defp transfer_parameter_value(<<?", rest::binary>>), do: quoted_string(rest)
+
+  defp transfer_parameter_value(<<char, _::binary>> = string) when is_tchar(char) do
+    {:ok, skip_token(string)}
+  end
+
+  defp transfer_parameter_value(_string), do: :error
+
+  # RFC 9110 5.6.4: qdtext and quoted-pair, after the opening quote.
+  defp quoted_string(<<?", rest::binary>>), do: {:ok, rest}
+
+  defp quoted_string(<<?\\, char, rest::binary>>)
+       when char == 9 or char in 32..126 or char in 128..255,
+       do: quoted_string(rest)
+
+  defp quoted_string(<<char, rest::binary>>)
+       when char in [9, 32, 33] or char in 35..91 or char in 93..126 or char in 128..255,
+       do: quoted_string(rest)
+
+  defp quoted_string(_string), do: :error
+
+  defp skip_token(<<char, rest::binary>>) when is_tchar(char), do: skip_token(rest)
+  defp skip_token(rest), do: rest
+
+  defp take_token_downcase(<<char, rest::binary>>, acc) when is_tchar(char),
+    do: take_token_downcase(rest, <<acc::binary, downcase_ascii_char(char)>>)
+
+  defp take_token_downcase(rest, acc), do: {acc, rest}
+
   defp split_into_downcase_tokens(string) do
     case token_list_downcase(string) do
       {:ok, []} -> {:error, :empty_token_list}

test/mint/http1/conn_test.exs

@@ -1207,6 +1207,43 @@
     assert done == {:done, ref}
   end
 
+  test "chunked framing applies after a transfer coding with parameters", %{conn: conn} do
+    {:ok, conn, ref1} = HTTP1.request(conn, "GET", "/", [], nil)
+    {:ok, conn, ref2} = HTTP1.request(conn, "GET", "/", [], nil)
+
+    response =
+      "HTTP/1.1 200 OK\r\ntransfer-encoding: custom; level=1 ;name=\"a, b\", chunked\r\n\r\n" <>
+        "5\r\nhello\r\n0\r\n\r\n" <>
+        "HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok"
+
+    assert {:ok, conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, response})
+
+    assert [
+             {:status, ^ref1, 200},
+             {:headers, ^ref1, _},
+             {:data, ^ref1, "hello"},
+             {:done, ^ref1},
+             {:status, ^ref2, 200},
+             {:headers, ^ref2, _},
+             {:data, ^ref2, "ok"},
+             {:done, ^ref2}
+           ] = responses
+
+    assert HTTP1.open?(conn)
+  end
+
+  test "the chunked transfer coding with parameters is an error", %{conn: conn} do
+    {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil)
+
+    response =
+      "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked; level=1\r\n\r\n5\r\nhello\r\n0\r\n\r\n"
+
+    assert {:error, conn, %HTTPError{reason: {:invalid_token_list, "chunked; level=1"}},
+            [{:status, ^ref, 200}]} = HTTP1.stream(conn, {:tcp, conn.socket, response})
+
+    refute HTTP1.open?(conn)
+  end
+
   test "close/1", %{conn: conn} do
     assert HTTP1.open?(conn)
     assert {:ok, conn} = HTTP1.close(conn)
@@ -1863,6 +1900,65 @@
       assert HTTP1.open?(conn)
     end
 
+    test "chunked is appended to a transfer coding with parameters",
+         %{conn: conn, server_socket: server_socket, port: port} do
+      {:ok, conn, ref} =
+        HTTP1.request(conn, "GET", "/", [{"transfer-encoding", "gzip;q=1"}], :stream)
+
+      assert receive_request_string(server_socket) ==
+               request_string("""
+               GET / HTTP/1.1
+               host: localhost:#{port}
+               user-agent: #{mint_user_agent()}
+               transfer-encoding: gzip;q=1,chunked
+
+               \
+               """)
+
+      {:ok, conn} = HTTP1.stream_request_body(conn, ref, "hello")
+      assert receive_request_string(server_socket) == "5\r\nhello\r\n"
+    end
+
+    test "chunked is appended to the last of several transfer-encoding headers",
+         %{conn: conn, server_socket: server_socket, port: port} do
+      headers = [{"transfer-encoding", "custom;p=1"}, {"transfer-encoding", "gzip"}]
+      {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", headers, :stream)
+
+      assert receive_request_string(server_socket) ==
+               request_string("""
+               GET / HTTP/1.1
+               host: localhost:#{port}
+               user-agent: #{mint_user_agent()}
+               transfer-encoding: custom;p=1
+               transfer-encoding: gzip,chunked
+
+               \
+               """)
+
+      {:ok, conn} = HTTP1.stream_request_body(conn, ref, "hello")
+      assert receive_request_string(server_socket) == "5\r\nhello\r\n"
+    end
+
+    test "transfer-encoding headers are combined before they're parsed",
+         %{conn: conn, server_socket: server_socket, port: port} do
+      headers = [{"transfer-encoding", ~s(custom;p="a)}, {"transfer-encoding", ~s(b", chunked)}]
+      {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", headers, :stream)
+
+      assert receive_request_string(server_socket) ==
+               request_string("""
+               GET / HTTP/1.1
+               host: localhost:#{port}
+               user-agent: #{mint_user_agent()}
+               transfer-encoding: custom;p="a
+               transfer-encoding: b", chunked
+
+               \
+               """)
+
+      {:ok, conn} = HTTP1.stream_request_body(conn, ref, "hello")
+      assert receive_request_string(server_socket) == "hello"
+    end
+
     test "transfer-encoding is not set to chunked if already set to identity",
          %{conn: conn, server_socket: server_socket, port: port} do
       {:ok, conn, ref} =

test/mint/http1/parse_test.exs

@@ -164,6 +164,29 @@ defmodule Mint.HTTP1.ParseTest do
     assert transfer_encoding_header("") == {:error, :empty_token_list}
   end
 
+  test "transfer_encoding_header/1 with transfer coding parameters" do
+    assert transfer_encoding_header("Custom;Level=1") == {:ok, ["custom"]}
+
+    assert transfer_encoding_header("custom ; a=1 ;b = x, chunked") ==
+             {:ok, ["custom", "chunked"]}
+
+    assert transfer_encoding_header(~s(custom; a="x, \\"y\\"; z", gzip)) ==
+             {:ok, ["custom", "gzip"]}
+
+    for value <- [
+          "custom;",
+          "custom; a",
+          "custom; a=",
+          "custom; =1",
+          "custom; a=1 b",
+          ~s(custom; a="x),
+          "chunked; a=1",
+          "Chunked;a=1, gzip"
+        ] do
+      assert transfer_encoding_header(value) == {:error, {:invalid_token_list, value}}
+    end
+  end
+
   describe "token_list_downcase/1" do
     property "returns an empty list if there's no token" do
       no_tokens_generator = string([?\s, ?\t, ?,])