diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index 2e5a2325..6fed3027 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -1358,17 +1358,14 @@ defmodule Mint.HTTP1 do Headers.has?(headers, "content-length") -> {:ok, headers, :identity} - found = Headers.find(headers, "transfer-encoding") -> - {raw_name, value} = found + Headers.has?(headers, "transfer-encoding") -> + values = for {_name, "transfer-encoding", value} <- headers, do: value - with {:ok, tokens} <- Parse.transfer_encoding_header(value) do - if "chunked" in tokens or "identity" in tokens do + with {:ok, codings} <- transfer_codings(values) do + if "chunked" in codings or "identity" in codings do {:ok, headers, :identity} else - headers = - Headers.replace(headers, raw_name, "transfer-encoding", value <> ",chunked") - - {:ok, headers, :chunked} + {:ok, append_chunked_coding(headers), :chunked} end end @@ -1393,6 +1390,25 @@ defmodule Mint.HTTP1 do :identity} end + # Adds chunked as the final transfer coding, after the codings of the last + # Transfer-Encoding field. + defp append_chunked_coding(headers) do + headers = Enum.reverse(headers) + {name, "transfer-encoding", value} = List.keyfind(headers, "transfer-encoding", 1) + + headers + |> Headers.replace(name, "transfer-encoding", value <> ",chunked") + |> Enum.reverse() + end + + # RFC 9110 5.3: the field lines are combined with ", " into one field value, + # so a quoted string can span two of them. + defp transfer_codings(values) do + values + |> Enum.join(", ") + |> Parse.transfer_encoding_header() + end + defp wrap_error(reason) do %HTTPError{reason: reason, module: __MODULE__} end diff --git a/lib/mint/http1/parse.ex b/lib/mint/http1/parse.ex index 09ec7b18..27fc590c 100644 --- a/lib/mint/http1/parse.ex +++ b/lib/mint/http1/parse.ex @@ -135,10 +135,84 @@ defmodule Mint.HTTP1.Parse do split_into_downcase_tokens(string) end + # RFC 9110 10.1.4: transfer-coding = token *( OWS ";" OWS transfer-parameter ) + # and transfer-parameter = token BWS "=" BWS ( token / quoted-string ). Only the + # coding names are returned. RFC 9112 7.1: chunked defines no parameters and + # their presence should be treated as an error. def transfer_encoding_header(string) do - split_into_downcase_tokens(string) + case transfer_coding_list(string, []) do + {:ok, []} -> {:error, :empty_token_list} + {:ok, codings} -> {:ok, codings} + :error -> {:error, {:invalid_token_list, string}} + end + end + + defp transfer_coding_list(<<>>, acc), do: {:ok, :lists.reverse(acc)} + + defp transfer_coding_list(<>, acc) + when is_whitespace(char) or is_comma(char), + do: transfer_coding_list(rest, acc) + + defp transfer_coding_list(<> = string, acc) when is_tchar(char) do + {coding, rest} = take_token_downcase(string, <<>>) + transfer_parameters(trim_leading_whitespace(rest), [coding | acc]) end + defp transfer_coding_list(_string, _acc), do: :error + + # Parses the parameters of the last coding in acc, then the list separator. + defp transfer_parameters(<<>>, acc), do: {:ok, :lists.reverse(acc)} + + defp transfer_parameters(<>, acc) when is_comma(char), + do: transfer_coding_list(rest, acc) + + defp transfer_parameters(<>, [coding | _] = acc) when coding != "chunked" do + with {:ok, rest} <- transfer_parameter(trim_leading_whitespace(rest)), + do: transfer_parameters(trim_leading_whitespace(rest), acc) + end + + defp transfer_parameters(_string, _acc), do: :error + + defp transfer_parameter(<> = string) when is_tchar(char) do + rest = skip_token(string) + + case trim_leading_whitespace(rest) do + <> -> transfer_parameter_value(trim_leading_whitespace(rest)) + _other -> :error + end + end + + defp transfer_parameter(_string), do: :error + + defp transfer_parameter_value(<>), do: quoted_string(rest) + + defp transfer_parameter_value(<> = string) when is_tchar(char) do + {:ok, skip_token(string)} + end + + defp transfer_parameter_value(_string), do: :error + + # RFC 9110 5.6.4: qdtext and quoted-pair, after the opening quote. + defp quoted_string(<>), do: {:ok, rest} + + defp quoted_string(<>) + when char == 9 or char in 32..126 or char in 128..255, + do: quoted_string(rest) + + defp quoted_string(<>) + when char in [9, 32, 33] or char in 35..91 or char in 93..126 or char in 128..255, + do: quoted_string(rest) + + defp quoted_string(_string), do: :error + + defp skip_token(<>) when is_tchar(char), do: skip_token(rest) + defp skip_token(rest), do: rest + + defp take_token_downcase(<>, acc) when is_tchar(char), + do: take_token_downcase(rest, <>) + + defp take_token_downcase(rest, acc), do: {acc, rest} + defp split_into_downcase_tokens(string) do case token_list_downcase(string) do {:ok, []} -> {:error, :empty_token_list} diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index a3620eb4..cb1fe45a 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -1207,6 +1207,43 @@ defmodule Mint.HTTP1Test do assert done == {:done, ref} end + test "chunked framing applies after a transfer coding with parameters", %{conn: conn} do + {:ok, conn, ref1} = HTTP1.request(conn, "GET", "/", [], nil) + {:ok, conn, ref2} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: custom; level=1 ;name=\"a, b\", chunked\r\n\r\n" <> + "5\r\nhello\r\n0\r\n\r\n" <> + "HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok" + + assert {:ok, conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + + assert [ + {:status, ^ref1, 200}, + {:headers, ^ref1, _}, + {:data, ^ref1, "hello"}, + {:done, ^ref1}, + {:status, ^ref2, 200}, + {:headers, ^ref2, _}, + {:data, ^ref2, "ok"}, + {:done, ^ref2} + ] = responses + + assert HTTP1.open?(conn) + end + + test "the chunked transfer coding with parameters is an error", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + response = + "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked; level=1\r\n\r\n5\r\nhello\r\n0\r\n\r\n" + + assert {:error, conn, %HTTPError{reason: {:invalid_token_list, "chunked; level=1"}}, + [{:status, ^ref, 200}]} = HTTP1.stream(conn, {:tcp, conn.socket, response}) + + refute HTTP1.open?(conn) + end + test "close/1", %{conn: conn} do assert HTTP1.open?(conn) assert {:ok, conn} = HTTP1.close(conn) @@ -1863,6 +1900,65 @@ defmodule Mint.HTTP1Test do assert HTTP1.open?(conn) end + test "chunked is appended to a transfer coding with parameters", + %{conn: conn, server_socket: server_socket, port: port} do + {:ok, conn, ref} = + HTTP1.request(conn, "GET", "/", [{"transfer-encoding", "gzip;q=1"}], :stream) + + assert receive_request_string(server_socket) == + request_string(""" + GET / HTTP/1.1 + host: localhost:#{port} + user-agent: #{mint_user_agent()} + transfer-encoding: gzip;q=1,chunked + + \ + """) + + {:ok, conn} = HTTP1.stream_request_body(conn, ref, "hello") + assert receive_request_string(server_socket) == "5\r\nhello\r\n" + end + + test "chunked is appended to the last of several transfer-encoding headers", + %{conn: conn, server_socket: server_socket, port: port} do + headers = [{"transfer-encoding", "custom;p=1"}, {"transfer-encoding", "gzip"}] + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", headers, :stream) + + assert receive_request_string(server_socket) == + request_string(""" + GET / HTTP/1.1 + host: localhost:#{port} + user-agent: #{mint_user_agent()} + transfer-encoding: custom;p=1 + transfer-encoding: gzip,chunked + + \ + """) + + {:ok, conn} = HTTP1.stream_request_body(conn, ref, "hello") + assert receive_request_string(server_socket) == "5\r\nhello\r\n" + end + + test "transfer-encoding headers are combined before they're parsed", + %{conn: conn, server_socket: server_socket, port: port} do + headers = [{"transfer-encoding", ~s(custom;p="a)}, {"transfer-encoding", ~s(b", chunked)}] + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", headers, :stream) + + assert receive_request_string(server_socket) == + request_string(""" + GET / HTTP/1.1 + host: localhost:#{port} + user-agent: #{mint_user_agent()} + transfer-encoding: custom;p="a + transfer-encoding: b", chunked + + \ + """) + + {:ok, conn} = HTTP1.stream_request_body(conn, ref, "hello") + assert receive_request_string(server_socket) == "hello" + end + test "transfer-encoding is not set to chunked if already set to identity", %{conn: conn, server_socket: server_socket, port: port} do {:ok, conn, ref} = diff --git a/test/mint/http1/parse_test.exs b/test/mint/http1/parse_test.exs index 8cbe0dbd..6986e915 100644 --- a/test/mint/http1/parse_test.exs +++ b/test/mint/http1/parse_test.exs @@ -164,6 +164,29 @@ defmodule Mint.HTTP1.ParseTest do assert transfer_encoding_header("") == {:error, :empty_token_list} end + test "transfer_encoding_header/1 with transfer coding parameters" do + assert transfer_encoding_header("Custom;Level=1") == {:ok, ["custom"]} + + assert transfer_encoding_header("custom ; a=1 ;b = x, chunked") == + {:ok, ["custom", "chunked"]} + + assert transfer_encoding_header(~s(custom; a="x, \\"y\\"; z", gzip)) == + {:ok, ["custom", "gzip"]} + + for value <- [ + "custom;", + "custom; a", + "custom; a=", + "custom; =1", + "custom; a=1 b", + ~s(custom; a="x), + "chunked; a=1", + "Chunked;a=1, gzip" + ] do + assert transfer_encoding_header(value) == {:error, {:invalid_token_list, value}} + end + end + describe "token_list_downcase/1" do property "returns an empty list if there's no token" do no_tokens_generator = string([?\s, ?\t, ?,])