diff --git a/lib/async/http/body/pipe.rb b/lib/async/http/body/pipe.rb index da221778..d9883646 100644 --- a/lib/async/http/body/pipe.rb +++ b/lib/async/http/body/pipe.rb @@ -33,6 +33,12 @@ def to_io @tail end + # Close the socket and stop reading from the input. Unlike {close}, the writer forwards any data already written to the socket before closing the output normally. + def finish + @reader&.stop + @tail.close + end + # Close the pipe and stop the reader and writer tasks. def close @reader&.stop diff --git a/lib/async/http/protocol/http2/client.rb b/lib/async/http/protocol/http2/client.rb index bf1ca4a8..ba295cff 100644 --- a/lib/async/http/protocol/http2/client.rb +++ b/lib/async/http/protocol/http2/client.rb @@ -43,11 +43,31 @@ def call(request) response = create_response write_request(response, request) - read_response(response) + + begin + read_response(response) + received = true + ensure + # If the caller stopped waiting for the response (e.g. its timeout expired) while the request body is still being sent, cancel the stream. Otherwise the task sending the body would keep running, and for a body which is never finished, such as a CONNECT tunnel, forever. + cancel_stream(response.stream) unless received + end return response end + # Cancel a stream whose request body is still being sent. The caller has stopped waiting, and sending the reset can block (e.g. the remote peer is not reading), so it is sent by a separate task. + # @parameter stream [Stream] The stream to cancel. + def cancel_stream(stream) + return unless stream.sending? + + Async(transient: true, annotation: "Cancelling #{stream}.") do + stream.send_reset_stream(::Protocol::HTTP2::Error::CANCEL) if stream.sending? + rescue => error + # The stream or the connection was closed in the meantime: + Console.debug(self, "Failed to cancel stream!", error) + end + end + # Write a request to the remote server via the given response stream. # @parameter response [Response] The response stream to write through. # @parameter request [Protocol::HTTP::Request] The request to send. diff --git a/lib/async/http/protocol/http2/output.rb b/lib/async/http/protocol/http2/output.rb index ea7019ac..a9b93bc5 100644 --- a/lib/async/http/protocol/http2/output.rb +++ b/lib/async/http/protocol/http2/output.rb @@ -135,7 +135,7 @@ def passthrough(task) # chunk.clear unless chunk.frozen? # GC.start end - rescue Async::Cancel, StandardError => error + rescue Async::Stop, StandardError => error raise ensure # Ensure the body we are reading from is fully closed: diff --git a/lib/async/http/protocol/http2/response.rb b/lib/async/http/protocol/http2/response.rb index 79411400..b6c69f89 100644 --- a/lib/async/http/protocol/http2/response.rb +++ b/lib/async/http/protocol/http2/response.rb @@ -170,10 +170,10 @@ def wait @stream.wait end - # Close this response as quickly as possible. If the response body is still active, cancel the HTTP/2 exchange rather than draining it. + # Close this response as quickly as possible. If the response body is still active, cancel the HTTP/2 exchange rather than draining it. An error also cancels the exchange if the response has no body (left) but the request is still being sent. # @parameter error [Exception | Nil] The error which closed the response. def close(error = nil) - if @body && !@stream.closed? + if (@body || error) && !@stream.closed? code = error ? ::Protocol::HTTP2::Error::INTERNAL_ERROR : ::Protocol::HTTP2::Error::CANCEL @stream.send_reset_stream(code) end diff --git a/lib/async/http/protocol/http2/stream.rb b/lib/async/http/protocol/http2/stream.rb index 2ca4207c..572e9eef 100644 --- a/lib/async/http/protocol/http2/stream.rb +++ b/lib/async/http/protocol/http2/stream.rb @@ -160,6 +160,11 @@ def send_body(body, trailer = nil) @output.start end + # @returns [Boolean] Whether the body given to {send_body} is still being sent. + def sending? + !@output.nil? + end + # Called when the output terminates normally. def finish_output(error = nil) return if self.closed? diff --git a/lib/async/http/proxy.rb b/lib/async/http/proxy.rb index 4f295f7e..66d1cf6e 100644 --- a/lib/async/http/proxy.rb +++ b/lib/async/http/proxy.rb @@ -15,14 +15,171 @@ module HTTP class Proxy # Raised when a CONNECT tunnel through a proxy cannot be established. class ConnectFailure < StandardError + # The maximum number of bytes of the response body to read, see {body}. + MAXIMUM_BODY_SIZE = 1024 * 8 + # Initialize the failure with the unsuccessful response. # @parameter response [Protocol::HTTP::Response] The failed response from the proxy. - def initialize(response) + # @parameter body [String | Nil] The beginning of the response body, if any. + def initialize(response, body = nil) super "Failed to connect: #{response.status}" @response = response + @body = body end + # @attribute [Protocol::HTTP::Response] The failed response from the proxy. It has already been closed. attr :response + + # @attribute [String | Nil] Up to {MAXIMUM_BODY_SIZE} bytes of the response body, which may explain why the proxy rejected the connection. + # + # The body is read before this error is raised, so a proxy which neither finishes the body nor closes the connection (e.g. an HTTP/1 response without `content-length` on a persistent connection) will delay the failure until the caller's own timeout, if any. + attr :body + end + + # A CONNECT tunnel, which is closed once its socket has been closed. + # + # Closing the socket of a {Body::Pipe} only ends the CONNECT request body, so the CONNECT response would remain open (and the proxy connection busy) until the proxy closes its end of the tunnel. Instead, the CONNECT response is closed as soon as the socket is closed, and the CONNECT request is closed once any data already written to the socket has been sent. HTTP/2 then resets the stream and HTTP/1 closes the connection, neither of which can be reused after a tunnel anyway. + class Tunnel + # The default time in seconds to wait for data already written to the socket to be sent to the proxy, after the socket is closed. + CLOSE_TIMEOUT = 10 + + # Extends the socket of a tunnel so that closing it closes the entire tunnel. These methods can be called from any thread. + module Closable + # Close the socket, and then the tunnel. + def close + if @tunnel + # Any other descriptor for the socket (e.g. from `clone`) would keep it open, and the tunnel is about to be closed, so make sure they observe the end of the stream: + begin + self.shutdown(::Socket::SHUT_RDWR) + rescue IOError, SystemCallError + # The socket is already closed or shut down. + end + end + + super + ensure + close_tunnel + end + + # Close the reading side of the socket, and then the tunnel if the writing side is also closed. + def close_read + super + ensure + close_tunnel if closed? + end + + # Close the writing side of the socket, and then the tunnel if the reading side is also closed. + def close_write + super + ensure + close_tunnel if closed? + end + + # A copy has its own file descriptor, so closing it must not close the tunnel. + def initialize_copy(other) + super + + @tunnel = nil + end + + private + + def close_tunnel + if tunnel = @tunnel + @tunnel = nil + tunnel.close + end + end + end + + # @parameter timeout [Numeric] The time in seconds to wait for data already written to the socket to be sent to the proxy, after the socket is closed. + def initialize(timeout = CLOSE_TIMEOUT) + @timeout = timeout + + # What has happened to the tunnel: `:sent`, `:received` and `:closed`. A queue can be pushed to from any thread, and its timeout doesn't interrupt the waiting task with an exception, which Ruby 3.3.6 doesn't handle while waiting on a condition variable. + @events = ::Thread::Queue.new + + @input = Body::Writable.new + + # The connection closes the request body once it has finished sending it: + @request_body = ::Protocol::HTTP::Body::Completable.new(@input, proc{@events.push(:sent)}) + + @response = nil + @pipe = nil + end + + # @attribute [Protocol::HTTP::Body::Readable] The body of the CONNECT request, which is everything written to the socket. + attr :request_body + + # Open the tunnel using a successful CONNECT response, and start a task which closes the tunnel once its socket is closed. + # @parameter response [Protocol::HTTP::Response] The CONNECT response. + # @returns [Socket] The socket for the tunnel, which closes the tunnel when closed. + def open(response, parent: Task.current) + @response = response + + # The response has no body if the proxy has already finished it. The pipe closes the response body once it has finished receiving it: + response_body = response.body || ::Protocol::HTTP::Body::Readable.new + response_body = ::Protocol::HTTP::Body::Completable.new(response_body, proc{@events.push(:received)}) + + @pipe = Body::Pipe.new(response_body, @input, task: parent) + + parent.async(transient: true, annotation: "#{self.class} closer.", &self.method(:closer)) + + socket = @pipe.to_io + socket.extend(Closable) + socket.instance_variable_set(:@tunnel, self) + + return socket + end + + # Close the tunnel. The tunnel is closed by its own task, so unlike stopping the tasks of the pipe, this can be called from any thread. + def close + @events.push(:closed) + end + + # Close the request body of a tunnel which was not opened, so that the connection stops waiting for it. + def abort + @input.close + end + + private + + # Wait until one of the given events has happened. + # @returns [Symbol | Nil] The event which happened, or `nil` if the timeout expired first. + def wait_for(*events, timeout: nil) + deadline = Clock.now + timeout if timeout + + until event = events.find{|event| @happened.include?(event)} + if deadline + remaining = deadline - Clock.now + return nil unless remaining > 0 and happened = @events.pop(timeout: remaining) + else + happened = @events.pop + end + + @happened << happened + + # Once both directions have finished, there is nothing left to close: + @happened << :finished if @happened.include?(:sent) and @happened.include?(:received) + end + + return event + end + + def closer(task) + @happened = [] + + # If the tunnel finishes without the socket being closed (e.g. each side of the socket was closed separately), there is nothing to do: + return unless wait_for(:closed, :finished) == :closed + + # Close the CONNECT response and the socket. The pipe forwards any data already written to the socket, and then ends the CONNECT request: + @pipe.finish + + unless wait_for(:sent, timeout: @timeout) + # The proxy has stopped reading, so give up, which resets an HTTP/2 stream. An HTTP/1 connection which is blocked writing to the proxy is not interrupted: + @response.close(::IO::TimeoutError.new("Timed out while closing the tunnel!")) + end + end end # Extends {Async::HTTP::Client} with proxy capabilities. @@ -76,10 +233,19 @@ def self.endpoint(client, endpoint, headers = nil) # @param client [Async::HTTP::Client] the client which will be used as a proxy server. # @param address [String] the address to connect to. # @param headers [Array] an optional list of headers to use when establishing the connection. - def initialize(client, address, headers = nil) + # @param close_timeout [Numeric] the time in seconds to wait for data already written to a tunnel to be sent to the proxy, after the tunnel is closed. + def initialize(client, address, headers = nil, close_timeout: Tunnel::CLOSE_TIMEOUT, **options) + # Headers given as a trailing hash, e.g. `Proxy.new(client, address, "proxy-authorization" => token)`, are received as keyword arguments: + if headers.nil? + headers = options unless options.empty? + elsif !options.empty? + raise ArgumentError, "unknown keywords: #{options.keys.map(&:inspect).join(', ')}" + end + @client = client @address = address @headers = ::Protocol::HTTP::Headers[headers].freeze + @close_timeout = close_timeout end attr :client @@ -90,28 +256,40 @@ def close end # Establish a TCP connection to the specified host. + # + # The tunnel is closed when the returned socket is closed, or when the block exits. Any data already written to the socket is sent to the proxy first, unless that takes longer than the close timeout. # @return [Socket] a connected bi-directional socket. def connect(&block) - input = Body::Writable.new + tunnel = Tunnel.new(@close_timeout) - response = @client.connect(authority: @address, headers: @headers, body: input) + begin + response = @client.connect(authority: @address, headers: @headers, body: tunnel.request_body) + ensure + # If the request failed, e.g. the proxy closed the connection or the caller's timeout expired, a task may still be waiting to read the request body, and would never finish: + tunnel.abort unless response + end if response.success? - pipe = Body::Pipe.new(response.body, input) + socket = tunnel.open(response) - return pipe.to_io unless block_given? + return socket unless block_given? begin - yield pipe.to_io + yield socket ensure - pipe.close + socket.close end else # This ensures we don't leave a response dangling: - input.close - response.close + tunnel.abort + + begin + body = read_failure_body(response) + ensure + response.close + end - raise ConnectFailure, response + raise ConnectFailure.new(response, body) end end @@ -119,6 +297,28 @@ def connect(&block) def wrap_endpoint(endpoint) Endpoint.new(endpoint.url, self, **endpoint.options) end + + private + + # Proxies often explain why they rejected a connection in the response body, so read the beginning of it. Failing to do so shouldn't hide the original failure. + def read_failure_body(response) + body = response.body + return nil unless body + + buffer = String.new + + begin + # A single chunk can be much larger than the limit, so only append what is still needed: + while (remaining = ConnectFailure::MAXIMUM_BODY_SIZE - buffer.bytesize) > 0 and chunk = body.read + buffer << chunk.byteslice(0, remaining) + end + rescue IOError, SystemCallError, ::Protocol::HTTP::Error => error + # Anything else, e.g. `Async::TimeoutError`, should propagate to the caller: + Console.debug(self, "Failed to read proxy response body!", error) + end + + return buffer + end end Client.prepend(Proxy::Client) diff --git a/releases.md b/releases.md index f56fb22f..e4686783 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,16 @@ # Releases +## Unreleased + + - Closing the socket returned by `Async::HTTP::Proxy#connect` (without a block) now closes the entire CONNECT tunnel, rather than only ending the CONNECT request body. The CONNECT response is closed immediately, and the CONNECT request is closed once any data already written to the socket has been forwarded, after which HTTP/2 proxies receive `RST_STREAM(NO_ERROR)` and HTTP/1 proxy connections are closed. `Async::HTTP::Client#close` for the proxy client no longer waits for the proxy to close its end of the tunnel. The block form of `Async::HTTP::Proxy#connect` closes the tunnel the same way when the block exits, and no longer discards data written immediately before the block exits. + - The tunnel is closed by a background task, so the socket can be closed from any thread. Closing each side of the socket separately (`close_write` and `close_read`) also closes the tunnel. Closing a copy of the socket (e.g. `Socket#clone`) only closes the copy, and closing the socket itself ends the stream for any copies. + - Added the `close_timeout:` option to `Async::HTTP::Proxy.new` (10 seconds by default), which limits how long a closed tunnel waits for data already written to the socket to be sent. After that, an HTTP/2 proxy which has stopped reading receives `RST_STREAM`. An HTTP/1 proxy connection which is blocked writing is not interrupted. + - `Async::HTTP::Proxy::ConnectFailure#body` exposes up to 8 KiB of the proxy's response body, which often explains why the connection was rejected. I/O and protocol errors while reading it are ignored, but other errors, such as `Async::TimeoutError`, propagate to the caller. + - `Async::HTTP::Proxy#connect` now closes the CONNECT request body when the request fails without a response, e.g. the proxy closes the connection or the caller's timeout expires. Previously, the task writing the request body was left waiting forever on HTTP/1 proxy connections. + - HTTP/2 clients now send `RST_STREAM(CANCEL)` when the caller stops waiting for the response headers (e.g. its timeout expires) while the request body is still being sent. Previously, the task sending the body kept running, and never finished if the body was never finished, as for a CONNECT tunnel. The reset is sent by a separate task, so the caller is not delayed if the connection can't be written to. + - Closing an HTTP/2 response with an error now resets the stream even if the response has no body (left), as the request may still be being sent. + - Fixed `uninitialized constant Async::Cancel` when the task sending an HTTP/2 request or response body is stopped or fails with async versions before `Async::Cancel` was introduced. + ## v0.105.0 - Respond with `400 Bad Request` for any `Protocol::HTTP::BadRequest` raised while parsing HTTP/1 or HTTP/2 requests, include the exception class name without reflecting request data, and avoid reporting them as unhandled server errors. diff --git a/test/async/http/body/pipe.rb b/test/async/http/body/pipe.rb index 3d043b0a..1542e394 100644 --- a/test/async/http/body/pipe.rb +++ b/test/async/http/body/pipe.rb @@ -54,6 +54,49 @@ def before end end + with "#finish" do + include Sus::Fixtures::Async::ReactorContext + + let(:output) {Async::HTTP::Body::Writable.new} + let(:pipe) {subject.new(input, output)} + + it "forwards data written to the socket before closing the output" do + pipe.to_io.write(data) + pipe.finish + + expect(pipe.to_io).to be(:closed?) + expect(input).to be(:closed?) + + expect(output.read).to be == data + expect(output.read).to be_nil + end + end + + with "#close" do + include Sus::Fixtures::Async::ReactorContext + + let(:output) {Async::HTTP::Body::Writable.new} + let(:pipe) {subject.new(input, output)} + + it "closes the input when forwarding to the closed socket fails" do + pipe.to_io.close + + # Closing the socket only ends the output: + expect(output.read).to be_nil + expect(input).not.to be(:closed?) + + input.write(data) + + Async::Task.current.with_timeout(1) do + Async::Task.current.yield until input.closed? + end + + expect(input).to be(:closed?) + ensure + pipe.close + end + end + with "reactor going out of scope" do it "finishes" do # ensures pipe background tasks are transient diff --git a/test/async/http/protocol/http2.rb b/test/async/http/protocol/http2.rb index be6d4cf9..a26ebfa3 100644 --- a/test/async/http/protocol/http2.rb +++ b/test/async/http/protocol/http2.rb @@ -125,6 +125,74 @@ end end + with "a response which never arrives" do + let(:app) do + Protocol::HTTP::Middleware.for do |request| + sleep + end + end + + it "cancels the stream if the caller stops waiting while the request body is being sent" do + body = Async::HTTP::Body::Writable.new + + expect do + Async::Task.current.with_timeout(0.1) do + client.post("/", {}, body) + end + end.to raise_exception(Async::TimeoutError) + + connection = client.pool.acquire + + expect(connection.streams).to be(:empty?) + expect(connection).to be(:reusable?) + + # The task sending the body has finished, and closed it: + expect(body).to be(:closed?) + ensure + client.pool.release(connection) if connection + end + end + + with "a response which never arrives while the connection is busy writing" do + let(:app) do + Protocol::HTTP::Middleware.for do |request| + sleep + end + end + + let(:delay) {0.5} + + it "doesn't wait for the stream to be cancelled" do + body = Async::HTTP::Body::Writable.new + + connection = client.pool.acquire + client.pool.release(connection) + + # Once the request has been sent, prevent anything else from being written, as if the remote end had stopped reading: + writer = Async do + sleep(0.05) + connection.synchronize{sleep(delay)} + end + + clock = Async::Clock.start + + expect do + Async::Task.current.with_timeout(0.1) do + client.post("/", {}, body) + end + end.to raise_exception(Async::TimeoutError) + + expect(clock.total).to be < (delay / 2.0) + + # The stream is cancelled once the connection can be written to again: + writer.wait + sleep(0.01) + + expect(connection.streams).to be(:empty?) + expect(body).to be(:closed?) + end + end + with "host header" do let(:app) do Protocol::HTTP::Middleware.for do |request| diff --git a/test/async/http/proxy.rb b/test/async/http/proxy.rb index 34b99df3..a4fa98aa 100644 --- a/test/async/http/proxy.rb +++ b/test/async/http/proxy.rb @@ -25,6 +25,56 @@ end end + with ".new" do + let(:app) do + Protocol::HTTP::Middleware.for do |request| + Protocol::HTTP::Response[403, {}, [request.headers["proxy-authorization"].to_s]] + end + end + + it "accepts headers as a trailing hash" do + proxy = Async::HTTP::Proxy.new(client, "localhost:1", "proxy-authorization" => "secret") + + begin + proxy.connect + rescue Async::HTTP::Proxy::ConnectFailure => error + end + + expect(error.body).to be == "secret" + + current_task = Async::Task.current + current_task.with_timeout(1) do + current_task.yield while proxy.client.pool.busy? + end + ensure + proxy&.close + end + + it "accepts headers and options" do + proxy = Async::HTTP::Proxy.new(client, "localhost:1", {"proxy-authorization" => "secret"}, close_timeout: 1) + + begin + proxy.connect + rescue Async::HTTP::Proxy::ConnectFailure => error + end + + expect(error.body).to be == "secret" + + current_task = Async::Task.current + current_task.with_timeout(1) do + current_task.yield while proxy.client.pool.busy? + end + ensure + proxy&.close + end + + it "rejects unknown options" do + expect do + Async::HTTP::Proxy.new(client, "localhost:1", [], unknown: true) + end.to raise_exception(ArgumentError, message: be =~ /unknown/) + end + end + with ".proxied_client" do it "can construct valid client" do endpoint = Async::HTTP::Endpoint.parse("http://www.codeotaku.com") @@ -114,6 +164,131 @@ expect(proxy.client.pool).to be(:empty?) end + + it "closes the tunnel when the socket is closed from another thread" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + peer = proxy.connect + + peer.write(data) + expect(peer.readpartial(1024)).to be == data + + # This raises any error from the thread: + Thread.new{peer.close}.value + + expect(peer).to be(:closed?) + + current_task = Async::Task.current + current_task.with_timeout(1) do + current_task.yield while proxy.client.pool.busy? + end + ensure + proxy&.close + end + + it "doesn't close the tunnel when a copy of the socket is closed" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + peer = proxy.connect + + copy = peer.clone + copy.close + + expect(copy).to be(:closed?) + expect(peer).not.to be(:closed?) + + peer.write(data) + peer.close_write + + expect(peer.read).to be == data + + peer.close + + current_task = Async::Task.current + current_task.with_timeout(1) do + current_task.yield while proxy.client.pool.busy? + end + ensure + proxy&.close + end + + it "ends the stream for a copy of the socket when the socket is closed" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + peer = proxy.connect + copy = peer.clone + + peer.close + + Async::Task.current.with_timeout(1) do + expect(copy.read).to be == "" + end + + current_task = Async::Task.current + current_task.with_timeout(1) do + current_task.yield while proxy.client.pool.busy? + end + ensure + copy&.close + proxy&.close + end + + it "closes the tunnel when both sides of the socket are closed separately" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + current_task = Async::Task.current + existing_tasks = current_task.children.to_a + + peer = proxy.connect + + tasks = current_task.children.to_a - existing_tasks + closer = tasks.find{|task| task.annotation.to_s.include?("closer")} + expect(closer).to be(:alive?) + + peer.write(data) + peer.close_write + + expect(peer.read).to be == data + + peer.close_read + expect(peer).to be(:closed?) + + current_task.with_timeout(1) do + current_task.yield while closer.alive? or proxy.client.pool.busy? + end + + expect(closer).not.to be(:alive?) + ensure + proxy&.close + end + + it "stops waiting for the socket to be closed when the tunnel has finished" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + current_task = Async::Task.current + existing_tasks = current_task.children.to_a + + peer = proxy.connect + + tasks = current_task.children.to_a - existing_tasks + closer = tasks.find{|task| task.annotation.to_s.include?("closer")} + + peer.write(data) + peer.close_write + + # The remote end finishes too, and the socket is never closed: + expect(peer.read).to be == data + + current_task.with_timeout(1) do + current_task.yield while closer.alive? or proxy.client.pool.busy? + end + + expect(closer).not.to be(:alive?) + expect(peer).not.to be(:closed?) + ensure + peer&.close + proxy&.close + end end with "independent tunnel directions" do @@ -139,14 +314,13 @@ end end - it "closes the response when forwarding to the closed peer fails" do + it "keeps the response open when the request is closed" do proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) peer = proxy.connect expect(proxy.client.pool).to be(:busy?) - peer.close - peer = nil + peer.close_write current_task = Async::Task.current current_task.with_timeout(1) do @@ -157,6 +331,8 @@ write_response.resolve(true) + expect(peer.read).to be == "Hello World!" + current_task.with_timeout(1) do current_task.yield while proxy.client.pool.busy? end @@ -167,6 +343,333 @@ peer&.close proxy&.close end + + it "closes the response when the peer is closed" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + peer = proxy.connect + + expect(proxy.client.pool).to be(:busy?) + + peer.close + expect(peer).to be(:closed?) + + current_task = Async::Task.current + current_task.with_timeout(1) do + request_closed.wait + current_task.yield while proxy.client.pool.busy? + end + + # The remote end never finished the response, but the tunnel was closed anyway: + expect(write_response).not.to be(:resolved?) + expect(proxy.client.pool).not.to be(:busy?) + ensure + peer&.close + proxy&.close + end + end + + with "rejected CONNECT" do + let(:reason) {"Egress proxying is denied to host 'localhost'."} + + let(:app) do + Protocol::HTTP::Middleware.for do |request| + Protocol::HTTP::Response[403, {"content-type" => "text/plain"}, [reason]] + end + end + + it "exposes the response body" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + expect do + proxy.connect + end.to raise_exception(Async::HTTP::Proxy::ConnectFailure, message: be == "Failed to connect: 403") + + begin + proxy.connect + rescue Async::HTTP::Proxy::ConnectFailure => error + expect(error.response.status).to be == 403 + expect(error.body).to be == reason + end + + expect(error).to be_a(Async::HTTP::Proxy::ConnectFailure) + + current_task = Async::Task.current + current_task.with_timeout(1) do + current_task.yield while proxy.client.pool.busy? + end + ensure + proxy&.close + end + + with "large response body" do + let(:reason) {"x" * (Async::HTTP::Proxy::ConnectFailure::MAXIMUM_BODY_SIZE * 4)} + + it "reads a bounded amount of the response body" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + begin + proxy.connect + rescue Async::HTTP::Proxy::ConnectFailure => error + expect(error.body).to be == reason.byteslice(0, Async::HTTP::Proxy::ConnectFailure::MAXIMUM_BODY_SIZE) + end + + expect(error).to be_a(Async::HTTP::Proxy::ConnectFailure) + + current_task = Async::Task.current + current_task.with_timeout(1) do + current_task.yield while proxy.client.pool.busy? + end + ensure + proxy&.close + end + end + + with "response body which never finishes" do + let(:body) {Async::HTTP::Body::Writable.new} + + let(:app) do + Protocol::HTTP::Middleware.for do |request| + body.write(reason) + + Protocol::HTTP::Response[403, {"content-type" => "text/plain"}, body] + end + end + + it "propagates the caller's timeout" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + expect do + Async::Task.current.with_timeout(0.1) do + proxy.connect + end + end.to raise_exception(Async::TimeoutError) + + current_task = Async::Task.current + current_task.with_timeout(1) do + current_task.yield while proxy.client.pool.busy? + end + ensure + body.close + proxy&.close + end + end + end + + with "CONNECT which fails without a response" do + # Call {Async::HTTP::Proxy#connect}, which is expected to fail, and return the tasks it started, after giving them a chance to finish. + def tasks_started_by_connect(proxy, timeout: 0.5) + current_task = Async::Task.current + existing_tasks = current_task.children.to_a + + begin + yield + rescue => error + end + + expect(error).not.to be_nil + + # The connection's own background tasks are transient, and stop when the connection is closed: + tasks = (current_task.children.to_a - existing_tasks).reject(&:transient?) + + # Give the tasks a chance to finish: + clock = Async::Clock.start + sleep(0.001) while tasks.any?(&:alive?) and clock.total < timeout + + return error, tasks + end + + with "remote end which closes the connection" do + let(:server_class) do + Class.new(Async::HTTP::Server) do + # Read the beginning of the request, then close the connection without responding: + def accept(peer, address, task: Async::Task.current) + peer.readpartial(1024) + ensure + peer.close + end + end + end + + def make_server(endpoint) + server_class.new(middleware, endpoint) + end + + # The request must not be retried, as each attempt would be closed the same way: + def retries + 0 + end + + it "doesn't leave the request running" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + error, tasks = tasks_started_by_connect(proxy) do + proxy.connect + end + + expect(error).to be_a(EOFError) + expect(tasks.select(&:alive?)).to be(:empty?) + expect(client.pool).not.to be(:busy?) + ensure + proxy&.close + end + end + + with "remote end which never responds" do + let(:app) do + Protocol::HTTP::Middleware.for do |request| + sleep + end + end + + it "doesn't leave the request running after the caller's timeout" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + error, tasks = tasks_started_by_connect(proxy) do + Async::Task.current.with_timeout(0.1) do + proxy.connect + end + end + + expect(error).to be_a(Async::TimeoutError) + expect(tasks.select(&:alive?)).to be(:empty?) + + # The caller has already handled the error, so it must not also fail the task writing the request: + expect(tasks.select(&:failed?)).to be(:empty?) + expect(client.pool).not.to be(:busy?) + ensure + proxy&.close + end + end + end + + with "data written immediately before closing" do + let(:received) {Async::Promise.new} + + let(:app) do + Protocol::HTTP::Middleware.for do |request| + Async::HTTP::Body::Hijack.response(request, 200, {}) do |stream| + buffer = String.new + + begin + while chunk = stream.read_partial + buffer << chunk + end + ensure + received.resolve(buffer) + end + ensure + stream.close + end + end + end + + def wait_until_released(proxy) + current_task = Async::Task.current + current_task.with_timeout(1) do + received.wait + current_task.yield while proxy.client.pool.busy? + end + end + + it "forwards the data before closing the tunnel" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + peer = proxy.connect + peer.write("QUIT\r\n") + peer.close + + wait_until_released(proxy) + + expect(received.value).to be == "QUIT\r\n" + ensure + proxy&.close + end + + it "forwards the data before closing the tunnel using a block" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 1) + + proxy.connect do |peer| + peer.write("QUIT\r\n") + end + + wait_until_released(proxy) + + expect(received.value).to be == "QUIT\r\n" + ensure + proxy&.close + end + end + + with "remote end which closes the tunnel slowly" do + let(:delay) {1} + + let(:app) do + Protocol::HTTP::Middleware.for do |request| + Async::HTTP::Body::Hijack.response(request, 200, {}) do |stream| + stream.read_until("\r\n\r\n") + + stream.write("HTTP/1.1 200 OK\r\ncontent-length: 12\r\n\r\nHello World!") + stream.flush + + # Wait for the client to close its side of the tunnel: + while stream.read_partial + end + + sleep(delay) + ensure + stream.close + end + end + end + + it "closes the tunnel without waiting for the remote end" do + proxied_endpoint = client.proxied_endpoint(Async::HTTP::Endpoint.parse("http://localhost")) + proxied_client = Async::HTTP::Client.new(proxied_endpoint) + + response = proxied_client.get("/") + expect(response.read).to be == "Hello World!" + + start_time = Async::Clock.now + + proxied_client.close + + # The tunnel is released once the request has been forwarded to the proxy: + current_task = Async::Task.current + current_task.with_timeout(delay) do + current_task.yield while client.pool.busy? + end + + client.close + + expect(Async::Clock.now - start_time).to be < (delay / 4.0) + expect(client.pool).to be(:empty?) + end + + it "closes the tunnel without waiting for the remote end using a block" do + proxy = Async::HTTP::Proxy.tcp(client, "localhost", 80) + + start_time = nil + + proxy.connect do |peer| + peer.write("GET / HTTP/1.1\r\nhost: localhost\r\n\r\n") + + buffer = String.new + buffer << peer.readpartial(1024) until buffer.end_with?("Hello World!") + + start_time = Async::Clock.now + end + + # The tunnel is released once the request has been forwarded to the proxy: + current_task = Async::Task.current + current_task.with_timeout(delay) do + current_task.yield while client.pool.busy? + end + + proxy.close + + expect(Async::Clock.now - start_time).to be < (delay / 4.0) + expect(client.pool).to be(:empty?) + end end with "proxied client" do @@ -305,4 +808,87 @@ describe Async::HTTP::Protocol::HTTP2 do it_behaves_like AProxy + + with "remote end which never reads from the tunnel" do + include Sus::Fixtures::Async::HTTP::ServerContext + + let(:protocol) {subject} + + let(:app) do + Protocol::HTTP::Middleware.for do |request| + Async::HTTP::Body::Hijack.response(request, 200, {}) do |stream| + sleep + ensure + stream.close + end + end + end + + let(:close_timeout) {0.1} + + # More than the flow control window of the stream, so it can't all be sent unless the remote end reads it: + let(:data) {"x" * (Async::HTTP::Protocol::HTTP2::SERVER_SETTINGS[Protocol::HTTP2::Settings::INITIAL_WINDOW_SIZE] * 2)} + + def wait_until_released(proxy) + current_task = Async::Task.current + current_task.with_timeout(close_timeout * 20) do + current_task.yield while proxy.client.pool.busy? + end + end + + with "response which has already finished" do + let(:app) do + Protocol::HTTP::Middleware.for do |request| + # Keep the request open, without reading it: + hold << request + + Protocol::HTTP::Response[200, {}, nil] + end + end + + let(:hold) {[]} + + it "resets the stream after the close timeout" do + proxy = Async::HTTP::Proxy.new(client, "localhost:1", close_timeout: close_timeout) + + peer = proxy.connect + peer.write(data) + peer.close + + wait_until_released(proxy) + + expect(proxy.client.pool).not.to be(:busy?) + ensure + proxy&.close + end + end + + it "resets the stream after the close timeout" do + proxy = Async::HTTP::Proxy.new(client, "localhost:1", close_timeout: close_timeout) + + peer = proxy.connect + peer.write(data) + peer.close + + wait_until_released(proxy) + + expect(proxy.client.pool).not.to be(:busy?) + ensure + proxy&.close + end + + it "resets the stream after the close timeout using a block" do + proxy = Async::HTTP::Proxy.new(client, "localhost:1", close_timeout: close_timeout) + + proxy.connect do |peer| + peer.write(data) + end + + wait_until_released(proxy) + + expect(proxy.client.pool).not.to be(:busy?) + ensure + proxy&.close + end + end end