From 64ca28517c3ca76e7c89a358000bd5ac528733e2 Mon Sep 17 00:00:00 2001 From: bneradt Date: Wed, 20 May 2026 15:30:31 -0500 Subject: [PATCH 1/2] HTTP/3 via OpenSSL 3.5 + quiche Fedora now ships OpenSSL 3.5 with the third-party QUIC TLS callback API, but quiche still links against the older quictls/BoringSSL symbols. ATS therefore could not use the system OpenSSL library for downstream HTTP/3 without dragging in a different TLS stack. This adds CMake detection for the OpenSSL callback API and provides a private compatibility layer that maps quiche's legacy hooks onto SSL_set_quic_tls_cbs. This requires static quiche in that mode so ATS resolves the shim symbols locally and links the final binaries against the system OpenSSL libraries. This also relaxes verifier-only HTTP/3 AuTest gates that do not execute curl, so those tests can run when ATS has QUIC support but the installed curl lacks HTTP/3. Unknown unidirectional HTTP/3 stream types and aliased frame-type interests also exposed gaps in shared H3 handling. This discards ignored stream data and de-duplicates handler registration so buffered input does not grow indefinitely and MAX_PUSH_ID handlers run once. --- CMakeLists.txt | 25 +- cmake/CheckOpenSSLHasQuicTlsCbs.cmake | 55 ++ cmake/Findquiche.cmake | 27 + src/iocore/net/quic/CMakeLists.txt | 11 + src/iocore/net/quic/OpenSSLQuicCompat.cc | 493 ++++++++++++++++++ src/proxy/http3/Http3App.cc | 9 +- src/proxy/http3/Http3FrameCounter.cc | 3 +- src/proxy/http3/Http3FrameDispatcher.cc | 8 +- src/proxy/http3/Http3ProtocolEnforcer.cc | 4 +- .../http3/test/test_Http3FrameDispatcher.cc | 47 ++ 10 files changed, 673 insertions(+), 9 deletions(-) create mode 100644 cmake/CheckOpenSSLHasQuicTlsCbs.cmake create mode 100644 src/iocore/net/quic/OpenSSLQuicCompat.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index ea333aa4dcf..f806c0ea1e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -283,6 +283,7 @@ pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) include(CheckOpenSSLIsBoringSSL) include(CheckOpenSSLIsQuictls) include(CheckOpenSSLIsAwsLc) +include(CheckOpenSSLHasQuicTlsCbs) find_package(OpenSSL REQUIRED) check_openssl_is_boringssl(SSLLIB_IS_BORINGSSL BORINGSSL_VERSION "${OPENSSL_INCLUDE_DIR}") check_openssl_is_awslc(SSLLIB_IS_AWSLC AWSLC_VERSION "${OPENSSL_INCLUDE_DIR}") @@ -315,6 +316,16 @@ if(NOT SSLLIB_IS_BORINGSSL add_compile_definitions(OPENSSL_API_COMPAT=10002 OPENSSL_IS_OPENSSL3) endif() +check_openssl_has_quic_tls_cbs(SSLLIB_HAS_QUIC_TLS_CBS "${OPENSSL_INCLUDE_DIR}") +if(SSLLIB_HAS_QUIC_TLS_CBS + AND NOT SSLLIB_IS_BORINGSSL + AND NOT SSLLIB_IS_AWSLC + AND NOT SSLLIB_IS_QUICTLS +) + set(TS_OPENSSL_QUIC_TLS_CBS_COMPAT TRUE) + add_compile_definitions(TS_OPENSSL_QUIC_TLS_CBS_COMPAT) +endif() + if(ENABLE_PROFILER) find_package(profiler REQUIRED) set(TS_HAS_PROFILER ${profiler_FOUND}) @@ -331,12 +342,20 @@ elseif(TS_HAS_MIMALLOC) endif() if(ENABLE_QUICHE) + if(TS_OPENSSL_QUIC_TLS_CBS_COMPAT) + set(quiche_USE_STATIC TRUE) + endif() find_package(quiche REQUIRED) set(TS_HAS_QUICHE ${quiche_FOUND}) set(TS_USE_QUIC ${TS_HAS_QUICHE}) - if(NOT SSLLIB_IS_BORINGSSL AND NOT SSLLIB_IS_QUICTLS) - message(FATAL_ERROR "Use of BoringSSL or OPENSSL/QUICTLS is required if quiche is used.") + if(NOT SSLLIB_IS_BORINGSSL + AND NOT SSLLIB_IS_QUICTLS + AND NOT TS_OPENSSL_QUIC_TLS_CBS_COMPAT + ) + message( + FATAL_ERROR "Use of BoringSSL, OPENSSL/QUICTLS, or OpenSSL QUIC TLS callbacks is required if quiche is used." + ) endif() if(SSLLIB_IS_QUICTLS) @@ -345,6 +364,8 @@ if(ENABLE_QUICHE) message( "WARNING - Using quictls requires using a special version of quiche. Make sure quictls is supported in quiche." ) + elseif(TS_OPENSSL_QUIC_TLS_CBS_COMPAT) + message(STATUS "Using OpenSSL QUIC TLS callbacks compatibility for quiche") endif() endif() diff --git a/cmake/CheckOpenSSLHasQuicTlsCbs.cmake b/cmake/CheckOpenSSLHasQuicTlsCbs.cmake new file mode 100644 index 00000000000..3aec5cf4cee --- /dev/null +++ b/cmake/CheckOpenSSLHasQuicTlsCbs.cmake @@ -0,0 +1,55 @@ +####################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +####################### + +function(CHECK_OPENSSL_HAS_QUIC_TLS_CBS OUT_VAR OPENSSL_INCLUDE_DIR) + set(CHECK_PROGRAM + " + #include + #include + + int main() { + OSSL_FUNC_SSL_QUIC_TLS_crypto_send_fn *crypto_send = nullptr; + OSSL_FUNC_SSL_QUIC_TLS_crypto_recv_rcd_fn *crypto_recv = nullptr; + OSSL_FUNC_SSL_QUIC_TLS_crypto_release_rcd_fn *crypto_release = nullptr; + OSSL_FUNC_SSL_QUIC_TLS_yield_secret_fn *yield_secret = nullptr; + OSSL_FUNC_SSL_QUIC_TLS_got_transport_params_fn *got_transport_params = nullptr; + OSSL_FUNC_SSL_QUIC_TLS_alert_fn *alert = nullptr; + const OSSL_DISPATCH callbacks[] = { + {OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_SEND, reinterpret_cast(crypto_send)}, + {OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RECV_RCD, reinterpret_cast(crypto_recv)}, + {OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RELEASE_RCD, reinterpret_cast(crypto_release)}, + {OSSL_FUNC_SSL_QUIC_TLS_YIELD_SECRET, reinterpret_cast(yield_secret)}, + {OSSL_FUNC_SSL_QUIC_TLS_GOT_TRANSPORT_PARAMS, reinterpret_cast(got_transport_params)}, + {OSSL_FUNC_SSL_QUIC_TLS_ALERT, reinterpret_cast(alert)}, + {0, nullptr}, + }; + + return callbacks[0].function_id == 0 || + SSL_set_quic_tls_cbs == nullptr || + SSL_set_quic_tls_transport_params == nullptr; + } + " + ) + set(CMAKE_REQUIRED_INCLUDES "${OPENSSL_INCLUDE_DIR}") + set(CMAKE_REQUIRED_LIBRARIES OpenSSL::SSL OpenSSL::Crypto) + include(CheckCXXSourceCompiles) + check_cxx_source_compiles("${CHECK_PROGRAM}" ${OUT_VAR}) + set(${OUT_VAR} + ${${OUT_VAR}} + PARENT_SCOPE + ) +endfunction() diff --git a/cmake/Findquiche.cmake b/cmake/Findquiche.cmake index fcb3c61f276..4cead8bbdfb 100644 --- a/cmake/Findquiche.cmake +++ b/cmake/Findquiche.cmake @@ -28,7 +28,24 @@ # quiche::quiche # +if(quiche_USE_STATIC + AND quiche_LIBRARY + AND NOT quiche_LIBRARY MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$" +) + unset(quiche_LIBRARY CACHE) +endif() + +if(quiche_USE_STATIC) + set(_quiche_ORIGINAL_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX}) +endif() + find_library(quiche_LIBRARY NAMES quiche) + +if(quiche_USE_STATIC) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${_quiche_ORIGINAL_FIND_LIBRARY_SUFFIXES}) +endif() + find_path( quiche_INCLUDE_DIR NAMES quiche.h @@ -40,8 +57,18 @@ mark_as_advanced(quiche_FOUND quiche_LIBRARY quiche_INCLUDE_DIR) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(quiche REQUIRED_VARS quiche_LIBRARY quiche_INCLUDE_DIR) +if(quiche_USE_STATIC + AND quiche_FOUND + AND NOT quiche_LIBRARY MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$" +) + message(FATAL_ERROR "Static quiche was requested, but ${quiche_LIBRARY} is not a static library") +endif() + if(quiche_FOUND) set(quiche_INCLUDE_DIRS "${quiche_INCLUDE_DIR}") + if(quiche_USE_STATIC) + message(STATUS "Using static quiche library: ${quiche_LIBRARY}") + endif() endif() if(quiche_FOUND AND NOT TARGET quiche::quiche) diff --git a/src/iocore/net/quic/CMakeLists.txt b/src/iocore/net/quic/CMakeLists.txt index 4151917c017..b3f1b7cb1d4 100644 --- a/src/iocore/net/quic/CMakeLists.txt +++ b/src/iocore/net/quic/CMakeLists.txt @@ -34,3 +34,14 @@ add_library( add_library(ts::quic ALIAS quic) target_link_libraries(quic PUBLIC ts::inkevent ts::inknet ts::tscore OpenSSL::Crypto OpenSSL::SSL quiche::quiche) + +if(TS_OPENSSL_QUIC_TLS_CBS_COMPAT) + add_library(openssl_quic_compat STATIC OpenSSLQuicCompat.cc) + add_library(ts::openssl_quic_compat ALIAS openssl_quic_compat) + target_link_libraries(openssl_quic_compat PUBLIC OpenSSL::SSL OpenSSL::Crypto) + set_property( + TARGET quiche::quiche + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES ts::openssl_quic_compat + ) +endif() diff --git a/src/iocore/net/quic/OpenSSLQuicCompat.cc b/src/iocore/net/quic/OpenSSLQuicCompat.cc new file mode 100644 index 00000000000..7afafff280e --- /dev/null +++ b/src/iocore/net/quic/OpenSSLQuicCompat.cc @@ -0,0 +1,493 @@ +/** @file + + Bridges quiche's legacy QUIC TLS callback API to OpenSSL 3.5's third-party + QUIC TLS callback API. + + This compatibility layer exports the quictls/BoringSSL-style symbols that + quiche expects while ATS links against upstream OpenSSL 3.5. It stores + per-SSL callback state in OpenSSL ex-data and translates CRYPTO data, + encryption secrets, alerts, and transport parameters between the two APIs. + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +enum ssl_encryption_level_t { + ssl_encryption_initial = 0, + ssl_encryption_early_data, + ssl_encryption_handshake, + ssl_encryption_application, +}; + +struct ssl_quic_method_st { + int (*set_encryption_secrets)(SSL *ssl, ssl_encryption_level_t level, uint8_t const *read_secret, uint8_t const *write_secret, + size_t secret_len); + int (*add_handshake_data)(SSL *ssl, ssl_encryption_level_t level, uint8_t const *data, size_t len); + int (*flush_flight)(SSL *ssl); + int (*send_alert)(SSL *ssl, ssl_encryption_level_t level, uint8_t alert); +}; + +using SSL_QUIC_METHOD = ssl_quic_method_st; + +namespace +{ + +constexpr auto read_secret_direction = 0; +constexpr auto write_secret_direction = 1; +constexpr auto quic_level_count = 4; + +struct PendingSecret { + std::vector read; + std::vector write; + bool have_read{false}; + bool have_write{false}; + bool delivered{false}; +}; + +struct QuicCompatState { + SSL_QUIC_METHOD const *method{nullptr}; + std::array>, quic_level_count> crypto_data; + std::array secrets; + ssl_encryption_level_t read_level{ssl_encryption_initial}; + ssl_encryption_level_t write_level{ssl_encryption_initial}; + ssl_encryption_level_t active_recv_level{ssl_encryption_initial}; + bool active_recv{false}; + std::vector local_transport_params; + std::vector peer_transport_params; +}; + +void +free_quic_ex_data(void *, void *ptr, CRYPTO_EX_DATA *, int, long, void *) +{ + delete static_cast(ptr); +} + +int +quic_ex_data_index() +{ + static int index = SSL_get_ex_new_index(0, nullptr, nullptr, nullptr, free_quic_ex_data); + + return index; +} + +bool +is_valid_level(ssl_encryption_level_t level) +{ + auto index = static_cast(level); + + return index >= 0 && index < quic_level_count; +} + +size_t +level_index(ssl_encryption_level_t level) +{ + return static_cast(level); +} + +uint8_t const * +data_or_null(std::vector const &data) +{ + return data.empty() ? nullptr : data.data(); +} + +bool +assign_bytes(std::vector &dst, uint8_t const *data, size_t len) +{ + if (len == 0) { + dst.clear(); + return true; + } + if (data == nullptr) { + return false; + } + + dst.assign(data, data + len); + return true; +} + +QuicCompatState * +get_state(SSL const *ssl) +{ + if (ssl == nullptr) { + return nullptr; + } + + int const index = quic_ex_data_index(); + return index < 0 ? nullptr : static_cast(SSL_get_ex_data(ssl, index)); +} + +QuicCompatState * +get_or_create_state(SSL *ssl) +{ + if (ssl == nullptr) { + return nullptr; + } + + if (auto *state = get_state(ssl); state != nullptr) { + return state; + } + + auto *state = new QuicCompatState; + int const index = quic_ex_data_index(); + if (index < 0 || SSL_set_ex_data(ssl, index, state) != 1) { + delete state; + return nullptr; + } + + return state; +} + +bool +compat_level_from_protection(uint32_t prot_level, ssl_encryption_level_t &level) +{ + switch (prot_level) { + case OSSL_RECORD_PROTECTION_LEVEL_NONE: + level = ssl_encryption_initial; + return true; + case OSSL_RECORD_PROTECTION_LEVEL_EARLY: + level = ssl_encryption_early_data; + return true; + case OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE: + level = ssl_encryption_handshake; + return true; + case OSSL_RECORD_PROTECTION_LEVEL_APPLICATION: + level = ssl_encryption_application; + return true; + default: + return false; + } +} + +bool +deliver_0rtt_secret(SSL *ssl, QuicCompatState &state, ssl_encryption_level_t level, int direction, + std::vector const &secret) +{ + if (state.method == nullptr || state.method->set_encryption_secrets == nullptr) { + return false; + } + + bool const is_server = SSL_is_server(ssl) == 1; + if (direction == read_secret_direction && is_server) { + return state.method->set_encryption_secrets(ssl, level, data_or_null(secret), nullptr, secret.size()) == 1; + } + + if (direction == write_secret_direction && !is_server) { + return state.method->set_encryption_secrets(ssl, level, nullptr, data_or_null(secret), secret.size()) == 1; + } + + return true; +} + +bool +deliver_paired_secrets(SSL *ssl, QuicCompatState &state, ssl_encryption_level_t level) +{ + if (state.method == nullptr || state.method->set_encryption_secrets == nullptr) { + return false; + } + + auto &pending = state.secrets[level_index(level)]; + if (!pending.have_read || !pending.have_write || pending.delivered) { + return true; + } + if (pending.read.size() != pending.write.size()) { + return false; + } + + int const result = + state.method->set_encryption_secrets(ssl, level, data_or_null(pending.read), data_or_null(pending.write), pending.read.size()); + if (result == 1) { + pending.delivered = true; + } + + return result == 1; +} + +int +crypto_send_cb(SSL *ssl, unsigned char const *buf, size_t buf_len, size_t *consumed, void *) +{ + auto *state = get_state(ssl); + if (state == nullptr || state->method == nullptr || state->method->add_handshake_data == nullptr) { + return 0; + } + + if (consumed != nullptr) { + *consumed = 0; + } + + static constexpr uint8_t empty_data = 0; + if (buf == nullptr && buf_len > 0) { + return 0; + } + + auto const *data = buf_len == 0 ? &empty_data : reinterpret_cast(buf); + if (state->method->add_handshake_data(ssl, state->write_level, data, buf_len) != 1) { + return 0; + } + if (state->method->flush_flight != nullptr && state->method->flush_flight(ssl) != 1) { + return 0; + } + + if (consumed != nullptr) { + *consumed = buf_len; + } + + return 1; +} + +int +crypto_recv_rcd_cb(SSL *ssl, unsigned char const **buf, size_t *bytes_read, void *) +{ + auto *state = get_state(ssl); + if (state == nullptr || buf == nullptr || bytes_read == nullptr) { + return 0; + } + + *buf = nullptr; + *bytes_read = 0; + + if (state->active_recv) { + auto &active_queue = state->crypto_data[level_index(state->active_recv_level)]; + if (!active_queue.empty()) { + *buf = active_queue.front().data(); + *bytes_read = active_queue.front().size(); + } + return 1; + } + + auto &queue = state->crypto_data[level_index(state->read_level)]; + if (queue.empty()) { + return 1; + } + + state->active_recv = true; + state->active_recv_level = state->read_level; + *buf = queue.front().data(); + *bytes_read = queue.front().size(); + + return 1; +} + +int +crypto_release_rcd_cb(SSL *ssl, size_t bytes_read, void *) +{ + auto *state = get_state(ssl); + if (state == nullptr || !state->active_recv) { + return 1; + } + + auto &queue = state->crypto_data[level_index(state->active_recv_level)]; + if (!queue.empty()) { + if (bytes_read >= queue.front().size()) { + queue.pop_front(); + } else { + queue.front().erase(queue.front().begin(), queue.front().begin() + bytes_read); + } + } + state->active_recv = false; + + return 1; +} + +int +yield_secret_cb(SSL *ssl, uint32_t prot_level, int direction, unsigned char const *secret, size_t secret_len, void *) +{ + auto *state = get_state(ssl); + if (state == nullptr) { + return 0; + } + + ssl_encryption_level_t level = ssl_encryption_initial; + if (!compat_level_from_protection(prot_level, level)) { + return 0; + } + + if (direction == read_secret_direction) { + state->read_level = level; + } else if (direction == write_secret_direction) { + state->write_level = level; + } else { + return 0; + } + + std::vector secret_copy; + if (!assign_bytes(secret_copy, reinterpret_cast(secret), secret_len)) { + return 0; + } + + if (level == ssl_encryption_early_data) { + return deliver_0rtt_secret(ssl, *state, level, direction, secret_copy) ? 1 : 0; + } + + auto &pending = state->secrets[level_index(level)]; + if (direction == read_secret_direction) { + pending.read = std::move(secret_copy); + pending.have_read = true; + } else { + pending.write = std::move(secret_copy); + pending.have_write = true; + } + pending.delivered = false; + + return deliver_paired_secrets(ssl, *state, level) ? 1 : 0; +} + +int +got_transport_params_cb(SSL *ssl, unsigned char const *params, size_t params_len, void *) +{ + auto *state = get_state(ssl); + if (state == nullptr) { + return 0; + } + + if (!assign_bytes(state->peer_transport_params, reinterpret_cast(params), params_len)) { + return 0; + } + + return 1; +} + +int +alert_cb(SSL *ssl, unsigned char alert_code, void *) +{ + auto *state = get_state(ssl); + if (state == nullptr || state->method == nullptr || state->method->send_alert == nullptr) { + return 0; + } + + return state->method->send_alert(ssl, state->write_level, alert_code); +} + +OSSL_DISPATCH const quic_tls_callbacks[] = { + {OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_SEND, reinterpret_cast(crypto_send_cb) }, + {OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RECV_RCD, reinterpret_cast(crypto_recv_rcd_cb) }, + {OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RELEASE_RCD, reinterpret_cast(crypto_release_rcd_cb) }, + {OSSL_FUNC_SSL_QUIC_TLS_YIELD_SECRET, reinterpret_cast(yield_secret_cb) }, + {OSSL_FUNC_SSL_QUIC_TLS_GOT_TRANSPORT_PARAMS, reinterpret_cast(got_transport_params_cb)}, + {OSSL_FUNC_SSL_QUIC_TLS_ALERT, reinterpret_cast(alert_cb) }, + {0, nullptr }, +}; + +} // namespace + +extern "C" int +SSL_set_quic_method(SSL *ssl, SSL_QUIC_METHOD const *quic_method) +{ + auto *state = get_or_create_state(ssl); + if (state == nullptr) { + return 0; + } + + state->method = quic_method; + + return SSL_set_quic_tls_cbs(ssl, quic_tls_callbacks, nullptr); +} + +extern "C" int +SSL_set_quic_transport_params(SSL *ssl, uint8_t const *params, size_t params_len) +{ + auto *state = get_or_create_state(ssl); + if (state == nullptr) { + return 0; + } + + if (!assign_bytes(state->local_transport_params, params, params_len)) { + return 0; + } + return SSL_set_quic_tls_transport_params(ssl, data_or_null(state->local_transport_params), state->local_transport_params.size()); +} + +extern "C" void +SSL_get_peer_quic_transport_params(SSL const *ssl, uint8_t const **out_params, size_t *out_params_len) +{ + auto const *state = get_state(ssl); + if (out_params != nullptr) { + *out_params = state == nullptr ? nullptr : data_or_null(state->peer_transport_params); + } + if (out_params_len != nullptr) { + *out_params_len = state == nullptr ? 0 : state->peer_transport_params.size(); + } +} + +extern "C" ssl_encryption_level_t +SSL_quic_write_level(SSL const *ssl) +{ + auto const *state = get_state(ssl); + + return state == nullptr ? ssl_encryption_initial : state->write_level; +} + +extern "C" int +SSL_provide_quic_data(SSL *ssl, ssl_encryption_level_t level, uint8_t const *data, size_t len) +{ + auto *state = get_or_create_state(ssl); + if (state == nullptr || !is_valid_level(level)) { + return 0; + } + + auto &records = state->crypto_data[level_index(level)]; + records.emplace_back(); + if (!assign_bytes(records.back(), data, len)) { + records.pop_back(); + return 0; + } + + return 1; +} + +extern "C" int +SSL_process_quic_post_handshake(SSL *ssl) +{ + auto *state = get_state(ssl); + if (state == nullptr) { + return 0; + } + + bool has_crypto_data = false; + for (auto const &queue : state->crypto_data) { + if (!queue.empty()) { + has_crypto_data = true; + break; + } + } + if (!has_crypto_data) { + return 1; + } + + unsigned char data = 0; + size_t bytes_read = 0; + int const result = SSL_read_ex(ssl, &data, 0, &bytes_read); + + if (result == 1) { + return 1; + } + + int const error = SSL_get_error(ssl, result); + + return error == SSL_ERROR_WANT_READ || error == SSL_ERROR_WANT_WRITE; +} + +extern "C" void +SSL_set_quic_use_legacy_codepoint(SSL *, int) +{ +} diff --git a/src/proxy/http3/Http3App.cc b/src/proxy/http3/Http3App.cc index 1f3d1a3172a..cca7960303f 100644 --- a/src/proxy/http3/Http3App.cc +++ b/src/proxy/http3/Http3App.cc @@ -296,8 +296,13 @@ Http3App::_handle_uni_stream_on_read_ready(int /* event */, VIO *vio) // Recipients of unknown stream types MUST either abort reading of the stream or discard incoming data without further // processing. If reading is aborted, the recipient SHOULD use the H3_STREAM_CREATION_ERROR error code or a reserved error code // (Section 8.1). The recipient MUST NOT consider unknown stream types to be a connection error of any kind. - error = std::make_unique(Http3ErrorClass::STREAM, Http3ErrorCode::H3_STREAM_CREATION_ERROR, "Stream type unkown"); - Dbg(dbg_ctl, "UNKNOWN stream [%" PRIu64 "] error: %hu, %s", adapter->stream().id(), error->get_code(), error->msg); + IOBufferReader *reader = vio->get_reader(); + if (reader != nullptr) { + int64_t const bytes_to_discard = reader->read_avail(); + + reader->consume(bytes_to_discard); + vio->ndone += bytes_to_discard; + } break; } default: diff --git a/src/proxy/http3/Http3FrameCounter.cc b/src/proxy/http3/Http3FrameCounter.cc index 1ac83210d2b..d9696735a8f 100644 --- a/src/proxy/http3/Http3FrameCounter.cc +++ b/src/proxy/http3/Http3FrameCounter.cc @@ -30,8 +30,7 @@ Http3FrameCounter::interests() { return {Http3FrameType::DATA, Http3FrameType::HEADERS, Http3FrameType::X_RESERVED_1, Http3FrameType::CANCEL_PUSH, Http3FrameType::SETTINGS, Http3FrameType::PUSH_PROMISE, Http3FrameType::X_RESERVED_2, Http3FrameType::GOAWAY, - Http3FrameType::X_RESERVED_3, Http3FrameType::X_RESERVED_4, Http3FrameType::MAX_PUSH_ID, Http3FrameType::X_MAX_DEFINED, - Http3FrameType::UNKNOWN}; + Http3FrameType::X_RESERVED_3, Http3FrameType::X_RESERVED_4, Http3FrameType::MAX_PUSH_ID, Http3FrameType::UNKNOWN}; } Http3ErrorUPtr diff --git a/src/proxy/http3/Http3FrameDispatcher.cc b/src/proxy/http3/Http3FrameDispatcher.cc index fb0e7970865..f697cb4eadf 100644 --- a/src/proxy/http3/Http3FrameDispatcher.cc +++ b/src/proxy/http3/Http3FrameDispatcher.cc @@ -42,8 +42,14 @@ DbgCtl dbg_ctl_http3{"http3"}; void Http3FrameDispatcher::add_handler(Http3FrameHandler *handler) { + bool registered[256] = {}; + for (Http3FrameType t : handler->interests()) { - this->_handlers[static_cast(t)].push_back(handler); + auto const type = static_cast(t); + if (!registered[type]) { + this->_handlers[type].push_back(handler); + registered[type] = true; + } } } diff --git a/src/proxy/http3/Http3ProtocolEnforcer.cc b/src/proxy/http3/Http3ProtocolEnforcer.cc index a37cb543ae4..06121d3aa99 100644 --- a/src/proxy/http3/Http3ProtocolEnforcer.cc +++ b/src/proxy/http3/Http3ProtocolEnforcer.cc @@ -29,8 +29,8 @@ Http3ProtocolEnforcer::interests() { return {Http3FrameType::DATA, Http3FrameType::HEADERS, Http3FrameType::X_RESERVED_1, Http3FrameType::CANCEL_PUSH, Http3FrameType::SETTINGS, Http3FrameType::PUSH_PROMISE, Http3FrameType::X_RESERVED_2, Http3FrameType::GOAWAY, - Http3FrameType::X_RESERVED_3, Http3FrameType::X_RESERVED_4, Http3FrameType::MAX_PUSH_ID, Http3FrameType::X_MAX_DEFINED, - Http3FrameType::RESERVED, Http3FrameType::UNKNOWN}; + Http3FrameType::X_RESERVED_3, Http3FrameType::X_RESERVED_4, Http3FrameType::MAX_PUSH_ID, Http3FrameType::RESERVED, + Http3FrameType::UNKNOWN}; } Http3ErrorUPtr diff --git a/src/proxy/http3/test/test_Http3FrameDispatcher.cc b/src/proxy/http3/test/test_Http3FrameDispatcher.cc index 37560e9a157..87116fa4728 100644 --- a/src/proxy/http3/test/test_Http3FrameDispatcher.cc +++ b/src/proxy/http3/test/test_Http3FrameDispatcher.cc @@ -27,6 +27,28 @@ #include "proxy/http3/Http3ProtocolEnforcer.h" #include "Mock.h" +namespace +{ +class AliasedInterestsFrameHandler : public Http3FrameHandler +{ +public: + std::vector + interests() override + { + return {Http3FrameType::MAX_PUSH_ID, Http3FrameType::X_MAX_DEFINED}; + } + + Http3ErrorUPtr + handle_frame(std::shared_ptr /* frame ATS_UNUSED */, Http3StreamType /* s_type ATS_UNUSED */) override + { + ++this->total_frame_received; + return nullptr; + } + + int total_frame_received = 0; +}; +} // namespace + TEST_CASE("Http3FrameHandler dispatch", "[http3]") { Http3FrameDispatcher http3FrameDispatcher; @@ -100,6 +122,31 @@ TEST_CASE("Http3FrameHandler dispatch", "[http3]") free_MIOBuffer(buf); } +TEST_CASE("Http3FrameHandler aliased interests", "[http3]") +{ + Http3FrameDispatcher http3_frame_dispatcher; + AliasedInterestsFrameHandler handler; + http3_frame_dispatcher.add_handler(&handler); + + MIOBuffer *buf = new_MIOBuffer(BUFFER_SIZE_INDEX_512); + IOBufferReader *reader = buf->alloc_reader(); + uint64_t nread = 0; + uint8_t input[] = { + 0x0d, // Type: MAX_PUSH_ID + 0x01, // Length + 0x01, // Push ID + }; + + buf->write(input, sizeof(input)); + + auto error = http3_frame_dispatcher.on_read_ready(0, Http3StreamType::CONTROL, *reader, nread); + CHECK(!error); + CHECK(handler.total_frame_received == 1); + CHECK(nread == sizeof(input)); + + free_MIOBuffer(buf); +} + TEST_CASE("control stream tests", "[http3]") { Http3FrameDispatcher http3FrameDispatcher; From 293b207e014b096207a82b77f6177f624f9e0bd1 Mon Sep 17 00:00:00 2001 From: bneradt Date: Tue, 26 May 2026 15:43:32 -0500 Subject: [PATCH 2/2] HTTP/3 via OpenSSL QUIC OpenSSL 3.5 can terminate QUIC connections directly, but ATS only had a quiche-backed HTTP/3 listener. Operators who want to use the system OpenSSL QUIC stack needed a separate downstream backend without changing the existing quiche path or origin HTTP/3 scope. This adds an optional ENABLE_OPENSSL_QUIC backend that uses OpenSSL's native QUIC listener and stream APIs for downstream HTTP/3. This keeps the backend mutually exclusive with quiche, exposes TS_HAS_OPENSSL_QUIC, and shares ATS's existing HTTP/3 stream handling above the transport. This also installs native-QUIC TLS callbacks for ALPN and SNI certificate selection before ATS has a QUIC NetVC to bind. OpenSSL native QUIC does not make a selected SSL_CTX certificate active via SSL_set_SSL_CTX alone, so this applies the selected cert, key, and chain to the connection SSL. This also broadens client-side HTTP/3 tests to run with either backend, keeps H3 streams open across informational responses, and hardens transaction cleanup when OpenSSL closes stream state before ATS finishes teardown. This caches stream identifiers, declines listener-time QUIC tickets until a NetVC is bound, and adds focused H3 lifecycle and session-ticket coverage. --- CMakeLists.txt | 23 + CMakePresets.json | 6 +- cmake/CheckOpenSSLHasNativeQuic.cmake | 59 + .../iocore/net/QUICMultiCertConfigLoader.h | 14 +- include/iocore/net/TLSSNISupport.h | 1 + include/iocore/net/quic/QUICConfig.h | 7 + include/iocore/net/quic/QUICStream.h | 20 +- include/iocore/net/quic/QUICStreamManager.h | 10 +- include/proxy/http3/Http3Frame.h | 3 + include/proxy/http3/Http3HeaderFramer.h | 23 +- include/proxy/http3/Http3HeaderVIOAdaptor.h | 1 - include/proxy/http3/Http3Transaction.h | 10 +- include/tscore/ink_config.h.cmake.in | 1 + src/iocore/net/CMakeLists.txt | 22 +- src/iocore/net/OpenSSLQUICNetProcessor.cc | 125 ++ src/iocore/net/OpenSSLQUICNetVConnection.cc | 1062 +++++++++++++++++ src/iocore/net/OpenSSLQUICPacketHandler.cc | 630 ++++++++++ src/iocore/net/P_QUICNetProcessor.h | 10 +- src/iocore/net/P_QUICNetVConnection.h | 56 +- src/iocore/net/P_QUICPacketHandler.h | 22 +- src/iocore/net/QUICMultiCertConfigLoader.cc | 243 +++- src/iocore/net/QUICNetVConnection.cc | 29 +- src/iocore/net/SSLSessionTicket.cc | 8 + src/iocore/net/TLSSNISupport.cc | 10 + src/iocore/net/quic/CMakeLists.txt | 9 +- src/iocore/net/quic/QUICConfig.cc | 23 + src/iocore/net/quic/QUICGlobals.cc | 3 +- src/iocore/net/quic/QUICStream.cc | 56 +- src/iocore/net/quic/QUICStreamManager.cc | 28 +- src/proxy/http3/Http3DataFramer.cc | 2 +- src/proxy/http3/Http3Frame.cc | 6 + src/proxy/http3/Http3HeaderFramer.cc | 30 + src/proxy/http3/Http3Transaction.cc | 15 +- .../http3/test/test_Http3FrameDispatcher.cc | 2 +- src/traffic_layout/info.cc | 1 + src/traffic_server/traffic_server.cc | 2 +- tests/gold_tests/h3/h3_go_client.test.py | 2 +- tests/gold_tests/headers/via.test.py | 2 +- 38 files changed, 2491 insertions(+), 85 deletions(-) create mode 100644 cmake/CheckOpenSSLHasNativeQuic.cmake create mode 100644 src/iocore/net/OpenSSLQUICNetProcessor.cc create mode 100644 src/iocore/net/OpenSSLQUICNetVConnection.cc create mode 100644 src/iocore/net/OpenSSLQUICPacketHandler.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index f806c0ea1e3..e926d8a8995 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,6 +215,7 @@ set(ENABLE_TPROXY 'X' where X is a number to use as the IP_TRANSPARENT sockopt, anything else to enable." ) +option(ENABLE_OPENSSL_QUIC "Use OpenSSL native QUIC (default OFF)") option(ENABLE_QUICHE "Use quiche (default OFF)") option(ENABLE_EXAMPLE "Build example directory (default OFF)") @@ -284,6 +285,7 @@ include(CheckOpenSSLIsBoringSSL) include(CheckOpenSSLIsQuictls) include(CheckOpenSSLIsAwsLc) include(CheckOpenSSLHasQuicTlsCbs) +include(CheckOpenSSLHasNativeQuic) find_package(OpenSSL REQUIRED) check_openssl_is_boringssl(SSLLIB_IS_BORINGSSL BORINGSSL_VERSION "${OPENSSL_INCLUDE_DIR}") check_openssl_is_awslc(SSLLIB_IS_AWSLC AWSLC_VERSION "${OPENSSL_INCLUDE_DIR}") @@ -326,6 +328,8 @@ if(SSLLIB_HAS_QUIC_TLS_CBS add_compile_definitions(TS_OPENSSL_QUIC_TLS_CBS_COMPAT) endif() +check_openssl_has_native_quic(SSLLIB_HAS_NATIVE_QUIC "${OPENSSL_INCLUDE_DIR}") + if(ENABLE_PROFILER) find_package(profiler REQUIRED) set(TS_HAS_PROFILER ${profiler_FOUND}) @@ -341,6 +345,25 @@ elseif(TS_HAS_MIMALLOC) link_libraries(mimalloc) endif() +if(ENABLE_OPENSSL_QUIC AND ENABLE_QUICHE) + message(FATAL_ERROR "ENABLE_OPENSSL_QUIC and ENABLE_QUICHE are mutually exclusive QUIC backends") +endif() + +if(ENABLE_OPENSSL_QUIC) + if(NOT SSLLIB_HAS_NATIVE_QUIC) + message(FATAL_ERROR "OpenSSL native QUIC support requires OpenSSL 3.5 or newer with OSSL_QUIC_server_method") + endif() + if(SSLLIB_IS_BORINGSSL + OR SSLLIB_IS_AWSLC + OR SSLLIB_IS_QUICTLS + ) + message(FATAL_ERROR "OpenSSL native QUIC support requires upstream OpenSSL 3.5 or newer") + endif() + set(TS_HAS_OPENSSL_QUIC TRUE) + set(TS_USE_QUIC TRUE) + message(STATUS "Using OpenSSL native QUIC") +endif() + if(ENABLE_QUICHE) if(TS_OPENSSL_QUIC_TLS_CBS_COMPAT) set(quiche_USE_STATIC TRUE) diff --git a/CMakePresets.json b/CMakePresets.json index bfa5cbb8de4..a4eff644ea8 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -191,7 +191,8 @@ "wamr_ROOT": "/opt", "ENABLE_OTEL_TRACER": "ON", "ENABLE_WASM_WAMR": "ON", - "ENABLE_CRIPTS": "ON" + "ENABLE_CRIPTS": "ON", + "ENABLE_OPENSSL_QUIC": "ON" } }, { @@ -207,7 +208,7 @@ "name": "ci-fedora-quiche", "displayName": "CI Fedora Quiche", "description": "CI Pipeline config for Fedora Linux (quiche build)", - "inherits": ["ci"], + "inherits": ["ci-fedora"], "cacheVariables": { "OPENSSL_ROOT_DIR": "/opt/h3-tools-boringssl/boringssl", "quiche_ROOT": "/opt/h3-tools-boringssl/quiche", @@ -217,6 +218,7 @@ "ENABLE_OTEL_TRACER": "ON", "ENABLE_WASM_WAMR": "ON", "CMAKE_INSTALL_PREFIX": "/tmp/ats-quiche", + "ENABLE_OPENSSL_QUIC": "OFF", "ENABLE_QUICHE": "ON" } }, diff --git a/cmake/CheckOpenSSLHasNativeQuic.cmake b/cmake/CheckOpenSSLHasNativeQuic.cmake new file mode 100644 index 00000000000..28c96acb71c --- /dev/null +++ b/cmake/CheckOpenSSLHasNativeQuic.cmake @@ -0,0 +1,59 @@ +####################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +####################### + +function(CHECK_OPENSSL_HAS_NATIVE_QUIC OUT_VAR OPENSSL_INCLUDE_DIR) + set(CHECK_PROGRAM + " + #include + #include + #include + + int main() { + const SSL_METHOD *method = OSSL_QUIC_server_method(); + SSL *ssl = nullptr; + uint64_t value = 0; + return method == nullptr || + SSL_new_listener == nullptr || + SSL_listen == nullptr || + SSL_handle_events == nullptr || + SSL_accept_connection == nullptr || + SSL_accept_stream == nullptr || + SSL_new_stream == nullptr || + SSL_stream_conclude == nullptr || + SSL_get_stream_id == nullptr || + SSL_get_stream_type == nullptr || + SSL_get_stream_read_state == nullptr || + SSL_get_stream_write_buf_avail(ssl, &value) || + SSL_get_conn_close_info == nullptr || + SSL_shutdown_ex == nullptr || + SSL_set_default_stream_mode == nullptr || + SSL_set_blocking_mode == nullptr || + SSL_set_event_handling_mode(ssl, SSL_VALUE_EVENT_HANDLING_MODE_EXPLICIT) || + SSL_set_feature_request_uint(ssl, SSL_VALUE_QUIC_IDLE_TIMEOUT, value) || + SSL_set_incoming_stream_policy == nullptr; + } + " + ) + set(CMAKE_REQUIRED_INCLUDES "${OPENSSL_INCLUDE_DIR}") + set(CMAKE_REQUIRED_LIBRARIES OpenSSL::SSL OpenSSL::Crypto) + include(CheckCXXSourceCompiles) + check_cxx_source_compiles("${CHECK_PROGRAM}" ${OUT_VAR}) + set(${OUT_VAR} + ${${OUT_VAR}} + PARENT_SCOPE + ) +endfunction() diff --git a/include/iocore/net/QUICMultiCertConfigLoader.h b/include/iocore/net/QUICMultiCertConfigLoader.h index 4dad3313a63..6ae299a510f 100644 --- a/include/iocore/net/QUICMultiCertConfigLoader.h +++ b/include/iocore/net/QUICMultiCertConfigLoader.h @@ -25,6 +25,9 @@ #include "iocore/net/SSLMultiCertConfigLoader.h" #include "iocore/eventsystem/ConfigProcessor.h" +#include "tscore/ink_config.h" + +#include class QUICCertConfig { @@ -40,6 +43,10 @@ class QUICCertConfig static int _config_id; }; +#if TS_HAS_OPENSSL_QUIC +std::string_view quic_sni_server_name(SSL *ssl); +#endif + class QUICMultiCertConfigLoader : public SSLMultiCertConfigLoader { public: @@ -48,7 +55,12 @@ class QUICMultiCertConfigLoader : public SSLMultiCertConfigLoader virtual SSL_CTX *default_server_ssl_ctx() override; private: - const char *_debug_tag() const override; + const char *_debug_tag() const override; +#if TS_HAS_OPENSSL_QUIC + virtual void _set_handshake_callbacks(SSL_CTX *ctx) override; + virtual bool _set_alpn_callback(SSL_CTX *ctx) override; + virtual bool _set_curves(SSL_CTX *ctx) override; +#endif virtual bool _setup_session_cache(SSL_CTX *ctx) override; virtual bool _set_cipher_suites_for_legacy_versions(SSL_CTX *ctx) override; virtual bool _set_info_callback(SSL_CTX *ctx) override; diff --git a/include/iocore/net/TLSSNISupport.h b/include/iocore/net/TLSSNISupport.h index 3e30f7aaf27..17ff9ed28e5 100644 --- a/include/iocore/net/TLSSNISupport.h +++ b/include/iocore/net/TLSSNISupport.h @@ -123,6 +123,7 @@ class TLSSNISupport * @return True if the servername was set successfully, false otherwise. */ bool set_sni_server_name(SSL *ssl, char const *name); + void set_sni_server_name(std::string_view name); /** * Get the server name in SNI diff --git a/include/iocore/net/quic/QUICConfig.h b/include/iocore/net/quic/QUICConfig.h index 1ebec4f6aa0..b0b1ea25b2f 100644 --- a/include/iocore/net/quic/QUICConfig.h +++ b/include/iocore/net/quic/QUICConfig.h @@ -24,7 +24,11 @@ #pragma once #include +#include "tscore/ink_config.h" + +#if TS_HAS_QUICHE #include +#endif #include "iocore/eventsystem/ConfigProcessor.h" #include "iocore/net/SSLTypes.h" @@ -89,7 +93,9 @@ class QUICConfigParams : public ConfigInfo bool disable_http_0_9() const; +#if TS_HAS_QUICHE quiche_cc_algorithm get_cc_algorithm() const; +#endif private: static int _connection_table_size; @@ -163,3 +169,4 @@ class QUICConfig }; SSL_CTX *quic_new_ssl_ctx(); +SSL_CTX *quic_new_server_ssl_ctx(); diff --git a/include/iocore/net/quic/QUICStream.h b/include/iocore/net/quic/QUICStream.h index ef84c9cb6aa..e57c49b6dc5 100644 --- a/include/iocore/net/quic/QUICStream.h +++ b/include/iocore/net/quic/QUICStream.h @@ -31,11 +31,25 @@ #include "iocore/net/quic/QUICConnection.h" #include "iocore/net/quic/QUICDebugNames.h" -#include +#include +#include class QUICStreamAdapter; class QUICStreamStateListener; +class QUICStreamIO +{ +public: + using ErrorCode = uint64_t; + + virtual ~QUICStreamIO() = default; + + virtual int64_t read_stream(QUICStreamId stream_id, uint8_t *buf, size_t len, bool &fin, ErrorCode &error_code) = 0; + virtual bool stream_read_finished(QUICStreamId stream_id) = 0; + virtual int64_t stream_write_capacity(QUICStreamId stream_id) = 0; + virtual int64_t write_stream(QUICStreamId stream_id, uint8_t const *buf, size_t len, bool fin, ErrorCode &error_code) = 0; +}; + /** * @brief QUIC Stream * TODO: This is similar to Http2Stream. Need to think some integration. @@ -61,8 +75,8 @@ class QUICStream void stop_sending(QUICStreamErrorUPtr error); void reset(QUICStreamErrorUPtr error); - void receive_data(quiche_conn *quiche_con); - void send_data(quiche_conn *quiche_con); + void receive_data(QUICStreamIO &stream_io); + int64_t send_data(QUICStreamIO &stream_io); /* * QUICApplication need to call one of these functions when it process VC_EVENT_* diff --git a/include/iocore/net/quic/QUICStreamManager.h b/include/iocore/net/quic/QUICStreamManager.h index 78985a7d3c4..c79704a9b73 100644 --- a/include/iocore/net/quic/QUICStreamManager.h +++ b/include/iocore/net/quic/QUICStreamManager.h @@ -48,10 +48,10 @@ class QUICStreamManager : public QUICStreamStateListener QUICStream *create_stream(QUICStreamId stream_id, QUICConnectionError &err); - QUICConnectionErrorUPtr create_uni_stream(QUICStreamId new_stream_id); - QUICConnectionErrorUPtr create_bidi_stream(QUICStreamId new_stream_id); - QUICConnectionErrorUPtr delete_stream(QUICStreamId new_stream_id); - void reset_stream(QUICStreamId stream_id, QUICStreamErrorUPtr error); + virtual QUICConnectionErrorUPtr create_uni_stream(QUICStreamId &new_stream_id); + virtual QUICConnectionErrorUPtr create_bidi_stream(QUICStreamId &new_stream_id); + QUICConnectionErrorUPtr delete_stream(QUICStreamId new_stream_id); + void reset_stream(QUICStreamId stream_id, QUICStreamErrorUPtr error); void set_default_application(QUICApplication *app); @@ -63,4 +63,6 @@ class QUICStreamManager : public QUICStreamStateListener protected: QUICContext *_context = nullptr; QUICApplicationMap *_app_map = nullptr; + QUICStreamId _next_local_bidi_stream_id{0}; + QUICStreamId _next_local_uni_stream_id{0}; }; diff --git a/include/proxy/http3/Http3Frame.h b/include/proxy/http3/Http3Frame.h index cd8d5e860e2..a0255496385 100644 --- a/include/proxy/http3/Http3Frame.h +++ b/include/proxy/http3/Http3Frame.h @@ -98,6 +98,9 @@ class Http3DataFrame : public Http3Frame // Head of IOBufferBlock chain to send Ptr _whole_frame; uint64_t _payload_len = 0; + +protected: + bool _parse() override; }; // diff --git a/include/proxy/http3/Http3HeaderFramer.h b/include/proxy/http3/Http3HeaderFramer.h index 548f88368dc..6dcd4a7d3ae 100644 --- a/include/proxy/http3/Http3HeaderFramer.h +++ b/include/proxy/http3/Http3HeaderFramer.h @@ -44,16 +44,21 @@ class Http3HeaderFramer : public Http3FrameGenerator Http3FrameUPtr generate_frame() override; bool is_done() const override; + void reset(); + bool is_final_header_sent() const; + private: - Http3Transaction *_transaction = nullptr; - VIO *_source_vio = nullptr; - QPACK *_qpack = nullptr; - MIOBuffer *_header_block = nullptr; - IOBufferReader *_header_block_reader = nullptr; - uint64_t _header_block_len = 0; - uint64_t _header_block_wrote = 0; - uint64_t _stream_id = 0; - bool _sent_all_data = false; + Http3Transaction *_transaction = nullptr; + VIO *_source_vio = nullptr; + QPACK *_qpack = nullptr; + MIOBuffer *_header_block = nullptr; + IOBufferReader *_header_block_reader = nullptr; + uint64_t _header_block_len = 0; + uint64_t _header_block_wrote = 0; + uint64_t _stream_id = 0; + bool _sent_all_data = false; + bool _is_current_header_final = false; + bool _is_final_header_sent = false; HTTPParser _http_parser; HTTPHdr _header; VersionConverter _hvc; diff --git a/include/proxy/http3/Http3HeaderVIOAdaptor.h b/include/proxy/http3/Http3HeaderVIOAdaptor.h index 59a02b9a15e..13b30f2314d 100644 --- a/include/proxy/http3/Http3HeaderVIOAdaptor.h +++ b/include/proxy/http3/Http3HeaderVIOAdaptor.h @@ -28,7 +28,6 @@ #include "proxy/http3/Http3FrameHandler.h" class HQTransaction; - class Http3HeaderVIOAdaptor : public Continuation, public Http3FrameHandler { public: diff --git a/include/proxy/http3/Http3Transaction.h b/include/proxy/http3/Http3Transaction.h index b253ecbb8be..5adb9bda173 100644 --- a/include/proxy/http3/Http3Transaction.h +++ b/include/proxy/http3/Http3Transaction.h @@ -130,8 +130,10 @@ class Http3Transaction : public HQTransaction Http3Transaction(Http3Session *session, QUICStreamVCAdapter::IOInfo &info); virtual ~Http3Transaction(); - int state_stream_open(int event, Event *data) override; - int state_stream_closed(int event, Event *data) override; + VIO *do_io_write(Continuation *c = nullptr, int64_t nbytes = INT64_MAX, IOBufferReader *buf = nullptr, + bool owner = false) override; + int state_stream_open(int event, Event *data) override; + int state_stream_closed(int event, Event *data) override; void do_io_close(int lerrno = -1) override; void on_header_decode_complete(); @@ -152,8 +154,8 @@ class Http3Transaction : public HQTransaction Http3FrameDispatcher _frame_dispatcher; Http3FrameCollector _frame_collector; Http3ProtocolEnforcer *_protocol_enforcer = nullptr; - Http3FrameGenerator *_header_framer = nullptr; - Http3FrameGenerator *_data_framer = nullptr; + Http3HeaderFramer *_header_framer = nullptr; + Http3DataFramer *_data_framer = nullptr; Http3HeaderVIOAdaptor *_header_handler = nullptr; Http3StreamDataVIOAdaptor *_data_handler = nullptr; }; diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index 73c8b860fb9..3898e3e7dc1 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -145,6 +145,7 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@; #cmakedefine01 TS_HAS_JEMALLOC #cmakedefine01 TS_HAS_MIMALLOC #cmakedefine01 TS_HAS_PROFILER +#cmakedefine01 TS_HAS_OPENSSL_QUIC #cmakedefine01 TS_HAS_QUICHE #cmakedefine01 TS_HAS_SO_MARK #cmakedefine01 TS_HAS_SO_PEERCRED diff --git a/src/iocore/net/CMakeLists.txt b/src/iocore/net/CMakeLists.txt index d1d5875ab46..6ad1804a912 100644 --- a/src/iocore/net/CMakeLists.txt +++ b/src/iocore/net/CMakeLists.txt @@ -81,18 +81,17 @@ if(TS_USE_QUIC) add_subdirectory(quic) target_sources( - inknet - PRIVATE QUICClosedConCollector.cc - QUICMultiCertConfigLoader.cc - QUICNet.cc - QUICNetProcessor.cc - QUICNetVConnection.cc - QUICNextProtocolAccept.cc - QUICPacketHandler.cc - QUICSupport.cc + inknet PRIVATE QUICClosedConCollector.cc QUICMultiCertConfigLoader.cc QUICNextProtocolAccept.cc QUICSupport.cc ) - target_link_libraries(inknet PUBLIC quiche::quiche ts::quic) + if(TS_HAS_OPENSSL_QUIC) + target_sources(inknet PRIVATE OpenSSLQUICNetProcessor.cc OpenSSLQUICNetVConnection.cc OpenSSLQUICPacketHandler.cc) + elseif(TS_HAS_QUICHE) + target_sources(inknet PRIVATE QUICNet.cc QUICNetProcessor.cc QUICNetVConnection.cc QUICPacketHandler.cc) + target_link_libraries(inknet PUBLIC quiche::quiche) + endif() + + target_link_libraries(inknet PUBLIC ts::quic) endif() if(BUILD_REGRESSION_TESTING OR BUILD_TESTING) @@ -168,6 +167,9 @@ if(BUILD_TESTING) ) if(TS_USE_QUIC) list(APPEND LINK_GROUP_LIBS quic http3) + if(TS_HAS_QUICHE) + list(APPEND LINK_GROUP_LIBS quiche::quiche) + endif() endif() if(CMAKE_LINK_GROUP_USING_RESCAN_SUPPORTED OR CMAKE_CXX_LINK_GROUP_USING_RESCAN_SUPPORTED) string(JOIN "," LINK_GROUP_LIBS_CSV ${LINK_GROUP_LIBS}) diff --git a/src/iocore/net/OpenSSLQUICNetProcessor.cc b/src/iocore/net/OpenSSLQUICNetProcessor.cc new file mode 100644 index 00000000000..bbaa5625516 --- /dev/null +++ b/src/iocore/net/OpenSSLQUICNetProcessor.cc @@ -0,0 +1,125 @@ +/** @file + + OpenSSL native QUIC NetProcessor support. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "P_Net.h" +#include "P_QUICNetProcessor.h" +#include "P_QUICPacketHandler.h" +#include "P_QUICNetVConnection.h" +#include "P_UnixNet.h" +#include "iocore/net/quic/QUICConfig.h" +#include "iocore/net/quic/QUICGlobals.h" +#include "iocore/net/QUICMultiCertConfigLoader.h" + +#include + +QUICNetProcessor quic_NetProcessor; + +namespace +{ +DbgCtl dbg_ctl_quic_ps{"quic_ps"}; +DbgCtl dbg_ctl_iocore_net_processor{"iocore_net_processor"}; +} // end anonymous namespace + +QUICNetProcessor::QUICNetProcessor() {} + +QUICNetProcessor::~QUICNetProcessor() {} + +void +QUICNetProcessor::init() +{ +} + +int +QUICNetProcessor::start(int, size_t /* stacksize ATS_UNUSED */) +{ + QUIC::init(); + QUICConfig::startup(); + QUICCertConfig::startup(); + + return 0; +} + +NetAccept * +QUICNetProcessor::createNetAccept(NetProcessor::AcceptOptions const &opt) +{ + return new QUICPacketHandlerIn(opt); +} + +NetVConnection * +QUICNetProcessor::allocate_vc(EThread *t) +{ + QUICNetVConnection *vc = nullptr; + + if (t) { + vc = THREAD_ALLOC(quicNetVCAllocator, t); + new (vc) QUICNetVConnection(); + } else if (likely(vc = quicNetVCAllocator.alloc())) { + new (vc) QUICNetVConnection(); + vc->from_accept_thread = true; + } + + if (vc != nullptr) { + vc->ep.syscall = false; + } + + return vc; +} + +Action * +QUICNetProcessor::connect_re(Continuation *cont, sockaddr const * /* remote_addr ATS_UNUSED */, + NetVCOptions const & /* opt ATS_UNUSED */) +{ + Dbg(dbg_ctl_quic_ps, "OpenSSL native QUIC origin connections are not supported"); + cont->handleEvent(NET_EVENT_OPEN_FAILED, reinterpret_cast(-ENOTSUP)); + return ACTION_IO_ERROR; +} + +Action * +QUICNetProcessor::main_accept(Continuation *cont, SOCKET fd, AcceptOptions const &opt) +{ + Dbg(dbg_ctl_iocore_net_processor, "NetProcessor::main_accept - port %d,recv_bufsize %d, send_bufsize %d, sockopt 0x%0x", + opt.local_port, opt.recv_bufsize, opt.send_bufsize, opt.sockopt_flags); + + IpEndpoint accept_ip; + NetAccept *na = createNetAccept(opt); + + Metrics::Gauge::increment(net_rsb.accepts_currently_open); + + if (opt.localhost_only) { + accept_ip.setToLoopback(opt.ip_family); + } else if (opt.local_ip.isValid()) { + accept_ip.assign(opt.local_ip); + } else { + accept_ip.setToAnyAddr(opt.ip_family); + } + ink_assert(0 < opt.local_port && opt.local_port < 65536); + accept_ip.network_order_port() = htons(opt.local_port); + + na->server.sock = UnixSocket{fd}; + ats_ip_copy(&na->server.accept_addr, &accept_ip); + + na->action_ = new NetAcceptAction(cont, &na->server); + na->init_accept(); + + return na->action_.get(); +} diff --git a/src/iocore/net/OpenSSLQUICNetVConnection.cc b/src/iocore/net/OpenSSLQUICNetVConnection.cc new file mode 100644 index 00000000000..9a06fb94d26 --- /dev/null +++ b/src/iocore/net/OpenSSLQUICNetVConnection.cc @@ -0,0 +1,1062 @@ +/** @file + + OpenSSL native QUIC NetVConnection support. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "P_SSLUtils.h" +#include "P_QUICNetVConnection.h" +#include "P_QUICPacketHandler.h" +#include "P_UnixNet.h" +#include "api/APIHook.h" +#include "iocore/eventsystem/EThread.h" +#include "iocore/net/QUICMultiCertConfigLoader.h" +#include "iocore/net/SSLAPIHooks.h" +#include "iocore/net/quic/QUICApplicationMap.h" +#include "iocore/net/quic/QUICEvents.h" +#include "iocore/net/quic/QUICGlobals.h" +#include "iocore/net/quic/QUICStream.h" +#include "iocore/net/quic/QUICStreamManager.h" +#include "tscore/ink_config.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace +{ +constexpr ink_hrtime OPENSSL_QUIC_EVENT_INTERVAL = HRTIME_MSECONDS(2); + +DbgCtl dbg_ctl_quic_net{"quic_net"}; +DbgCtl dbg_ctl_v_quic_net{"v_quic_net"}; + +class OpenSSLQUICStreamManager : public QUICStreamManager +{ +public: + OpenSSLQUICStreamManager(QUICContext *context, QUICApplicationMap *app_map, QUICNetVConnection *vc) + : QUICStreamManager(context, app_map), _vc(vc) + { + } + + QUICConnectionErrorUPtr + create_uni_stream(QUICStreamId &new_stream_id) override + { + return this->_vc->create_openssl_stream(SSL_STREAM_FLAG_UNI | SSL_STREAM_FLAG_NO_BLOCK, new_stream_id); + } + + QUICConnectionErrorUPtr + create_bidi_stream(QUICStreamId &new_stream_id) override + { + return this->_vc->create_openssl_stream(SSL_STREAM_FLAG_NO_BLOCK, new_stream_id); + } + +private: + QUICNetVConnection *_vc = nullptr; +}; + +} // end anonymous namespace + +#define QUICConDebug(fmt, ...) Dbg(dbg_ctl_quic_net, "[%s] " fmt, this->cids().data(), ##__VA_ARGS__) +#define QUICConVDebug(fmt, ...) Dbg(dbg_ctl_v_quic_net, "[%s] " fmt, this->cids().data(), ##__VA_ARGS__) + +ClassAllocator quicNetVCAllocator("quicNetVCAllocator"); + +QUICNetVConnection::QUICNetVConnection() +{ + this->_set_service(static_cast(this)); + this->_set_service(static_cast(this)); + this->_set_service(static_cast(this)); + this->_set_service(static_cast(this)); + this->_set_service(static_cast(this)); + this->_set_service(static_cast(this)); + this->_set_service(static_cast(this)); +} + +QUICNetVConnection::~QUICNetVConnection() {} + +void +QUICNetVConnection::init(SSL *ssl, QUICPacketHandler *packet_handler) +{ + SET_HANDLER((NetVConnHandler)&QUICNetVConnection::acceptEvent); + + this->_ssl = ssl; + this->_packet_handler = packet_handler; + this->_quic_connection_id.randomize(); + this->_initial_source_connection_id = this->_quic_connection_id; + this->_cid_text = this->_quic_connection_id.hex(); + + SSL_set_ex_data(ssl, QUIC::ssl_quic_qc_index, static_cast(this)); + SSL_set_blocking_mode(ssl, 0); + SSL_set_event_handling_mode(ssl, SSL_VALUE_EVENT_HANDLING_MODE_IMPLICIT); + SSL_set_incoming_stream_policy(ssl, SSL_INCOMING_STREAM_POLICY_ACCEPT, 0); + SSL_set_default_stream_mode(ssl, SSL_DEFAULT_STREAM_MODE_NONE); + + QUICConfig::scoped_config params; + SSL_set_feature_request_uint(ssl, SSL_VALUE_QUIC_IDLE_TIMEOUT, params->no_activity_timeout_in()); + SSL_set_feature_request_uint(ssl, SSL_VALUE_QUIC_STREAM_BIDI_LOCAL_AVAIL, params->initial_max_streams_bidi_in()); + SSL_set_feature_request_uint(ssl, SSL_VALUE_QUIC_STREAM_UNI_LOCAL_AVAIL, params->initial_max_streams_uni_in()); + + this->_bindSSLObject(); + if (auto const servername = quic_sni_server_name(ssl); !servername.empty()) { + this->set_sni_server_name(servername); + } +} + +void +QUICNetVConnection::set_quic_endpoints(IpEndpoint const &local, IpEndpoint const &remote) +{ + this->local_addr = local; + this->remote_addr = remote; + this->got_local_addr = true; + this->got_remote_addr = true; +} + +QUICConnectionErrorUPtr +QUICNetVConnection::create_openssl_stream(uint64_t flags, QUICStreamId &new_stream_id) +{ + if (this->_ssl == nullptr) { + return std::make_unique(QUICTransErrorCode::INTERNAL_ERROR, "QUIC connection is not initialized"); + } + + SSL *stream_ssl = SSL_new_stream(this->_ssl, flags); + if (stream_ssl == nullptr) { + return std::make_unique(QUICTransErrorCode::INTERNAL_ERROR, "Failed to create QUIC stream"); + } + + SSL_set_blocking_mode(stream_ssl, 0); + SSL_set_event_handling_mode(stream_ssl, SSL_VALUE_EVENT_HANDLING_MODE_IMPLICIT); + SSL_set_mode(stream_ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_ENABLE_PARTIAL_WRITE); + + new_stream_id = SSL_get_stream_id(stream_ssl); + this->_openssl_streams.emplace(new_stream_id, stream_ssl); + + [[maybe_unused]] QUICConnectionError err; + this->_stream_manager->create_stream(new_stream_id, err); + + return nullptr; +} + +void +QUICNetVConnection::free() +{ + this->free_thread(this_ethread()); +} + +void +QUICNetVConnection::remove_connection_ids() +{ +} + +void +QUICNetVConnection::destroy(EThread *t) +{ + QUICConDebug("Destroy connection"); + if (from_accept_thread) { + quicNetVCAllocator.free(this); + } else { + THREAD_FREE(this, quicNetVCAllocator, t); + } +} + +void +QUICNetVConnection::set_local_addr() +{ +} + +void +QUICNetVConnection::free_thread(EThread * /* t ATS_UNUSED */) +{ + QUICConDebug("Free connection"); + + this->_unschedule_openssl_event(); + + for (auto &[stream_id, stream_ssl] : this->_openssl_streams) { + SSL_free(stream_ssl); + } + this->_openssl_streams.clear(); + + if (this->_ssl != nullptr) { + this->_unbindSSLObject(); + SSL_free(this->_ssl); + this->_ssl = nullptr; + } + + this->_application_map.reset(); + this->_stream_manager.reset(); + + super::clear(); + ALPNSupport::clear(); + TLSBasicSupport::clear(); + TLSEventSupport::clear(); + TLSCertSwitchSupport::_clear(); + + if (this->_packet_handler != nullptr) { + this->_packet_handler->close_connection(this); + this->_packet_handler = nullptr; + } +} + +void +QUICNetVConnection::reenable(VIO * /* vio ATS_UNUSED */) +{ +} + +int +QUICNetVConnection::state_handshake(int event, Event *data) +{ + if (data == this->_packet_write_ready) { + this->_packet_write_ready = nullptr; + } + + switch (event) { + case EVENT_IMMEDIATE: + case EVENT_INTERVAL: + case QUIC_EVENT_PACKET_READ_READY: + case QUIC_EVENT_PACKET_WRITE_READY: + this->_handle_openssl_events(); + break; + case VC_EVENT_EOS: + case VC_EVENT_ERROR: + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_INACTIVITY_TIMEOUT: + this->_unschedule_openssl_event(); + this->_propagate_event(event); + this->closed = 1; + break; + default: + QUICConDebug("Unhandled event: %d", event); + break; + } + + if (this->closed != 1 && SSL_is_init_finished(this->_ssl)) { + this->_switch_to_established_state(); + this->_handle_openssl_events(); + } + + if (this->closed != 1 && this->_openssl_connection_closed()) { + this->_schedule_closing_event(); + } else if (this->closed != 1) { + this->_schedule_openssl_event(); + } + + return EVENT_DONE; +} + +int +QUICNetVConnection::state_established(int event, Event *data) +{ + if (this->_ssl == nullptr) { + return EVENT_DONE; + } + + if (data == this->_packet_write_ready) { + this->_packet_write_ready = nullptr; + } + + switch (event) { + case EVENT_IMMEDIATE: + case EVENT_INTERVAL: + case QUIC_EVENT_PACKET_READ_READY: + case QUIC_EVENT_PACKET_WRITE_READY: + this->_handle_openssl_events(); + break; + case VC_EVENT_EOS: + case VC_EVENT_ERROR: + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_INACTIVITY_TIMEOUT: + this->_unschedule_openssl_event(); + this->_propagate_event(event); + this->closed = 1; + break; + default: + QUICConDebug("Unhandled event: %d", event); + break; + } + + if (this->closed != 1 && this->_openssl_connection_closed()) { + this->_schedule_closing_event(); + } else if (this->closed != 1) { + this->_schedule_openssl_event(); + } + + return EVENT_DONE; +} + +void +QUICNetVConnection::_switch_to_established_state() +{ + QUICConDebug("Enter state_connection_established"); + this->_record_tls_handshake_end_time(); + this->_update_end_of_handshake_stats(); + SET_HANDLER((NetVConnHandler)&QUICNetVConnection::state_established); + this->_handshake_completed = true; + this->_start_application(); +} + +void +QUICNetVConnection::_start_application() +{ + if (this->_application_started) { + return; + } + + this->_application_started = true; + + unsigned char const *app_name = nullptr; + unsigned int app_name_len = 0; + SSL_get0_alpn_selected(this->_ssl, &app_name, &app_name_len); + + if (app_name == nullptr || app_name_len == 0) { + app_name = reinterpret_cast(IP_PROTO_TAG_HTTP_QUIC.data()); + app_name_len = IP_PROTO_TAG_HTTP_QUIC.size(); + } + + this->_negotiated_alpn.assign(reinterpret_cast(app_name), app_name_len); + this->set_negotiated_protocol_id(this->_negotiated_alpn); + + if (netvc_context == NET_VCONNECTION_IN) { + if (this->setSelectedProtocol(app_name, app_name_len)) { + this->endpoint()->handleEvent(NET_EVENT_ACCEPT, this); + } + } else { + this->action_.continuation->handleEvent(NET_EVENT_OPEN, this); + } +} + +void +QUICNetVConnection::_propagate_event(int event) +{ + QUICConVDebug("Propagating: %d", event); + if (this->read.vio.cont && this->read.vio.mutex == this->read.vio.cont->mutex) { + this->read.vio.cont->handleEvent(event, &this->read.vio); + } else if (this->write.vio.cont && this->write.vio.mutex == this->write.vio.cont->mutex) { + this->write.vio.cont->handleEvent(event, &this->write.vio); + } else { + QUICConVDebug("Session does not exist"); + } +} + +bool +QUICNetVConnection::shouldDestroy() +{ + return this->refcount() == 0; +} + +VIO * +QUICNetVConnection::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) +{ + auto vio = super::do_io_read(c, nbytes, buf); + this->read.enabled = 1; + return vio; +} + +VIO * +QUICNetVConnection::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool /* owner ATS_UNUSED */) +{ + auto vio = super::do_io_write(c, nbytes, buf); + this->write.enabled = 1; + this->_schedule_openssl_event(false); + return vio; +} + +int +QUICNetVConnection::acceptEvent(int event, Event *e) +{ + EThread *t = (e == nullptr) ? this_ethread() : e->ethread; + NetHandler *h = get_NetHandler(t); + + MUTEX_TRY_LOCK(lock, h->mutex, t); + if (!lock.is_locked()) { + if (event == EVENT_NONE) { + t->schedule_in(this, HRTIME_MSECONDS(net_retry_delay)); + return EVENT_DONE; + } else { + e->schedule_in(HRTIME_MSECONDS(net_retry_delay)); + return EVENT_CONT; + } + } + + this->_context = std::make_unique(this); + this->_application_map = std::make_unique(); + this->_stream_manager = std::make_unique(this->_context.get(), this->_application_map.get(), this); + + ink_assert(this->thread == this_ethread()); + + if (h->startIO(this) < 0) { + this->free_thread(t); + return EVENT_DONE; + } + + this->read.enabled = 1; + this->write.enabled = 1; + + SET_HANDLER((NetVConnHandler)&QUICNetVConnection::state_handshake); + + nh->startCop(this); + this->set_default_inactivity_timeout(0); + + if (inactivity_timeout_in) { + this->set_inactivity_timeout(inactivity_timeout_in); + } + + if (active_timeout_in) { + set_active_timeout(active_timeout_in); + } + + action_.continuation->handleEvent(NET_EVENT_ACCEPT, this); + this->_schedule_openssl_event(false); + + return EVENT_DONE; +} + +int +QUICNetVConnection::connectUp(EThread * /* t ATS_UNUSED */, int /* fd ATS_UNUSED */) +{ + return 0; +} + +QUICStreamManager * +QUICNetVConnection::stream_manager() +{ + return this->_stream_manager.get(); +} + +void +QUICNetVConnection::close_quic_connection(QUICConnectionErrorUPtr error) +{ + if (this->_ssl != nullptr) { + SSL_SHUTDOWN_EX_ARGS args = {}; + if (error != nullptr) { + args.quic_error_code = error->code; + args.quic_reason = error->msg; + } + SSL_shutdown_ex(this->_ssl, SSL_SHUTDOWN_FLAG_NO_BLOCK, &args, sizeof(args)); + } +} + +void +QUICNetVConnection::reset_quic_connection() +{ + if (this->_ssl != nullptr) { + SSL_shutdown_ex(this->_ssl, SSL_SHUTDOWN_FLAG_RAPID | SSL_SHUTDOWN_FLAG_NO_BLOCK, nullptr, 0); + } +} + +void +QUICNetVConnection::handle_received_packet(UDPPacket * /* packet ATS_UNUSED */) +{ +} + +void +QUICNetVConnection::ping() +{ +} + +QUICConnectionId +QUICNetVConnection::peer_connection_id() const +{ + return {}; +} + +QUICConnectionId +QUICNetVConnection::original_connection_id() const +{ + return {}; +} + +QUICConnectionId +QUICNetVConnection::first_connection_id() const +{ + return {}; +} + +QUICConnectionId +QUICNetVConnection::retry_source_connection_id() const +{ + return {}; +} + +QUICConnectionId +QUICNetVConnection::initial_source_connection_id() const +{ + return this->_initial_source_connection_id; +} + +QUICConnectionId +QUICNetVConnection::connection_id() const +{ + return this->_quic_connection_id; +} + +std::string_view +QUICNetVConnection::cids() const +{ + return this->_cid_text; +} + +QUICFiveTuple const +QUICNetVConnection::five_tuple() const +{ + return QUICFiveTuple(this->remote_addr, this->local_addr, IPPROTO_UDP); +} + +uint32_t +QUICNetVConnection::pmtu() const +{ + return 0; +} + +NetVConnectionContext_t +QUICNetVConnection::direction() const +{ + return NET_VCONNECTION_IN; +} + +QUICVersion +QUICNetVConnection::negotiated_version() const +{ + return QUIC_SUPPORTED_VERSIONS[0]; +} + +std::string_view +QUICNetVConnection::negotiated_application_name() const +{ + return this->_negotiated_alpn; +} + +void +QUICNetVConnection::on_stream_updated() +{ + this->_schedule_openssl_event(false); +} + +bool +QUICNetVConnection::is_closed() const +{ + return this->_openssl_connection_closed(); +} + +bool +QUICNetVConnection::is_at_anti_amplification_limit() const +{ + return false; +} + +bool +QUICNetVConnection::is_address_validation_completed() const +{ + return true; +} + +bool +QUICNetVConnection::is_handshake_completed() const +{ + return this->_handshake_completed; +} + +void +QUICNetVConnection::net_read_io(NetHandler * /* nh ATS_UNUSED */) +{ + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + this->handleEvent(QUIC_EVENT_PACKET_READ_READY, nullptr); +} + +int64_t +QUICNetVConnection::load_buffer_and_write(int64_t /* towrite ATS_UNUSED */, MIOBufferAccessor & /* buf ATS_UNUSED */, + int64_t & /* total_written ATS_UNUSED */, int & /* needs ATS_UNUSED */) +{ + return 0; +} + +void +QUICNetVConnection::_bindSSLObject() +{ + TLSBasicSupport::bind(this->_ssl, this); + TLSEventSupport::bind(this->_ssl, this); + ALPNSupport::bind(this->_ssl, this); + TLSSessionResumptionSupport::bind(this->_ssl, this); + TLSSNISupport::bind(this->_ssl, this); + TLSCertSwitchSupport::bind(this->_ssl, this); + QUICSupport::bind(this->_ssl, this); +} + +void +QUICNetVConnection::_unbindSSLObject() +{ + TLSBasicSupport::unbind(this->_ssl); + TLSEventSupport::unbind(this->_ssl); + ALPNSupport::unbind(this->_ssl); + TLSSessionResumptionSupport::unbind(this->_ssl); + TLSSNISupport::unbind(this->_ssl); + TLSCertSwitchSupport::unbind(this->_ssl); + QUICSupport::unbind(this->_ssl); +} + +void +QUICNetVConnection::_schedule_packet_write_ready(bool delay) +{ + this->_schedule_openssl_event(delay); +} + +void +QUICNetVConnection::_unschedule_packet_write_ready() +{ + this->_unschedule_openssl_event(); +} + +void +QUICNetVConnection::_close_packet_write_ready(Event *data) +{ + if (this->_packet_write_ready == data) { + this->_packet_write_ready = nullptr; + } +} + +void +QUICNetVConnection::_schedule_quiche_timeout() +{ +} + +void +QUICNetVConnection::_unschedule_quiche_timeout() +{ +} + +void +QUICNetVConnection::_close_quiche_timeout(Event * /* data ATS_UNUSED */) +{ +} + +void +QUICNetVConnection::_schedule_closing_event() +{ + QUICConDebug("Scheduling closing event"); + SSL_CONN_CLOSE_INFO info = {}; + if (this->_ssl != nullptr && SSL_get_conn_close_info(this->_ssl, &info, sizeof(info)) == 1) { + QUICConDebug("QUIC close info: error_code=%" PRIu64 " frame_type=%" PRIu64 " flags=0x%x reason=\"%.*s\"", info.error_code, + info.frame_type, info.flags, static_cast(info.reason_len), info.reason == nullptr ? "" : info.reason); + if (info.error_code == OSSL_QUIC_LOCAL_ERR_IDLE_TIMEOUT) { + QUICConDebug("QUIC Idle timeout detected"); + this->thread->schedule_imm(this, VC_EVENT_INACTIVITY_TIMEOUT); + return; + } + } + + this->thread->schedule_imm(this, VC_EVENT_EOS); +} + +void +QUICNetVConnection::_handle_read_ready() +{ + this->_handle_openssl_events(); +} + +void +QUICNetVConnection::_handle_write_ready() +{ + this->_handle_openssl_events(); +} + +void +QUICNetVConnection::_handle_interval() +{ + this->_handle_openssl_events(); +} + +void +QUICNetVConnection::_handle_openssl_events() +{ + if (this->_ssl == nullptr) { + return; + } + + if (SSL_handle_events(this->_ssl) != 1) { + QUICConDebug("SSL_handle_events failed: %s", ERR_error_string(ERR_peek_error(), nullptr)); + } + + if (this->_stream_manager != nullptr && this->_application_started) { + this->_accept_openssl_streams(); + this->_process_openssl_streams(); + } + + this->netActivity(); +} + +void +QUICNetVConnection::_accept_openssl_streams() +{ + while (SSL *stream_ssl = SSL_accept_stream(this->_ssl, SSL_ACCEPT_STREAM_NO_BLOCK)) { + SSL_set_blocking_mode(stream_ssl, 0); + SSL_set_event_handling_mode(stream_ssl, SSL_VALUE_EVENT_HANDLING_MODE_IMPLICIT); + SSL_set_mode(stream_ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_ENABLE_PARTIAL_WRITE); + + QUICStreamId stream_id = SSL_get_stream_id(stream_ssl); + this->_openssl_streams.emplace(stream_id, stream_ssl); + + if (this->_stream_manager->find_stream(stream_id) == nullptr) { + [[maybe_unused]] QUICConnectionError err; + this->_stream_manager->create_stream(stream_id, err); + } + } +} + +void +QUICNetVConnection::_process_openssl_streams() +{ + std::vector stream_ids; + stream_ids.reserve(this->_openssl_streams.size()); + for (auto const &entry : this->_openssl_streams) { + stream_ids.push_back(entry.first); + } + + for (QUICStreamId stream_id : stream_ids) { + auto stream_it = this->_openssl_streams.find(stream_id); + if (stream_it == this->_openssl_streams.end()) { + continue; + } + + SSL *stream_ssl = stream_it->second; + QUICStream *stream = static_cast(this->_stream_manager->find_stream(stream_id)); + if (stream == nullptr) { + continue; + } + + int stream_type = SSL_get_stream_type(stream_ssl); + if ((stream_type & SSL_STREAM_TYPE_READ) != 0) { + stream->receive_data(*this); + } + if ((stream_type & SSL_STREAM_TYPE_WRITE) != 0 || stream->has_data_to_send()) { + if (stream->has_data_to_send()) { + while (stream->has_data_to_send() && stream->send_data(*this) > 0) {} + } else { + stream->send_data(*this); + } + } + } +} + +void +QUICNetVConnection::_schedule_openssl_event(bool delay) +{ + if (!delay && this->_packet_write_ready != nullptr) { + this->_packet_write_ready->cancel(); + this->_packet_write_ready = nullptr; + } + + if (this->_packet_write_ready == nullptr && this->thread != nullptr) { + if (delay) { + this->_packet_write_ready = this->thread->schedule_in(this, OPENSSL_QUIC_EVENT_INTERVAL); + } else { + this->_packet_write_ready = this->thread->schedule_imm(this, QUIC_EVENT_PACKET_WRITE_READY); + } + } +} + +void +QUICNetVConnection::_unschedule_openssl_event() +{ + if (this->_packet_write_ready != nullptr) { + this->_packet_write_ready->cancel(); + this->_packet_write_ready = nullptr; + } +} + +bool +QUICNetVConnection::_openssl_connection_closed() const +{ + if (this->_ssl == nullptr) { + return true; + } + + SSL_CONN_CLOSE_INFO info = {}; + return SSL_get_conn_close_info(this->_ssl, &info, sizeof(info)) == 1; +} + +int +QUICNetVConnection::populate_protocol(std::string_view *results, int n) const +{ + int retval = 0; + if (n > retval) { + results[retval++] = IP_PROTO_TAG_QUIC; + if (n > retval) { + results[retval++] = IP_PROTO_TAG_TLS_1_3; + if (n > retval) { + retval += super::populate_protocol(results + retval, n - retval); + } + } + } + return retval; +} + +char const * +QUICNetVConnection::protocol_contains(std::string_view prefix) const +{ + char const *retval = nullptr; + if (prefix.size() <= IP_PROTO_TAG_QUIC.size() && strncmp(IP_PROTO_TAG_QUIC.data(), prefix.data(), prefix.size()) == 0) { + retval = IP_PROTO_TAG_QUIC.data(); + } else if (prefix.size() <= IP_PROTO_TAG_TLS_1_3.size() && + strncmp(IP_PROTO_TAG_TLS_1_3.data(), prefix.data(), prefix.size()) == 0) { + retval = IP_PROTO_TAG_TLS_1_3.data(); + } else { + retval = super::protocol_contains(prefix); + } + return retval; +} + +QUICConnection * +QUICNetVConnection::get_quic_connection() +{ + return static_cast(this); +} + +int64_t +QUICNetVConnection::read_stream(QUICStreamId stream_id, uint8_t *buf, size_t len, bool &fin, QUICStreamIO::ErrorCode &error_code) +{ + auto it = this->_openssl_streams.find(stream_id); + if (it == this->_openssl_streams.end()) { + error_code = ENOENT; + return -1; + } + + SSL_handle_events(it->second); + + size_t read_len = 0; + fin = false; + if (SSL_read_ex(it->second, buf, len, &read_len) == 1) { + fin = this->stream_read_finished(stream_id); + return read_len; + } + + int ssl_error = SSL_get_error(it->second, 0); + if (ssl_error == SSL_ERROR_WANT_READ || ssl_error == SSL_ERROR_WANT_WRITE) { + return 0; + } + if (ssl_error == SSL_ERROR_ZERO_RETURN) { + fin = true; + return 0; + } + + error_code = ssl_error; + return -1; +} + +bool +QUICNetVConnection::stream_read_finished(QUICStreamId stream_id) +{ + auto it = this->_openssl_streams.find(stream_id); + if (it == this->_openssl_streams.end()) { + return true; + } + + int state = SSL_get_stream_read_state(it->second); + return state == SSL_STREAM_STATE_FINISHED || state == SSL_STREAM_STATE_RESET_REMOTE || state == SSL_STREAM_STATE_CONN_CLOSED; +} + +int64_t +QUICNetVConnection::stream_write_capacity(QUICStreamId stream_id) +{ + auto it = this->_openssl_streams.find(stream_id); + if (it == this->_openssl_streams.end()) { + return -1; + } + + uint64_t avail = 0; + if (SSL_get_stream_write_buf_avail(it->second, &avail) == 1) { + return std::min(avail, 16 * 1024); + } + + return 0; +} + +int64_t +QUICNetVConnection::write_stream(QUICStreamId stream_id, uint8_t const *buf, size_t len, bool fin, + QUICStreamIO::ErrorCode &error_code) +{ + auto it = this->_openssl_streams.find(stream_id); + if (it == this->_openssl_streams.end()) { + error_code = ENOENT; + return -1; + } + + SSL_handle_events(it->second); + + if (len == 0 && fin) { + SSL_stream_conclude(it->second, 0); + return 0; + } + + size_t written_len = 0; + const uint64_t flags = fin ? SSL_WRITE_FLAG_CONCLUDE : 0; + if (SSL_write_ex2(it->second, buf, len, flags, &written_len) == 1) { + return written_len; + } + + int ssl_error = SSL_get_error(it->second, 0); + if (ssl_error == SSL_ERROR_WANT_READ || ssl_error == SSL_ERROR_WANT_WRITE) { + return 0; + } + + error_code = ssl_error; + QUICConDebug("Failed to write %zu bytes on QUIC stream %" PRIu64 ": ssl_error=%d, openssl_error=%s", len, stream_id, ssl_error, + ERR_error_string(ERR_peek_error(), nullptr)); + return -1; +} + +void +QUICNetVConnection::reenable(int event) +{ + this->_is_verifying_cert = false; + + if (event == TS_EVENT_ERROR) { + this->_is_cert_verified = false; + } +} + +Continuation * +QUICNetVConnection::getContinuationForTLSEvents() +{ + return this; +} + +EThread * +QUICNetVConnection::getThreadForTLSEvents() +{ + return this->thread; +} + +Ptr +QUICNetVConnection::getMutexForTLSEvents() +{ + return this->nh->mutex; +} + +bool +QUICNetVConnection::_isReadyToTransferData() const +{ + return this->_handshake_completed; +} + +SSL * +QUICNetVConnection::_get_ssl_object() const +{ + return this->_ssl; +} + +ssl_curve_id +QUICNetVConnection::_get_tls_curve() const +{ + if (getIsResumedFromSessionCache()) { + return getSSLCurveNID(); + } else { + return SSLGetCurveNID(this->_ssl); + } +} + +std::string_view +QUICNetVConnection::_get_tls_group() const +{ + if (getIsResumedFromSessionCache()) { + return getSSLGroupName(); + } else { + return SSLGetGroupName(this->_ssl); + } +} + +int +QUICNetVConnection::_verify_certificate(X509_STORE_CTX * /* ctx ATS_UNUSED */) +{ + TSEvent eventId; + TSHttpHookID hookId; + APIHook *hook = nullptr; + + if (get_context() == NET_VCONNECTION_IN) { + eventId = TS_EVENT_SSL_VERIFY_CLIENT; + hookId = TS_SSL_VERIFY_CLIENT_HOOK; + } else { + eventId = TS_EVENT_SSL_VERIFY_SERVER; + hookId = TS_SSL_VERIFY_SERVER_HOOK; + } + hook = SSLAPIHooks::instance()->get(TSSslHookInternalID(hookId)); + if (hook != nullptr) { + this->_is_verifying_cert = true; + WEAK_SCOPED_MUTEX_LOCK(lock, hook->m_cont->mutex, this_ethread()); + hook->invoke(eventId, this); + } + + ink_assert(this->_is_verifying_cert == false); + if (this->_is_cert_verified) { + return 1; + } + + return 0; +} + +in_port_t +QUICNetVConnection::_get_local_port() +{ + return this->get_local_port(); +} + +IpEndpoint const & +QUICNetVConnection::_getLocalEndpoint() +{ + return this->local_addr; +} + +bool +QUICNetVConnection::_isTryingRenegotiation() const +{ + return this->_handshake_completed; +} + +shared_SSL_CTX +QUICNetVConnection::_lookupContextByName(std::string const &servername, SSLCertContextType ctxType) +{ + shared_SSL_CTX ctx = nullptr; + QUICCertConfig::scoped_config lookup; + SSLCertContext *cc = lookup->find(servername, ctxType); + + if (cc && cc->getCtx()) { + ctx = cc->getCtx(); + } + + return ctx; +} + +shared_SSL_CTX +QUICNetVConnection::_lookupContextByIP() +{ + shared_SSL_CTX ctx = nullptr; + QUICCertConfig::scoped_config lookup; + QUICFiveTuple five_tuple = this->five_tuple(); + IpEndpoint ip = five_tuple.destination(); + SSLCertContext *cc = lookup->find(ip); + + if (cc && cc->getCtx()) { + ctx = cc->getCtx(); + } + + return ctx; +} diff --git a/src/iocore/net/OpenSSLQUICPacketHandler.cc b/src/iocore/net/OpenSSLQUICPacketHandler.cc new file mode 100644 index 00000000000..f62766b9746 --- /dev/null +++ b/src/iocore/net/OpenSSLQUICPacketHandler.cc @@ -0,0 +1,630 @@ +/** @file + + OpenSSL native QUIC listener support. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "P_QUICPacketHandler.h" +#include "P_QUICNetProcessor.h" +#include "P_QUICNetVConnection.h" +#include "P_SSLCertLookup.h" +#include "P_UnixNet.h" +#include "iocore/net/QUICMultiCertConfigLoader.h" +#include "iocore/net/quic/QUICConfig.h" +#include "tscore/ink_atomic.h" +#include "tscore/ink_sock.h" + +#include +#include +#include +#include + +#include +#include + +namespace +{ +constexpr ink_hrtime OPENSSL_QUIC_EVENT_INTERVAL = HRTIME_MSECONDS(2); + +DbgCtl dbg_ctl_openssl_quic{"openssl_quic"}; + +struct PeerCaptureBioState { + BIO *inner{nullptr}; + IpEndpoint last_peer; + bool have_last_peer{false}; +}; + +thread_local PeerCaptureBioState *active_peer_capture = nullptr; + +void +free_quic_peer_ex_data(void *, void *ptr, CRYPTO_EX_DATA *, int, long, void *) +{ + delete static_cast(ptr); +} + +int +quic_peer_ex_data_index() +{ + static int index = SSL_get_ex_new_index(0, nullptr, nullptr, nullptr, free_quic_peer_ex_data); + + return index; +} + +bool +bio_addr_to_ip_endpoint(IpEndpoint &endpoint, BIO_ADDR const *bio_addr) +{ + if (bio_addr == nullptr) { + return false; + } + + in_port_t const port = BIO_ADDR_rawport(bio_addr); + if (BIO_ADDR_family(bio_addr) == AF_INET) { + in_addr_t addr = 0; + size_t len = sizeof(addr); + + if (BIO_ADDR_rawaddress(bio_addr, &addr, &len) != 1 || len != sizeof(addr)) { + return false; + } + + ats_ip4_set(&endpoint, addr, port); + return true; + } + + if (BIO_ADDR_family(bio_addr) == AF_INET6) { + in6_addr addr; + size_t len = sizeof(addr); + + if (BIO_ADDR_rawaddress(bio_addr, &addr, &len) != 1 || len != sizeof(addr)) { + return false; + } + + ats_ip6_set(&endpoint, addr, port); + return true; + } + + return false; +} + +PeerCaptureBioState * +peer_capture_state(BIO *bio) +{ + return static_cast(BIO_get_data(bio)); +} + +BIO_MSG * +bio_msg_at(BIO_MSG *msg, size_t stride, size_t index) +{ + return reinterpret_cast(reinterpret_cast(msg) + (stride * index)); +} + +void +update_last_peer(PeerCaptureBioState &state, BIO_MSG const &msg) +{ + IpEndpoint peer; + if (msg.peer != nullptr && bio_addr_to_ip_endpoint(peer, msg.peer)) { + state.last_peer = peer; + state.have_last_peer = true; + active_peer_capture = &state; + } +} + +void +copy_retry_flags(BIO *bio, BIO *inner) +{ + BIO_clear_retry_flags(bio); + BIO_set_flags(bio, BIO_get_retry_flags(inner)); + BIO_set_retry_reason(bio, BIO_get_retry_reason(inner)); +} + +int +peer_capture_bio_create(BIO *bio) +{ + BIO_set_init(bio, 0); + BIO_set_data(bio, nullptr); + BIO_set_shutdown(bio, 1); + + return 1; +} + +int +peer_capture_bio_destroy(BIO *bio) +{ + if (bio == nullptr) { + return 0; + } + + auto *state = peer_capture_state(bio); + if (state != nullptr) { + if (active_peer_capture == state) { + active_peer_capture = nullptr; + } + BIO_free(state->inner); + delete state; + BIO_set_data(bio, nullptr); + } + BIO_set_init(bio, 0); + + return 1; +} + +int +peer_capture_bio_write(BIO *bio, char const *data, int len) +{ + auto *state = peer_capture_state(bio); + if (state == nullptr || state->inner == nullptr) { + return 0; + } + + int const result = BIO_write(state->inner, data, len); + copy_retry_flags(bio, state->inner); + + return result; +} + +int +peer_capture_bio_write_ex(BIO *bio, char const *data, size_t len, size_t *written) +{ + auto *state = peer_capture_state(bio); + if (state == nullptr || state->inner == nullptr) { + return 0; + } + + int const result = BIO_write_ex(state->inner, data, len, written); + copy_retry_flags(bio, state->inner); + + return result; +} + +int +peer_capture_bio_sendmmsg(BIO *bio, BIO_MSG *msg, size_t stride, size_t num_msg, uint64_t flags, size_t *num_processed) +{ + auto *state = peer_capture_state(bio); + if (state == nullptr || state->inner == nullptr) { + return 0; + } + + int const result = BIO_sendmmsg(state->inner, msg, stride, num_msg, flags, num_processed); + copy_retry_flags(bio, state->inner); + + return result; +} + +int +peer_capture_bio_read(BIO *bio, char *data, int len) +{ + auto *state = peer_capture_state(bio); + if (state == nullptr || state->inner == nullptr) { + return 0; + } + + int const result = BIO_read(state->inner, data, len); + copy_retry_flags(bio, state->inner); + + return result; +} + +int +peer_capture_bio_read_ex(BIO *bio, char *data, size_t len, size_t *read_bytes) +{ + auto *state = peer_capture_state(bio); + if (state == nullptr || state->inner == nullptr) { + return 0; + } + + int const result = BIO_read_ex(state->inner, data, len, read_bytes); + copy_retry_flags(bio, state->inner); + + return result; +} + +int +peer_capture_bio_recvmmsg(BIO *bio, BIO_MSG *msg, size_t stride, size_t num_msg, uint64_t flags, size_t *num_processed) +{ + auto *state = peer_capture_state(bio); + if (state == nullptr || state->inner == nullptr) { + return 0; + } + + size_t const read_limit = std::min(num_msg, 1); + + int const result = BIO_recvmmsg(state->inner, msg, stride, read_limit, flags, num_processed); + copy_retry_flags(bio, state->inner); + + if (result == 1 && num_processed != nullptr && *num_processed > 0) { + update_last_peer(*state, *bio_msg_at(msg, stride, 0)); + } + + return result; +} + +long +peer_capture_bio_ctrl(BIO *bio, int cmd, long larg, void *parg) +{ + auto *state = peer_capture_state(bio); + if (state == nullptr || state->inner == nullptr) { + return 0; + } + + return BIO_ctrl(state->inner, cmd, larg, parg); +} + +long +peer_capture_bio_callback_ctrl(BIO *bio, int cmd, BIO_info_cb *cb) +{ + auto *state = peer_capture_state(bio); + if (state == nullptr || state->inner == nullptr) { + return 0; + } + + return BIO_callback_ctrl(state->inner, cmd, cb); +} + +BIO_METHOD * +peer_capture_bio_method() +{ + static BIO_METHOD *method = []() -> BIO_METHOD * { + BIO_METHOD *m = BIO_meth_new(BIO_TYPE_SOURCE_SINK | BIO_get_new_index(), "ATS OpenSSL QUIC peer capture"); + if (m == nullptr) { + return nullptr; + } + + BIO_meth_set_create(m, peer_capture_bio_create); + BIO_meth_set_destroy(m, peer_capture_bio_destroy); + BIO_meth_set_write(m, peer_capture_bio_write); + BIO_meth_set_write_ex(m, peer_capture_bio_write_ex); + BIO_meth_set_sendmmsg(m, peer_capture_bio_sendmmsg); + BIO_meth_set_read(m, peer_capture_bio_read); + BIO_meth_set_read_ex(m, peer_capture_bio_read_ex); + BIO_meth_set_recvmmsg(m, peer_capture_bio_recvmmsg); + BIO_meth_set_ctrl(m, peer_capture_bio_ctrl); + BIO_meth_set_callback_ctrl(m, peer_capture_bio_callback_ctrl); + + return m; + }(); + + return method; +} + +BIO * +new_peer_capture_bio(int fd) +{ + BIO *inner = BIO_new_dgram(fd, BIO_NOCLOSE); + if (inner == nullptr) { + return nullptr; + } + + BIO *bio = BIO_new(peer_capture_bio_method()); + if (bio == nullptr) { + BIO_free(inner); + return nullptr; + } + + auto *state = new PeerCaptureBioState; + state->inner = inner; + BIO_set_data(bio, state); + BIO_set_init(bio, 1); + + return bio; +} + +int +new_pending_quic_connection_cb(SSL_CTX *, SSL *new_ssl, void *) +{ + auto *capture = active_peer_capture; + if (capture == nullptr || !capture->have_last_peer) { + return 1; + } + + int const index = quic_peer_ex_data_index(); + if (index < 0) { + return 0; + } + + delete static_cast(SSL_get_ex_data(new_ssl, index)); + auto *peer = new IpEndpoint(capture->last_peer); + if (SSL_set_ex_data(new_ssl, index, peer) != 1) { + delete peer; + return 0; + } + + return 1; +} + +bool +get_bio_peer_addr(BIO *bio, IpEndpoint &remote_addr) +{ + if (bio == nullptr) { + return false; + } + + BIO_ADDR *peer_addr = BIO_ADDR_new(); + if (peer_addr == nullptr) { + return false; + } + + bool const success = BIO_dgram_get_peer(bio, peer_addr) == 1 && bio_addr_to_ip_endpoint(remote_addr, peer_addr); + BIO_ADDR_free(peer_addr); + + return success; +} + +bool +get_quic_peer_addr(SSL *ssl, IpEndpoint &remote_addr) +{ + int const index = quic_peer_ex_data_index(); + if (index >= 0) { + auto *peer = static_cast(SSL_get_ex_data(ssl, index)); + if (peer != nullptr) { + remote_addr = *peer; + return true; + } + } + + return get_bio_peer_addr(SSL_get_rbio(ssl), remote_addr) || get_bio_peer_addr(SSL_get_wbio(ssl), remote_addr); +} + +} // end anonymous namespace + +QUICPacketHandler::QUICPacketHandler() +{ + this->_closed_con_collector = std::make_unique(); + this->_closed_con_collector->mutex = new_ProxyMutex(); +} + +QUICPacketHandler::~QUICPacketHandler() +{ + if (this->_collector_event != nullptr) { + this->_collector_event->cancel(); + this->_collector_event = nullptr; + } +} + +void +QUICPacketHandler::close_connection(QUICNetVConnection *conn) +{ + int isin = ink_atomic_swap(&conn->in_closed_queue, 1); + if (!isin) { + this->_closed_con_collector->closedQueue.push(conn); + } +} + +void +QUICPacketHandler::send_packet(UDPConnection * /* udp_con ATS_UNUSED */, IpEndpoint & /* addr ATS_UNUSED */, + Ptr /* udp_payload ATS_UNUSED */, uint16_t /* segment_size ATS_UNUSED */, + struct timespec * /* send_at_hint ATS_UNUSED */) +{ +} + +QUICPacketHandlerIn::QUICPacketHandlerIn(NetProcessor::AcceptOptions const &opt) : NetAccept(opt), QUICPacketHandler() +{ + this->mutex = new_ProxyMutex(); +} + +QUICPacketHandlerIn::~QUICPacketHandlerIn() +{ + if (this->_event != nullptr) { + this->_event->cancel(); + this->_event = nullptr; + } + if (this->_listener != nullptr) { + SSL_free(this->_listener); + this->_listener = nullptr; + } +} + +NetProcessor * +QUICPacketHandlerIn::getNetProcessor() const +{ + return &quic_NetProcessor; +} + +NetAccept * +QUICPacketHandlerIn::clone() const +{ + return new QUICPacketHandlerIn(this->opt); +} + +int +QUICPacketHandlerIn::acceptEvent(int /* event ATS_UNUSED */, void * /* data ATS_UNUSED */) +{ + this->setThreadAffinity(this_ethread()); + + if (this->_collector_event == nullptr) { + this->_collector_event = this_ethread()->schedule_every(this->_closed_con_collector.get(), HRTIME_MSECONDS(100)); + } + + if (this->_listener == nullptr && !this->_start_listener()) { + return EVENT_DONE; + } + + this->_event = nullptr; + this->_service_listener(); + this->_schedule_event(); + + return EVENT_CONT; +} + +void +QUICPacketHandlerIn::init_accept(EThread *t) +{ + SET_HANDLER(&QUICPacketHandlerIn::acceptEvent); + + if (t == nullptr) { + t = eventProcessor.assign_thread(ET_NET); + } + + if (!this->action_->continuation->mutex) { + this->action_->continuation->mutex = t->mutex; + this->action_->mutex = t->mutex; + } + + this->mutex = get_NetHandler(t)->mutex; + t->schedule_imm(this); +} + +Continuation * +QUICPacketHandlerIn::_get_continuation() +{ + return this; +} + +void +QUICPacketHandlerIn::_recv_packet(int /* event ATS_UNUSED */, UDPPacket * /* udpPacket ATS_UNUSED */) +{ +} + +bool +QUICPacketHandlerIn::_start_listener() +{ + if (!this->server.sock.is_ok()) { + this->server.sock = UnixSocket{this->server.accept_addr.sa.sa_family, SOCK_DGRAM, 0}; + if (!this->server.sock.is_ok()) { + Error("failed to create OpenSSL QUIC UDP socket: %s", strerror(errno)); + return false; + } + + if (this->server.accept_addr.sa.sa_family == AF_INET6 && this->server.sock.enable_option(IPPROTO_IPV6, IPV6_V6ONLY) < 0) { + Error("failed to set IPV6_V6ONLY on OpenSSL QUIC socket: %s", strerror(errno)); + return false; + } + if (this->server.sock.enable_option(SOL_SOCKET, SO_REUSEADDR) < 0) { + Error("failed to set SO_REUSEADDR on OpenSSL QUIC socket: %s", strerror(errno)); + return false; + } + if (this->server.sock.bind(&this->server.accept_addr.sa, ats_ip_size(&this->server.accept_addr.sa)) < 0) { + Error("failed to bind OpenSSL QUIC socket: %s", strerror(errno)); + return false; + } + } + + if (this->server.sock.set_nonblocking() < 0) { + Error("failed to make OpenSSL QUIC socket nonblocking: %s", strerror(errno)); + return false; + } + if (this->opt.recv_bufsize > 0) { + this->server.sock.set_rcvbuf_size(this->opt.recv_bufsize); + } + if (this->opt.send_bufsize > 0) { + this->server.sock.set_sndbuf_size(this->opt.send_bufsize); + } + + QUICCertConfig::scoped_config server_cert; + QUICConfig::scoped_config params; + auto default_ctx = server_cert->defaultContext(); + uint64_t listener_flags = 0; + if (!params->stateless_retry()) { + listener_flags |= SSL_LISTENER_FLAG_NO_VALIDATE; + } + + SSL_CTX_set_new_pending_conn_cb(default_ctx.get(), new_pending_quic_connection_cb, nullptr); + + this->_listener = SSL_new_listener(default_ctx.get(), listener_flags); + if (this->_listener == nullptr) { + Error("failed to create OpenSSL QUIC listener"); + return false; + } + + BIO *bio = new_peer_capture_bio(this->server.sock.get_fd()); + if (bio == nullptr) { + Error("failed to create OpenSSL QUIC peer capture BIO"); + return false; + } + SSL_set_bio(this->_listener, bio, bio); + if (SSL_get_rbio(this->_listener) == nullptr || SSL_get_wbio(this->_listener) == nullptr) { + Error("failed to configure OpenSSL QUIC listener socket"); + return false; + } + SSL_set_blocking_mode(this->_listener, 0); + SSL_set_event_handling_mode(this->_listener, SSL_VALUE_EVENT_HANDLING_MODE_EXPLICIT); + + SSL_set_feature_request_uint(this->_listener, SSL_VALUE_QUIC_IDLE_TIMEOUT, params->no_activity_timeout_in()); + SSL_set_feature_request_uint(this->_listener, SSL_VALUE_QUIC_STREAM_BIDI_LOCAL_AVAIL, params->initial_max_streams_bidi_in()); + SSL_set_feature_request_uint(this->_listener, SSL_VALUE_QUIC_STREAM_UNI_LOCAL_AVAIL, params->initial_max_streams_uni_in()); + SSL_set_incoming_stream_policy(this->_listener, SSL_INCOMING_STREAM_POLICY_ACCEPT, 0); + + if (SSL_listen(this->_listener) != 1) { + Error("failed to start OpenSSL QUIC listener"); + return false; + } + + Dbg(dbg_ctl_openssl_quic, "OpenSSL QUIC listener started on fd %d", this->server.sock.get_fd()); + return true; +} + +void +QUICPacketHandlerIn::_service_listener() +{ + SSL_handle_events(this->_listener); + + while (SSL *ssl = SSL_accept_connection(this->_listener, SSL_ACCEPT_CONNECTION_NO_BLOCK)) { + SSL_set_blocking_mode(ssl, 0); + SSL_set_event_handling_mode(ssl, SSL_VALUE_EVENT_HANDLING_MODE_IMPLICIT); + SSL_set_incoming_stream_policy(ssl, SSL_INCOMING_STREAM_POLICY_ACCEPT, 0); + SSL_set_default_stream_mode(ssl, SSL_DEFAULT_STREAM_MODE_NONE); + + auto *vc = static_cast(getNetProcessor()->allocate_vc(this_ethread())); + if (vc == nullptr) { + SSL_free(ssl); + continue; + } + + IpEndpoint remote_addr; + if (!get_quic_peer_addr(ssl, remote_addr)) { + remote_addr.setToAnyAddr(this->server.accept_addr.sa.sa_family); + } + + vc->init(ssl, this); + vc->id = net_next_connection_number(); + vc->set_quic_endpoints(this->server.accept_addr, remote_addr); + vc->submit_time = ink_get_hrtime(); + vc->thread = this_ethread(); + vc->mutex = get_NetHandler(this_ethread())->mutex; + vc->action_ = *this->action_; + vc->set_is_transparent(this->opt.f_inbound_transparent); + vc->set_context(NET_VCONNECTION_IN); + vc->options.ip_proto = NetVCOptions::USE_UDP; + vc->options.ip_family = this->server.accept_addr.sa.sa_family; + this_ethread()->schedule_imm(vc, EVENT_NONE, nullptr); + } +} + +void +QUICPacketHandlerIn::_schedule_event() +{ + if (this->_event == nullptr) { + this->_event = this_ethread()->schedule_in(this, OPENSSL_QUIC_EVENT_INTERVAL); + } +} + +void +QUICPacketHandlerOut::init(QUICNetVConnection * /* vc ATS_UNUSED */) +{ +} + +Continuation * +QUICPacketHandlerOut::_get_continuation() +{ + return this; +} + +void +QUICPacketHandlerOut::_recv_packet(int /* event ATS_UNUSED */, UDPPacket * /* udp_packet ATS_UNUSED */) +{ +} diff --git a/src/iocore/net/P_QUICNetProcessor.h b/src/iocore/net/P_QUICNetProcessor.h index 677b1fececb..21d2e91dc8a 100644 --- a/src/iocore/net/P_QUICNetProcessor.h +++ b/src/iocore/net/P_QUICNetProcessor.h @@ -43,7 +43,11 @@ #include "P_UnixNetProcessor.h" #include "iocore/net/quic/QUICConnectionTable.h" +#include "tscore/ink_config.h" + +#if TS_HAS_QUICHE #include +#endif class UnixNetVConnection; struct NetAccept; @@ -68,17 +72,21 @@ class QUICNetProcessor : public UnixNetProcessor Action *main_accept(Continuation *cont, SOCKET fd, AcceptOptions const &opt) override; +#if TS_HAS_QUICHE off_t quicPollCont_offset; +#endif protected: - NetAccept *createNetAccept(const NetProcessor::AcceptOptions &opt) override; + NetAccept *createNetAccept(NetProcessor::AcceptOptions const &opt) override; private: QUICNetProcessor(const QUICNetProcessor &); QUICNetProcessor &operator=(const QUICNetProcessor &); +#if TS_HAS_QUICHE QUICConnectionTable *_ctable = nullptr; quiche_config *_quiche_config = nullptr; +#endif }; extern QUICNetProcessor quic_NetProcessor; diff --git a/src/iocore/net/P_QUICNetVConnection.h b/src/iocore/net/P_QUICNetVConnection.h index 89af3685e4e..bb8064ecb49 100644 --- a/src/iocore/net/P_QUICNetVConnection.h +++ b/src/iocore/net/P_QUICNetVConnection.h @@ -46,11 +46,19 @@ #include "iocore/net/quic/QUICConnection.h" #include "iocore/net/quic/QUICConnectionTable.h" #include "iocore/net/quic/QUICContext.h" +#include "iocore/net/quic/QUICStream.h" #include "iocore/net/quic/QUICStreamManager.h" +#include "tscore/ink_config.h" #include "tscore/List.h" #include +#include +#include +#include + +#if TS_HAS_QUICHE #include +#endif class EThread; class QUICPacketHandler; @@ -66,16 +74,23 @@ class QUICNetVConnection : public UnixNetVConnection, public TLSCertSwitchSupport, public TLSEventSupport, public TLSBasicSupport, - public QUICSupport + public QUICSupport, + public QUICStreamIO { using super = UnixNetVConnection; ///< Parent type. public: QUICNetVConnection(); ~QUICNetVConnection(); +#if TS_HAS_OPENSSL_QUIC + void init(SSL *ssl, QUICPacketHandler *packet_handler); + void set_quic_endpoints(IpEndpoint const &local, IpEndpoint const &remote); + QUICConnectionErrorUPtr create_openssl_stream(uint64_t flags, QUICStreamId &new_stream_id); +#elif TS_HAS_QUICHE void init(QUICVersion version, QUICConnectionId peer_cid, QUICConnectionId original_cid, UDPConnection *, QUICPacketHandler *); void init(QUICVersion version, QUICConnectionId peer_cid, QUICConnectionId original_cid, QUICConnectionId first_cid, QUICConnectionId retry_cid, UDPConnection *, quiche_conn *, QUICPacketHandler *, QUICConnectionTable *ctable, SSL *); +#endif // Event handlers int acceptEvent(int event, Event *e); @@ -103,7 +118,7 @@ class QUICNetVConnection : public UnixNetVConnection, // NetVConnection int populate_protocol(std::string_view *results, int n) const override; - const char *protocol_contains(std::string_view tag) const override; + char const *protocol_contains(std::string_view tag) const override; // QUICConnection QUICStreamManager *stream_manager() override; @@ -120,20 +135,27 @@ class QUICNetVConnection : public UnixNetVConnection, QUICConnectionId initial_source_connection_id() const override; QUICConnectionId connection_id() const override; std::string_view cids() const override; - const QUICFiveTuple five_tuple() const override; + QUICFiveTuple const five_tuple() const override; uint32_t pmtu() const override; NetVConnectionContext_t direction() const override; QUICVersion negotiated_version() const override; std::string_view negotiated_application_name() const override; + void on_stream_updated() override; bool is_closed() const override; bool is_at_anti_amplification_limit() const override; bool is_address_validation_completed() const override; bool is_handshake_completed() const override; - void on_stream_updated() override; // QUICSupport QUICConnection *get_quic_connection() override; + // QUICStreamIO + int64_t read_stream(QUICStreamId stream_id, uint8_t *buf, size_t len, bool &fin, QUICStreamIO::ErrorCode &error_code) override; + bool stream_read_finished(QUICStreamId stream_id) override; + int64_t stream_write_capacity(QUICStreamId stream_id) override; + int64_t write_stream(QUICStreamId stream_id, uint8_t const *buf, size_t len, bool fin, + QUICStreamIO::ErrorCode &error_code) override; + // QUICNetVConnection int in_closed_queue = 0; @@ -167,11 +189,11 @@ class QUICNetVConnection : public UnixNetVConnection, in_port_t _get_local_port() override; // TLSSessionResumptionSupport - const IpEndpoint &_getLocalEndpoint() override; + IpEndpoint const &_getLocalEndpoint() override; // TLSCertSwitchSupport bool _isTryingRenegotiation() const override; - shared_SSL_CTX _lookupContextByName(const std::string &servername, SSLCertContextType ctxType) override; + shared_SSL_CTX _lookupContextByName(std::string const &servername, SSLCertContextType ctxType) override; shared_SSL_CTX _lookupContextByIP() override; // TLSEventSupport @@ -199,9 +221,16 @@ class QUICNetVConnection : public UnixNetVConnection, QUICConnectionId _initial_source_connection_id; // src cid used for Initial packet QUICConnectionId _quic_connection_id; // src cid in local - UDPConnection *_udp_con = nullptr; - quiche_conn *_quiche_con = nullptr; - QUICConnectionTable *_ctable = nullptr; + QUICConnectionTable *_ctable = nullptr; + +#if TS_HAS_OPENSSL_QUIC + std::unordered_map _openssl_streams; + std::string _cid_text; + std::string _negotiated_alpn; +#elif TS_HAS_QUICHE + UDPConnection *_udp_con = nullptr; + quiche_conn *_quiche_con = nullptr; +#endif void _bindSSLObject(); void _unbindSSLObject(); @@ -222,6 +251,15 @@ class QUICNetVConnection : public UnixNetVConnection, void _handle_write_ready(); void _handle_interval(); +#if TS_HAS_OPENSSL_QUIC + void _handle_openssl_events(); + void _accept_openssl_streams(); + void _process_openssl_streams(); + void _schedule_openssl_event(bool delay = true); + void _unschedule_openssl_event(); + bool _openssl_connection_closed() const; +#endif + void _propagate_event(int event); void _switch_to_established_state(); diff --git a/src/iocore/net/P_QUICPacketHandler.h b/src/iocore/net/P_QUICPacketHandler.h index f64cfc63105..47283c4058f 100644 --- a/src/iocore/net/P_QUICPacketHandler.h +++ b/src/iocore/net/P_QUICPacketHandler.h @@ -26,8 +26,13 @@ #include "P_NetAccept.h" #include "P_QUICClosedConCollector.h" #include "iocore/net/UDPConnection.h" +#include "tscore/ink_config.h" +#if TS_HAS_OPENSSL_QUIC +#include +#elif TS_HAS_QUICHE #include +#endif class QUICNetVConnection; class QUICConnectionTable; @@ -54,7 +59,11 @@ class QUICPacketHandler class QUICPacketHandlerIn : public NetAccept, public QUICPacketHandler { public: - QUICPacketHandlerIn(const NetProcessor::AcceptOptions &opt, QUICConnectionTable &ctable, quiche_config &config); +#if TS_HAS_OPENSSL_QUIC + QUICPacketHandlerIn(NetProcessor::AcceptOptions const &opt); +#elif TS_HAS_QUICHE + QUICPacketHandlerIn(NetProcessor::AcceptOptions const &opt, QUICConnectionTable &ctable, quiche_config &config); +#endif ~QUICPacketHandlerIn(); // NetAccept @@ -68,10 +77,21 @@ class QUICPacketHandlerIn : public NetAccept, public QUICPacketHandler Continuation *_get_continuation() override; private: +#if TS_HAS_OPENSSL_QUIC + SSL *_listener = nullptr; + Event *_event = nullptr; +#elif TS_HAS_QUICHE QUICConnectionTable &_ctable; quiche_config &_quiche_config; +#endif void _recv_packet(int event, UDPPacket *udpPacket) override; + +#if TS_HAS_OPENSSL_QUIC + bool _start_listener(); + void _service_listener(); + void _schedule_event(); +#endif }; class QUICPacketHandlerOut : public Continuation, public QUICPacketHandler diff --git a/src/iocore/net/QUICMultiCertConfigLoader.cc b/src/iocore/net/QUICMultiCertConfigLoader.cc index 2e7049643e8..556b581485c 100644 --- a/src/iocore/net/QUICMultiCertConfigLoader.cc +++ b/src/iocore/net/QUICMultiCertConfigLoader.cc @@ -24,9 +24,13 @@ #include "P_SSLCertLookup.h" #include "P_SSLConfig.h" #include "iocore/net/QUICMultiCertConfigLoader.h" +#include "iocore/net/TLSSNISupport.h" #include "iocore/net/quic/QUICConfig.h" #include "mgmt/config/ConfigContextDiags.h" +#include +#include + int QUICCertConfig::_config_id = 0; // @@ -88,8 +92,245 @@ QUICCertConfig::release(SSLCertLookup *lookup) SSL_CTX * QUICMultiCertConfigLoader::default_server_ssl_ctx() { - return quic_new_ssl_ctx(); + return quic_new_server_ssl_ctx(); +} + +#if TS_HAS_OPENSSL_QUIC +namespace +{ +DbgCtl dbg_ctl_quic{"quic"}; +} // end anonymous namespace + +namespace +{ +void +free_quic_sni_ex_data(void *, void *ptr, CRYPTO_EX_DATA *, int, long, void *) +{ + delete static_cast(ptr); +} + +int +quic_sni_ex_data_index() +{ + static int index = SSL_get_ex_new_index(0, nullptr, nullptr, nullptr, free_quic_sni_ex_data); + + return index; +} + +void +remember_quic_sni(SSL *ssl, std::string_view servername) +{ + if (ssl == nullptr || servername.empty()) { + return; + } + + auto const index = quic_sni_ex_data_index(); + if (index < 0) { + return; + } + + auto *stored = static_cast(SSL_get_ex_data(ssl, index)); + if (stored == nullptr) { + stored = new std::string; + if (SSL_set_ex_data(ssl, index, stored) != 1) { + delete stored; + return; + } + } + stored->assign(servername); + + if (auto *snis = TLSSNISupport::getInstance(ssl); snis != nullptr) { + snis->set_sni_server_name(servername); + } +} + +bool +apply_quic_sni_certificate(SSL *ssl, SSL_CTX *ctx) +{ + X509 *cert = SSL_CTX_get0_certificate(ctx); + EVP_PKEY *key = SSL_CTX_get0_privatekey(ctx); + + if (cert == nullptr && key == nullptr) { + return true; + } + if (cert == nullptr || key == nullptr) { + return false; + } + + if (SSL_use_certificate(ssl, cert) != 1 || SSL_use_PrivateKey(ssl, key) != 1) { + return false; + } + + STACK_OF(X509) *chain = nullptr; + if (SSL_CTX_get0_chain_certs(ctx, &chain) == 1 && chain != nullptr && SSL_set1_chain(ssl, chain) != 1) { + return false; + } + + return true; +} + +bool +select_quic_sni_context(SSL *ssl, std::string_view servername) +{ + if (servername.empty()) { + return true; + } + + QUICCertConfig::scoped_config lookup; + if (!lookup) { + Dbg(dbg_ctl_quic, "OpenSSL QUIC SNI context selection could not acquire cert lookup"); + return false; + } + + SSLCertContext *cert_context = lookup->find(std::string(servername)); + if (cert_context == nullptr) { + return true; + } + + shared_SSL_CTX const selected_ctx = cert_context->getCtx(); + if (selected_ctx == nullptr) { + Dbg(dbg_ctl_quic, "OpenSSL QUIC SNI context selection found null context for SNI '%.*s'", static_cast(servername.size()), + servername.data()); + return false; + } + + SSL_set_SSL_CTX(ssl, selected_ctx.get()); + if (!apply_quic_sni_certificate(ssl, selected_ctx.get())) { + Dbg(dbg_ctl_quic, "OpenSSL QUIC SNI context selection failed to apply SSL_CTX %p certificate to ssl=%p", selected_ctx.get(), + ssl); + return false; + } + + return true; +} + +std::string_view +servername_from_client_hello(SSL *ssl) +{ + unsigned char const *p = nullptr; + size_t remaining = 0; + if (SSL_client_hello_get0_ext(ssl, TLSEXT_TYPE_server_name, &p, &remaining) != 1 || remaining <= 2) { + return {}; + } + + size_t len = (static_cast(*p++) << 8); + len += *p++; + if (len + 2 != remaining) { + return {}; + } + remaining = len; + + if (remaining <= 3 || *p++ != TLSEXT_NAMETYPE_host_name) { + return {}; + } + --remaining; + + len = (static_cast(*p++) << 8); + len += *p++; + if (len + 2 > remaining) { + return {}; + } + + return {reinterpret_cast(p), len}; +} + +int +quic_client_hello_callback(SSL *ssl, int *, void *) +{ + auto const servername = servername_from_client_hello(ssl); + remember_quic_sni(ssl, servername); + + return select_quic_sni_context(ssl, servername) ? SSL_CLIENT_HELLO_SUCCESS : SSL_CLIENT_HELLO_ERROR; +} + +int +quic_servername_callback(SSL *ssl, int *, void *) +{ + char const *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + auto const sni = servername == nullptr ? std::string_view{} : std::string_view{servername}; + remember_quic_sni(ssl, sni); + + return select_quic_sni_context(ssl, sni) ? SSL_TLSEXT_ERR_OK : SSL_TLSEXT_ERR_ALERT_FATAL; +} +} // end anonymous namespace + +std::string_view +quic_sni_server_name(SSL *ssl) +{ + if (ssl == nullptr) { + return {}; + } + + auto const index = quic_sni_ex_data_index(); + if (index < 0) { + return {}; + } + + auto *stored = static_cast(SSL_get_ex_data(ssl, index)); + + return stored == nullptr ? std::string_view{} : std::string_view{*stored}; +} + +void +QUICMultiCertConfigLoader::_set_handshake_callbacks(SSL_CTX *ctx) +{ + Dbg(dbg_ctl_quic, "installing OpenSSL QUIC cert callback on SSL_CTX %p", ctx); + SSL_CTX_set_client_hello_cb(ctx, quic_client_hello_callback, nullptr); + SSL_CTX_set_tlsext_servername_callback(ctx, quic_servername_callback); + SSL_CTX_set_cert_cb( + ctx, + [](SSL *ssl, void *) -> int { + char const *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + auto const sni = servername == nullptr ? std::string_view{} : std::string_view{servername}; + remember_quic_sni(ssl, sni); + + return select_quic_sni_context(ssl, sni) ? 1 : 0; + }, + nullptr); +} + +namespace +{ +int +quic_alpn_select_callback(SSL * /* ssl ATS_UNUSED */, const unsigned char **out, unsigned char *outlen, const unsigned char *in, + unsigned int inlen, void * /* arg ATS_UNUSED */) +{ + static constexpr unsigned char server_protos[] = {2, 'h', '3', 5, 'h', '3', '-', '2', '9', 5, 'h', '3', '-', '2', '7'}; + + if (SSL_select_next_proto(const_cast(out), outlen, server_protos, sizeof(server_protos), in, inlen) == + OPENSSL_NPN_NEGOTIATED) { + return SSL_TLSEXT_ERR_OK; + } + + *out = nullptr; + *outlen = 0; + return SSL_TLSEXT_ERR_ALERT_FATAL; +} +} // end anonymous namespace + +bool +QUICMultiCertConfigLoader::_set_alpn_callback(SSL_CTX *ctx) +{ + SSL_CTX_set_alpn_select_cb(ctx, quic_alpn_select_callback, nullptr); + return true; +} + +bool +QUICMultiCertConfigLoader::_set_curves(SSL_CTX *ctx) +{ + QUICConfig::scoped_config params; + if (params->server_supported_groups() == nullptr) { + return true; + } + + if (!SSL_CTX_set1_groups_list(ctx, params->server_supported_groups())) { + Error("invalid QUIC groups list for server in records.config"); + return false; + } + + return true; } +#endif bool QUICMultiCertConfigLoader::_setup_session_cache(SSL_CTX * /* ctx ATS_UNUSED */) diff --git a/src/iocore/net/QUICNetVConnection.cc b/src/iocore/net/QUICNetVConnection.cc index 18a85913739..451b1fc1243 100644 --- a/src/iocore/net/QUICNetVConnection.cc +++ b/src/iocore/net/QUICNetVConnection.cc @@ -679,7 +679,7 @@ QUICNetVConnection::_handle_read_ready() [[maybe_unused]] QUICConnectionError err; stream = this->_stream_manager->create_stream(s, err); } - stream->receive_data(this->_quiche_con); + stream->receive_data(*this); } quiche_stream_iter_free(readable); } @@ -696,7 +696,7 @@ QUICNetVConnection::_handle_write_ready() [[maybe_unused]] QUICConnectionError err; stream = this->_stream_manager->create_stream(s, err); } - stream->send_data(this->_quiche_con); + stream->send_data(*this); } quiche_stream_iter_free(writable); } @@ -791,6 +791,31 @@ QUICNetVConnection::get_quic_connection() return static_cast(this); } +int64_t +QUICNetVConnection::read_stream(QUICStreamId stream_id, uint8_t *buf, size_t len, bool &fin, QUICStreamIO::ErrorCode &error_code) +{ + return quiche_conn_stream_recv(this->_quiche_con, stream_id, buf, len, &fin, &error_code); +} + +bool +QUICNetVConnection::stream_read_finished(QUICStreamId stream_id) +{ + return quiche_conn_stream_finished(this->_quiche_con, stream_id); +} + +int64_t +QUICNetVConnection::stream_write_capacity(QUICStreamId stream_id) +{ + return quiche_conn_stream_capacity(this->_quiche_con, stream_id); +} + +int64_t +QUICNetVConnection::write_stream(QUICStreamId stream_id, const uint8_t *buf, size_t len, bool fin, + QUICStreamIO::ErrorCode &error_code) +{ + return quiche_conn_stream_send(this->_quiche_con, stream_id, const_cast(buf), len, fin, &error_code); +} + void QUICNetVConnection::reenable(int event) { diff --git a/src/iocore/net/SSLSessionTicket.cc b/src/iocore/net/SSLSessionTicket.cc index 7d6674bd64c..58ff5fe46a8 100644 --- a/src/iocore/net/SSLSessionTicket.cc +++ b/src/iocore/net/SSLSessionTicket.cc @@ -53,6 +53,14 @@ ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, if (srs) { return srs->processSessionTicket(ssl, keyname, iv, cipher_ctx, hctx, enc); } else { +#if TS_HAS_OPENSSL_QUIC + if (SSL_is_quic(ssl) == 1) { + // OpenSSL native QUIC can parse client-offered tickets while servicing + // the listener, before ATS has accepted and bound a QUIC NetVC. + return 0; + } +#endif + // We could implement a default behavior that would have been done if this callback was not registered, but it's not necessary // at the moment because TLSSessionResumptionSupport is always available when the callback is registered. ink_assert(!"srs should be available"); diff --git a/src/iocore/net/TLSSNISupport.cc b/src/iocore/net/TLSSNISupport.cc index b6acd202b2b..55e0a673a4b 100644 --- a/src/iocore/net/TLSSNISupport.cc +++ b/src/iocore/net/TLSSNISupport.cc @@ -162,6 +162,16 @@ TLSSNISupport::set_sni_server_name(SSL *ssl, char const *name) return true; } +void +TLSSNISupport::set_sni_server_name(std::string_view name) +{ + if (name.empty()) { + this->_clear(); + } else { + this->_set_sni_server_name_buffer(name); + } +} + void TLSSNISupport::_clear() { diff --git a/src/iocore/net/quic/CMakeLists.txt b/src/iocore/net/quic/CMakeLists.txt index b3f1b7cb1d4..0d559c72dd5 100644 --- a/src/iocore/net/quic/CMakeLists.txt +++ b/src/iocore/net/quic/CMakeLists.txt @@ -26,16 +26,19 @@ add_library( QUICTypes.cc QUICIntUtil.cc QUICStream.cc - QUICStream.cc QUICStreamManager.cc QUICStreamAdapter.cc QUICStreamVCAdapter.cc ) add_library(ts::quic ALIAS quic) -target_link_libraries(quic PUBLIC ts::inkevent ts::inknet ts::tscore OpenSSL::Crypto OpenSSL::SSL quiche::quiche) +target_link_libraries(quic PUBLIC ts::inkevent ts::inknet ts::tscore OpenSSL::Crypto OpenSSL::SSL) + +if(TS_HAS_QUICHE) + target_link_libraries(quic PUBLIC quiche::quiche) +endif() -if(TS_OPENSSL_QUIC_TLS_CBS_COMPAT) +if(TS_OPENSSL_QUIC_TLS_CBS_COMPAT AND TS_HAS_QUICHE) add_library(openssl_quic_compat STATIC OpenSSLQuicCompat.cc) add_library(ts::openssl_quic_compat ALIAS openssl_quic_compat) target_link_libraries(openssl_quic_compat PUBLIC OpenSSL::SSL OpenSSL::Crypto) diff --git a/src/iocore/net/quic/QUICConfig.cc b/src/iocore/net/quic/QUICConfig.cc index 9905ae06889..7d9fe28927a 100644 --- a/src/iocore/net/quic/QUICConfig.cc +++ b/src/iocore/net/quic/QUICConfig.cc @@ -23,6 +23,9 @@ #include "iocore/net/quic/QUICConfig.h" +#if TS_HAS_OPENSSL_QUIC +#include +#endif #include #include "mgmt/config/ConfigContextDiags.h" @@ -54,6 +57,24 @@ quic_new_ssl_ctx() return ssl_ctx; } +SSL_CTX * +quic_new_server_ssl_ctx() +{ +#if TS_HAS_OPENSSL_QUIC + SSL_CTX *ssl_ctx = SSL_CTX_new(OSSL_QUIC_server_method()); + if (ssl_ctx == nullptr) { + return nullptr; + } + + SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION); + SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION); + + return ssl_ctx; +#else + return quic_new_ssl_ctx(); +#endif +} + /** ALPN and SNI should be set to SSL object with NETVC_OPTIONS **/ @@ -429,6 +450,7 @@ QUICConfigParams::disable_http_0_9() const return this->_disable_http_0_9; } +#if TS_HAS_QUICHE quiche_cc_algorithm QUICConfigParams::get_cc_algorithm() const { @@ -441,6 +463,7 @@ QUICConfigParams::get_cc_algorithm() const return QUICHE_CC_RENO; } } +#endif // // QUICConfig diff --git a/src/iocore/net/quic/QUICGlobals.cc b/src/iocore/net/quic/QUICGlobals.cc index b44366f8d2b..57d7b4d067d 100644 --- a/src/iocore/net/quic/QUICGlobals.cc +++ b/src/iocore/net/quic/QUICGlobals.cc @@ -54,8 +54,7 @@ QUIC::init() int QUIC::ssl_client_new_session([[maybe_unused]] SSL *ssl, [[maybe_unused]] SSL_SESSION *session) { -#if TS_HAS_QUICHE -#else +#if !TS_HAS_OPENSSL_QUIC && !TS_HAS_QUICHE QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); const char *session_file = qtls->session_file(); auto file = BIO_new_file(session_file, "w"); diff --git a/src/iocore/net/quic/QUICStream.cc b/src/iocore/net/quic/QUICStream.cc index cdc427c9af8..9f8de91fa0a 100644 --- a/src/iocore/net/quic/QUICStream.cc +++ b/src/iocore/net/quic/QUICStream.cc @@ -118,36 +118,43 @@ QUICStream::on_eos() } void -QUICStream::receive_data(quiche_conn *quiche_con) +QUICStream::receive_data(QUICStreamIO &stream_io) { uint8_t buf[4096]; - bool fin; - ssize_t read_len = 0; - [[maybe_unused]] ErrorCode error_code{0}; // Only set if QUICHE_ERR_STREAM_STOPPED(-15) or QUICHE_ERR_STREAM_RESET(-16) are - // returned by quiche_conn_stream_recv. + bool fin = false; + bool delivered = false; + ssize_t read_len = 0; + [[maybe_unused]] ErrorCode error_code{0}; - while ((read_len = quiche_conn_stream_recv(quiche_con, this->_id, buf, sizeof(buf), &fin, &error_code)) > 0) { + while ((read_len = stream_io.read_stream(this->_id, buf, sizeof(buf), fin, error_code)) > 0) { this->_adapter->write(this->_received_bytes, buf, read_len, fin); this->_received_bytes += read_len; + delivered = true; } - this->_has_no_more_data = quiche_conn_stream_finished(quiche_con, this->_id); - this->_adapter->encourge_read(); + if (read_len == 0 && fin && !this->_has_no_more_data) { + this->_adapter->write(this->_received_bytes, buf, 0, true); + delivered = true; + } + this->_has_no_more_data = stream_io.stream_read_finished(this->_id); + + if (delivered) { + this->_adapter->encourge_read(); + } } -void -QUICStream::send_data(quiche_conn *quiche_con) +int64_t +QUICStream::send_data(QUICStreamIO &stream_io) { bool fin = false; ssize_t len = 0; - [[maybe_unused]] ErrorCode error_code{0}; // Only set if QUICHE_ERR_STREAM_STOPPED(-15) or QUICHE_ERR_STREAM_RESET(-16) are - // returned by quiche_conn_stream_send. - size_t written_this_event = 0; + [[maybe_unused]] ErrorCode error_code{0}; + size_t written_this_event = 0; while (written_this_event < MAX_STREAM_SEND_BYTES_PER_EVENT) { - len = quiche_conn_stream_capacity(quiche_con, this->_id); + len = stream_io.stream_write_capacity(this->_id); if (len <= 0) { - return; + return written_this_event; } if (!this->_pending_send_block) { @@ -156,13 +163,14 @@ QUICStream::send_data(quiche_conn *quiche_con) if (!this->_pending_send_block) { if (!this->_sent_fin && this->_adapter->is_eos() && this->_adapter->total_len() == this->_sent_bytes) { static constexpr uint8_t empty_data = 0; - ssize_t written_len = quiche_conn_stream_send(quiche_con, this->_id, &empty_data, 0, true, &error_code); + ssize_t written_len = stream_io.write_stream(this->_id, &empty_data, 0, true, error_code); if (written_len >= 0) { this->_sent_fin = true; + return written_this_event + static_cast(written_len); } } this->_adapter->encourge_write(); - return; + return written_this_event; } this->_pending_send_fin = this->_adapter->total_len() == this->_sent_bytes + this->_pending_send_block->size(); } @@ -177,28 +185,30 @@ QUICStream::send_data(quiche_conn *quiche_con) } if (block->size() > 0 || fin) { - ssize_t written_len = quiche_conn_stream_send(quiche_con, this->_id, reinterpret_cast(block->start()), - block->size(), fin, &error_code); + ssize_t written_len = + stream_io.write_stream(this->_id, reinterpret_cast(block->start()), block->size(), fin, error_code); if (written_len >= 0) { this->_adapter->consume(written_len); this->_sent_bytes += written_len; - written_this_event += written_len; + written_this_event += static_cast(written_len); if (written_len >= block->size()) { this->_pending_send_block = nullptr; this->_pending_send_fin = false; this->_sent_fin = fin; } else { block->consume(written_len); - return; + return written_this_event; } if (!this->has_data_to_send()) { this->_adapter->encourge_write(); - return; + return written_this_event; } continue; } } this->_adapter->encourge_write(); - return; + return written_this_event; } + + return written_this_event; } diff --git a/src/iocore/net/quic/QUICStreamManager.cc b/src/iocore/net/quic/QUICStreamManager.cc index b3b82692fa4..4fbeb7b870f 100644 --- a/src/iocore/net/quic/QUICStreamManager.cc +++ b/src/iocore/net/quic/QUICStreamManager.cc @@ -113,14 +113,38 @@ QUICStreamManager::create_stream(QUICStreamId stream_id, QUICConnectionError & / } QUICConnectionErrorUPtr -QUICStreamManager::create_uni_stream(QUICStreamId /* new_stream_id ATS_UNUSED */) +QUICStreamManager::create_uni_stream(QUICStreamId &new_stream_id) { + if (this->_next_local_uni_stream_id == 0) { + this->_next_local_uni_stream_id = + static_cast(this->_context->connection_info()->direction() == NET_VCONNECTION_IN ? QUICStreamType::SERVER_UNI : + QUICStreamType::CLIENT_UNI); + } + + new_stream_id = this->_next_local_uni_stream_id; + this->_next_local_uni_stream_id += 4; + + [[maybe_unused]] QUICConnectionError err; + this->create_stream(new_stream_id, err); + return nullptr; } QUICConnectionErrorUPtr -QUICStreamManager::create_bidi_stream(QUICStreamId /* new_stream_id ATS_UNUSED */) +QUICStreamManager::create_bidi_stream(QUICStreamId &new_stream_id) { + if (this->_next_local_bidi_stream_id == 0) { + this->_next_local_bidi_stream_id = + static_cast(this->_context->connection_info()->direction() == NET_VCONNECTION_IN ? QUICStreamType::SERVER_BIDI : + QUICStreamType::CLIENT_BIDI); + } + + new_stream_id = this->_next_local_bidi_stream_id; + this->_next_local_bidi_stream_id += 4; + + [[maybe_unused]] QUICConnectionError err; + this->create_stream(new_stream_id, err); + return nullptr; } diff --git a/src/proxy/http3/Http3DataFramer.cc b/src/proxy/http3/Http3DataFramer.cc index f9e52be630d..17d626b2518 100644 --- a/src/proxy/http3/Http3DataFramer.cc +++ b/src/proxy/http3/Http3DataFramer.cc @@ -53,5 +53,5 @@ Http3DataFramer::generate_frame() bool Http3DataFramer::is_done() const { - return this->_source_vio->ntodo() == 0; + return this->_transaction->is_response_header_sent() && this->_source_vio->ntodo() == 0; } diff --git a/src/proxy/http3/Http3Frame.cc b/src/proxy/http3/Http3Frame.cc index 6fc7d5b5441..2c15ee03ef3 100644 --- a/src/proxy/http3/Http3Frame.cc +++ b/src/proxy/http3/Http3Frame.cc @@ -266,6 +266,12 @@ Http3DataFrame::payload_length() const return this->_length; } +bool +Http3DataFrame::_parse() +{ + return this->_reader->read_avail() >= static_cast(this->_length); +} + IOBufferReader * Http3DataFrame::data() const { diff --git a/src/proxy/http3/Http3HeaderFramer.cc b/src/proxy/http3/Http3HeaderFramer.cc index f514fda7b8f..94410be0227 100644 --- a/src/proxy/http3/Http3HeaderFramer.cc +++ b/src/proxy/http3/Http3HeaderFramer.cc @@ -73,6 +73,9 @@ Http3HeaderFramer::generate_frame() if (this->_header_block_len == this->_header_block_wrote) { this->_sent_all_data = true; + if (this->_is_current_header_final) { + this->_is_final_header_sent = true; + } } return frame; } else { @@ -86,6 +89,31 @@ Http3HeaderFramer::is_done() const return this->_sent_all_data; } +void +Http3HeaderFramer::reset() +{ + this->_header.destroy(); + if (this->_header_block != nullptr) { + free_MIOBuffer(this->_header_block); + } + + this->_header_block = nullptr; + this->_header_block_reader = nullptr; + this->_header_block_len = 0; + this->_header_block_wrote = 0; + this->_sent_all_data = false; + this->_is_current_header_final = false; + + http_parser_clear(&this->_http_parser); + http_parser_init(&this->_http_parser); +} + +bool +Http3HeaderFramer::is_final_header_sent() const +{ + return this->_is_final_header_sent; +} + void Http3HeaderFramer::_generate_header_block() { @@ -104,6 +132,8 @@ Http3HeaderFramer::_generate_header_block() switch (parse_result) { case ParseResult::DONE: { + this->_is_current_header_final = + this->_transaction->direction() == NET_VCONNECTION_OUT || !this->_header.expect_final_response(); this->_hvc.convert(this->_header, 1, 3); this->_header_block = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); diff --git a/src/proxy/http3/Http3Transaction.cc b/src/proxy/http3/Http3Transaction.cc index e1f336f25c9..084ea0d566f 100644 --- a/src/proxy/http3/Http3Transaction.cc +++ b/src/proxy/http3/Http3Transaction.cc @@ -463,7 +463,8 @@ HQTransaction::_is_write_buffer_flushed() SCOPED_MUTEX_LOCK(lock, this->_info.write_vio->mutex, this_ethread()); - return this->_info.write_vio->ntodo() == 0; + IOBufferReader *reader = this->_info.write_vio->get_reader(); + return reader == nullptr || reader->read_avail() == 0; } bool @@ -539,6 +540,16 @@ Http3Transaction::~Http3Transaction() this->_data_handler = nullptr; } +VIO * +Http3Transaction::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) +{ + if (c != nullptr && nbytes > 0 && buf != nullptr) { + this->_header_framer->reset(); + } + + return super::do_io_write(c, nbytes, buf, owner); +} + int Http3Transaction::state_stream_open(int event, Event *edata) { @@ -675,7 +686,7 @@ Http3Transaction::on_header_decode_complete() bool Http3Transaction::is_response_header_sent() const { - return this->_header_framer->is_done(); + return this->_header_framer->is_final_header_sent(); } bool diff --git a/src/proxy/http3/test/test_Http3FrameDispatcher.cc b/src/proxy/http3/test/test_Http3FrameDispatcher.cc index 87116fa4728..dec273aff4a 100644 --- a/src/proxy/http3/test/test_Http3FrameDispatcher.cc +++ b/src/proxy/http3/test/test_Http3FrameDispatcher.cc @@ -114,7 +114,7 @@ TEST_CASE("Http3FrameHandler dispatch", "[http3]") total_nread += nread; CHECK(!error); } - CHECK(handler.total_frame_received == 5); + CHECK(handler.total_frame_received == 1); CHECK(total_nread == 19); } } diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index e77c2926d3e..9b1e6c331c1 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -160,6 +160,7 @@ produce_features(bool json) print_feature("TS_USE_HWLOC", TS_USE_HWLOC, json); print_feature("TS_USE_TLS13", TS_USE_TLS13, json); print_feature("TS_USE_QUIC", TS_USE_QUIC, json); + print_feature("TS_HAS_OPENSSL_QUIC", TS_HAS_OPENSSL_QUIC, json); print_feature("TS_HAS_QUICHE", TS_HAS_QUICHE, json); print_feature("TS_HAS_SO_PEERCRED", TS_HAS_SO_PEERCRED, json); print_feature("TS_USE_REMOTE_UNWINDING", TS_USE_REMOTE_UNWINDING, json); diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index fd2f837a8a8..5ee457b5be0 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -77,7 +77,7 @@ extern "C" int plock(int); #include "Crash.h" #include "tscore/signals.h" #include "../iocore/net/P_Net.h" -#if TS_HAS_QUICHE +#if TS_USE_QUIC == 1 #include "../iocore/net/P_QUICNetProcessor.h" #endif #include "../iocore/net/P_UDPNet.h" diff --git a/tests/gold_tests/h3/h3_go_client.test.py b/tests/gold_tests/h3/h3_go_client.test.py index d99c1979e1e..5feee435000 100644 --- a/tests/gold_tests/h3/h3_go_client.test.py +++ b/tests/gold_tests/h3/h3_go_client.test.py @@ -103,7 +103,7 @@ def _configure_client(self): tr.Setup.Copy("go_h3_client") tr.Processes.Default.StartBefore(self._server) tr.Processes.Default.StartBefore(self._ts) - tr.Processes.Default.Env['GOFLAGS'] = '-mod=readonly' + tr.Processes.Default.Env['GOFLAGS'] = '-mod=readonly -modcacherw' tr.Processes.Default.Env['GOCACHE'] = os.path.join(tr.RunDirectory, 'gocache') tr.Processes.Default.Env['GOMODCACHE'] = os.path.join(tr.RunDirectory, 'gomodcache') tr.Processes.Default.Env['GOTOOLCHAIN'] = 'local' diff --git a/tests/gold_tests/headers/via.test.py b/tests/gold_tests/headers/via.test.py index 841736340ec..0a9f527cf07 100644 --- a/tests/gold_tests/headers/via.test.py +++ b/tests/gold_tests/headers/via.test.py @@ -115,7 +115,7 @@ tr.StillRunningAfter = ts # HTTP 3 - if Condition.HasATSFeature('TS_HAS_QUICHE') and Condition.HasCurlFeature('http3'): + if Condition.HasATSFeature('TS_USE_QUIC') and Condition.HasCurlFeature('http3'): tr = Test.AddTestRun() tr.MakeCurlCommand( '--verbose --ipv4 --http3 --insecure --header "Host: www.example.com" https://localhost:{}'.format(