From eea61721ef8c6a55aff8238829cd5094cd6f3e08 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 10 Sep 2026 00:11:14 -0400 Subject: [PATCH 01/35] refactor: Remove unreferenced function Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 15 --------------- src/AsyncTCP.h | 1 - 2 files changed, 16 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 5a172e18..94187892 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1050,21 +1050,6 @@ void AsyncClient::_error(int8_t err) { } } -// In LwIP Thread -int8_t AsyncClient::_lwip_fin(tcp_pcb *pcb, int8_t err) { - if (!_pcb || pcb != _pcb) { - async_tcp_log_d("0x%08" PRIx32 " != 0x%08" PRIx32, (uint32_t)pcb, (uint32_t)_pcb); - return ERR_OK; - } - _reset_tcp_callbacks(_pcb, this); - if (tcp_close(_pcb) != ERR_OK) { - tcp_abort(_pcb); - } - _pcb = NULL; - return ERR_OK; -} - -// In Async Thread int8_t AsyncClient::_fin(tcp_pcb *pcb, int8_t err) { close(); return ERR_OK; diff --git a/src/AsyncTCP.h b/src/AsyncTCP.h index 536008e3..ab1eaf63 100644 --- a/src/AsyncTCP.h +++ b/src/AsyncTCP.h @@ -306,7 +306,6 @@ class AsyncClient { int8_t _poll(tcp_pcb *pcb); int8_t _sent(tcp_pcb *pcb, uint16_t len); int8_t _fin(tcp_pcb *pcb, int8_t err); - int8_t _lwip_fin(tcp_pcb *pcb, int8_t err); void _dns_found(ip_addr_t *ipaddr); }; From b1af47e00e6caacd2770bc9f525aff2e393a6311 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 10 Sep 2026 00:11:49 -0400 Subject: [PATCH 02/35] fix: Null-guard setKeepAlive Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 94187892..c1e16d23 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1195,6 +1195,9 @@ bool AsyncClient::getNoDelay() { } void AsyncClient::setKeepAlive(uint32_t ms, uint8_t cnt) { + if (!_pcb) { + return; + } if (ms != 0) { _pcb->so_options |= SOF_KEEPALIVE; // Turn on TCP Keepalive for the given pcb // Set the time between keepalive messages in milli-seconds From e9d02bc3f2bae388b29169be6d2b0255966b7a07 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 13 Sep 2026 09:59:54 -0400 Subject: [PATCH 03/35] fix: Initialise the list's element count Co-Authored-By: Claude Opus 5 --- src/AsyncTCPSimpleIntrusiveList.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AsyncTCPSimpleIntrusiveList.h b/src/AsyncTCPSimpleIntrusiveList.h index 589268ce..919d1496 100644 --- a/src/AsyncTCPSimpleIntrusiveList.h +++ b/src/AsyncTCPSimpleIntrusiveList.h @@ -29,7 +29,7 @@ template class SimpleIntrusiveList { public: // Object methods - SimpleIntrusiveList() : _head(nullptr), _tail(&_head) {} + SimpleIntrusiveList() : _head(nullptr), _tail(&_head), _size(0) {} ~SimpleIntrusiveList() { clear(); } From 498bb30a13974735169c64c2bd57b75688f932e4 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 10 Sep 2026 00:11:49 -0400 Subject: [PATCH 04/35] fix: Close the bound pcb when AsyncServer fails to listen LwIP frees the original only on success, so the port stayed reserved for the rest of the boot and every later begin() on it failed with ERR_USE. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index c1e16d23..c90df59f 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1532,11 +1532,19 @@ void AsyncServer::begin() { } static uint8_t backlog = 5; - _pcb = _tcp_listen_with_backlog(_pcb, backlog); - if (!_pcb) { + tcp_pcb *listen_pcb = _tcp_listen_with_backlog(_pcb, backlog); + if (!listen_pcb) { + // LwIP frees the original pcb only on success; ours is still bound, and holds + // the port reserved until we release it. async_tcp_log_e("listen_pcb == NULL"); + tcp_core_guard tcg; + if (tcp_close(_pcb) != ERR_OK) { + tcp_abort(_pcb); + } + _pcb = nullptr; return; } + _pcb = listen_pcb; tcp_core_guard tcg; tcp_arg(_pcb, (void *)this); tcp_accept(_pcb, &AsyncTCP_detail::tcp_accept); From 7befe25a549c71b0fc7794596ba9d21fa6fd9d1f Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 10 Sep 2026 00:12:16 -0400 Subject: [PATCH 05/35] fix: Report an aborted connection to LwIP as aborted ERR_OK after tcp_abort() sends LwIP into tcp_receive() on a freed pcb; any error but ERR_ABRT makes it abort a second time. Allocating the event before the AsyncClient also keeps a client off the failure path, which we could not destroy from this thread anyway. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 62 ++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index c90df59f..c38d82e3 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1564,45 +1564,45 @@ void AsyncServer::end() { // runs on LwIP thread int8_t AsyncTCP_detail::tcp_accept(void *arg, tcp_pcb *pcb, int8_t err) { + // Note LwIP return value semantics: + // - ERR_OK: delivery successful + // - ERR_ABRT: the connection was aborted, discard your reference + // - Anything else: abort the connection + if (!pcb) { async_tcp_log_e("_accept failed: pcb is NULL"); return ERR_ABRT; } + auto server = reinterpret_cast(arg); - if (server->_connect_cb) { - AsyncClient *c = new (std::nothrow) AsyncClient(pcb); - if (c && c->pcb()) { - c->setNoDelay(server->_noDelay); - - lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_ACCEPT, c}; - if (e) { - e->accept.server = server; - - queue_mutex_guard guard; - _prepend_async_event(e); - return ERR_OK; // success - } + if (!server->_connect_cb) { + async_tcp_log_e("_accept failed: no onConnect callback"); + tcp_abort(pcb); + return ERR_ABRT; + } - // Couldn't allocate accept event - // We can't let the client object call in to close, as we're on the LWIP thread; it could deadlock trying to RPC to itself - c->_pcb = nullptr; - tcp_abort(pcb); - async_tcp_log_e("_accept failed: couldn't accept client"); - return ERR_ABRT; - } - if (c) { - // Couldn't complete setup - // pcb has already been aborted - delete c; - pcb = nullptr; - async_tcp_log_e("_accept failed: couldn't complete setup"); - return ERR_ABRT; - } + // Queue the accept event for processing by the async task. + lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_ACCEPT, nullptr}; + if (!e) { + async_tcp_log_e("_accept failed: couldn't allocate event"); + tcp_abort(pcb); + return ERR_ABRT; + } + + AsyncClient *c = new (std::nothrow) AsyncClient(pcb); + if (!c) { + _free_event(e); async_tcp_log_e("_accept failed: couldn't allocate client"); - } else { - async_tcp_log_e("_accept failed: no onConnect callback"); + tcp_abort(pcb); + return ERR_ABRT; } - tcp_abort(pcb); + + c->setNoDelay(server->_noDelay); + e->client = c; + e->accept.server = server; + + queue_mutex_guard guard; + _prepend_async_event(e); return ERR_OK; } From ef9750f724199c243131fd1983abb0168181a41b Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 10 Sep 2026 00:12:16 -0400 Subject: [PATCH 06/35] fix: Set up and tear down the listening socket in one transaction each begin() entered the LwIP context five times over, so a pcb could be seen bound but not listening, and _pcb reached callers in stages. end() was the same in reverse, letting an accept queue its event after the sweep - _remove_events_for_client() keys on the client, so ~AsyncServer() then called _accepted() on freed memory. end() disposes of the detached chain outside its transaction, because that closes pcbs. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 212 +++++++++++++++++++++++++++-------------------- 1 file changed, 122 insertions(+), 90 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index c38d82e3..f84d380b 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -108,6 +108,9 @@ struct tcp_core_guard { */ #define CONFIG_ASYNC_TCP_POLL_TIMER 1 +// Depth of the pending-connection queue handed to tcp_listen_with_backlog() +#define ASYNCTCP_LISTEN_BACKLOG 5 + /* * TCP/IP Event Task * */ @@ -295,6 +298,36 @@ static size_t _remove_events_for_client(AsyncClient *client) { return count; }; +// Called from AsyncServer::end() on the application thread, so it can destroy any +// clients that were accepted but never delivered to onClient. +/* + Detach this server's queued accept events. Safe to call inside the LwIP context, and + that is where it belongs: the accept callback that would add more runs on the same + thread, so once this returns from there, nothing further can arrive. + + Disposing of the chain is a separate step because it cannot be done from that context - + destroying an accepted client closes its pcb, which is an api call. +*/ +static lwip_tcp_event_packet_t *_detach_events_for_server(AsyncServer *server) { + queue_mutex_guard guard; + return _async_queue.remove_if([=](lwip_tcp_event_packet_t &pkt) { + return (pkt.event == LWIP_TCP_ACCEPT) && (pkt.accept.server == server); + }); +} + +// Must not run in LwIP context; see above. +static size_t _dispose_event_chain(lwip_tcp_event_packet_t *chain) { + size_t count = 0; + while (chain) { + ++count; + auto t = chain; + chain = t->next; + delete t->client; + _free_event(t); + } + return count; +} + void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { if (e->client == NULL) { // do nothing when arg is NULL @@ -568,11 +601,6 @@ typedef struct { uint16_t port; tcp_connected_fn cb; } connect; - struct { - ip_addr_t *addr; - uint16_t port; - } bind; - uint8_t backlog; }; } tcp_api_call_t; @@ -724,50 +752,6 @@ static esp_err_t _tcp_connect(tcp_pcb *pcb, ip_addr_t *addr, uint16_t port, tcp_ return msg.err; } -static err_t _tcp_bind_api(struct tcpip_api_call_data *api_call_msg) { - tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; - tcp_pcb *pcb = *msg->pcb; - msg->err = tcp_bind(pcb, msg->bind.addr, msg->bind.port); - if (msg->err != ERR_OK) { - // Close the pcb on behalf of the server without an extra round-trip through the LwIP lock - if (tcp_close(pcb) != ERR_OK) { - tcp_abort(pcb); - } - *msg->pcb = nullptr; // PCB is now owned by LwIP - } - return msg->err; -} - -static esp_err_t _tcp_bind(tcp_pcb **pcb, ip_addr_t *addr, uint16_t port) { - if (!pcb || !*pcb) { - return ESP_FAIL; - } - tcp_api_call_t msg; - msg.pcb = pcb; - msg.bind.addr = addr; - msg.bind.port = port; - tcpip_api_call(_tcp_bind_api, (struct tcpip_api_call_data *)&msg); - return msg.err; -} - -static err_t _tcp_listen_api(struct tcpip_api_call_data *api_call_msg) { - tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; - msg->err = 0; - *msg->pcb = tcp_listen_with_backlog(*msg->pcb, msg->backlog); - return msg->err; -} - -static tcp_pcb *_tcp_listen_with_backlog(tcp_pcb *pcb, uint8_t backlog) { - if (!pcb) { - return NULL; - } - tcp_api_call_t msg; - msg.pcb = &pcb; - msg.backlog = backlog ? backlog : 0xFF; - tcpip_api_call(_tcp_listen_api, (struct tcpip_api_call_data *)&msg); - return pcb; -} - /* Async TCP Client */ @@ -1509,57 +1493,105 @@ void AsyncServer::begin() { async_tcp_log_e("failed to start task"); return; } - int8_t err; - { - tcp_core_guard tcg; + + /* + Allocate, bind, listen and hook up the accept callback as one transaction. This used + to be five separate entries into the LwIP context, three of them taking only the core + lock - which does not exist without CONFIG_LWIP_TCPIP_CORE_LOCKING - so the LwIP thread + could see a pcb bound but not listening, or listening with no accept callback and no + argument, and _pcb itself was published in stages. + */ + struct begin_call { + struct tcpip_api_call_data call; + AsyncServer *server; + err_t err; + } args; + args.server = this; + args.err = ERR_OK; + + tcpip_api_call( + +[](struct tcpip_api_call_data *c) -> err_t { + begin_call *a = reinterpret_cast(c); + AsyncServer *server = a->server; + #if LWIP_IPV4 && LWIP_IPV6 - _pcb = tcp_new_ip_type(_addr.type); + tcp_pcb *pcb = tcp_new_ip_type(server->_addr.type); #else - _pcb = tcp_new_ip_type(IPADDR_TYPE_ANY); + tcp_pcb *pcb = tcp_new_ip_type(IPADDR_TYPE_ANY); #endif - } - if (!_pcb) { - async_tcp_log_e("_pcb == NULL"); - return; - } + if (!pcb) { + a->err = ERR_MEM; + return a->err; + } - err = _tcp_bind(&_pcb, &_addr, _port); + a->err = tcp_bind(pcb, &server->_addr, server->_port); + if (a->err != ERR_OK) { + // Never registered, so nothing can reach it; dispose of it here. + if (tcp_close(pcb) != ERR_OK) { + tcp_abort(pcb); + } + return a->err; + } - if (err != ERR_OK) { - // pcb was closed by _tcp_bind - async_tcp_log_e("bind error: %d", err); - return; - } + tcp_pcb *listen_pcb = tcp_listen_with_backlog(pcb, ASYNCTCP_LISTEN_BACKLOG); + if (!listen_pcb) { + // LwIP frees the original only on success; ours is still bound, and holds the + // port reserved until we release it. + if (tcp_close(pcb) != ERR_OK) { + tcp_abort(pcb); + } + a->err = ERR_MEM; + return a->err; + } - static uint8_t backlog = 5; - tcp_pcb *listen_pcb = _tcp_listen_with_backlog(_pcb, backlog); - if (!listen_pcb) { - // LwIP frees the original pcb only on success; ours is still bound, and holds - // the port reserved until we release it. - async_tcp_log_e("listen_pcb == NULL"); - tcp_core_guard tcg; - if (tcp_close(_pcb) != ERR_OK) { - tcp_abort(_pcb); - } - _pcb = nullptr; - return; + // Published last, and with its callbacks already attached: an accept cannot arrive + // against a server that is not ready for it. + tcp_arg(listen_pcb, server); + tcp_accept(listen_pcb, &AsyncTCP_detail::tcp_accept); + server->_pcb = listen_pcb; + a->err = ERR_OK; + return a->err; + }, + &args.call + ); + + if (args.err != ERR_OK) { + async_tcp_log_e("begin failed: %d", (int)args.err); } - _pcb = listen_pcb; - tcp_core_guard tcg; - tcp_arg(_pcb, (void *)this); - tcp_accept(_pcb, &AsyncTCP_detail::tcp_accept); } void AsyncServer::end() { - if (_pcb) { - tcp_core_guard tcg; - tcp_arg(_pcb, NULL); - tcp_accept(_pcb, NULL); - if (tcp_close(_pcb) != ERR_OK) { - tcp_abort(_pcb); - } - _pcb = NULL; - } + // The mirror of begin(): stop listening and take the queue with it, in one transaction. + // Detaching the events is safe here because the accept callback that would add more runs + // on this same thread. Disposing of them is not - destroying an accepted client closes + // its pcb, which is an api call - so the chain goes back out to the caller. + struct end_call { + struct tcpip_api_call_data call; + AsyncServer *server; + lwip_tcp_event_packet_t *events; + } args; + args.server = this; + args.events = nullptr; + + tcpip_api_call( + +[](struct tcpip_api_call_data *c) -> err_t { + end_call *a = reinterpret_cast(c); + AsyncServer *server = a->server; + if (server->_pcb) { + tcp_arg(server->_pcb, NULL); + tcp_accept(server->_pcb, NULL); + if (tcp_close(server->_pcb) != ERR_OK) { + tcp_abort(server->_pcb); + } + server->_pcb = NULL; // PCB is now the property of LwIP + } + a->events = _detach_events_for_server(server); + return ERR_OK; + }, + &args.call + ); + + _dispose_event_chain(args.events); } // runs on LwIP thread From 6107cca0f2b783e928ea5261aedc28d34ae88779 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 10 Sep 2026 00:13:47 -0400 Subject: [PATCH 07/35] fix: Reset per-connection state when a pcb is adopted _adopt() is the single point where a pcb is taken on. connect() adopts immediately rather than at _connected(): the pcb used to sit unreferenced until the connect completed, so close(), abort() and the destructor could not find it. - Fold _close() into close(); the ackLater() flush now covers the RX timeout and destructor paths. - Dispose of the pcb when tcp_connect() fails; LwIP neither frees nor registers it. - Skip poll processing while connecting, now that polls can reach an unestablished client. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 102 +++++++++++++++++++++++++++++++---------------- src/AsyncTCP.h | 2 +- 2 files changed, 69 insertions(+), 35 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index f84d380b..3009fbb8 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -735,16 +735,34 @@ static esp_err_t _tcp_abort(tcp_pcb **pcb, AsyncClient *client) { static err_t _tcp_connect_api(struct tcpip_api_call_data *api_call_msg) { tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; - msg->err = tcp_connect(*msg->pcb, msg->connect.addr, msg->connect.port, msg->connect.cb); + tcp_pcb *pcb = *msg->pcb; + // Non-zero only if the caller handed us an already-bound pcb + u16_t bound_port = pcb->local_port; + + msg->err = tcp_connect(pcb, msg->connect.addr, msg->connect.port, msg->connect.cb); + if (msg->err != ERR_OK) { + // Failure - clean up + _reset_tcp_callbacks(pcb, nullptr); + if (bound_port == 0) { + // tcp_connect() may have taken a local port without adding the pcb to + // tcp_bound_pcbs. Clear it, or tcp_close() will try to remove the pcb from a + // list it was never on - which trips an assert on an empty list. + pcb->local_port = 0; + } + if (tcp_close(pcb) != ERR_OK) { + tcp_abort(pcb); + } + *msg->pcb = nullptr; // PCB is now the property of LwIP + } return msg->err; } -static esp_err_t _tcp_connect(tcp_pcb *pcb, ip_addr_t *addr, uint16_t port, tcp_connected_fn cb) { - if (!pcb) { +static esp_err_t _tcp_connect(tcp_pcb **pcb, ip_addr_t *addr, uint16_t port, tcp_connected_fn cb) { + if (!pcb || !*pcb) { return ESP_FAIL; } tcp_api_call_t msg; - msg.pcb = &pcb; // cannot be invalidated by LwIP at this point + msg.pcb = pcb; msg.connect.addr = addr; msg.connect.port = port; msg.connect.cb = cb; @@ -757,19 +775,17 @@ static esp_err_t _tcp_connect(tcp_pcb *pcb, ip_addr_t *addr, uint16_t port, tcp_ */ AsyncClient::AsyncClient(tcp_pcb *pcb) - : _connect_cb(0), _connect_cb_arg(0), _discard_cb(0), _discard_cb_arg(0), _sent_cb(0), _sent_cb_arg(0), _error_cb(0), _error_cb_arg(0), _recv_cb(0), - _recv_cb_arg(0), _pb_cb(0), _pb_cb_arg(0), _timeout_cb(0), _timeout_cb_arg(0), _poll_cb(0), _poll_cb_arg(0), _ack_pcb(true), _tx_last_packet(0), - _rx_timeout(0), _rx_last_ack(0), _ack_timeout(CONFIG_ASYNC_TCP_MAX_ACK_TIME), _connect_port(0) { - _pcb = pcb; - if (_pcb) { - _rx_last_packet = millis(); - _bind_tcp_callbacks(_pcb, this); + : _pcb(nullptr), _connect_cb(0), _connect_cb_arg(0), _discard_cb(0), _discard_cb_arg(0), _sent_cb(0), _sent_cb_arg(0), _error_cb(0), _error_cb_arg(0), + _recv_cb(0), _recv_cb_arg(0), _pb_cb(0), _pb_cb_arg(0), _timeout_cb(0), _timeout_cb_arg(0), _poll_cb(0), _poll_cb_arg(0), _ack_pcb(true), + _tx_last_packet(0), _rx_ack_len(0), _rx_last_packet(0), _rx_timeout(0), _rx_last_ack(0), _ack_timeout(CONFIG_ASYNC_TCP_MAX_ACK_TIME), _connect_port(0) { + if (pcb) { + _adopt(pcb); } } AsyncClient::~AsyncClient() { if (_pcb) { - _close(); + close(); } } @@ -851,11 +867,17 @@ bool AsyncClient::connect(ip_addr_t addr, uint16_t port) { async_tcp_log_e("pcb == NULL"); return false; } - _bind_tcp_callbacks(pcb, this); + // Take ownership now: until _connected() arrives the pcb is only reachable + // through us, so close(), abort() and ~AsyncClient() must be able to find it. + _adopt(pcb); } - esp_err_t err = _tcp_connect(pcb, &addr, port, (tcp_connected_fn)&_tcp_connected); - return err == ESP_OK; + if (_tcp_connect(&_pcb, &addr, port, (tcp_connected_fn)&_tcp_connected) == ESP_OK) { + return true; + } + // _pcb is now NULL; no callbacks are raised, as we are returning failure + async_tcp_log_d("connect failed"); + return false; } #ifdef ARDUINO @@ -922,10 +944,17 @@ bool AsyncClient::connect(const char *host, uint16_t port) { } void AsyncClient::close() { - if (_pcb) { + if (_pcb && _rx_ack_len) { + // Ack anything withheld by ackLater() before we drop the connection _tcp_recved(&_pcb, _rx_ack_len); } - _close(); + _rx_ack_len = 0; + int8_t err = _tcp_close(&_pcb, this); + // _pcb is now NULL + if ((err == ERR_OK) && _discard_cb) { + // _pcb was closed here + async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, this)); + } } int8_t AsyncClient::abort() { @@ -997,28 +1026,28 @@ void AsyncClient::ackPacket(struct pbuf *pb) { * Main Private Methods * */ -int8_t AsyncClient::_close() { - // ets_printf("X: 0x%08x\n", (uint32_t)this); - int8_t err = _tcp_close(&_pcb, this); - // _pcb is now NULL - if ((err == ERR_OK) && _discard_cb) { - // _pcb was closed here - async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, this)); - } - return err; -} - /* * Private Callbacks * */ -int8_t AsyncClient::_connected(tcp_pcb *pcb, int8_t err) { - _pcb = reinterpret_cast(pcb); - if (_pcb) { - _rx_last_packet = millis(); - } +// Adopt a pcb and reset all per-connection state. Callers must already be +// serialized against the LwIP core (accept callback, or tcp_core_guard). +void AsyncClient::_adopt(tcp_pcb *pcb) { + _pcb = pcb; + _rx_ack_len = 0; _tx_last_packet = 0; _rx_last_ack = 0; + _rx_last_packet = millis(); + _bind_tcp_callbacks(pcb, this); +} + +int8_t AsyncClient::_connected(tcp_pcb *pcb, int8_t err) { + if (pcb != _pcb) { + // Stale event for a pcb we no longer own + async_tcp_log_d("%p != %p", (const void *)pcb, (const void *)_pcb); + return ERR_OK; + } + _rx_last_packet = millis(); if (_connect_cb) { async_tcp_log_elapsed("onConnect", _connect_cb(_connect_cb_arg, this)); } @@ -1082,6 +1111,11 @@ int8_t AsyncClient::_poll(tcp_pcb *pcb) { return ERR_OK; } + if (_pcb->state < ESTABLISHED) { + // Still connecting; LwIP handles the SYN timeout itself + return ERR_OK; + } + uint32_t now = millis(); // ACK Timeout @@ -1099,7 +1133,7 @@ int8_t AsyncClient::_poll(tcp_pcb *pcb) { // RX Timeout if (_rx_timeout && (now - _rx_last_packet) >= (_rx_timeout * 1000)) { async_tcp_log_d("rx timeout %d", pcb->state); - _close(); + close(); return ERR_OK; } // Everything is fine diff --git a/src/AsyncTCP.h b/src/AsyncTCP.h index ab1eaf63..87976bc3 100644 --- a/src/AsyncTCP.h +++ b/src/AsyncTCP.h @@ -300,7 +300,7 @@ class AsyncClient { uint32_t _ack_timeout; uint16_t _connect_port; - int8_t _close(); + void _adopt(tcp_pcb *pcb); int8_t _connected(tcp_pcb *pcb, int8_t err); void _error(int8_t err); int8_t _poll(tcp_pcb *pcb); From 5a5c7ae9c40f181d776a025815e9353e777e6024 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 10 Sep 2026 00:14:41 -0400 Subject: [PATCH 08/35] fix: Report DNS failures instead of connecting to 0.0.0.0 The event carries the address inline, so &e->dns.addr was never null and the failure branch was dead. An any-address answer is treated as failure too, on both routes - a cached name resolves synchronously and never reaches the callback. - Report ERR_CONN when the inner connect() fails; an attempt that returned true now ends in exactly one callback. - Drop dns.name: it points into LwIP's DNS table and dangles by dispatch. Never read. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 45 +++++++++++++++++++++++++++++---------------- src/AsyncTCP.h | 2 +- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 3009fbb8..3007fe60 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -158,8 +158,9 @@ struct lwip_tcp_event_packet_t { AsyncServer *server; } accept; struct { - const char *name; ip_addr_t addr; + uint16_t port; + bool resolved; } dns; }; @@ -178,6 +179,7 @@ class AsyncTCP_detail { static void __attribute__((visibility("internal"))) tcp_error(void *arg, int8_t err); static int8_t __attribute__((visibility("internal"))) tcp_poll(void *arg, struct tcp_pcb *pcb); static int8_t __attribute__((visibility("internal"))) tcp_accept(void *arg, tcp_pcb *pcb, int8_t err); + static void __attribute__((visibility("internal"))) tcp_dns_found(const char *name, const ip_addr_t *ipaddr, void *arg); }; // Guard class for the global queue @@ -355,8 +357,8 @@ void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { // ets_printf("A: 0x%08x 0x%08x\n", e->client, e->accept.client); e->accept.server->_accepted(e->client); } else if (e->event == LWIP_TCP_DNS) { - // ets_printf("D: 0x%08x %s = %s\n", e->client, e->dns.name, ipaddr_ntoa(&e->dns.addr)); - e->client->_dns_found(&e->dns.addr); + // ets_printf("D: 0x%08x = %s\n", e->client, ipaddr_ntoa(&e->dns.addr)); + e->client->_dns_found(e->dns.resolved, &e->dns.addr, e->dns.port); } _free_event(e); } @@ -557,8 +559,10 @@ void AsyncTCP_detail::tcp_error(void *arg, int8_t err) { _send_async_event(e); } -static void _tcp_dns_found(const char *name, ip_addr_t *ipaddr, void *arg) { +void AsyncTCP_detail::tcp_dns_found(const char *name, const ip_addr_t *ipaddr, void *arg) { // ets_printf("+DNS: name=%s ipaddr=0x%08x arg=%x\n", name, ipaddr, arg); + // 'name' points into LwIP's DNS table, which the next lookup recycles; don't keep it. + (void)name; auto client = reinterpret_cast(arg); lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_DNS, client}; @@ -567,7 +571,8 @@ static void _tcp_dns_found(const char *name, ip_addr_t *ipaddr, void *arg) { return; } - e->dns.name = name; + e->dns.port = client->_connect_port; + e->dns.resolved = (ipaddr != nullptr); if (ipaddr) { memcpy(&e->dns.addr, ipaddr, sizeof(ip_addr_t)); } else { @@ -919,10 +924,17 @@ bool AsyncClient::connect(const char *host, uint16_t port) { err_t err; { tcp_core_guard tcg; - err = dns_gethostbyname(host, &addr, (dns_found_callback)&_tcp_dns_found, this); + err = dns_gethostbyname(host, &addr, (dns_found_callback)&AsyncTCP_detail::tcp_dns_found, this); } if (err == ERR_OK) { + // Answered from the cache, or the name was a literal address + // Validate the resolved address (same rules as tcp_dns_found) + if (ip_addr_isany_val(addr)) { + async_tcp_log_d("lookup returned the any-address"); + return false; + } + // We can delegate to the direct connection function here. #if ESP_IDF_VERSION_MAJOR < 5 #if LWIP_IPV6 if (addr.type == IPADDR_TYPE_V6) { @@ -1143,17 +1155,18 @@ int8_t AsyncClient::_poll(tcp_pcb *pcb) { return ERR_OK; } -void AsyncClient::_dns_found(ip_addr_t *ipaddr) { - if (ipaddr) { - connect(*ipaddr, _connect_port); - } else { - if (_error_cb) { - async_tcp_log_elapsed("onError", _error_cb(_error_cb_arg, this, -55)); - } - if (_discard_cb) { - async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, this)); - } +void AsyncClient::_dns_found(bool resolved, ip_addr_t *ipaddr, uint16_t port) { + if (resolved && ip_addr_isany_val(*ipaddr)) { + // A resolver that answers with the any-address - a sinkhole, or a blocklist - has + // not given us anywhere to go. Report it as a failed lookup rather than dialling it. + resolved = false; + } + if (resolved && connect(*ipaddr, port)) { + return; } + // connect() reported success to the caller, so this attempt owes them exactly one + // of onConnect or onError. + _error(resolved ? ERR_CONN : -55); } /* diff --git a/src/AsyncTCP.h b/src/AsyncTCP.h index 87976bc3..b4015a78 100644 --- a/src/AsyncTCP.h +++ b/src/AsyncTCP.h @@ -306,7 +306,7 @@ class AsyncClient { int8_t _poll(tcp_pcb *pcb); int8_t _sent(tcp_pcb *pcb, uint16_t len); int8_t _fin(tcp_pcb *pcb, int8_t err); - void _dns_found(ip_addr_t *ipaddr); + void _dns_found(bool resolved, ip_addr_t *ipaddr, uint16_t port); }; class AsyncServer { From b615bdc3a96917f617418633da9560b844bf61cf Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 10 Sep 2026 00:28:50 -0400 Subject: [PATCH 09/35] refactor: Move client state into a shared implementation object Three parties held AsyncClient as a raw pointer with no shared ownership, and two holes could not be closed: the async task drops the queue mutex before invoking a callback, and deleting a client inside onData destroyed the std::function that was running. State and handlers move to AsyncClientImpl, shared by the facade and every queued event. LwIP's pointers are deliberately not owners - ~AsyncClient() closes unconditionally and the destructor only asserts the binding is clear, since it may run on any thread. - _facade is nulled under the queue mutex; dispatch is gated on it, so a detached implementation just drains. - The queue mutex is created by AsyncClient and AsyncServer construction, ahead of every user. - The facade's forwarders are always_inline: these builds have no LTO. - close()'s ack folds into the close transaction, so the peer is no longer RST over data we had in fact processed. - _error() and close() hold a reference across their callbacks and re-read _facade. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 984 +++++++++++++++++++++++++++++++++++------------ src/AsyncTCP.h | 50 +-- 2 files changed, 751 insertions(+), 283 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 3007fe60..c930cb38 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -5,6 +5,17 @@ #include "AsyncTCPLogging.h" #include "AsyncTCPSimpleIntrusiveList.h" +#include +#include + +/** + * Utility macro + * + * Most API functions are delegated to the impl class. Ensure the wrapped functions are always inlined + * to avoid the extra function call overhead. +*/ +#define ASYNCTCP_ALWAYS_INLINE __attribute__((always_inline)) inline + /** * LibreTiny specific configurations */ @@ -129,7 +140,7 @@ typedef enum { struct lwip_tcp_event_packet_t { lwip_tcp_event_packet_t *next; lwip_tcp_event_t event; - AsyncClient *client; + std::shared_ptr impl; // keeps it alive for the life of the event union { struct { tcp_pcb *pcb; @@ -164,7 +175,7 @@ struct lwip_tcp_event_packet_t { } dns; }; - inline lwip_tcp_event_packet_t(lwip_tcp_event_t _event, AsyncClient *_client) : next(nullptr), event(_event), client(_client){}; + inline lwip_tcp_event_packet_t(lwip_tcp_event_t _event, std::shared_ptr _impl) : next(nullptr), event(_event), impl(std::move(_impl)){}; }; // Detail class for interacting with AsyncClient internals, but without exposing the API @@ -187,6 +198,20 @@ namespace { static SemaphoreHandle_t _async_queue_mutex = nullptr; +// The queue is only reachable through an AsyncClient or an AsyncServer, so creating the +// mutex in their constructors puts it ahead of every user. Unsynchronized, in the same +// way _start_async_task() always has been: the first of those can only be constructed by +// application code, before the async task exists and before LwIP points at anything. +static bool _init_queue_mutex() { + if (!_async_queue_mutex) { + _async_queue_mutex = xSemaphoreCreateMutex(); + if (!_async_queue_mutex) { + async_tcp_log_e("Failed to create the queue mutex"); + } + } + return _async_queue_mutex != nullptr; +} + class queue_mutex_guard { bool holds_mutex; @@ -206,6 +231,139 @@ class queue_mutex_guard { static SimpleIntrusiveList _async_queue; static TaskHandle_t _async_service_task_handle = NULL; +/* + Client implementation, held by shared_ptr. + + The AsyncClient facade is created and destroyed by the application, but LwIP and the + event queue both hold pointers that outlive it. Putting the state here and sharing + ownership of it means neither has to care: the facade and each queued event own the + implementation, and it lives until the last of them lets go. + + Because the callbacks live here rather than on the facade, they stay in scope while + being called, ensuring that any captures remain alive until the callback completes, + even if the AsyncClient object is destroyed. + + The context passed to LwIP callbacks are deliberately not owners -- instead this class + owns them and ensures that all are released before destruction. + + _facade is nulled when the application's object is destroyed. Callback dispatch is + always gated on it, so a detached implementation runs no user code and any leftover + events simply drain. +*/ +class AsyncClientImpl : public std::enable_shared_from_this { +public: + explicit AsyncClientImpl(AsyncClient *facade); + ~AsyncClientImpl(); + + AsyncClientImpl(const AsyncClientImpl &) = delete; + AsyncClientImpl &operator=(const AsyncClientImpl &) = delete; + + AsyncClient *_facade; // nulled once the application's object is destroyed + + tcp_pcb *_pcb; + + AcConnectHandler _connect_cb; + void *_connect_cb_arg; + AcConnectHandler _discard_cb; + void *_discard_cb_arg; + AcAckHandler _sent_cb; + void *_sent_cb_arg; + AcErrorHandler _error_cb; + void *_error_cb_arg; + AcDataHandler _recv_cb; + void *_recv_cb_arg; + AcPacketHandler _pb_cb; + void *_pb_cb_arg; + AcTimeoutHandler _timeout_cb; + void *_timeout_cb_arg; + AcConnectHandler _poll_cb; + void *_poll_cb_arg; + + bool _ack_pcb; + uint32_t _tx_last_packet; + uint32_t _rx_ack_len; + uint32_t _rx_last_packet; + uint32_t _rx_timeout; + uint32_t _rx_last_ack; + uint32_t _ack_timeout; + uint16_t _connect_port; + bool _dns_pending; // a lookup is outstanding and holds a reference + uint32_t _in_callback_ack_len; // bytes handed to onData but not yet acked + + ASYNCTCP_ALWAYS_INLINE bool connect(ip_addr_t addr, uint16_t port); + ASYNCTCP_ALWAYS_INLINE bool connect(const char *host, uint16_t port); +#ifdef ARDUINO + ASYNCTCP_ALWAYS_INLINE bool connect(const IPAddress &ip, uint16_t port); +#if LWIP_IPV6 && ESP_IDF_VERSION_MAJOR < 5 + ASYNCTCP_ALWAYS_INLINE bool connect(const IPv6Address &ip, uint16_t port); +#endif +#endif + ASYNCTCP_ALWAYS_INLINE void close(); + ASYNCTCP_ALWAYS_INLINE int8_t abort(); + ASYNCTCP_ALWAYS_INLINE bool free(); + + ASYNCTCP_ALWAYS_INLINE bool canSend() const; + ASYNCTCP_ALWAYS_INLINE size_t space() const; + ASYNCTCP_ALWAYS_INLINE size_t add(const char *data, size_t size, uint8_t apiflags); + ASYNCTCP_ALWAYS_INLINE bool send(); + ASYNCTCP_ALWAYS_INLINE size_t write(const char *data, size_t size, uint8_t apiflags); + + ASYNCTCP_ALWAYS_INLINE uint8_t state() const; + ASYNCTCP_ALWAYS_INLINE bool connecting() const; + ASYNCTCP_ALWAYS_INLINE bool connected() const; + ASYNCTCP_ALWAYS_INLINE bool disconnecting() const; + ASYNCTCP_ALWAYS_INLINE bool disconnected() const; + ASYNCTCP_ALWAYS_INLINE bool freeable() const; + void _abandonResolve(); + + ASYNCTCP_ALWAYS_INLINE uint16_t getMss() const; + ASYNCTCP_ALWAYS_INLINE uint32_t getRxTimeout() const; + ASYNCTCP_ALWAYS_INLINE void setRxTimeout(uint32_t timeout); + ASYNCTCP_ALWAYS_INLINE uint32_t getAckTimeout() const; + ASYNCTCP_ALWAYS_INLINE void setAckTimeout(uint32_t timeout); + ASYNCTCP_ALWAYS_INLINE void setNoDelay(bool nodelay) const; + ASYNCTCP_ALWAYS_INLINE bool getNoDelay(); + ASYNCTCP_ALWAYS_INLINE void setKeepAlive(uint32_t ms, uint8_t cnt); + + ASYNCTCP_ALWAYS_INLINE uint32_t getRemoteAddress() const; + ASYNCTCP_ALWAYS_INLINE uint16_t getRemotePort() const; + ASYNCTCP_ALWAYS_INLINE uint32_t getLocalAddress() const; + ASYNCTCP_ALWAYS_INLINE uint16_t getLocalPort() const; + ASYNCTCP_ALWAYS_INLINE ip4_addr_t getRemoteAddress4() const; + ASYNCTCP_ALWAYS_INLINE ip4_addr_t getLocalAddress4() const; +#if LWIP_IPV6 + ASYNCTCP_ALWAYS_INLINE ip6_addr_t getRemoteAddress6() const; + ASYNCTCP_ALWAYS_INLINE ip6_addr_t getLocalAddress6() const; +#ifdef ARDUINO +#if ESP_IDF_VERSION_MAJOR < 5 + ASYNCTCP_ALWAYS_INLINE IPv6Address remoteIP6() const; + ASYNCTCP_ALWAYS_INLINE IPv6Address localIP6() const; +#else + ASYNCTCP_ALWAYS_INLINE IPAddress remoteIP6() const; + ASYNCTCP_ALWAYS_INLINE IPAddress localIP6() const; +#endif +#endif +#endif +#ifdef ARDUINO + ASYNCTCP_ALWAYS_INLINE IPAddress remoteIP() const; + ASYNCTCP_ALWAYS_INLINE IPAddress localIP() const; +#endif + + ASYNCTCP_ALWAYS_INLINE void ackPacket(struct pbuf *pb); + ASYNCTCP_ALWAYS_INLINE size_t ack(size_t len); + ASYNCTCP_ALWAYS_INLINE void ackLater(); + ASYNCTCP_ALWAYS_INLINE const char *stateToString() const; + + void _adopt(tcp_pcb *pcb); + int8_t _connected(tcp_pcb *pcb, int8_t err); + void _error(int8_t err); + int8_t _poll(tcp_pcb *pcb); + int8_t _sent(tcp_pcb *pcb, uint16_t len); + int8_t _fin(tcp_pcb *pcb, int8_t err); + int8_t _recv(tcp_pcb *pcb, pbuf *pb, int8_t err); + void _dns_found(bool resolved, ip_addr_t *ipaddr, uint16_t port); +}; + static uint32_t _xor_shift_state = 31; // any nonzero seed will do static uint32_t _xor_shift_next() { uint32_t x = _xor_shift_state; @@ -219,7 +377,7 @@ static void _free_event(lwip_tcp_event_packet_t *evpkt) { if ((evpkt->event == LWIP_TCP_RECV) && (evpkt->recv.pb != nullptr)) { pbuf_free(evpkt->recv.pb); } - delete evpkt; + delete evpkt; // releases the event's reference to the implementation } static inline void _send_async_event(lwip_tcp_event_packet_t *e) { @@ -239,15 +397,22 @@ static inline void _prepend_async_event(lwip_tcp_event_packet_t *e) { } static inline lwip_tcp_event_packet_t *_get_async_event() { - queue_mutex_guard guard; - while (1) { - lwip_tcp_event_packet_t *e = _async_queue.pop_front(); + lwip_tcp_event_packet_t *result = nullptr; + // Discarded events are freed once the mutex is released, to keep pbuf_free() out of + // the critical section. + lwip_tcp_event_packet_t *discarded = nullptr; - if ((!e) || (e->event != LWIP_TCP_POLL)) { - return e; - } + { + queue_mutex_guard guard; + while (1) { + lwip_tcp_event_packet_t *e = _async_queue.pop_front(); + + if ((!e) || (e->event != LWIP_TCP_POLL)) { + result = e; + break; + } - /* + /* Let's try to coalesce two (or more) consecutive poll events into one this usually happens with poor implemented user-callbacks that are runs too long and makes poll events to stack in the queue if consecutive user callback for a same connection runs longer that poll time then it will fill the queue with events until it deadlocks. @@ -255,14 +420,16 @@ static inline lwip_tcp_event_packet_t *_get_async_event() { It won't be effective if user would run multiple simultaneous long running callbacks due to message interleaving. todo: implement some kind of fair dequeuing or (better) simply punish user for a bad designed callbacks by resetting hog connections */ - for (lwip_tcp_event_packet_t *next_pkt = _async_queue.begin(); next_pkt && (next_pkt->client == e->client) && (next_pkt->event == LWIP_TCP_POLL); - next_pkt = _async_queue.begin()) { - // if the next event that will come is a poll event for the same connection, we can discard it and continue - _free_event(_async_queue.pop_front()); - async_tcp_log_d("coalescing polls, network congestion or async callbacks might be too slow!"); - } + for (lwip_tcp_event_packet_t *next_pkt = _async_queue.begin(); next_pkt && (next_pkt->impl == e->impl) && (next_pkt->event == LWIP_TCP_POLL); + next_pkt = _async_queue.begin()) { + // if the next event that will come is a poll event for the same connection, we can discard it and continue + auto dup = _async_queue.pop_front(); + dup->next = discarded; + discarded = dup; + async_tcp_log_d("coalescing polls, network congestion or async callbacks might be too slow!"); + } - /* + /* now we have to decide if to proceed with poll callback handler or discard it? poor designed apps using asynctcp without proper dataflow control could flood the queue with interleaved pool/ack events. I.e. on each poll app would try to generate more data to send, which in turn results in additional ack event triggering chain effect @@ -271,22 +438,32 @@ static inline lwip_tcp_event_packet_t *_get_async_event() { Let's discard poll events processing using linear-increasing probability curve when queue size grows over 3/4 Poll events are periodic and connection could get another chance next time */ - if (_async_queue.size() > (_xor_shift_next() % CONFIG_ASYNC_TCP_QUEUE_SIZE / 4 + CONFIG_ASYNC_TCP_QUEUE_SIZE * 3 / 4)) { - _free_event(e); - async_tcp_log_d("discarding poll due to queue congestion"); - continue; + if (_async_queue.size() > (_xor_shift_next() % CONFIG_ASYNC_TCP_QUEUE_SIZE / 4 + CONFIG_ASYNC_TCP_QUEUE_SIZE * 3 / 4)) { + e->next = discarded; + discarded = e; + async_tcp_log_d("discarding poll due to queue congestion"); + continue; + } + + result = e; + break; } + } - return e; + while (discarded) { + auto t = discarded; + discarded = t->next; + _free_event(t); } + return result; } -static size_t _remove_events_for_client(AsyncClient *client) { +static size_t _remove_events_for_client(AsyncClientImpl *client) { lwip_tcp_event_packet_t *removed_event_chain; { queue_mutex_guard guard; removed_event_chain = _async_queue.remove_if([=](lwip_tcp_event_packet_t &pkt) { - return pkt.client == client; + return pkt.impl.get() == client; }); } @@ -324,41 +501,43 @@ static size_t _dispose_event_chain(lwip_tcp_event_packet_t *chain) { ++count; auto t = chain; chain = t->next; - delete t->client; + delete t->impl->_facade; _free_event(t); } return count; } void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { - if (e->client == NULL) { - // do nothing when arg is NULL + // A detached implementation has no facade to hand to the user's callbacks, so its + // events simply drain. The reference the event holds keeps it alive until then. + if ((e->impl == NULL) || (e->impl->_facade == NULL)) { // ets_printf("event arg == NULL: 0x%08x\n", e->recv.pcb); } else if (e->event == LWIP_TCP_RECV) { // ets_printf("-R: 0x%08x\n", e->recv.pcb); - e->client->_recv(e->recv.pcb, e->recv.pb, e->recv.err); + e->impl->_recv(e->recv.pcb, e->recv.pb, e->recv.err); e->recv.pb = nullptr; // given to client } else if (e->event == LWIP_TCP_FIN) { // ets_printf("-F: 0x%08x\n", e->fin.pcb); - e->client->_fin(e->fin.pcb, e->fin.err); + e->impl->_fin(e->fin.pcb, e->fin.err); } else if (e->event == LWIP_TCP_SENT) { // ets_printf("-S: 0x%08x\n", e->sent.pcb); - e->client->_sent(e->sent.pcb, e->sent.len); + e->impl->_sent(e->sent.pcb, e->sent.len); } else if (e->event == LWIP_TCP_POLL) { // ets_printf("-P: 0x%08x\n", e->poll.pcb); - e->client->_poll(e->poll.pcb); + e->impl->_poll(e->poll.pcb); } else if (e->event == LWIP_TCP_ERROR) { - // ets_printf("-E: 0x%08x %d\n", e->client, e->error.err); - e->client->_error(e->error.err); + // ets_printf("-E: 0x%08x %d\n", e->impl, e->error.err); + e->impl->_error(e->error.err); } else if (e->event == LWIP_TCP_CONNECTED) { - // ets_printf("C: 0x%08x 0x%08x %d\n", e->client, e->connected.pcb, e->connected.err); - e->client->_connected(e->connected.pcb, e->connected.err); + // ets_printf("C: 0x%08x 0x%08x %d\n", e->impl, e->connected.pcb, e->connected.err); + e->impl->_connected(e->connected.pcb, e->connected.err); } else if (e->event == LWIP_TCP_ACCEPT) { - // ets_printf("A: 0x%08x 0x%08x\n", e->client, e->accept.client); - e->accept.server->_accepted(e->client); + // ets_printf("A: 0x%08x 0x%08x\n", e->impl, e->accept.server); + // Gated above, so the facade is still alive + e->accept.server->_accepted(e->impl->_facade); } else if (e->event == LWIP_TCP_DNS) { - // ets_printf("D: 0x%08x = %s\n", e->client, ipaddr_ntoa(&e->dns.addr)); - e->client->_dns_found(e->dns.resolved, &e->dns.addr, e->dns.port); + // ets_printf("D: 0x%08x = %s\n", e->impl, ipaddr_ntoa(&e->dns.addr)); + e->impl->_dns_found(e->dns.resolved, &e->dns.addr, e->dns.port); } _free_event(e); } @@ -417,11 +596,8 @@ static bool customTaskCreateUniversal( } static bool _start_async_task() { - if (!_async_queue_mutex) { - _async_queue_mutex = xSemaphoreCreateMutex(); - if (!_async_queue_mutex) { - return false; - } + if (!_init_queue_mutex()) { + return false; } if (!_async_service_task_handle) { @@ -439,7 +615,8 @@ static bool _start_async_task() { * LwIP Callbacks * */ -static void _bind_tcp_callbacks(tcp_pcb *pcb, AsyncClient *client) { +// Attach an AsyncClient to a TCP PCB by setting the appropriate LwIP callbacks and argument. +static void _bind_tcp_callbacks(tcp_pcb *pcb, AsyncClientImpl *client) { tcp_arg(pcb, client); tcp_recv(pcb, &AsyncTCP_detail::tcp_recv); tcp_sent(pcb, &AsyncTCP_detail::tcp_sent); @@ -447,7 +624,7 @@ static void _bind_tcp_callbacks(tcp_pcb *pcb, AsyncClient *client) { tcp_poll(pcb, &AsyncTCP_detail::tcp_poll, CONFIG_ASYNC_TCP_POLL_TIMER); } -static void _reset_tcp_callbacks(tcp_pcb *pcb, AsyncClient *client) { +static void _reset_tcp_callbacks(tcp_pcb *pcb, AsyncClientImpl *client) { tcp_arg(pcb, NULL); tcp_sent(pcb, NULL); tcp_recv(pcb, NULL); @@ -458,10 +635,11 @@ static void _reset_tcp_callbacks(tcp_pcb *pcb, AsyncClient *client) { } } +// Callback when a connection is established. pcb is already bound to the client. static int8_t _tcp_connected(void *arg, tcp_pcb *pcb, int8_t err) { // ets_printf("+C: 0x%08x\n", pcb); - AsyncClient *client = reinterpret_cast(arg); - lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_CONNECTED, client}; + AsyncClientImpl *client = reinterpret_cast(arg); + lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_CONNECTED, client->shared_from_this()}; if (!e) { async_tcp_log_e("Failed to allocate event packet"); return ERR_MEM; @@ -485,8 +663,8 @@ int8_t AsyncTCP_detail::tcp_poll(void *arg, struct tcp_pcb *pcb) { } // ets_printf("+P: 0x%08x\n", pcb); - AsyncClient *client = reinterpret_cast(arg); - lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_POLL, client}; + AsyncClientImpl *client = reinterpret_cast(arg); + lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_POLL, client->shared_from_this()}; if (!e) { async_tcp_log_e("Failed to allocate event packet"); return ERR_MEM; @@ -499,8 +677,8 @@ int8_t AsyncTCP_detail::tcp_poll(void *arg, struct tcp_pcb *pcb) { } int8_t AsyncTCP_detail::tcp_recv(void *arg, struct tcp_pcb *pcb, struct pbuf *pb, int8_t err) { - AsyncClient *client = reinterpret_cast(arg); - lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_RECV, client}; + AsyncClientImpl *client = reinterpret_cast(arg); + lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_RECV, client->shared_from_this()}; if (!e) { async_tcp_log_e("Failed to allocate event packet"); return ERR_MEM; @@ -524,8 +702,8 @@ int8_t AsyncTCP_detail::tcp_recv(void *arg, struct tcp_pcb *pcb, struct pbuf *pb int8_t AsyncTCP_detail::tcp_sent(void *arg, struct tcp_pcb *pcb, uint16_t len) { // ets_printf("+S: 0x%08x\n", pcb); - AsyncClient *client = reinterpret_cast(arg); - lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_SENT, client}; + AsyncClientImpl *client = reinterpret_cast(arg); + lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_SENT, client->shared_from_this()}; if (!e) { async_tcp_log_e("Failed to allocate event packet"); return ERR_MEM; @@ -540,47 +718,72 @@ int8_t AsyncTCP_detail::tcp_sent(void *arg, struct tcp_pcb *pcb, uint16_t len) { void AsyncTCP_detail::tcp_error(void *arg, int8_t err) { // ets_printf("+E: 0x%08x\n", arg); - AsyncClient *client = reinterpret_cast(arg); - if (client && client->_pcb) { - // The pcb has already been freed by LwIP; do not attempt to clear the callbacks! - _remove_events_for_client(client); - client->_pcb = nullptr; + AsyncClientImpl *client = reinterpret_cast(arg); + if ((client == nullptr) || (client->_pcb == nullptr)) { + // Not the client that owns this pcb: there is nothing to report, and LwIP's + // reference was already released when we forgot the pcb. + async_tcp_log_e("error callback for a client with no pcb"); + return; } + // The pcb has already been freed by LwIP; do not attempt to clear the callbacks! + _remove_events_for_client(client); + // enqueue event to be processed in the async task for the user callback - lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_ERROR, client}; + lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_ERROR, client->shared_from_this()}; if (!e) { async_tcp_log_e("Failed to allocate event packet"); - return; + } else { + e->error.err = err; } - e->error.err = err; - queue_mutex_guard guard; - _send_async_event(e); + { + queue_mutex_guard guard; + client->_pcb = nullptr; + _send_async_event(e); // no-op if the allocation failed + } + // Nothing may touch the client past this point. Clearing _pcb above released the + // binding, so ~AsyncClient() is free to drop the last reference as soon as it reads + // _pcb as null - and it can only reach that read through the mutex just released. } void AsyncTCP_detail::tcp_dns_found(const char *name, const ip_addr_t *ipaddr, void *arg) { // ets_printf("+DNS: name=%s ipaddr=0x%08x arg=%x\n", name, ipaddr, arg); // 'name' points into LwIP's DNS table, which the next lookup recycles; don't keep it. (void)name; - auto client = reinterpret_cast(arg); + // Adopt ownership of the callback state from the argument passed by LwIP. + std::unique_ptr> callback_state(reinterpret_cast *>(arg)); + // Validate that our client still exists. + auto client = callback_state->lock(); + if (!client) { + return; // the implementation was destroyed while the lookup was in flight + } + + bool wanted; + { + queue_mutex_guard guard; + wanted = client->_dns_pending; + client->_dns_pending = false; + } + if (!wanted) { + return; // close() or abort() gave up on this lookup while it was in flight + } lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_DNS, client}; if (!e) { async_tcp_log_e("Failed to allocate event packet"); - return; - } - - e->dns.port = client->_connect_port; - e->dns.resolved = (ipaddr != nullptr); - if (ipaddr) { - memcpy(&e->dns.addr, ipaddr, sizeof(ip_addr_t)); } else { - memset(&e->dns.addr, 0, sizeof(e->dns.addr)); + e->dns.port = client->_connect_port; + e->dns.resolved = (ipaddr != nullptr); + if (ipaddr) { + memcpy(&e->dns.addr, ipaddr, sizeof(ip_addr_t)); + } else { + memset(&e->dns.addr, 0, sizeof(e->dns.addr)); + } } queue_mutex_guard guard; - _send_async_event(e); + _send_async_event(e); // no-op if the allocation failed } /* @@ -589,12 +792,18 @@ void AsyncTCP_detail::tcp_dns_found(const char *name, const ip_addr_t *ipaddr, v #include "lwip/priv/tcpip_priv.h" +/* + One transaction's arguments. The client-side calls address the pcb through `client`, + which is the only thing that knows which pcb is current; `pcb` is for the server, whose + listening socket is not owned by an implementation object. +*/ typedef struct { struct tcpip_api_call_data call; + AsyncClientImpl *client; tcp_pcb **pcb; int8_t err; union { - AsyncClient *close; + size_t close_ack; // bytes to tcp_recved() before closing, in the same pass struct { const char *data; size_t size; @@ -612,18 +821,18 @@ typedef struct { static err_t _tcp_output_api(struct tcpip_api_call_data *api_call_msg) { tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; msg->err = ERR_CONN; - if (*msg->pcb) { - msg->err = tcp_output(*msg->pcb); + if (msg->client->_pcb) { + msg->err = tcp_output(msg->client->_pcb); } return msg->err; } -static esp_err_t _tcp_output(tcp_pcb **pcb) { - if (!pcb || !*pcb) { +static esp_err_t _tcp_output(AsyncClientImpl *client) { + if (!client->_pcb) { return ERR_CONN; } tcp_api_call_t msg; - msg.pcb = pcb; + msg.client = client; tcpip_api_call(_tcp_output_api, (struct tcpip_api_call_data *)&msg); return msg.err; } @@ -631,18 +840,18 @@ static esp_err_t _tcp_output(tcp_pcb **pcb) { static err_t _tcp_write_api(struct tcpip_api_call_data *api_call_msg) { tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; msg->err = ERR_CONN; - if (*msg->pcb) { - msg->err = tcp_write(*msg->pcb, msg->write.data, msg->write.size, msg->write.apiflags); + if (msg->client->_pcb) { + msg->err = tcp_write(msg->client->_pcb, msg->write.data, msg->write.size, msg->write.apiflags); } return msg->err; } -static esp_err_t _tcp_write(tcp_pcb **pcb, const char *data, size_t size, uint8_t apiflags) { - if (!pcb || !*pcb) { +static esp_err_t _tcp_write(AsyncClientImpl *client, const char *data, size_t size, uint8_t apiflags) { + if (!client->_pcb) { return ERR_CONN; } tcp_api_call_t msg; - msg.pcb = pcb; + msg.client = client; msg.write.data = data; msg.write.size = size; msg.write.apiflags = apiflags; @@ -653,19 +862,19 @@ static esp_err_t _tcp_write(tcp_pcb **pcb, const char *data, size_t size, uint8_ static err_t _tcp_recved_api(struct tcpip_api_call_data *api_call_msg) { tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; msg->err = ERR_CONN; - if (*msg->pcb) { + if (msg->client->_pcb) { msg->err = 0; - tcp_recved(*msg->pcb, msg->received); + tcp_recved(msg->client->_pcb, msg->received); } return msg->err; } -static esp_err_t _tcp_recved(tcp_pcb **pcb, size_t len) { - if (!pcb || !*pcb) { +static esp_err_t _tcp_recved(AsyncClientImpl *client, size_t len) { + if (!client->_pcb) { return ERR_CONN; } tcp_api_call_t msg; - msg.pcb = pcb; + msg.client = client; msg.received = len; tcpip_api_call(_tcp_recved_api, (struct tcpip_api_call_data *)&msg); return msg.err; @@ -681,59 +890,63 @@ static err_t _tcp_close_api(struct tcpip_api_call_data *api_call_msg) { // thread has committed to closing/destructing the AsyncClient object. tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; + AsyncClientImpl *client = msg->client; msg->err = ERR_CONN; - if (*msg->pcb) { - tcp_pcb *pcb = *msg->pcb; - _reset_tcp_callbacks(pcb, msg->close); + if (client->_pcb) { + tcp_pcb *pcb = client->_pcb; + // Ack anything the application withheld, in this same pass. Done as a separate call + // the LwIP thread could run in between, and the peer would get an RST rather than a + // FIN for data that was in fact processed. + if (msg->close_ack) { + tcp_recved(pcb, msg->close_ack); + } + _reset_tcp_callbacks(pcb, client); if (tcp_close(pcb) != ERR_OK) { // We do not permit failure here: abandon the pcb anyways. tcp_abort(pcb); } msg->err = ERR_OK; - *msg->pcb = nullptr; // PCB is now the property of LwIP + client->_pcb = nullptr; // PCB is now the property of LwIP } else { // Ensure there is not an error event queued for this client - if (_remove_events_for_client(msg->close)) { + if (_remove_events_for_client(client)) { msg->err = ERR_OK; // dispose needs to be run } } return msg->err; } -static esp_err_t _tcp_close(tcp_pcb **pcb, AsyncClient *client) { +static esp_err_t _tcp_close(AsyncClientImpl *client, size_t ack) { tcp_api_call_t msg; - msg.pcb = pcb; - msg.close = client; + msg.client = client; + msg.close_ack = ack; tcpip_api_call(_tcp_close_api, (struct tcpip_api_call_data *)&msg); return msg.err; } static err_t _tcp_abort_api(struct tcpip_api_call_data *api_call_msg) { - // Like close(), we must ensure that the queue is cleared of any events referencing the AsyncClient. + // Like close(), we must ensure that the queue is cleared of any events referencing the + // AsyncClient. // ERR_ABRT: the pcb was aborted. // ERR_OK: the pcb was already gone, but a queued error event was purged, so the // caller must still run the discard callback (dispose needs to run). // ERR_CONN: nothing to do (pcb already null and no queued events). tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; - if (*msg->pcb) { - _reset_tcp_callbacks(*msg->pcb, msg->close); - tcp_abort(*msg->pcb); - *msg->pcb = nullptr; // PCB is now the property of LwIP + AsyncClientImpl *client = msg->client; + if (client->_pcb) { + _reset_tcp_callbacks(client->_pcb, client); + tcp_abort(client->_pcb); + client->_pcb = nullptr; // PCB is now the property of LwIP msg->err = ERR_ABRT; } else { - // Ensure there is not an error event queued for this client - msg->err = _remove_events_for_client(msg->close) ? ERR_OK : ERR_CONN; + msg->err = _remove_events_for_client(client) ? ERR_OK : ERR_CONN; } return msg->err; } -static esp_err_t _tcp_abort(tcp_pcb **pcb, AsyncClient *client) { - if (!pcb || !*pcb) { - return ERR_CONN; - } +static esp_err_t _tcp_abort(AsyncClientImpl *client) { tcp_api_call_t msg; - msg.pcb = pcb; - msg.close = client; + msg.client = client; tcpip_api_call(_tcp_abort_api, (struct tcpip_api_call_data *)&msg); return msg.err; } @@ -779,19 +992,32 @@ static esp_err_t _tcp_connect(tcp_pcb **pcb, ip_addr_t *addr, uint16_t port, tcp Async TCP Client */ -AsyncClient::AsyncClient(tcp_pcb *pcb) - : _pcb(nullptr), _connect_cb(0), _connect_cb_arg(0), _discard_cb(0), _discard_cb_arg(0), _sent_cb(0), _sent_cb_arg(0), _error_cb(0), _error_cb_arg(0), - _recv_cb(0), _recv_cb_arg(0), _pb_cb(0), _pb_cb_arg(0), _timeout_cb(0), _timeout_cb_arg(0), _poll_cb(0), _poll_cb_arg(0), _ack_pcb(true), - _tx_last_packet(0), _rx_ack_len(0), _rx_last_packet(0), _rx_timeout(0), _rx_last_ack(0), _ack_timeout(CONFIG_ASYNC_TCP_MAX_ACK_TIME), _connect_port(0) { +AsyncClientImpl::AsyncClientImpl(AsyncClient *facade) + : _facade(facade), _pcb(nullptr), _connect_cb(0), _connect_cb_arg(0), _discard_cb(0), _discard_cb_arg(0), _sent_cb(0), _sent_cb_arg(0), _error_cb(0), + _error_cb_arg(0), _recv_cb(0), _recv_cb_arg(0), _pb_cb(0), _pb_cb_arg(0), _timeout_cb(0), _timeout_cb_arg(0), _poll_cb(0), _poll_cb_arg(0), _ack_pcb(true), + _tx_last_packet(0), _rx_ack_len(0), _rx_last_packet(0), _rx_timeout(0), _rx_last_ack(0), _ack_timeout(CONFIG_ASYNC_TCP_MAX_ACK_TIME), _connect_port(0), + _dns_pending(false), _in_callback_ack_len(0) {} + +AsyncClientImpl::~AsyncClientImpl() { + // Every path that drops the last reference must ensure the binding was cleared first. + // This can be called from the LwIP thread, so any cleanup must have been performed already. + assert(!_pcb); +} + +AsyncClient::AsyncClient(tcp_pcb *pcb) : _impl(std::make_shared(this)) { + _init_queue_mutex(); if (pcb) { - _adopt(pcb); + _impl->_adopt(pcb); } } AsyncClient::~AsyncClient() { - if (_pcb) { - close(); + if (_impl->_pcb) { + _impl->close(); } + // Detach: queued events and LwIP callbacks may still reach the implementation, but + // must not run user code against a facade that no longer exists. + _impl->_facade = nullptr; } /* @@ -799,62 +1025,26 @@ AsyncClient::~AsyncClient() { * */ bool AsyncClient::operator==(const AsyncClient &other) const { - return _pcb == other._pcb; + return _impl->_pcb == other._impl->_pcb; } /* * Callback Setters * */ -void AsyncClient::onConnect(AcConnectHandler cb, void *arg) { - _connect_cb = cb; - _connect_cb_arg = arg; -} - -void AsyncClient::onDisconnect(AcConnectHandler cb, void *arg) { - _discard_cb = cb; - _discard_cb_arg = arg; -} - -void AsyncClient::onAck(AcAckHandler cb, void *arg) { - _sent_cb = cb; - _sent_cb_arg = arg; -} - -void AsyncClient::onError(AcErrorHandler cb, void *arg) { - _error_cb = cb; - _error_cb_arg = arg; -} - -void AsyncClient::onData(AcDataHandler cb, void *arg) { - _recv_cb = cb; - _recv_cb_arg = arg; -} - -void AsyncClient::onPacket(AcPacketHandler cb, void *arg) { - _pb_cb = cb; - _pb_cb_arg = arg; -} - -void AsyncClient::onTimeout(AcTimeoutHandler cb, void *arg) { - _timeout_cb = cb; - _timeout_cb_arg = arg; -} - -void AsyncClient::onPoll(AcConnectHandler cb, void *arg) { - _poll_cb = cb; - _poll_cb_arg = arg; -} - /* * Main Public Methods * */ -bool AsyncClient::connect(ip_addr_t addr, uint16_t port) { +bool AsyncClientImpl::connect(ip_addr_t addr, uint16_t port) { if (_pcb) { async_tcp_log_d("already connected, state %d", _pcb->state); return false; } + if (_dns_pending) { + async_tcp_log_d("name resolution in progress"); + return false; + } if (!_start_async_task()) { async_tcp_log_e("failed to start task"); return false; @@ -880,13 +1070,14 @@ bool AsyncClient::connect(ip_addr_t addr, uint16_t port) { if (_tcp_connect(&_pcb, &addr, port, (tcp_connected_fn)&_tcp_connected) == ESP_OK) { return true; } - // _pcb is now NULL; no callbacks are raised, as we are returning failure + // _pcb is now NULL and the api call cleared tcp_arg(), so the binding is gone. No + // callbacks are raised: we are returning failure. async_tcp_log_d("connect failed"); return false; } #ifdef ARDUINO -bool AsyncClient::connect(const IPAddress &ip, uint16_t port) { +bool AsyncClientImpl::connect(const IPAddress &ip, uint16_t port) { ip_addr_t addr; #if ESP_IDF_VERSION_MAJOR < 5 #if LWIP_IPV4 && LWIP_IPV6 @@ -905,7 +1096,7 @@ bool AsyncClient::connect(const IPAddress &ip, uint16_t port) { #endif #if LWIP_IPV6 && ESP_IDF_VERSION_MAJOR < 5 -bool AsyncClient::connect(const IPv6Address &ip, uint16_t port) { +bool AsyncClientImpl::connect(const IPv6Address &ip, uint16_t port) { auto ipaddr = static_cast(ip); ip_addr_t addr = IPADDR6_INIT(ipaddr[0], ipaddr[1], ipaddr[2], ipaddr[3]); @@ -913,18 +1104,42 @@ bool AsyncClient::connect(const IPv6Address &ip, uint16_t port) { } #endif -bool AsyncClient::connect(const char *host, uint16_t port) { +bool AsyncClientImpl::connect(const char *host, uint16_t port) { ip_addr_t addr; + if (_pcb) { + async_tcp_log_d("already connected, state %d", _pcb->state); + return false; + } + if (_dns_pending) { + // LwIP coalesces a duplicate lookup onto the outstanding one anyway, so a retry + // never bought anything. Let the attempt already in flight run to completion. + async_tcp_log_d("name resolution already in progress"); + return true; + } if (!_start_async_task()) { async_tcp_log_e("failed to start task"); return false; } + std::unique_ptr> callback_state(new (std::nothrow) std::weak_ptr(shared_from_this())); + if (!callback_state) { + async_tcp_log_e("Failed to allocate the lookup context"); + return false; + } + err_t err; { tcp_core_guard tcg; - err = dns_gethostbyname(host, &addr, (dns_found_callback)&AsyncTCP_detail::tcp_dns_found, this); + err = dns_gethostbyname(host, &addr, (dns_found_callback)&AsyncTCP_detail::tcp_dns_found, callback_state.get()); + } + + if (err == ERR_INPROGRESS) { + _connect_port = port; + _dns_pending = true; + // DNS lookup in progress + callback_state.release(); // now tcp_dns_found's to destroy + return true; } if (err == ERR_OK) { @@ -947,30 +1162,34 @@ bool AsyncClient::connect(const char *host, uint16_t port) { #else return connect(addr, port); #endif - } else if (err == ERR_INPROGRESS) { - _connect_port = port; - return true; } async_tcp_log_d("error: %d", err); return false; } -void AsyncClient::close() { - if (_pcb && _rx_ack_len) { - // Ack anything withheld by ackLater() before we drop the connection - _tcp_recved(&_pcb, _rx_ack_len); - } +void AsyncClientImpl::close() { + _abandonResolve(); + // Ack anything withheld by ackLater(), plus the packet onData is holding right now if + // we are being called from inside it. Without this the peer gets an RST rather than a + // FIN for data the application did in fact process. + const size_t pending = _rx_ack_len + _in_callback_ack_len; _rx_ack_len = 0; - int8_t err = _tcp_close(&_pcb, this); + _in_callback_ack_len = 0; + int8_t err = _tcp_close(this, pending); // _pcb is now NULL if ((err == ERR_OK) && _discard_cb) { - // _pcb was closed here - async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, this)); + // _pcb was closed here. Same hazard as _error(): the application may destroy its + // AsyncClient from inside the callback, and close() is reached from the calling task + // where the facade can be the only holder. Nothing follows the call today, so this + // costs nothing and stops that being load-bearing. + auto self = shared_from_this(); + async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, _facade)); } } -int8_t AsyncClient::abort() { - int8_t err = _tcp_abort(&_pcb, this); +int8_t AsyncClientImpl::abort() { + _abandonResolve(); + int8_t err = _tcp_abort(this); // _pcb is now NULL // LwIP invokes the error callback when abort is issued; preserve this semantic. // This will also trigger the dispose callback. @@ -981,14 +1200,14 @@ int8_t AsyncClient::abort() { return err; } -size_t AsyncClient::space() const { +size_t AsyncClientImpl::space() const { if ((_pcb != NULL) && (_pcb->state == ESTABLISHED)) { return tcp_sndbuf(_pcb); } return 0; } -size_t AsyncClient::add(const char *data, size_t size, uint8_t apiflags) { +size_t AsyncClientImpl::add(const char *data, size_t size, uint8_t apiflags) { if (!_pcb || size == 0 || data == NULL) { return 0; } @@ -998,42 +1217,46 @@ size_t AsyncClient::add(const char *data, size_t size, uint8_t apiflags) { } size_t will_send = (room < size) ? room : size; int8_t err = ERR_OK; - err = _tcp_write(&_pcb, data, will_send, apiflags); + err = _tcp_write(this, data, will_send, apiflags); if (err != ERR_OK) { return 0; } return will_send; } -bool AsyncClient::send() { +bool AsyncClientImpl::send() { auto backup = _tx_last_packet; _tx_last_packet = millis(); - if (_tcp_output(&_pcb) == ERR_OK) { + if (_tcp_output(this) == ERR_OK) { return true; } _tx_last_packet = backup; return false; } -size_t AsyncClient::ack(size_t len) { +size_t AsyncClientImpl::ack(size_t len) { if (len > _rx_ack_len) { len = _rx_ack_len; } if (len) { - _tcp_recved(&_pcb, len); + _tcp_recved(this, len); } _rx_ack_len -= len; return len; } -void AsyncClient::ackPacket(struct pbuf *pb) { +void AsyncClientImpl::ackPacket(struct pbuf *pb) { if (!pb) { return; } - _tcp_recved(&_pcb, pb->len); + _tcp_recved(this, pb->len); pbuf_free(pb); } +void AsyncClientImpl::ackLater() { + _ack_pcb = false; +} + /* * Main Private Methods * */ @@ -1042,9 +1265,16 @@ void AsyncClient::ackPacket(struct pbuf *pb) { * Private Callbacks * */ +// LwIP has no way to cancel a lookup, so record that we no longer want the answer. +// tcp_dns_found() drops it and releases LwIP's reference when it eventually fires. +void AsyncClientImpl::_abandonResolve() { + queue_mutex_guard guard; + _dns_pending = false; +} + // Adopt a pcb and reset all per-connection state. Callers must already be // serialized against the LwIP core (accept callback, or tcp_core_guard). -void AsyncClient::_adopt(tcp_pcb *pcb) { +void AsyncClientImpl::_adopt(tcp_pcb *pcb) { _pcb = pcb; _rx_ack_len = 0; _tx_last_packet = 0; @@ -1053,7 +1283,7 @@ void AsyncClient::_adopt(tcp_pcb *pcb) { _bind_tcp_callbacks(pcb, this); } -int8_t AsyncClient::_connected(tcp_pcb *pcb, int8_t err) { +int8_t AsyncClientImpl::_connected(tcp_pcb *pcb, int8_t err) { if (pcb != _pcb) { // Stale event for a pcb we no longer own async_tcp_log_d("%p != %p", (const void *)pcb, (const void *)_pcb); @@ -1061,34 +1291,37 @@ int8_t AsyncClient::_connected(tcp_pcb *pcb, int8_t err) { } _rx_last_packet = millis(); if (_connect_cb) { - async_tcp_log_elapsed("onConnect", _connect_cb(_connect_cb_arg, this)); + async_tcp_log_elapsed("onConnect", _connect_cb(_connect_cb_arg, _facade)); } return ERR_OK; } -void AsyncClient::_error(int8_t err) { +void AsyncClientImpl::_error(int8_t err) { + // Hold a reference to self to ensure this remains valid throughout the user callbacks. + auto self = shared_from_this(); + // Run error callback if (_error_cb) { - async_tcp_log_elapsed("onError", _error_cb(_error_cb_arg, this, err)); + async_tcp_log_elapsed("onError", _error_cb(_error_cb_arg, _facade, err)); } - if (_discard_cb) { - async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, this)); + if (_facade && _discard_cb) { + async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, _facade)); } } -int8_t AsyncClient::_fin(tcp_pcb *pcb, int8_t err) { +int8_t AsyncClientImpl::_fin(tcp_pcb *pcb, int8_t err) { close(); return ERR_OK; } -int8_t AsyncClient::_sent(tcp_pcb *pcb, uint16_t len) { +int8_t AsyncClientImpl::_sent(tcp_pcb *pcb, uint16_t len) { _rx_last_ack = _rx_last_packet = millis(); if (_sent_cb) { - async_tcp_log_elapsed("onAck", _sent_cb(_sent_cb_arg, this, len, (_rx_last_packet - _tx_last_packet))); + async_tcp_log_elapsed("onAck", _sent_cb(_sent_cb_arg, _facade, len, (_rx_last_packet - _tx_last_packet))); } return ERR_OK; } -int8_t AsyncClient::_recv(tcp_pcb *pcb, pbuf *pb, int8_t err) { +int8_t AsyncClientImpl::_recv(tcp_pcb *pcb, pbuf *pb, int8_t err) { while (pb != NULL) { _rx_last_packet = millis(); // we should not ack before we assimilate the data @@ -1097,23 +1330,37 @@ int8_t AsyncClient::_recv(tcp_pcb *pcb, pbuf *pb, int8_t err) { pb = b->next; b->next = NULL; if (_pb_cb) { - async_tcp_log_elapsed("onPacket", _pb_cb(_pb_cb_arg, this, b)); + // The callback owns b now, and must ackPacket() or free it + async_tcp_log_elapsed("onPacket", _pb_cb(_pb_cb_arg, _facade, b)); } else { + // Visible to close(), so closing from inside onData still acks this packet + _in_callback_ack_len = b->len; if (_recv_cb) { - async_tcp_log_elapsed("onData", _recv_cb(_recv_cb_arg, this, b->payload, b->len)); + async_tcp_log_elapsed("onData", _recv_cb(_recv_cb_arg, _facade, b->payload, b->len)); } - if (!_ack_pcb) { - _rx_ack_len += b->len; - } else if (_pcb) { - _tcp_recved(&_pcb, b->len); + if (_in_callback_ack_len) { // still ours - close() did not take it + if (!_ack_pcb) { + _rx_ack_len += _in_callback_ack_len; + } else if (_pcb) { + _tcp_recved(this, _in_callback_ack_len); + } + _in_callback_ack_len = 0; } pbuf_free(b); } + // Stop if the callback closed the connection or destroyed the client. We outlive + // the facade, but there is nobody left to hand the remaining packets to. + if (!_pcb || !_facade) { + break; + } + } + if (pb) { + pbuf_free(pb); // release whatever we did not get to } return ERR_OK; } -int8_t AsyncClient::_poll(tcp_pcb *pcb) { +int8_t AsyncClientImpl::_poll(tcp_pcb *pcb) { if (!_pcb) { // async_tcp_log_d("pcb is NULL"); return ERR_OK; @@ -1137,7 +1384,7 @@ int8_t AsyncClient::_poll(tcp_pcb *pcb) { if (last_tx_is_after_last_ack && (now - _tx_last_packet) >= _ack_timeout) { async_tcp_log_d("ack timeout %d", pcb->state); if (_timeout_cb) { - async_tcp_log_elapsed("onTimeout", _timeout_cb(_timeout_cb_arg, this, (now - _tx_last_packet))); + async_tcp_log_elapsed("onTimeout", _timeout_cb(_timeout_cb_arg, _facade, (now - _tx_last_packet))); } return ERR_OK; } @@ -1150,12 +1397,12 @@ int8_t AsyncClient::_poll(tcp_pcb *pcb) { } // Everything is fine if (_poll_cb) { - async_tcp_log_elapsed("onPoll", _poll_cb(_poll_cb_arg, this)); + async_tcp_log_elapsed("onPoll", _poll_cb(_poll_cb_arg, _facade)); } return ERR_OK; } -void AsyncClient::_dns_found(bool resolved, ip_addr_t *ipaddr, uint16_t port) { +void AsyncClientImpl::_dns_found(bool resolved, ip_addr_t *ipaddr, uint16_t port) { if (resolved && ip_addr_isany_val(*ipaddr)) { // A resolver that answers with the any-address - a sinkhole, or a blocklist - has // not given us anywhere to go. Report it as a failed lookup rather than dialling it. @@ -1173,7 +1420,7 @@ void AsyncClient::_dns_found(bool resolved, ip_addr_t *ipaddr, uint16_t port) { * Public Helper Methods * */ -bool AsyncClient::free() { +bool AsyncClientImpl::free() { if (!_pcb) { return true; } @@ -1183,7 +1430,7 @@ bool AsyncClient::free() { return false; } -size_t AsyncClient::write(const char *data, size_t size, uint8_t apiflags) { +size_t AsyncClientImpl::write(const char *data, size_t size, uint8_t apiflags) { size_t will_send = add(data, size, apiflags); if (!will_send || !send()) { return 0; @@ -1191,23 +1438,23 @@ size_t AsyncClient::write(const char *data, size_t size, uint8_t apiflags) { return will_send; } -void AsyncClient::setRxTimeout(uint32_t timeout) { +void AsyncClientImpl::setRxTimeout(uint32_t timeout) { _rx_timeout = timeout; } -uint32_t AsyncClient::getRxTimeout() const { +uint32_t AsyncClientImpl::getRxTimeout() const { return _rx_timeout; } -uint32_t AsyncClient::getAckTimeout() const { +uint32_t AsyncClientImpl::getAckTimeout() const { return _ack_timeout; } -void AsyncClient::setAckTimeout(uint32_t timeout) { +void AsyncClientImpl::setAckTimeout(uint32_t timeout) { _ack_timeout = timeout; } -void AsyncClient::setNoDelay(bool nodelay) const { +void AsyncClientImpl::setNoDelay(bool nodelay) const { if (!_pcb) { return; } @@ -1218,14 +1465,14 @@ void AsyncClient::setNoDelay(bool nodelay) const { } } -bool AsyncClient::getNoDelay() { +bool AsyncClientImpl::getNoDelay() { if (!_pcb) { return false; } return tcp_nagle_disabled(_pcb); } -void AsyncClient::setKeepAlive(uint32_t ms, uint8_t cnt) { +void AsyncClientImpl::setKeepAlive(uint32_t ms, uint8_t cnt) { if (!_pcb) { return; } @@ -1240,14 +1487,14 @@ void AsyncClient::setKeepAlive(uint32_t ms, uint8_t cnt) { } } -uint16_t AsyncClient::getMss() const { +uint16_t AsyncClientImpl::getMss() const { if (!_pcb) { return 0; } return tcp_mss(_pcb); } -uint32_t AsyncClient::getRemoteAddress() const { +uint32_t AsyncClientImpl::getRemoteAddress() const { if (!_pcb) { return 0; } @@ -1259,7 +1506,7 @@ uint32_t AsyncClient::getRemoteAddress() const { } #if LWIP_IPV6 -ip6_addr_t AsyncClient::getRemoteAddress6() const { +ip6_addr_t AsyncClientImpl::getRemoteAddress6() const { if (_pcb && _pcb->remote_ip.type == IPADDR_TYPE_V6) { return _pcb->remote_ip.u_addr.ip6; } else { @@ -1269,7 +1516,7 @@ ip6_addr_t AsyncClient::getRemoteAddress6() const { } } -ip6_addr_t AsyncClient::getLocalAddress6() const { +ip6_addr_t AsyncClientImpl::getLocalAddress6() const { if (_pcb && _pcb->local_ip.type == IPADDR_TYPE_V6) { return _pcb->local_ip.u_addr.ip6; } else { @@ -1280,15 +1527,15 @@ ip6_addr_t AsyncClient::getLocalAddress6() const { } #ifdef ARDUINO #if ESP_IDF_VERSION_MAJOR < 5 -IPv6Address AsyncClient::remoteIP6() const { +IPv6Address AsyncClientImpl::remoteIP6() const { return IPv6Address(getRemoteAddress6().addr); } -IPv6Address AsyncClient::localIP6() const { +IPv6Address AsyncClientImpl::localIP6() const { return IPv6Address(getLocalAddress6().addr); } #else -IPAddress AsyncClient::remoteIP6() const { +IPAddress AsyncClientImpl::remoteIP6() const { if (!_pcb) { return IPAddress(IPType::IPv6); } @@ -1297,7 +1544,7 @@ IPAddress AsyncClient::remoteIP6() const { return ip; } -IPAddress AsyncClient::localIP6() const { +IPAddress AsyncClientImpl::localIP6() const { if (!_pcb) { return IPAddress(IPType::IPv6); } @@ -1309,14 +1556,14 @@ IPAddress AsyncClient::localIP6() const { #endif #endif -uint16_t AsyncClient::getRemotePort() const { +uint16_t AsyncClientImpl::getRemotePort() const { if (!_pcb) { return 0; } return _pcb->remote_port; } -uint32_t AsyncClient::getLocalAddress() const { +uint32_t AsyncClientImpl::getLocalAddress() const { if (!_pcb) { return 0; } @@ -1327,14 +1574,14 @@ uint32_t AsyncClient::getLocalAddress() const { #endif } -uint16_t AsyncClient::getLocalPort() const { +uint16_t AsyncClientImpl::getLocalPort() const { if (!_pcb) { return 0; } return _pcb->local_port; } -ip4_addr_t AsyncClient::getRemoteAddress4() const { +ip4_addr_t AsyncClientImpl::getRemoteAddress4() const { #if LWIP_IPV4 && LWIP_IPV6 if (_pcb && _pcb->remote_ip.type == IPADDR_TYPE_V4) { return _pcb->remote_ip.u_addr.ip4; @@ -1351,7 +1598,7 @@ ip4_addr_t AsyncClient::getRemoteAddress4() const { } } -ip4_addr_t AsyncClient::getLocalAddress4() const { +ip4_addr_t AsyncClientImpl::getLocalAddress4() const { #if LWIP_IPV4 && LWIP_IPV6 if (_pcb && _pcb->local_ip.type == IPADDR_TYPE_V4) { return _pcb->local_ip.u_addr.ip4; @@ -1369,7 +1616,7 @@ ip4_addr_t AsyncClient::getLocalAddress4() const { } #ifdef ARDUINO -IPAddress AsyncClient::remoteIP() const { +IPAddress AsyncClientImpl::remoteIP() const { #if ESP_IDF_VERSION_MAJOR < 5 return IPAddress(getRemoteAddress()); #else @@ -1382,7 +1629,7 @@ IPAddress AsyncClient::remoteIP() const { #endif } -IPAddress AsyncClient::localIP() const { +IPAddress AsyncClientImpl::localIP() const { #if ESP_IDF_VERSION_MAJOR < 5 return IPAddress(getLocalAddress()); #else @@ -1396,49 +1643,55 @@ IPAddress AsyncClient::localIP() const { } #endif -uint8_t AsyncClient::state() const { +uint8_t AsyncClientImpl::state() const { if (!_pcb) { return 0; } return _pcb->state; } -bool AsyncClient::connected() const { +bool AsyncClientImpl::connected() const { if (!_pcb) { return false; } return _pcb->state == ESTABLISHED; } -bool AsyncClient::connecting() const { +bool AsyncClientImpl::connecting() const { + if (_dns_pending) { + return true; // resolving the hostname + } if (!_pcb) { return false; } return _pcb->state > CLOSED && _pcb->state < ESTABLISHED; } -bool AsyncClient::disconnecting() const { +bool AsyncClientImpl::disconnecting() const { if (!_pcb) { return false; } return _pcb->state > ESTABLISHED && _pcb->state < TIME_WAIT; } -bool AsyncClient::disconnected() const { +bool AsyncClientImpl::disconnected() const { + if (_dns_pending) { + return false; // resolving the hostname + } if (!_pcb) { return true; } return _pcb->state == CLOSED || _pcb->state == TIME_WAIT; } -bool AsyncClient::freeable() const { +bool AsyncClientImpl::freeable() const { if (!_pcb) { return true; } return _pcb->state == CLOSED || _pcb->state > ESTABLISHED; } -bool AsyncClient::canSend() const { +bool AsyncClientImpl::canSend() const { return space() > 0; } @@ -1465,7 +1718,7 @@ const char *AsyncClient::errorToString(int8_t error) { } } -const char *AsyncClient::stateToString() const { +const char *AsyncClientImpl::stateToString() const { switch (state()) { case 0: return "Closed"; case 1: return "Listen"; @@ -1482,15 +1735,260 @@ const char *AsyncClient::stateToString() const { } } +/* + * AsyncClient - the application-facing facade. + * + * Every method below forwards to the reference-counted implementation. _impl is never + * null for a live AsyncClient: it is created in the constructor and only released in + * the destructor. + */ + +bool AsyncClient::connect(ip_addr_t addr, uint16_t port) { + return _impl->connect(addr, port); +} + +bool AsyncClient::connect(const char *host, uint16_t port) { + return _impl->connect(host, port); +} + +#ifdef ARDUINO +bool AsyncClient::connect(const IPAddress &ip, uint16_t port) { + return _impl->connect(ip, port); +} +#if LWIP_IPV6 && ESP_IDF_VERSION_MAJOR < 5 +bool AsyncClient::connect(const IPv6Address &ip, uint16_t port) { + return _impl->connect(ip, port); +} +#endif +#endif + +void AsyncClient::close() { + _impl->close(); +} + +int8_t AsyncClient::abort() { + return _impl->abort(); +} + +bool AsyncClient::free() { + return _impl->free(); +} + +bool AsyncClient::canSend() const { + return _impl->canSend(); +} + +size_t AsyncClient::space() const { + return _impl->space(); +} + +size_t AsyncClient::add(const char *data, size_t size, uint8_t apiflags) { + return _impl->add(data, size, apiflags); +} + +bool AsyncClient::send() { + return _impl->send(); +} + +size_t AsyncClient::write(const char *data, size_t size, uint8_t apiflags) { + return _impl->write(data, size, apiflags); +} + +uint8_t AsyncClient::state() const { + return _impl->state(); +} + +bool AsyncClient::connecting() const { + return _impl->connecting(); +} + +bool AsyncClient::connected() const { + return _impl->connected(); +} + +bool AsyncClient::disconnecting() const { + return _impl->disconnecting(); +} + +bool AsyncClient::disconnected() const { + return _impl->disconnected(); +} + +bool AsyncClient::freeable() const { + return _impl->freeable(); +} + +uint16_t AsyncClient::getMss() const { + return _impl->getMss(); +} + +uint32_t AsyncClient::getRxTimeout() const { + return _impl->getRxTimeout(); +} + +void AsyncClient::setRxTimeout(uint32_t timeout) { + _impl->setRxTimeout(timeout); +} + +uint32_t AsyncClient::getAckTimeout() const { + return _impl->getAckTimeout(); +} + +void AsyncClient::setAckTimeout(uint32_t timeout) { + _impl->setAckTimeout(timeout); +} + +void AsyncClient::setNoDelay(bool nodelay) const { + _impl->setNoDelay(nodelay); +} + +bool AsyncClient::getNoDelay() { + return _impl->getNoDelay(); +} + +void AsyncClient::setKeepAlive(uint32_t ms, uint8_t cnt) { + _impl->setKeepAlive(ms, cnt); +} + +uint32_t AsyncClient::getRemoteAddress() const { + return _impl->getRemoteAddress(); +} + +uint16_t AsyncClient::getRemotePort() const { + return _impl->getRemotePort(); +} + +uint32_t AsyncClient::getLocalAddress() const { + return _impl->getLocalAddress(); +} + +uint16_t AsyncClient::getLocalPort() const { + return _impl->getLocalPort(); +} + +ip4_addr_t AsyncClient::getRemoteAddress4() const { + return _impl->getRemoteAddress4(); +} + +ip4_addr_t AsyncClient::getLocalAddress4() const { + return _impl->getLocalAddress4(); +} + +#if LWIP_IPV6 +ip6_addr_t AsyncClient::getRemoteAddress6() const { + return _impl->getRemoteAddress6(); +} + +ip6_addr_t AsyncClient::getLocalAddress6() const { + return _impl->getLocalAddress6(); +} +#ifdef ARDUINO +#if ESP_IDF_VERSION_MAJOR < 5 +IPv6Address AsyncClient::remoteIP6() const { + return _impl->remoteIP6(); +} + +IPv6Address AsyncClient::localIP6() const { + return _impl->localIP6(); +} +#else +IPAddress AsyncClient::remoteIP6() const { + return _impl->remoteIP6(); +} + +IPAddress AsyncClient::localIP6() const { + return _impl->localIP6(); +} +#endif +#endif +#endif + +#ifdef ARDUINO +IPAddress AsyncClient::remoteIP() const { + return _impl->remoteIP(); +} + +IPAddress AsyncClient::localIP() const { + return _impl->localIP(); +} +#endif + +void AsyncClient::ackPacket(struct pbuf *pb) { + _impl->ackPacket(pb); +} + +size_t AsyncClient::ack(size_t len) { + return _impl->ack(len); +} + +void AsyncClient::ackLater() { + _impl->ackLater(); +} + +const char *AsyncClient::stateToString() const { + return _impl->stateToString(); +} + +int8_t AsyncClient::_recv(tcp_pcb *pcb, pbuf *pb, int8_t err) { + return _impl->_recv(pcb, pb, err); +} + +tcp_pcb *AsyncClient::pcb() { + return _impl->_pcb; +} + +void AsyncClient::onConnect(AcConnectHandler cb, void *arg) { + _impl->_connect_cb = cb; + _impl->_connect_cb_arg = arg; +} + +void AsyncClient::onDisconnect(AcConnectHandler cb, void *arg) { + _impl->_discard_cb = cb; + _impl->_discard_cb_arg = arg; +} + +void AsyncClient::onAck(AcAckHandler cb, void *arg) { + _impl->_sent_cb = cb; + _impl->_sent_cb_arg = arg; +} + +void AsyncClient::onError(AcErrorHandler cb, void *arg) { + _impl->_error_cb = cb; + _impl->_error_cb_arg = arg; +} + +void AsyncClient::onData(AcDataHandler cb, void *arg) { + _impl->_recv_cb = cb; + _impl->_recv_cb_arg = arg; +} + +void AsyncClient::onPacket(AcPacketHandler cb, void *arg) { + _impl->_pb_cb = cb; + _impl->_pb_cb_arg = arg; +} + +void AsyncClient::onTimeout(AcTimeoutHandler cb, void *arg) { + _impl->_timeout_cb = cb; + _impl->_timeout_cb_arg = arg; +} + +void AsyncClient::onPoll(AcConnectHandler cb, void *arg) { + _impl->_poll_cb = cb; + _impl->_poll_cb_arg = arg; +} + /* Async TCP Server */ AsyncServer::AsyncServer(ip_addr_t addr, uint16_t port) - : _port(port), _addr(addr), _noDelay(false), _pcb(nullptr), _connect_cb(nullptr), _connect_cb_arg(nullptr) {} + : _port(port), _addr(addr), _noDelay(false), _pcb(nullptr), _connect_cb(nullptr), _connect_cb_arg(nullptr) { + _init_queue_mutex(); +} #ifdef ARDUINO AsyncServer::AsyncServer(IPAddress addr, uint16_t port) : _port(port), _noDelay(false), _pcb(0), _connect_cb(0), _connect_cb_arg(0) { + _init_queue_mutex(); #if ESP_IDF_VERSION_MAJOR < 5 #if LWIP_IPV4 && LWIP_IPV6 _addr.type = IPADDR_TYPE_V4; @@ -1504,6 +2002,7 @@ AsyncServer::AsyncServer(IPAddress addr, uint16_t port) : _port(port), _noDelay( } #if ESP_IDF_VERSION_MAJOR < 5 && __has_include() && LWIP_IPV6 AsyncServer::AsyncServer(IPv6Address addr, uint16_t port) : _port(port), _noDelay(false), _pcb(0), _connect_cb(0), _connect_cb_arg(0) { + _init_queue_mutex(); #if LWIP_IPV4 && LWIP_IPV6 _addr.type = IPADDR_TYPE_V6; #endif @@ -1514,6 +2013,7 @@ AsyncServer::AsyncServer(IPv6Address addr, uint16_t port) : _port(port), _noDela #endif AsyncServer::AsyncServer(uint16_t port) : _port(port), _noDelay(false), _pcb(0), _connect_cb(0), _connect_cb_arg(0) { + _init_queue_mutex(); #if LWIP_IPV4 && LWIP_IPV6 _addr.type = IPADDR_TYPE_ANY; _addr.u_addr.ip4.addr = INADDR_ANY; @@ -1677,7 +2177,7 @@ int8_t AsyncTCP_detail::tcp_accept(void *arg, tcp_pcb *pcb, int8_t err) { } c->setNoDelay(server->_noDelay); - e->client = c; + e->impl = c->_impl; e->accept.server = server; queue_mutex_guard guard; diff --git a/src/AsyncTCP.h b/src/AsyncTCP.h index b4015a78..93b6d46d 100644 --- a/src/AsyncTCP.h +++ b/src/AsyncTCP.h @@ -17,6 +17,7 @@ #include "lwip/ip6_addr.h" #include "lwip/ip_addr.h" #include +#include #ifndef LIBRETINY #include "sdkconfig.h" @@ -72,6 +73,7 @@ typedef std::function AcTimeoutHandl struct tcp_pcb; class AsyncTCP_detail; +class AsyncClientImpl; class AsyncClient { public: @@ -256,57 +258,23 @@ class AsyncClient { // ack data that you have not acked using the method below size_t ack(size_t len); // will not ack the current packet. Call from onData - void ackLater() { - _ack_pcb = false; - } + void ackLater(); static const char *errorToString(int8_t error); const char *stateToString() const; int8_t _recv(tcp_pcb *pcb, pbuf *pb, int8_t err); - tcp_pcb *pcb() { - return _pcb; - } + tcp_pcb *pcb(); protected: friend class AsyncTCP_detail; friend class AsyncServer; - tcp_pcb *_pcb; - - AcConnectHandler _connect_cb; - void *_connect_cb_arg; - AcConnectHandler _discard_cb; - void *_discard_cb_arg; - AcAckHandler _sent_cb; - void *_sent_cb_arg; - AcErrorHandler _error_cb; - void *_error_cb_arg; - AcDataHandler _recv_cb; - void *_recv_cb_arg; - AcPacketHandler _pb_cb; - void *_pb_cb_arg; - AcTimeoutHandler _timeout_cb; - void *_timeout_cb_arg; - AcConnectHandler _poll_cb; - void *_poll_cb_arg; - - bool _ack_pcb; - uint32_t _tx_last_packet; - uint32_t _rx_ack_len; - uint32_t _rx_last_packet; - uint32_t _rx_timeout; - uint32_t _rx_last_ack; - uint32_t _ack_timeout; - uint16_t _connect_port; - - void _adopt(tcp_pcb *pcb); - int8_t _connected(tcp_pcb *pcb, int8_t err); - void _error(int8_t err); - int8_t _poll(tcp_pcb *pcb); - int8_t _sent(tcp_pcb *pcb, uint16_t len); - int8_t _fin(tcp_pcb *pcb, int8_t err); - void _dns_found(bool resolved, ip_addr_t *ipaddr, uint16_t port); + // Every scrap of state lives in the implementation object, which outlives this + // facade: queued events share ownership of it. That keeps the callbacks themselves + // alive for the duration of a call, so destroying an AsyncClient from inside one of + // its own callbacks is safe. + std::shared_ptr _impl; }; class AsyncServer { From fa8e38b4bb3bb7b573668bc399a7d01a5676f311 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 11 Sep 2026 20:15:59 -0400 Subject: [PATCH 10/35] refactor: Rework the name lookup around a context LwIP owns LwIP cannot cancel a lookup, and promises exactly one callback iff dns_gethostbyname() returns ERR_INPROGRESS. The callback argument is a heap-allocated weak_ptr owned by LwIP, transferred on ERR_INPROGRESS and nowhere else; weak, so a client destroyed mid-lookup is not pinned until the resolver times out. It doubles as the lookup's identity, so superseded lookups orphan themselves - a bare flag cannot tell connect(A), close(), connect(B) apart. - Token and query are published in one transaction, or the callback can arrive first and drop an answer connect() has reported success for. - The answer is dialled from the callback; only the failure path defers, because it runs user callbacks. - close() and abort() stand a lookup down inside their own transactions, so the token is written only in LwIP context and needs no lock. - A new target supersedes rather than being refused: refusing is the better API, but would strand existing callers. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 296 ++++++++++++++++++++++++++--------------------- 1 file changed, 163 insertions(+), 133 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index c930cb38..ebd3e358 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -133,8 +133,7 @@ typedef enum { LWIP_TCP_ERROR, LWIP_TCP_POLL, LWIP_TCP_ACCEPT, - LWIP_TCP_CONNECTED, - LWIP_TCP_DNS + LWIP_TCP_CONNECTED } lwip_tcp_event_t; struct lwip_tcp_event_packet_t { @@ -168,11 +167,6 @@ struct lwip_tcp_event_packet_t { struct { AsyncServer *server; } accept; - struct { - ip_addr_t addr; - uint16_t port; - bool resolved; - } dns; }; inline lwip_tcp_event_packet_t(lwip_tcp_event_t _event, std::shared_ptr _impl) : next(nullptr), event(_event), impl(std::move(_impl)){}; @@ -258,9 +252,21 @@ class AsyncClientImpl : public std::enable_shared_from_this { AsyncClientImpl(const AsyncClientImpl &) = delete; AsyncClientImpl &operator=(const AsyncClientImpl &) = delete; - AsyncClient *_facade; // nulled once the application's object is destroyed + /* + Members held under LwIP context + */ tcp_pcb *_pcb; + uint16_t _connect_port; + std::weak_ptr *_dns_token; // non-owning pointer to the most recent DNS lookup token + + /* + Not synchronized. The handlers are installed by the application before it connects + and read by the async task when it dispatches; the rest is connection state that the + async task owns in practice, touched from the calling task only through calls the + application is expected not to make from two tasks at once against one client. + */ + AsyncClient *_facade; // nulled once the application's object is destroyed AcConnectHandler _connect_cb; void *_connect_cb_arg; @@ -286,8 +292,6 @@ class AsyncClientImpl : public std::enable_shared_from_this { uint32_t _rx_timeout; uint32_t _rx_last_ack; uint32_t _ack_timeout; - uint16_t _connect_port; - bool _dns_pending; // a lookup is outstanding and holds a reference uint32_t _in_callback_ack_len; // bytes handed to onData but not yet acked ASYNCTCP_ALWAYS_INLINE bool connect(ip_addr_t addr, uint16_t port); @@ -314,7 +318,6 @@ class AsyncClientImpl : public std::enable_shared_from_this { ASYNCTCP_ALWAYS_INLINE bool disconnecting() const; ASYNCTCP_ALWAYS_INLINE bool disconnected() const; ASYNCTCP_ALWAYS_INLINE bool freeable() const; - void _abandonResolve(); ASYNCTCP_ALWAYS_INLINE uint16_t getMss() const; ASYNCTCP_ALWAYS_INLINE uint32_t getRxTimeout() const; @@ -361,7 +364,6 @@ class AsyncClientImpl : public std::enable_shared_from_this { int8_t _sent(tcp_pcb *pcb, uint16_t len); int8_t _fin(tcp_pcb *pcb, int8_t err); int8_t _recv(tcp_pcb *pcb, pbuf *pb, int8_t err); - void _dns_found(bool resolved, ip_addr_t *ipaddr, uint16_t port); }; static uint32_t _xor_shift_state = 31; // any nonzero seed will do @@ -535,9 +537,6 @@ void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { // ets_printf("A: 0x%08x 0x%08x\n", e->impl, e->accept.server); // Gated above, so the facade is still alive e->accept.server->_accepted(e->impl->_facade); - } else if (e->event == LWIP_TCP_DNS) { - // ets_printf("D: 0x%08x = %s\n", e->impl, ipaddr_ntoa(&e->dns.addr)); - e->impl->_dns_found(e->dns.resolved, &e->dns.addr, e->dns.port); } _free_event(e); } @@ -615,6 +614,9 @@ static bool _start_async_task() { * LwIP Callbacks * */ +// Defined with the other api calls below; tcp_dns_found() dials through it directly. +static err_t _tcp_connect_in_context(AsyncClientImpl *client, const ip_addr_t *addr, uint16_t port); + // Attach an AsyncClient to a TCP PCB by setting the appropriate LwIP callbacks and argument. static void _bind_tcp_callbacks(tcp_pcb *pcb, AsyncClientImpl *client) { tcp_arg(pcb, client); @@ -635,6 +637,13 @@ static void _reset_tcp_callbacks(tcp_pcb *pcb, AsyncClientImpl *client) { } } +// Stand down any outstanding name lookup. Run in LwIP context to ensure atomicity with the tcp_dns_found() callback. +static void _abandon_resolve(AsyncClientImpl *client) { + if (client) { + client->_dns_token = nullptr; + } +} + // Callback when a connection is established. pcb is already bound to the client. static int8_t _tcp_connected(void *arg, tcp_pcb *pcb, int8_t err) { // ets_printf("+C: 0x%08x\n", pcb); @@ -759,31 +768,36 @@ void AsyncTCP_detail::tcp_dns_found(const char *name, const ip_addr_t *ipaddr, v return; // the implementation was destroyed while the lookup was in flight } - bool wanted; - { - queue_mutex_guard guard; - wanted = client->_dns_pending; - client->_dns_pending = false; + // Validate that we're the most recent DNS lookup for this client. + if (client->_dns_token != callback_state.get()) { + return; // close(), abort() or a newer lookup superseded this query } - if (!wanted) { - return; // close() or abort() gave up on this lookup while it was in flight + const uint16_t port = client->_connect_port; + client->_dns_token = nullptr; // lookup complete + + // Check that client isn't already open? + if (client->_pcb) { + return; // connected by some other route while we were resolving + } + + // Treat any-address or blocklist responses as failed lookups. + const bool resolved = (ipaddr != nullptr) && !ip_addr_isany_val(*ipaddr); + + // Start the connection right away; we're already in LwIP context + if (resolved && _tcp_connect_in_context(client.get(), ipaddr, port) == ERR_OK) { + return; } - lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_DNS, client}; + // The original DNS-based connect() reported "success" to the caller, so we must notify + // them of the failure. Queue an error event packet. + lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_ERROR, client}; if (!e) { async_tcp_log_e("Failed to allocate event packet"); - } else { - e->dns.port = client->_connect_port; - e->dns.resolved = (ipaddr != nullptr); - if (ipaddr) { - memcpy(&e->dns.addr, ipaddr, sizeof(ip_addr_t)); - } else { - memset(&e->dns.addr, 0, sizeof(e->dns.addr)); - } + return; } - + e->error.err = resolved ? ERR_CONN : -55; queue_mutex_guard guard; - _send_async_event(e); // no-op if the allocation failed + _send_async_event(e); } /* @@ -793,14 +807,12 @@ void AsyncTCP_detail::tcp_dns_found(const char *name, const ip_addr_t *ipaddr, v #include "lwip/priv/tcpip_priv.h" /* - One transaction's arguments. The client-side calls address the pcb through `client`, - which is the only thing that knows which pcb is current; `pcb` is for the server, whose - listening socket is not owned by an implementation object. + Context structure for `tcpip_api_call`, ie. code running in LwIP context. + The pcb is always reached through `client`. */ typedef struct { struct tcpip_api_call_data call; AsyncClientImpl *client; - tcp_pcb **pcb; int8_t err; union { size_t close_ack; // bytes to tcp_recved() before closing, in the same pass @@ -811,10 +823,15 @@ typedef struct { } write; size_t received; struct { - ip_addr_t *addr; + const ip_addr_t *addr; uint16_t port; - tcp_connected_fn cb; } connect; + struct { + const char *host; + ip_addr_t *addr; + std::weak_ptr *token; + uint16_t port; + } dns; }; } tcp_api_call_t; @@ -881,17 +898,17 @@ static esp_err_t _tcp_recved(AsyncClientImpl *client, size_t len) { } static err_t _tcp_close_api(struct tcpip_api_call_data *api_call_msg) { - // Unlike the other calls, this is not a direct wrapper of the LwIP function; - // we perform the AsyncClient teardown interlocked safely with the LwIP task. + // Unlike the other calls, this is not a direct wrapper of the LwIP function; we perform + // the AsyncClient teardown interlocked safely with the LwIP task. - // As a postcondition, the queue must not have any events referencing - // the AsyncClient in api_call_msg->close. This is because it is possible for - // an error event to have been queued, clearing the pcb*, but after the async - // thread has committed to closing/destructing the AsyncClient object. + // As a postcondition, the queue must not have any events referencing this client. This + // is because it is possible for an error event to have been queued, clearing the pcb*, + // but after the async thread has committed to closing/destructing the AsyncClient. tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; AsyncClientImpl *client = msg->client; msg->err = ERR_CONN; + _abandon_resolve(client); if (client->_pcb) { tcp_pcb *pcb = client->_pcb; // Ack anything the application withheld, in this same pass. Done as a separate call @@ -933,6 +950,7 @@ static err_t _tcp_abort_api(struct tcpip_api_call_data *api_call_msg) { // ERR_CONN: nothing to do (pcb already null and no queued events). tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; AsyncClientImpl *client = msg->client; + _abandon_resolve(client); if (client->_pcb) { _reset_tcp_callbacks(client->_pcb, client); tcp_abort(client->_pcb); @@ -945,58 +963,108 @@ static err_t _tcp_abort_api(struct tcpip_api_call_data *api_call_msg) { } static esp_err_t _tcp_abort(AsyncClientImpl *client) { + // No early return on a null pcb. An outstanding lookup is a second thing LwIP is + // holding, and standing it down has to happen inside the interlock. tcp_api_call_t msg; msg.client = client; tcpip_api_call(_tcp_abort_api, (struct tcpip_api_call_data *)&msg); return msg.err; } -static err_t _tcp_connect_api(struct tcpip_api_call_data *api_call_msg) { - tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; - tcp_pcb *pcb = *msg->pcb; - // Non-zero only if the caller handed us an already-bound pcb - u16_t bound_port = pcb->local_port; +static err_t _tcp_connect_in_context(AsyncClientImpl *client, const ip_addr_t *addr, uint16_t port) { +#if LWIP_IPV4 && LWIP_IPV6 + tcp_pcb *pcb = tcp_new_ip_type(addr->type); +#else + tcp_pcb *pcb = tcp_new_ip_type(IPADDR_TYPE_V4); +#endif + if (!pcb) { + async_tcp_log_e("pcb == NULL"); + return ERR_MEM; + } + + // Take ownership of the pcb + client->_adopt(pcb); - msg->err = tcp_connect(pcb, msg->connect.addr, msg->connect.port, msg->connect.cb); - if (msg->err != ERR_OK) { + err_t err = tcp_connect(pcb, addr, port, (tcp_connected_fn)&_tcp_connected); + if (err != ERR_OK) { // Failure - clean up _reset_tcp_callbacks(pcb, nullptr); - if (bound_port == 0) { - // tcp_connect() may have taken a local port without adding the pcb to - // tcp_bound_pcbs. Clear it, or tcp_close() will try to remove the pcb from a - // list it was never on - which trips an assert on an empty list. - pcb->local_port = 0; - } + // Work around LwIP bug + // tcp_connect() may have taken a local port without adding the pcb to + // tcp_bound_pcbs. Clear it, or tcp_close() will try to remove the pcb from a list + // it was never on. + pcb->local_port = 0; if (tcp_close(pcb) != ERR_OK) { tcp_abort(pcb); } - *msg->pcb = nullptr; // PCB is now the property of LwIP + client->_pcb = nullptr; // PCB is now the property of LwIP } + + return err; +} + +static err_t _tcp_connect_api(struct tcpip_api_call_data *api_call_msg) { + tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; + msg->err = _tcp_connect_in_context(msg->client, msg->connect.addr, msg->connect.port); return msg->err; } -static esp_err_t _tcp_connect(tcp_pcb **pcb, ip_addr_t *addr, uint16_t port, tcp_connected_fn cb) { - if (!pcb || !*pcb) { - return ESP_FAIL; - } +static esp_err_t _tcp_connect(AsyncClientImpl *client, const ip_addr_t *addr, uint16_t port) { tcp_api_call_t msg; - msg.pcb = pcb; + msg.client = client; msg.connect.addr = addr; msg.connect.port = port; - msg.connect.cb = cb; tcpip_api_call(_tcp_connect_api, (struct tcpip_api_call_data *)&msg); return msg.err; } +/* + Start a name lookup and publish its token as one transaction. + + These cannot be separate steps. LwIP answers on its own thread, so a token published + after dns_gethostbyname() returns can arrive too late: the callback finds no token, + concludes it has been superseded, and drops an answer the caller is still waiting for - + leaving connect() having reported success that never resolves either way. + + Taking the LwIP core lock around just the lookup would not fix it, and not only because + the window would remain: that lock does not exist without CONFIG_LWIP_TCPIP_CORE_LOCKING, + where the interlock is the api call itself and nothing else. +*/ +static err_t _tcp_dns_start_api(struct tcpip_api_call_data *api_call_msg) { + tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; + AsyncClientImpl *client = msg->client; + + // Supersedes any lookup already outstanding; that one will find the token moved on. + client->_connect_port = msg->dns.port; + client->_dns_token = msg->dns.token; + + msg->err = dns_gethostbyname(msg->dns.host, msg->dns.addr, (dns_found_callback)&AsyncTCP_detail::tcp_dns_found, msg->dns.token); + if (msg->err != ERR_INPROGRESS) { + client->_dns_token = nullptr; // no callback is coming + } + return msg->err; +} + +static err_t _tcp_dns_start(AsyncClientImpl *client, const char *host, ip_addr_t *addr, uint16_t port, std::weak_ptr *token) { + tcp_api_call_t msg; + msg.client = client; + msg.dns.host = host; + msg.dns.addr = addr; + msg.dns.port = port; + msg.dns.token = token; + tcpip_api_call(_tcp_dns_start_api, (struct tcpip_api_call_data *)&msg); + return msg.err; +} + /* Async TCP Client */ AsyncClientImpl::AsyncClientImpl(AsyncClient *facade) - : _facade(facade), _pcb(nullptr), _connect_cb(0), _connect_cb_arg(0), _discard_cb(0), _discard_cb_arg(0), _sent_cb(0), _sent_cb_arg(0), _error_cb(0), - _error_cb_arg(0), _recv_cb(0), _recv_cb_arg(0), _pb_cb(0), _pb_cb_arg(0), _timeout_cb(0), _timeout_cb_arg(0), _poll_cb(0), _poll_cb_arg(0), _ack_pcb(true), - _tx_last_packet(0), _rx_ack_len(0), _rx_last_packet(0), _rx_timeout(0), _rx_last_ack(0), _ack_timeout(CONFIG_ASYNC_TCP_MAX_ACK_TIME), _connect_port(0), - _dns_pending(false), _in_callback_ack_len(0) {} + : _pcb(nullptr), _connect_port(0), _dns_token(nullptr), _facade(facade), _connect_cb(0), _connect_cb_arg(0), _discard_cb(0), _discard_cb_arg(0), _sent_cb(0), + _sent_cb_arg(0), _error_cb(0), _error_cb_arg(0), _recv_cb(0), _recv_cb_arg(0), _pb_cb(0), _pb_cb_arg(0), _timeout_cb(0), _timeout_cb_arg(0), _poll_cb(0), + _poll_cb_arg(0), _ack_pcb(true), _tx_last_packet(0), _rx_ack_len(0), _rx_last_packet(0), _rx_timeout(0), _rx_last_ack(0), + _ack_timeout(CONFIG_ASYNC_TCP_MAX_ACK_TIME), _in_callback_ack_len(0) {} AsyncClientImpl::~AsyncClientImpl() { // Every path that drops the last reference must ensure the binding was cleared first. @@ -1012,9 +1080,12 @@ AsyncClient::AsyncClient(tcp_pcb *pcb) : _impl(std::make_shared } AsyncClient::~AsyncClient() { - if (_impl->_pcb) { - _impl->close(); - } + // Unconditional. Whether there is anything to close is not knowable from here: _pcb + // and the lookup token are both written by the LwIP thread, and a name lookup can + // still turn into a bound pcb after we have looked. Deciding out here would be + // deciding whether we need the interlock without holding it; close() is cheap when + // there is nothing to do. + _impl->close(); // Detach: queued events and LwIP callbacks may still reach the implementation, but // must not run user code against a facade that no longer exists. _impl->_facade = nullptr; @@ -1041,7 +1112,7 @@ bool AsyncClientImpl::connect(ip_addr_t addr, uint16_t port) { async_tcp_log_d("already connected, state %d", _pcb->state); return false; } - if (_dns_pending) { + if (_dns_token) { async_tcp_log_d("name resolution in progress"); return false; } @@ -1050,32 +1121,15 @@ bool AsyncClientImpl::connect(ip_addr_t addr, uint16_t port) { return false; } - tcp_pcb *pcb; - { - tcp_core_guard tcg; -#if LWIP_IPV4 && LWIP_IPV6 - pcb = tcp_new_ip_type(addr.type); -#else - pcb = tcp_new_ip_type(IPADDR_TYPE_V4); -#endif - if (!pcb) { - async_tcp_log_e("pcb == NULL"); - return false; - } - // Take ownership now: until _connected() arrives the pcb is only reachable - // through us, so close(), abort() and ~AsyncClient() must be able to find it. - _adopt(pcb); - } - - if (_tcp_connect(&_pcb, &addr, port, (tcp_connected_fn)&_tcp_connected) == ESP_OK) { + // One pass through the LwIP context for the whole sequence; see + // _tcp_connect_in_context(). On failure it has already disposed of the pcb and + // cleared _pcb, and no callbacks were raised: we are returning failure. + if (_tcp_connect(this, &addr, port) == ERR_OK) { return true; } - // _pcb is now NULL and the api call cleared tcp_arg(), so the binding is gone. No - // callbacks are raised: we are returning failure. async_tcp_log_d("connect failed"); return false; } - #ifdef ARDUINO bool AsyncClientImpl::connect(const IPAddress &ip, uint16_t port) { ip_addr_t addr; @@ -1111,32 +1165,31 @@ bool AsyncClientImpl::connect(const char *host, uint16_t port) { async_tcp_log_d("already connected, state %d", _pcb->state); return false; } - if (_dns_pending) { - // LwIP coalesces a duplicate lookup onto the outstanding one anyway, so a retry - // never bought anything. Let the attempt already in flight run to completion. - async_tcp_log_d("name resolution already in progress"); - return true; - } + if (!_start_async_task()) { async_tcp_log_e("failed to start task"); return false; } + // A lookup already in flight is superseded rather than refused. Ignoring the new + // call would silently dial the *previous* host and port and report success for it; + // refusing would be the better API, but previous versions would (brokenly) accept + // this (and then leak). The classic case is a client retrying a "wedged" lookup -- + // it's not really wedged, LwIP guarantees a return, but the timeouts might be longer + // than a client might expect. + // To avoid this leak, we allocate a unique "lookup context" for each request, and + // validate it when the request completes. Ownership of the context lives in the + // LwIP DNS callback once triggered. std::unique_ptr> callback_state(new (std::nothrow) std::weak_ptr(shared_from_this())); if (!callback_state) { async_tcp_log_e("Failed to allocate the lookup context"); return false; } - err_t err; - { - tcp_core_guard tcg; - err = dns_gethostbyname(host, &addr, (dns_found_callback)&AsyncTCP_detail::tcp_dns_found, callback_state.get()); - } + // Initiate DNS transaction + const err_t err = _tcp_dns_start(this, host, &addr, port, callback_state.get()); if (err == ERR_INPROGRESS) { - _connect_port = port; - _dns_pending = true; // DNS lookup in progress callback_state.release(); // now tcp_dns_found's to destroy return true; @@ -1168,7 +1221,6 @@ bool AsyncClientImpl::connect(const char *host, uint16_t port) { } void AsyncClientImpl::close() { - _abandonResolve(); // Ack anything withheld by ackLater(), plus the packet onData is holding right now if // we are being called from inside it. Without this the peer gets an RST rather than a // FIN for data the application did in fact process. @@ -1188,7 +1240,6 @@ void AsyncClientImpl::close() { } int8_t AsyncClientImpl::abort() { - _abandonResolve(); int8_t err = _tcp_abort(this); // _pcb is now NULL // LwIP invokes the error callback when abort is issued; preserve this semantic. @@ -1265,15 +1316,8 @@ void AsyncClientImpl::ackLater() { * Private Callbacks * */ -// LwIP has no way to cancel a lookup, so record that we no longer want the answer. -// tcp_dns_found() drops it and releases LwIP's reference when it eventually fires. -void AsyncClientImpl::_abandonResolve() { - queue_mutex_guard guard; - _dns_pending = false; -} - // Adopt a pcb and reset all per-connection state. Callers must already be -// serialized against the LwIP core (accept callback, or tcp_core_guard). +// serialized against the LwIP core - called from inside a transaction, or from a callback. void AsyncClientImpl::_adopt(tcp_pcb *pcb) { _pcb = pcb; _rx_ack_len = 0; @@ -1402,20 +1446,6 @@ int8_t AsyncClientImpl::_poll(tcp_pcb *pcb) { return ERR_OK; } -void AsyncClientImpl::_dns_found(bool resolved, ip_addr_t *ipaddr, uint16_t port) { - if (resolved && ip_addr_isany_val(*ipaddr)) { - // A resolver that answers with the any-address - a sinkhole, or a blocklist - has - // not given us anywhere to go. Report it as a failed lookup rather than dialling it. - resolved = false; - } - if (resolved && connect(*ipaddr, port)) { - return; - } - // connect() reported success to the caller, so this attempt owes them exactly one - // of onConnect or onError. - _error(resolved ? ERR_CONN : -55); -} - /* * Public Helper Methods * */ @@ -1658,7 +1688,7 @@ bool AsyncClientImpl::connected() const { } bool AsyncClientImpl::connecting() const { - if (_dns_pending) { + if (_dns_token) { return true; // resolving the hostname } if (!_pcb) { @@ -1675,7 +1705,7 @@ bool AsyncClientImpl::disconnecting() const { } bool AsyncClientImpl::disconnected() const { - if (_dns_pending) { + if (_dns_token) { return false; // resolving the hostname } if (!_pcb) { From a7e53df44ab50b12ba144dc314d4d90b78382254 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 13 Sep 2026 10:00:08 -0400 Subject: [PATCH 11/35] fix: Stop delivering a chain if a callback replaces the connection A callback that reconnects leaves _pcb non-null, so the loop kept feeding the old connection's segments to onData against the new pcb. The in-callback ack has the same flaw, and tcp_recved() on a pcb in SYN_SENT draws an RST. Compare against the pcb the data arrived on. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index ebd3e358..02b8d582 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1382,19 +1382,26 @@ int8_t AsyncClientImpl::_recv(tcp_pcb *pcb, pbuf *pb, int8_t err) { if (_recv_cb) { async_tcp_log_elapsed("onData", _recv_cb(_recv_cb_arg, _facade, b->payload, b->len)); } - if (_in_callback_ack_len) { // still ours - close() did not take it + pbuf_free(b); + if (_pcb != pcb) { + // Bizarre as it might sound, it's actually possible for the user callback to reconnect. + _in_callback_ack_len = 0; + break; + } + if (_in_callback_ack_len) { if (!_ack_pcb) { _rx_ack_len += _in_callback_ack_len; - } else if (_pcb) { + } else { _tcp_recved(this, _in_callback_ack_len); } _in_callback_ack_len = 0; } - pbuf_free(b); } - // Stop if the callback closed the connection or destroyed the client. We outlive - // the facade, but there is nobody left to hand the remaining packets to. - if (!_pcb || !_facade) { + // Stop if the callback closed the connection, destroyed the client, or replaced the + // connection outright - a callback that reconnects leaves _pcb non-null but pointing + // at a different peer, and the rest of this chain belongs to the old one. Comparing + // against the pcb the data arrived on covers all three. + if (_pcb != pcb || !_facade) { break; } } From 4192390c1fece8cc46cdbab74691d373980dcc17 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 13 Sep 2026 10:54:57 -0400 Subject: [PATCH 12/35] fix: Build the accepted client's facade at delivery, not at accept Only _dispose_event_chain() knew to destroy an undelivered AsyncClient, so a peer that reset between accept and dispatch had its event purged and its client leaked. It cannot be destroyed where the event is freed either - every caller of _remove_events_for_client() runs on the LwIP thread. So tcp_accept() creates only the implementation, and the async task wraps a facade round it on delivery. - end() marks queued accepts instead of handing a chain back out of the transaction. - tcp_error() clears _pcb before the purge so it can drop the last reference, and stops early on a null _facade. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 115 +++++++++++++++++++++-------------------------- src/AsyncTCP.h | 5 +++ 2 files changed, 57 insertions(+), 63 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 02b8d582..04dc4cc3 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -460,13 +460,16 @@ static inline lwip_tcp_event_packet_t *_get_async_event() { return result; } -static size_t _remove_events_for_client(AsyncClientImpl *client) { +static size_t _remove_events_for_client(AsyncClientImpl *client, lwip_tcp_event_packet_t *terminal_event = nullptr) { lwip_tcp_event_packet_t *removed_event_chain; { queue_mutex_guard guard; removed_event_chain = _async_queue.remove_if([=](lwip_tcp_event_packet_t &pkt) { return pkt.impl.get() == client; }); + if (terminal_event) { + _send_async_event(terminal_event); + } } size_t count = 0; @@ -482,37 +485,33 @@ static size_t _remove_events_for_client(AsyncClientImpl *client) { // Called from AsyncServer::end() on the application thread, so it can destroy any // clients that were accepted but never delivered to onClient. /* - Detach this server's queued accept events. Safe to call inside the LwIP context, and - that is where it belongs: the accept callback that would add more runs on the same - thread, so once this returns from there, nothing further can arrive. - - Disposing of the chain is a separate step because it cannot be done from that context - - destroying an accepted client closes its pcb, which is an api call. + Orphan this server's queued accept events. Safe inside the LwIP context - the accept + callback that would add more runs on this same thread - and it leaves the cleanup to the + async task, which is where closing a connection is allowed. The connections are dropped + as though they never arrived; the application has never seen them. */ -static lwip_tcp_event_packet_t *_detach_events_for_server(AsyncServer *server) { +static void _orphan_events_for_server(AsyncServer *server) { queue_mutex_guard guard; - return _async_queue.remove_if([=](lwip_tcp_event_packet_t &pkt) { - return (pkt.event == LWIP_TCP_ACCEPT) && (pkt.accept.server == server); - }); -} - -// Must not run in LwIP context; see above. -static size_t _dispose_event_chain(lwip_tcp_event_packet_t *chain) { - size_t count = 0; - while (chain) { - ++count; - auto t = chain; - chain = t->next; - delete t->impl->_facade; - _free_event(t); + for (lwip_tcp_event_packet_t *pkt = _async_queue.begin(); pkt; pkt = pkt->next) { + if ((pkt->event == LWIP_TCP_ACCEPT) && (pkt->accept.server == server)) { + pkt->accept.server = nullptr; + } } - return count; } void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { - // A detached implementation has no facade to hand to the user's callbacks, so its - // events simply drain. The reference the event holds keeps it alive until then. - if ((e->impl == NULL) || (e->impl->_facade == NULL)) { + if (e->event == LWIP_TCP_ACCEPT) { + // Accept is checked first because we need to handle it before the client facade check, + // since we haven't constructed a facade for it yet. + AsyncClient *c = e->accept.server ? new (std::nothrow) AsyncClient(e->impl) : nullptr; + if (c) { + e->accept.server->_accepted(c); + } else { + e->impl->close(); // Not wanted or failed to create a client facade + } + } else if ((e->impl == NULL) || (e->impl->_facade == NULL)) { + // A detached implementation has no facade to hand to the user's callbacks, so its + // events simply drain. The reference the event holds keeps it alive until then. // ets_printf("event arg == NULL: 0x%08x\n", e->recv.pcb); } else if (e->event == LWIP_TCP_RECV) { // ets_printf("-R: 0x%08x\n", e->recv.pcb); @@ -533,10 +532,6 @@ void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { } else if (e->event == LWIP_TCP_CONNECTED) { // ets_printf("C: 0x%08x 0x%08x %d\n", e->impl, e->connected.pcb, e->connected.err); e->impl->_connected(e->connected.pcb, e->connected.err); - } else if (e->event == LWIP_TCP_ACCEPT) { - // ets_printf("A: 0x%08x 0x%08x\n", e->impl, e->accept.server); - // Gated above, so the facade is still alive - e->accept.server->_accepted(e->impl->_facade); } _free_event(e); } @@ -735,25 +730,21 @@ void AsyncTCP_detail::tcp_error(void *arg, int8_t err) { return; } - // The pcb has already been freed by LwIP; do not attempt to clear the callbacks! - _remove_events_for_client(client); + // LwIP has already freed the pcb; do not attempt to clear the callbacks. Clear the + // saved value in the client object to avoid future use. + client->_pcb = nullptr; - // enqueue event to be processed in the async task for the user callback + // Construct event packet first. This will hold the client in scope in case we have yet to + // process its ACCEPT event (ie. it has no facade yet, and so the ACCEPT event is the only reference). lwip_tcp_event_packet_t *e = new (std::nothrow) lwip_tcp_event_packet_t{LWIP_TCP_ERROR, client->shared_from_this()}; - if (!e) { - async_tcp_log_e("Failed to allocate event packet"); - } else { + if (e) { e->error.err = err; + } else { + async_tcp_log_e("Failed to allocate event packet"); } - { - queue_mutex_guard guard; - client->_pcb = nullptr; - _send_async_event(e); // no-op if the allocation failed - } - // Nothing may touch the client past this point. Clearing _pcb above released the - // binding, so ~AsyncClient() is free to drop the last reference as soon as it reads - // _pcb as null - and it can only reach that read through the mutex just released. + // Remove all pending events for this client, and send the terminal error event if we could allocate it. + _remove_events_for_client(client, e); } void AsyncTCP_detail::tcp_dns_found(const char *name, const ip_addr_t *ipaddr, void *arg) { @@ -1079,6 +1070,10 @@ AsyncClient::AsyncClient(tcp_pcb *pcb) : _impl(std::make_shared } } +AsyncClient::AsyncClient(std::shared_ptr impl) : _impl(std::move(impl)) { + _impl->_facade = this; +} + AsyncClient::~AsyncClient() { // Unconditional. Whether there is anything to close is not knowable from here: _pcb // and the lookup token are both written by the LwIP thread, and a name lookup can @@ -2145,17 +2140,14 @@ void AsyncServer::begin() { } void AsyncServer::end() { - // The mirror of begin(): stop listening and take the queue with it, in one transaction. - // Detaching the events is safe here because the accept callback that would add more runs - // on this same thread. Disposing of them is not - destroying an accepted client closes - // its pcb, which is an api call - so the chain goes back out to the caller. + // The mirror of begin(): stop listening and disown anything already queued, in one + // transaction. The queued connections are not torn down here - closing one is itself a + // transaction - so they are marked and the async task drops them when it gets to them. struct end_call { struct tcpip_api_call_data call; AsyncServer *server; - lwip_tcp_event_packet_t *events; } args; args.server = this; - args.events = nullptr; tcpip_api_call( +[](struct tcpip_api_call_data *c) -> err_t { @@ -2169,13 +2161,11 @@ void AsyncServer::end() { } server->_pcb = NULL; // PCB is now the property of LwIP } - a->events = _detach_events_for_server(server); + _orphan_events_for_server(server); return ERR_OK; }, &args.call ); - - _dispose_event_chain(args.events); } // runs on LwIP thread @@ -2205,16 +2195,13 @@ int8_t AsyncTCP_detail::tcp_accept(void *arg, tcp_pcb *pcb, int8_t err) { return ERR_ABRT; } - AsyncClient *c = new (std::nothrow) AsyncClient(pcb); - if (!c) { - _free_event(e); - async_tcp_log_e("_accept failed: couldn't allocate client"); - tcp_abort(pcb); - return ERR_ABRT; - } - - c->setNoDelay(server->_noDelay); - e->impl = c->_impl; + // Allocate and initialize the AsyncClient implementation for this accepted connection. + // The facade will be constructed by the async task when it processes this event -- if the connection + // errors out prior to accept running, the event will be discarded and the implementation destroyed + // as there's no API to inform the server of the error. + e->impl = std::make_shared(nullptr); + e->impl->_adopt(pcb); + e->impl->setNoDelay(server->_noDelay); e->accept.server = server; queue_mutex_guard guard; @@ -2225,6 +2212,8 @@ int8_t AsyncTCP_detail::tcp_accept(void *arg, tcp_pcb *pcb, int8_t err) { int8_t AsyncServer::_accepted(AsyncClient *client) { if (_connect_cb) { async_tcp_log_elapsed("onClient", _connect_cb(_connect_cb_arg, client)); + } else { + delete client; // nowhere to go } return ERR_OK; } diff --git a/src/AsyncTCP.h b/src/AsyncTCP.h index 93b6d46d..aea061d1 100644 --- a/src/AsyncTCP.h +++ b/src/AsyncTCP.h @@ -270,6 +270,11 @@ class AsyncClient { friend class AsyncTCP_detail; friend class AsyncServer; + // Adopts an implementation that already exists. A connection accepted by AsyncServer + // gets its implementation on the LwIP thread but its facade here, on the async task, + // because destroying a facade closes it and closing is a transaction. + explicit AsyncClient(std::shared_ptr impl); + // Every scrap of state lives in the implementation object, which outlives this // facade: queued events share ownership of it. That keeps the callbacks themselves // alive for the duration of a call, so destroying an AsyncClient from inside one of From d93a9c380258721e4e5f0d00d990cb5b086d4ace Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 13 Sep 2026 12:58:39 -0400 Subject: [PATCH 13/35] debug: Switch to assertions for impossible cases ASYNCTCP_ASSERT is off by default, against ESP-IDF's usual, and enabled by CONFIG_ASYNC_TCP_ASSERTIONS. --- Kconfig.projbuild | 6 ++++++ src/AsyncTCP.cpp | 26 ++++++++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Kconfig.projbuild b/Kconfig.projbuild index fcbbc2d7..d276fbd9 100644 --- a/Kconfig.projbuild +++ b/Kconfig.projbuild @@ -52,4 +52,10 @@ config ASYNC_TCP_PRIORITY default 10 help This configures the priority of the Async TCP task. + +config ASYNC_TCP_ASSERTIONS + bool "Async TCP extra assertions" + default "n" + help + Enable some additional runtime checks for conditions thought to be impossible. endmenu diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 04dc4cc3..1f6e4fdf 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -8,6 +8,16 @@ #include #include +/** + * Assertion macros + * ESP-IDF leaves assertions on by default; we prefer the opposite as performance in this layer is critical. + */ +#ifdef CONFIG_ASYNC_TCP_ASSERTIONS +#define ASYNCTCP_ASSERT(x) assert(x) +#else +#define ASYNCTCP_ASSERT(x) ((void)0) +#endif + /** * Utility macro * @@ -383,17 +393,15 @@ static void _free_event(lwip_tcp_event_packet_t *evpkt) { } static inline void _send_async_event(lwip_tcp_event_packet_t *e) { - if (e == nullptr) { - return; - } + ASYNCTCP_ASSERT(e != nullptr); + ASYNCTCP_ASSERT(e->impl != nullptr); _async_queue.push_back(e); xTaskNotifyGive(_async_service_task_handle); } static inline void _prepend_async_event(lwip_tcp_event_packet_t *e) { - if (e == nullptr) { - return; - } + ASYNCTCP_ASSERT(e != nullptr); + ASYNCTCP_ASSERT(e->impl != nullptr); _async_queue.push_front(e); xTaskNotifyGive(_async_service_task_handle); } @@ -500,6 +508,8 @@ static void _orphan_events_for_server(AsyncServer *server) { } void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { + ASYNCTCP_ASSERT(e->impl); + if (e->event == LWIP_TCP_ACCEPT) { // Accept is checked first because we need to handle it before the client facade check, // since we haven't constructed a facade for it yet. @@ -509,7 +519,7 @@ void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { } else { e->impl->close(); // Not wanted or failed to create a client facade } - } else if ((e->impl == NULL) || (e->impl->_facade == NULL)) { + } else if ((e->impl->_facade == NULL)) { // A detached implementation has no facade to hand to the user's callbacks, so its // events simply drain. The reference the event holds keeps it alive until then. // ets_printf("event arg == NULL: 0x%08x\n", e->recv.pcb); @@ -1060,7 +1070,7 @@ AsyncClientImpl::AsyncClientImpl(AsyncClient *facade) AsyncClientImpl::~AsyncClientImpl() { // Every path that drops the last reference must ensure the binding was cleared first. // This can be called from the LwIP thread, so any cleanup must have been performed already. - assert(!_pcb); + ASYNCTCP_ASSERT(!_pcb); } AsyncClient::AsyncClient(tcp_pcb *pcb) : _impl(std::make_shared(this)) { From 4d398a4030857f7cb8d006609893e9ebc53b2927 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 13 Sep 2026 13:24:46 -0400 Subject: [PATCH 14/35] fix: Move server state into a shared implementation object An accept event outlives its AsyncServer, and marking queued events cannot reach one the async task has already dequeued; waiting for that task deadlocks, since end() may be called from onClient. State moves to AsyncServerImpl, shared with the accept event, and LwIP is handed the implementation. - _facade plus a listening epoch stamped on each accept subsume the marking pass; _orphan_events_for_server() is gone. - The epoch, not _pcb, is what survives end() followed by begin(): _pcb is current state, not history. - AsyncServer is noncopyable and nonmovable now - the implementation holds one back-pointer to its facade. - Removes AsyncServer::_accept(), declared but never defined. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 192 ++++++++++++++++++++++++++++++++--------------- src/AsyncTCP.h | 25 +++--- 2 files changed, 149 insertions(+), 68 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 1f6e4fdf..b87df4b2 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -150,6 +150,9 @@ struct lwip_tcp_event_packet_t { lwip_tcp_event_packet_t *next; lwip_tcp_event_t event; std::shared_ptr impl; // keeps it alive for the life of the event + // Only an accept carries one. It keeps the server's implementation alive too, so a + // connection still in flight can always find out whether it still has somewhere to go. + std::shared_ptr server; union { struct { tcp_pcb *pcb; @@ -175,7 +178,7 @@ struct lwip_tcp_event_packet_t { tcp_pcb *pcb; } poll; struct { - AsyncServer *server; + uint32_t epoch; // the listening session this connection was accepted on } accept; }; @@ -376,6 +379,66 @@ class AsyncClientImpl : public std::enable_shared_from_this { int8_t _recv(tcp_pcb *pcb, pbuf *pb, int8_t err); }; +/* + Server implementation class + + This has the same basic pattern as AsyncClient: an underlying shared_ptr allows the + event dispatcher to hold the implementation in scope if it is destroyed by the application + while a connection is being constructed. + + _facade is the orphan indicator. ~AsyncServer() clears it, and a queued connection that + finds it null is dropped as though it never arrived. A connection accepted before end() + is dropped too, on the epoch rather than on _pcb - see below. + + One consequence of outliving the facade: the last reference can be dropped by a purge on + the LwIP thread, so this object's destructor - and with it the application's onClient + std::function - can run there. That is the same exposure AsyncClientImpl's callbacks + have, and the same caution applies: nothing captured by a handler may re-enter the + library from its own destructor. +*/ +class AsyncServerImpl : public std::enable_shared_from_this { +public: + AsyncServerImpl(AsyncServer *facade, ip_addr_t addr, uint16_t port) + : _pcb(nullptr), _epoch(0), _facade(facade), _addr(addr), _port(port), _noDelay(false), _connect_cb(nullptr), _connect_cb_arg(nullptr) { + _init_queue_mutex(); + } + ~AsyncServerImpl() { + ASYNCTCP_ASSERT(!_pcb); + } + + AsyncServerImpl(const AsyncServerImpl &) = delete; + AsyncServerImpl &operator=(const AsyncServerImpl &) = delete; + + // Written only inside the LwIP context; null once we have stopped listening. + tcp_pcb *_pcb; + + /* + Which listening session we are on. Bumped in LwIP context in end() to ensure any pending + connections do not get mishandled if a client should call end() then begin() before LwIP + can dispatch them. + */ + uint32_t _epoch; + + // Cleared by ~AsyncServer(). Read without a lock on the async task, on the same terms + // as AsyncClientImpl::_facade - see the note there. + AsyncServer *_facade; + + // Not synchronized: set up before begin() and read afterwards. + ip_addr_t _addr; + uint16_t _port; + bool _noDelay; + AcConnectHandler _connect_cb; + void *_connect_cb_arg; + + ASYNCTCP_ALWAYS_INLINE void onClient(AcConnectHandler cb, void *arg); + ASYNCTCP_ALWAYS_INLINE void begin(); + ASYNCTCP_ALWAYS_INLINE void end(); + ASYNCTCP_ALWAYS_INLINE void setNoDelay(bool nodelay); + ASYNCTCP_ALWAYS_INLINE bool getNoDelay() const; + ASYNCTCP_ALWAYS_INLINE uint8_t status() const; + int8_t _accepted(AsyncClient *client); +}; + static uint32_t _xor_shift_state = 31; // any nonzero seed will do static uint32_t _xor_shift_next() { uint32_t x = _xor_shift_state; @@ -490,32 +553,19 @@ static size_t _remove_events_for_client(AsyncClientImpl *client, lwip_tcp_event_ return count; }; -// Called from AsyncServer::end() on the application thread, so it can destroy any -// clients that were accepted but never delivered to onClient. -/* - Orphan this server's queued accept events. Safe inside the LwIP context - the accept - callback that would add more runs on this same thread - and it leaves the cleanup to the - async task, which is where closing a connection is allowed. The connections are dropped - as though they never arrived; the application has never seen them. -*/ -static void _orphan_events_for_server(AsyncServer *server) { - queue_mutex_guard guard; - for (lwip_tcp_event_packet_t *pkt = _async_queue.begin(); pkt; pkt = pkt->next) { - if ((pkt->event == LWIP_TCP_ACCEPT) && (pkt->accept.server == server)) { - pkt->accept.server = nullptr; - } - } -} - void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { ASYNCTCP_ASSERT(e->impl); if (e->event == LWIP_TCP_ACCEPT) { // Accept is checked first because we need to handle it before the client facade check, // since we haven't constructed a facade for it yet. - AsyncClient *c = e->accept.server ? new (std::nothrow) AsyncClient(e->impl) : nullptr; + AsyncServerImpl *server = e->server.get(); + const bool wanted = server && server->_facade // the application still has this server + && (server->_epoch == e->accept.epoch) // ...and has not stopped listening since + && e->impl->_pcb; // ...and the connection is still up + AsyncClient *c = wanted ? new (std::nothrow) AsyncClient(e->impl) : nullptr; if (c) { - e->accept.server->_accepted(c); + server->_accepted(c); } else { e->impl->close(); // Not wanted or failed to create a client facade } @@ -2023,57 +2073,73 @@ void AsyncClient::onPoll(AcConnectHandler cb, void *arg) { Async TCP Server */ -AsyncServer::AsyncServer(ip_addr_t addr, uint16_t port) - : _port(port), _addr(addr), _noDelay(false), _pcb(nullptr), _connect_cb(nullptr), _connect_cb_arg(nullptr) { - _init_queue_mutex(); -} +AsyncServer::AsyncServer(ip_addr_t addr, uint16_t port) : _impl(std::make_shared(this, addr, port)) {} #ifdef ARDUINO -AsyncServer::AsyncServer(IPAddress addr, uint16_t port) : _port(port), _noDelay(false), _pcb(0), _connect_cb(0), _connect_cb_arg(0) { - _init_queue_mutex(); +AsyncServer::AsyncServer(IPAddress addr, uint16_t port) { + ip_addr_t a; #if ESP_IDF_VERSION_MAJOR < 5 #if LWIP_IPV4 && LWIP_IPV6 - _addr.type = IPADDR_TYPE_V4; - _addr.u_addr.ip4.addr = addr; + a.type = IPADDR_TYPE_V4; + a.u_addr.ip4.addr = addr; #else - _addr.addr = addr; + a.addr = addr; #endif #else - addr.to_ip_addr_t(&_addr); + addr.to_ip_addr_t(&a); #endif + _impl = std::make_shared(this, a, port); } #if ESP_IDF_VERSION_MAJOR < 5 && __has_include() && LWIP_IPV6 -AsyncServer::AsyncServer(IPv6Address addr, uint16_t port) : _port(port), _noDelay(false), _pcb(0), _connect_cb(0), _connect_cb_arg(0) { - _init_queue_mutex(); +AsyncServer::AsyncServer(IPv6Address addr, uint16_t port) { + auto ipaddr = static_cast(addr); + ip_addr_t a = IPADDR6_INIT(ipaddr[0], ipaddr[1], ipaddr[2], ipaddr[3]); #if LWIP_IPV4 && LWIP_IPV6 - _addr.type = IPADDR_TYPE_V6; + a.type = IPADDR_TYPE_V6; #endif - auto ipaddr = static_cast(addr); - _addr = IPADDR6_INIT(ipaddr[0], ipaddr[1], ipaddr[2], ipaddr[3]); + _impl = std::make_shared(this, a, port); } #endif #endif -AsyncServer::AsyncServer(uint16_t port) : _port(port), _noDelay(false), _pcb(0), _connect_cb(0), _connect_cb_arg(0) { - _init_queue_mutex(); +AsyncServer::AsyncServer(uint16_t port) { + ip_addr_t a; #if LWIP_IPV4 && LWIP_IPV6 - _addr.type = IPADDR_TYPE_ANY; - _addr.u_addr.ip4.addr = INADDR_ANY; + a.type = IPADDR_TYPE_ANY; + a.u_addr.ip4.addr = INADDR_ANY; #else - _addr.addr = INADDR_ANY; + a.addr = INADDR_ANY; #endif + _impl = std::make_shared(this, a, port); } AsyncServer::~AsyncServer() { - end(); + _impl->end(); + // Detach: a connection accepted but not yet delivered still points at the implementation, + // and this is how it learns there is no longer a server to hand it to. + _impl->_facade = nullptr; } void AsyncServer::onClient(AcConnectHandler cb, void *arg) { - _connect_cb = cb; - _connect_cb_arg = arg; + _impl->onClient(cb, arg); } - void AsyncServer::begin() { + _impl->begin(); +} +void AsyncServer::end() { + _impl->end(); +} +void AsyncServer::setNoDelay(bool nodelay) { + _impl->setNoDelay(nodelay); +} +bool AsyncServer::getNoDelay() const { + return _impl->getNoDelay(); +} +uint8_t AsyncServer::status() const { + return _impl->status(); +} + +void AsyncServerImpl::begin() { if (_pcb) { return; } @@ -2092,7 +2158,7 @@ void AsyncServer::begin() { */ struct begin_call { struct tcpip_api_call_data call; - AsyncServer *server; + AsyncServerImpl *server; err_t err; } args; args.server = this; @@ -2101,7 +2167,7 @@ void AsyncServer::begin() { tcpip_api_call( +[](struct tcpip_api_call_data *c) -> err_t { begin_call *a = reinterpret_cast(c); - AsyncServer *server = a->server; + AsyncServerImpl *server = a->server; #if LWIP_IPV4 && LWIP_IPV6 tcp_pcb *pcb = tcp_new_ip_type(server->_addr.type); @@ -2149,20 +2215,20 @@ void AsyncServer::begin() { } } -void AsyncServer::end() { - // The mirror of begin(): stop listening and disown anything already queued, in one - // transaction. The queued connections are not torn down here - closing one is itself a - // transaction - so they are marked and the async task drops them when it gets to them. +void AsyncServerImpl::end() { + // The mirror of begin(): stop listening, and disown anything accepted but not fully constructed + // on the session we are closing. Those connections are not torn down here to avoid racing with + // the async thread; the async task drops them when it reaches them. struct end_call { struct tcpip_api_call_data call; - AsyncServer *server; + AsyncServerImpl *server; } args; args.server = this; tcpip_api_call( +[](struct tcpip_api_call_data *c) -> err_t { end_call *a = reinterpret_cast(c); - AsyncServer *server = a->server; + AsyncServerImpl *server = a->server; if (server->_pcb) { tcp_arg(server->_pcb, NULL); tcp_accept(server->_pcb, NULL); @@ -2171,7 +2237,9 @@ void AsyncServer::end() { } server->_pcb = NULL; // PCB is now the property of LwIP } - _orphan_events_for_server(server); + // Disown anything already accepted on the session we just closed. Bumped inside the + // transaction, so it is ordered against tcp_accept() rather than racing it. + ++server->_epoch; return ERR_OK; }, &args.call @@ -2190,7 +2258,7 @@ int8_t AsyncTCP_detail::tcp_accept(void *arg, tcp_pcb *pcb, int8_t err) { return ERR_ABRT; } - auto server = reinterpret_cast(arg); + auto server = reinterpret_cast(arg); if (!server->_connect_cb) { async_tcp_log_e("_accept failed: no onConnect callback"); tcp_abort(pcb); @@ -2212,14 +2280,15 @@ int8_t AsyncTCP_detail::tcp_accept(void *arg, tcp_pcb *pcb, int8_t err) { e->impl = std::make_shared(nullptr); e->impl->_adopt(pcb); e->impl->setNoDelay(server->_noDelay); - e->accept.server = server; + e->server = server->shared_from_this(); // Lock server impl in scope + e->accept.epoch = server->_epoch; // And remember in case it's end()ed queue_mutex_guard guard; _prepend_async_event(e); return ERR_OK; } -int8_t AsyncServer::_accepted(AsyncClient *client) { +int8_t AsyncServerImpl::_accepted(AsyncClient *client) { if (_connect_cb) { async_tcp_log_elapsed("onClient", _connect_cb(_connect_cb_arg, client)); } else { @@ -2228,15 +2297,20 @@ int8_t AsyncServer::_accepted(AsyncClient *client) { return ERR_OK; } -void AsyncServer::setNoDelay(bool nodelay) { +void AsyncServerImpl::onClient(AcConnectHandler cb, void *arg) { + _connect_cb = cb; + _connect_cb_arg = arg; +} + +void AsyncServerImpl::setNoDelay(bool nodelay) { _noDelay = nodelay; } -bool AsyncServer::getNoDelay() const { +bool AsyncServerImpl::getNoDelay() const { return _noDelay; } -uint8_t AsyncServer::status() const { +uint8_t AsyncServerImpl::status() const { if (!_pcb) { return 0; } diff --git a/src/AsyncTCP.h b/src/AsyncTCP.h index aea061d1..21cd0f0e 100644 --- a/src/AsyncTCP.h +++ b/src/AsyncTCP.h @@ -74,6 +74,7 @@ typedef std::function AcTimeoutHandl struct tcp_pcb; class AsyncTCP_detail; class AsyncClientImpl; +class AsyncServerImpl; class AsyncClient { public: @@ -293,6 +294,16 @@ class AsyncServer { #endif AsyncServer(uint16_t port); ~AsyncServer(); + + // Noncopyable: the implementation holds one back-pointer, so a second facade sharing it + // would leave one of them unreachable from it. + AsyncServer(const AsyncServer &) = delete; + AsyncServer &operator=(const AsyncServer &) = delete; + + // Nonmovable + AsyncServer(AsyncServer &&) = delete; + AsyncServer &operator=(AsyncServer &&) = delete; + void onClient(AcConnectHandler cb, void *arg); void begin(); void end(); @@ -303,15 +314,11 @@ class AsyncServer { protected: friend class AsyncTCP_detail; - uint16_t _port; - ip_addr_t _addr; - bool _noDelay; - tcp_pcb *_pcb; - AcConnectHandler _connect_cb; - void *_connect_cb_arg; - - int8_t _accept(tcp_pcb *newpcb, int8_t err); - int8_t _accepted(AsyncClient *client); + // As with AsyncClient, every scrap of state lives in the implementation object, which + // outlives this facade: a connection accepted but not yet delivered holds a reference to + // it. Destroying an AsyncServer clears the implementation's back-pointer, which is how + // a connection still in flight learns that there is no longer anywhere to deliver it. + std::shared_ptr _impl; }; #endif /* ASYNCTCP_H_ */ From f9a3a1ca00ad1e27203ef751175bd62b10f0bc0c Mon Sep 17 00:00:00 2001 From: Will Miles Date: Mon, 14 Sep 2026 00:00:25 -0400 Subject: [PATCH 15/35] refactor: Use metafunctor for core locking With core locking, inline the lambda under the lock; without it, delegate to tcpip_api_call with a functor on the stack. Path selection moves to LwIP's defines rather than the IDF's, tested by value: opt.h defines NO_SYS and LWIP_TCPIP_CORE_LOCKING unconditionally, so an #ifdef would send every build down the lock-free path. Neither route may be entered from the LwIP thread. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 182 +++++++++++++++++++++++++---------------------- 1 file changed, 95 insertions(+), 87 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index b87df4b2..d9d63097 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -7,6 +7,7 @@ #include #include +#include /** * Assertion macros @@ -71,6 +72,7 @@ extern "C" { #include "lwip/opt.h" #include "lwip/tcp.h" #include "lwip/tcpip.h" +#include "lwip/priv/tcpip_priv.h" } #if CONFIG_ASYNC_TCP_USE_WDT @@ -95,33 +97,65 @@ extern "C" { async_tcp_log_v("%s took %" PRIu32 " us", tag, micros() - s_time); \ } -// https://github.com/espressif/arduino-esp32/issues/10526 +/** + * LwIP locking + * + * Utilities for managing LwIP TCP core mutexing. Includes two features + * - tcp_core_guard: a guard class usable when LWIP_TCPIP_CORE_LOCKING exposes the mutex for direct usage + * - with_tcp_core_lock: a metafunction that runs a function inside the LwIP TCP core lock context, using the most efficient method available on the current build. + * + * https://github.com/espressif/arduino-esp32/issues/10526 + */ namespace { -#ifdef CONFIG_LWIP_TCPIP_CORE_LOCKING +#if LWIP_TCPIP_CORE_LOCKING struct tcp_core_guard { - bool do_lock; - inline tcp_core_guard() : do_lock(!sys_thread_tcpip(LWIP_CORE_LOCK_QUERY_HOLDER)) { - if (do_lock) { - LOCK_TCPIP_CORE(); - } + inline tcp_core_guard() { + LOCK_TCPIP_CORE(); } inline ~tcp_core_guard() { - if (do_lock) { - UNLOCK_TCPIP_CORE(); - } + UNLOCK_TCPIP_CORE(); } tcp_core_guard(const tcp_core_guard &) = delete; tcp_core_guard(tcp_core_guard &&) = delete; tcp_core_guard &operator=(const tcp_core_guard &) = delete; tcp_core_guard &operator=(tcp_core_guard &&) = delete; } __attribute__((unused)); -#else // CONFIG_LWIP_TCPIP_CORE_LOCKING +#else // LWIP_TCPIP_CORE_LOCKING struct tcp_core_guard { } __attribute__((unused)); -#endif // CONFIG_LWIP_TCPIP_CORE_LOCKING -} // anonymous namespace +#endif // LWIP_TCPIP_CORE_LOCKING -#define INVALID_CLOSED_SLOT -1 +/* + Locking metafunction for TCP core operations. + + Runs a function inside the LwIP context by whichever route is cheapest on this build. + Where the core lock exists we take it and call straight through, which avoids marshaling + the arguments through a message. Where direct locking is not available, we wrap the function + call with tcpip_api_call. +*/ +template err_t with_tcp_core_lock(T &&func) { +#if LWIP_TCPIP_CORE_LOCKING || NO_SYS + tcp_core_guard guard; + return func(); +#else + // The functor is reached by pointer: a reference member would leave the struct without + // a default constructor, and it only has to outlive the call. + struct call_data { + struct tcpip_api_call_data call; // must be first member + typename std::remove_reference::type *f; + } data; + data.f = &func; + return tcpip_api_call( + // The lambda here constructs a C function which perfectly forwards the call to the functor. + +[](struct tcpip_api_call_data *c) -> err_t { + return (*reinterpret_cast(c)->f)(); + }, + &data.call + ); +#endif +} + +} // anonymous namespace /* TCP poll interval is specified in terms of the TCP coarse timer interval, which is called twice a second @@ -403,6 +437,10 @@ class AsyncServerImpl : public std::enable_shared_from_this { _init_queue_mutex(); } ~AsyncServerImpl() { + // end() here would be wrong for the same reason ~AsyncClientImpl does not close: the + // last reference can be dropped by a purge on the LwIP thread, and end() is a + // transaction, which from there would wait for itself. Every path that releases the + // implementation has already stopped it listening. ASYNCTCP_ASSERT(!_pcb); } @@ -855,8 +893,6 @@ void AsyncTCP_detail::tcp_dns_found(const char *name, const ip_addr_t *ipaddr, v * TCP/IP API Calls * */ -#include "lwip/priv/tcpip_priv.h" - /* Context structure for `tcpip_api_call`, ie. code running in LwIP context. The pcb is always reached through `client`. @@ -2156,62 +2192,45 @@ void AsyncServerImpl::begin() { could see a pcb bound but not listening, or listening with no accept callback and no argument, and _pcb itself was published in stages. */ - struct begin_call { - struct tcpip_api_call_data call; - AsyncServerImpl *server; - err_t err; - } args; - args.server = this; - args.err = ERR_OK; - - tcpip_api_call( - +[](struct tcpip_api_call_data *c) -> err_t { - begin_call *a = reinterpret_cast(c); - AsyncServerImpl *server = a->server; - + auto err = with_tcp_core_lock([&]() -> err_t { #if LWIP_IPV4 && LWIP_IPV6 - tcp_pcb *pcb = tcp_new_ip_type(server->_addr.type); + tcp_pcb *pcb = tcp_new_ip_type(_addr.type); #else - tcp_pcb *pcb = tcp_new_ip_type(IPADDR_TYPE_ANY); + tcp_pcb *pcb = tcp_new_ip_type(IPADDR_TYPE_ANY); #endif - if (!pcb) { - a->err = ERR_MEM; - return a->err; - } + if (!pcb) { + return ERR_MEM; + } - a->err = tcp_bind(pcb, &server->_addr, server->_port); - if (a->err != ERR_OK) { - // Never registered, so nothing can reach it; dispose of it here. - if (tcp_close(pcb) != ERR_OK) { - tcp_abort(pcb); - } - return a->err; + auto err = tcp_bind(pcb, &_addr, _port); + if (err != ERR_OK) { + // Never registered, so nothing can reach it; dispose of it here. + if (tcp_close(pcb) != ERR_OK) { + tcp_abort(pcb); } + return err; + } - tcp_pcb *listen_pcb = tcp_listen_with_backlog(pcb, ASYNCTCP_LISTEN_BACKLOG); - if (!listen_pcb) { - // LwIP frees the original only on success; ours is still bound, and holds the - // port reserved until we release it. - if (tcp_close(pcb) != ERR_OK) { - tcp_abort(pcb); - } - a->err = ERR_MEM; - return a->err; + tcp_pcb *listen_pcb = tcp_listen_with_backlog(pcb, ASYNCTCP_LISTEN_BACKLOG); + if (!listen_pcb) { + // LwIP frees the original only on success; ours is still bound, and holds the + // port reserved until we release it. + if (tcp_close(pcb) != ERR_OK) { + tcp_abort(pcb); } + return ERR_MEM; + } - // Published last, and with its callbacks already attached: an accept cannot arrive - // against a server that is not ready for it. - tcp_arg(listen_pcb, server); - tcp_accept(listen_pcb, &AsyncTCP_detail::tcp_accept); - server->_pcb = listen_pcb; - a->err = ERR_OK; - return a->err; - }, - &args.call - ); + // Published last, and with its callbacks already attached: an accept cannot arrive + // against a server that is not ready for it. + tcp_arg(listen_pcb, this); + tcp_accept(listen_pcb, &AsyncTCP_detail::tcp_accept); + _pcb = listen_pcb; + return ERR_OK; + }); // end LwIP scope - if (args.err != ERR_OK) { - async_tcp_log_e("begin failed: %d", (int)args.err); + if (err != ERR_OK) { + async_tcp_log_e("begin failed: %d", (int)err); } } @@ -2219,31 +2238,20 @@ void AsyncServerImpl::end() { // The mirror of begin(): stop listening, and disown anything accepted but not fully constructed // on the session we are closing. Those connections are not torn down here to avoid racing with // the async thread; the async task drops them when it reaches them. - struct end_call { - struct tcpip_api_call_data call; - AsyncServerImpl *server; - } args; - args.server = this; - - tcpip_api_call( - +[](struct tcpip_api_call_data *c) -> err_t { - end_call *a = reinterpret_cast(c); - AsyncServerImpl *server = a->server; - if (server->_pcb) { - tcp_arg(server->_pcb, NULL); - tcp_accept(server->_pcb, NULL); - if (tcp_close(server->_pcb) != ERR_OK) { - tcp_abort(server->_pcb); - } - server->_pcb = NULL; // PCB is now the property of LwIP + with_tcp_core_lock([&]() -> err_t { + if (_pcb) { + tcp_arg(_pcb, NULL); + tcp_accept(_pcb, NULL); + if (tcp_close(_pcb) != ERR_OK) { + tcp_abort(_pcb); } - // Disown anything already accepted on the session we just closed. Bumped inside the - // transaction, so it is ordered against tcp_accept() rather than racing it. - ++server->_epoch; - return ERR_OK; - }, - &args.call - ); + _pcb = NULL; // PCB is now the property of LwIP + } + // Disown anything already accepted on the session we just closed. Bumped inside the + // transaction, so it is ordered against tcp_accept() rather than racing it. + ++_epoch; + return ERR_OK; + }); } // runs on LwIP thread From 192c487f408e5d1da7f9b61816abefdb7861953b Mon Sep 17 00:00:00 2001 From: Will Miles Date: Mon, 14 Sep 2026 14:44:58 -0400 Subject: [PATCH 16/35] perf: Stop timing callbacks that are never logged micros() is opaque, so the compiler cannot drop the pair around each user callback when the log itself compiles out - twenty-two relocations against it in a build with logging off. Gate the timing on the verbose level; CONFIG_ASYNC_TCP_LOG_ELAPSED forces it either way. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index d9d63097..6cb49af6 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -89,13 +89,24 @@ extern "C" { #define ASYNC_TCP_MAX_TASK_SLEEP portMAX_DELAY #endif +/** + * async_tcp_log_elapsed + * Log the time elapsed in a given statement (typically the user callbacks). Useful for performance monitoring and debugging. + * The complex gate is because micros() is opaque to the compiler and cannot be optimized away when unused, so we + * only enable this feature if verbose logging is enabled. + */ +#if defined(CONFIG_ASYNC_TCP_DEBUG) || defined(CONFIG_ASYNC_TCP_LOG_CUSTOM) || (defined(CORE_DEBUG_LEVEL) && (CORE_DEBUG_LEVEL >= 5)) \ + || (defined(LOG_LOCAL_LEVEL) && (LOG_LOCAL_LEVEL >= 5)) #define async_tcp_log_elapsed(tag, statement) \ { \ - [[maybe_unused]] \ const uint32_t s_time = micros(); \ statement; \ async_tcp_log_v("%s took %" PRIu32 " us", tag, micros() - s_time); \ } +#else +#define async_tcp_log_elapsed(tag, statement) \ + { statement; } +#endif /** * LwIP locking From 1bf164dc65ac935fb636bbbc2d90eef481ff292c Mon Sep 17 00:00:00 2001 From: Will Miles Date: Mon, 14 Sep 2026 17:06:14 -0400 Subject: [PATCH 17/35] fix: Size the write against the room available inside the lock space() was read on the calling task and the figure handed to the write transaction, so a window that shrank in between drew ERR_MEM and lost the whole send. The unlocked space() at the top stays as an advisory early-out. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 44 +++++++++++--------------------------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 6cb49af6..cf6d3a41 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -5,6 +5,7 @@ #include "AsyncTCPLogging.h" #include "AsyncTCPSimpleIntrusiveList.h" +#include #include #include #include @@ -914,11 +915,6 @@ typedef struct { int8_t err; union { size_t close_ack; // bytes to tcp_recved() before closing, in the same pass - struct { - const char *data; - size_t size; - uint8_t apiflags; - } write; size_t received; struct { const ip_addr_t *addr; @@ -952,28 +948,6 @@ static esp_err_t _tcp_output(AsyncClientImpl *client) { return msg.err; } -static err_t _tcp_write_api(struct tcpip_api_call_data *api_call_msg) { - tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; - msg->err = ERR_CONN; - if (msg->client->_pcb) { - msg->err = tcp_write(msg->client->_pcb, msg->write.data, msg->write.size, msg->write.apiflags); - } - return msg->err; -} - -static esp_err_t _tcp_write(AsyncClientImpl *client, const char *data, size_t size, uint8_t apiflags) { - if (!client->_pcb) { - return ERR_CONN; - } - tcp_api_call_t msg; - msg.client = client; - msg.write.data = data; - msg.write.size = size; - msg.write.apiflags = apiflags; - tcpip_api_call(_tcp_write_api, (struct tcpip_api_call_data *)&msg); - return msg.err; -} - static err_t _tcp_recved_api(struct tcpip_api_call_data *api_call_msg) { tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; msg->err = ERR_CONN; @@ -1364,17 +1338,21 @@ size_t AsyncClientImpl::add(const char *data, size_t size, uint8_t apiflags) { if (!_pcb || size == 0 || data == NULL) { return 0; } - size_t room = space(); - if (!room) { + // Early check outside of lock + if (space() == 0) { return 0; } - size_t will_send = (room < size) ? room : size; - int8_t err = ERR_OK; - err = _tcp_write(this, data, will_send, apiflags); + auto err = with_tcp_core_lock([&]() -> err_t { + if (!_pcb) { + return ERR_CONN; + } + size = std::min(size, space()); + return tcp_write(_pcb, data, size, apiflags); + }); if (err != ERR_OK) { return 0; } - return will_send; + return size; } bool AsyncClientImpl::send() { From 204ae5e3eb0d1625f1bf57adef3b1cd8fe861a96 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Tue, 15 Sep 2026 10:10:58 -0400 Subject: [PATCH 18/35] fix: Report every connection exactly once Whether onDisconnect arrived turned on the close transaction's result code, where ERR_OK meant both "a live pcb was closed" and "a queued terminal event was purged unseen". _dispose_owed tracks the obligation instead: armed wherever the application is promised a terminal notification, claimed by whichever path ends the connection. Fixes abort() during a lookup reporting nothing, a reset client destroyed before dispatch draining silently, and ~AsyncClient() reporting twice. - Both arming points sit in LwIP context; arming on the caller's side races the answer and can wedge the client against every later connect(). - connect() refuses while a notification is owed, and the hostname overload tests the token too, since superseding a lookup transfers the obligation. - _error() claims the notification before running onError, then suppresses onDisconnect if the callback started a new attempt. Co-Authored-By: Claude Opus 5 --- src/AsyncTCP.cpp | 92 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 26 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index cf6d3a41..4a6c9fe2 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -313,11 +313,11 @@ class AsyncClientImpl : public std::enable_shared_from_this { /* Members held under LwIP context - */ tcp_pcb *_pcb; uint16_t _connect_port; std::weak_ptr *_dns_token; // non-owning pointer to the most recent DNS lookup token + bool _dispose_owed; // a terminal notification is owed to the application /* Not synchronized. The handlers are installed by the application before it connects @@ -1069,7 +1069,8 @@ static err_t _tcp_connect_in_context(AsyncClientImpl *client, const ip_addr_t *a if (tcp_close(pcb) != ERR_OK) { tcp_abort(pcb); } - client->_pcb = nullptr; // PCB is now the property of LwIP + client->_pcb = nullptr; + // Note that _adopt() armed _dispose_owed; the caller will need to clear it if it is no longer needed. } return err; @@ -1078,6 +1079,10 @@ static err_t _tcp_connect_in_context(AsyncClientImpl *client, const ip_addr_t *a static err_t _tcp_connect_api(struct tcpip_api_call_data *api_call_msg) { tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; msg->err = _tcp_connect_in_context(msg->client, msg->connect.addr, msg->connect.port); + // On a failure, release _dispose_owed while we still hold the LwIP lock. + if (msg->err != ERR_OK) { + msg->client->_dispose_owed = false; + } return msg->err; } @@ -1107,11 +1112,23 @@ static err_t _tcp_dns_start_api(struct tcpip_api_call_data *api_call_msg) { AsyncClientImpl *client = msg->client; // Supersedes any lookup already outstanding; that one will find the token moved on. + if (client->_dns_token) { + // The displaced lookup will never report, so its obligation dies with it rather than + // passing to this attempt - which may answer from cache and never arm one of its own, + // and would then refuse itself on a notice nobody can ever deliver. + client->_dispose_owed = false; + } client->_connect_port = msg->dns.port; client->_dns_token = msg->dns.token; msg->err = dns_gethostbyname(msg->dns.host, msg->dns.addr, (dns_found_callback)&AsyncTCP_detail::tcp_dns_found, msg->dns.token); - if (msg->err != ERR_INPROGRESS) { + if (msg->err == ERR_INPROGRESS) { + // Reporting success promises onConnect or onError, and a dispose with it, even though no + // pcb exists yet (and may never). Armed here rather than on return: the answer can land + // the instant this transaction ends, and an error reported before the caller had armed it + // would consume a notification that was not yet owed - and then rearm it forever. + client->_dispose_owed = true; + } else { client->_dns_token = nullptr; // no callback is coming } return msg->err; @@ -1133,9 +1150,9 @@ static err_t _tcp_dns_start(AsyncClientImpl *client, const char *host, ip_addr_t */ AsyncClientImpl::AsyncClientImpl(AsyncClient *facade) - : _pcb(nullptr), _connect_port(0), _dns_token(nullptr), _facade(facade), _connect_cb(0), _connect_cb_arg(0), _discard_cb(0), _discard_cb_arg(0), _sent_cb(0), - _sent_cb_arg(0), _error_cb(0), _error_cb_arg(0), _recv_cb(0), _recv_cb_arg(0), _pb_cb(0), _pb_cb_arg(0), _timeout_cb(0), _timeout_cb_arg(0), _poll_cb(0), - _poll_cb_arg(0), _ack_pcb(true), _tx_last_packet(0), _rx_ack_len(0), _rx_last_packet(0), _rx_timeout(0), _rx_last_ack(0), + : _pcb(nullptr), _connect_port(0), _dns_token(nullptr), _dispose_owed(false), _facade(facade), _connect_cb(0), _connect_cb_arg(0), _discard_cb(0), + _discard_cb_arg(0), _sent_cb(0), _sent_cb_arg(0), _error_cb(0), _error_cb_arg(0), _recv_cb(0), _recv_cb_arg(0), _pb_cb(0), _pb_cb_arg(0), _timeout_cb(0), + _timeout_cb_arg(0), _poll_cb(0), _poll_cb_arg(0), _ack_pcb(true), _tx_last_packet(0), _rx_ack_len(0), _rx_last_packet(0), _rx_timeout(0), _rx_last_ack(0), _ack_timeout(CONFIG_ASYNC_TCP_MAX_ACK_TIME), _in_callback_ack_len(0) {} AsyncClientImpl::~AsyncClientImpl() { @@ -1156,11 +1173,9 @@ AsyncClient::AsyncClient(std::shared_ptr impl) : _impl(std::mov } AsyncClient::~AsyncClient() { - // Unconditional. Whether there is anything to close is not knowable from here: _pcb - // and the lookup token are both written by the LwIP thread, and a name lookup can - // still turn into a bound pcb after we have looked. Deciding out here would be - // deciding whether we need the interlock without holding it; close() is cheap when - // there is nothing to do. + // Unconditional, because whether there is anything to close is not knowable from here - + // _pcb and the lookup token are both written by the LwIP thread, and a name lookup can + // still turn into a bound pcb after we have looked. _impl->close(); // Detach: queued events and LwIP callbacks may still reach the implementation, but // must not run user code against a facade that no longer exists. @@ -1192,14 +1207,21 @@ bool AsyncClientImpl::connect(ip_addr_t addr, uint16_t port) { async_tcp_log_d("name resolution in progress"); return false; } + if (_dispose_owed) { + // A previous failed connection has not reported yet - the error event + // is still queued (otherwise we'd have _pcb). Treat this as "not ready + // for a new connection". + async_tcp_log_d("previous connection not yet reported"); + return false; + } if (!_start_async_task()) { async_tcp_log_e("failed to start task"); return false; } // One pass through the LwIP context for the whole sequence; see - // _tcp_connect_in_context(). On failure it has already disposed of the pcb and - // cleared _pcb, and no callbacks were raised: we are returning failure. + // _tcp_connect_in_context(). On failure it has already disposed of the pcb and cleared + // _pcb, and no callbacks were raised. if (_tcp_connect(this, &addr, port) == ERR_OK) { return true; } @@ -1241,6 +1263,15 @@ bool AsyncClientImpl::connect(const char *host, uint16_t port) { async_tcp_log_d("already connected, state %d", _pcb->state); return false; } + // For compatibility, calling `connect()` with a lookup outstanding supersedes + // the previous lookup. + if (_dispose_owed && !_dns_token) { + // The previous connection has errored out but its async event hasn't run. + // We can't cancel it -- it may already be pending on the async task. + // So we treat this as "not ready for a new connection". + async_tcp_log_d("previous connection not yet reported"); + return false; + } if (!_start_async_task()) { async_tcp_log_e("failed to start task"); @@ -1266,7 +1297,7 @@ bool AsyncClientImpl::connect(const char *host, uint16_t port) { const err_t err = _tcp_dns_start(this, host, &addr, port, callback_state.get()); if (err == ERR_INPROGRESS) { - // DNS lookup in progress + // DNS lookup in progress; _tcp_dns_start_api armed _dispose_owed inside the transaction callback_state.release(); // now tcp_dns_found's to destroy return true; } @@ -1303,15 +1334,15 @@ void AsyncClientImpl::close() { const size_t pending = _rx_ack_len + _in_callback_ack_len; _rx_ack_len = 0; _in_callback_ack_len = 0; - int8_t err = _tcp_close(this, pending); - // _pcb is now NULL - if ((err == ERR_OK) && _discard_cb) { - // _pcb was closed here. Same hazard as _error(): the application may destroy its - // AsyncClient from inside the callback, and close() is reached from the calling task - // where the facade can be the only holder. Nothing follows the call today, so this - // costs nothing and stops that being load-bearing. - auto self = shared_from_this(); - async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, _facade)); + _tcp_close(this, pending); + // _pcb is now NULL, all queued events are cleared, and any outstanding lookup has been + // stood down. Call the dispose callback if owed. + if (_facade && _dispose_owed) { + _dispose_owed = false; + if (_discard_cb) { + auto self = shared_from_this(); // ensure our object stays in scope during the callback + async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, _facade)); + } } } @@ -1319,9 +1350,9 @@ int8_t AsyncClientImpl::abort() { int8_t err = _tcp_abort(this); // _pcb is now NULL // LwIP invokes the error callback when abort is issued; preserve this semantic. - // This will also trigger the dispose callback. + // This will also trigger the dispose callback if otherwise owed (ie. error is already pending or canceling a DNS lookup). // If the pcb was previously invalidated by some other queued error, we've discarded that value; so we always send ERR_ABRT. - if (err != ERR_CONN) { + if (err != ERR_CONN || _dispose_owed) { _error(ERR_ABRT); } return err; @@ -1399,6 +1430,7 @@ void AsyncClientImpl::ackLater() { // Adopt a pcb and reset all per-connection state. Callers must already be // serialized against the LwIP core - called from inside a transaction, or from a callback. void AsyncClientImpl::_adopt(tcp_pcb *pcb) { + _dispose_owed = true; // a bound pcb owes the application a terminal notification _pcb = pcb; _rx_ack_len = 0; _tx_last_packet = 0; @@ -1423,11 +1455,19 @@ int8_t AsyncClientImpl::_connected(tcp_pcb *pcb, int8_t err) { void AsyncClientImpl::_error(int8_t err) { // Hold a reference to self to ensure this remains valid throughout the user callbacks. auto self = shared_from_this(); + // Claim the notification before running anything: onError may destroy the AsyncClient, + // and ~AsyncClient() closes, which would otherwise report this same connection again. + const bool owed = _dispose_owed; + _dispose_owed = false; // Run error callback if (_error_cb) { async_tcp_log_elapsed("onError", _error_cb(_error_cb_arg, _facade, err)); } - if (_facade && _discard_cb) { + // Things could have been changed under that callback: + // - User might have deleted the AsyncClient (_facade is now nullptr) + // - User might have reconnected the object (re-armed _dispose_owed) + // In either case, we skip the dispose. + if (owed && _facade && !_dispose_owed && _discard_cb) { async_tcp_log_elapsed("onDisconnect", _discard_cb(_discard_cb_arg, _facade)); } } From 6062df05cc1fe2ea6bd9bd4b5aab7adbb5186823 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 2 Oct 2026 22:55:17 -0400 Subject: [PATCH 19/35] style: Reformat async_tcp_log_elapsed Co-Authored-By: Claude Opus 5.5 --- src/AsyncTCP.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index bd1c368f..8a7519b6 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -98,10 +98,10 @@ extern "C" { */ #if defined(CONFIG_ASYNC_TCP_DEBUG) || defined(CONFIG_ASYNC_TCP_LOG_CUSTOM) || (defined(CORE_DEBUG_LEVEL) && (CORE_DEBUG_LEVEL >= 5)) \ || (defined(LOG_LOCAL_LEVEL) && (LOG_LOCAL_LEVEL >= 5)) -#define async_tcp_log_elapsed(tag, statement) \ - { \ - const uint32_t s_time = micros(); \ - statement; \ +#define async_tcp_log_elapsed(tag, statement) \ + { \ + const uint32_t s_time = micros(); \ + statement; \ async_tcp_log_v("%s took %" PRIu32 " us", tag, static_cast(micros()) - s_time); \ } #else From d5b31544af7e9e6f6ec36ba4656db16d3f4e67e0 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 2 Oct 2026 23:24:00 -0400 Subject: [PATCH 20/35] fix: Start the async task before adopting a caller's pcb AsyncClient(tcp_pcb*) bound LwIP callbacks without starting the task, so the first event notified a null task handle. If the task cannot start, the pcb is closed instead. Co-Authored-By: Claude Opus 5.5 --- src/AsyncTCP.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 8a7519b6..5dff836d 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1164,7 +1164,17 @@ AsyncClientImpl::~AsyncClientImpl() { AsyncClient::AsyncClient(tcp_pcb *pcb) : _impl(std::make_shared(this)) { _init_queue_mutex(); if (pcb) { - _impl->_adopt(pcb); + // The pcb's callbacks queue events, so the task must be there to take them. + if (_start_async_task()) { + _impl->_adopt(pcb); + } else { + // The pcb is ours, and nothing could deliver its events. Close rather than abort: + // in an accept callback, only the callback's return value can report ERR_ABRT. + async_tcp_log_e("failed to start task"); + if (tcp_close(pcb) != ERR_OK) { + async_tcp_log_e("failed to close pcb"); + } + } } } From 4b6bc38ff8a8a81589ab44084c94202e9a602f29 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 2 Oct 2026 23:24:01 -0400 Subject: [PATCH 21/35] fix: Tolerate a queue mutex that was never created Ensure that all paths require creation before any event can be queued; and skip the purge when it does not exist since nothing can have been queued. Co-Authored-By: Claude Opus 5.5 --- src/AsyncTCP.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 5dff836d..bf4ab529 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -246,15 +246,12 @@ class AsyncTCP_detail { static void __attribute__((visibility("internal"))) tcp_dns_found(const char *name, const ip_addr_t *ipaddr, void *arg); }; -// Guard class for the global queue namespace { - +// Global queue mutex static SemaphoreHandle_t _async_queue_mutex = nullptr; -// The queue is only reachable through an AsyncClient or an AsyncServer, so creating the -// mutex in their constructors puts it ahead of every user. Unsynchronized, in the same -// way _start_async_task() always has been: the first of those can only be constructed by -// application code, before the async task exists and before LwIP points at anything. +// Created by _start_async_task(), which every path that can queue an event passes first. +// Unsynchronized, as _start_async_task() always has been. static bool _init_queue_mutex() { if (!_async_queue_mutex) { _async_queue_mutex = xSemaphoreCreateMutex(); @@ -265,6 +262,7 @@ static bool _init_queue_mutex() { return _async_queue_mutex != nullptr; } +// Guard class for the global queue class queue_mutex_guard { bool holds_mutex; @@ -445,9 +443,7 @@ class AsyncClientImpl : public std::enable_shared_from_this { class AsyncServerImpl : public std::enable_shared_from_this { public: AsyncServerImpl(AsyncServer *facade, ip_addr_t addr, uint16_t port) - : _pcb(nullptr), _epoch(0), _facade(facade), _addr(addr), _port(port), _noDelay(false), _connect_cb(nullptr), _connect_cb_arg(nullptr) { - _init_queue_mutex(); - } + : _pcb(nullptr), _epoch(0), _facade(facade), _addr(addr), _port(port), _noDelay(false), _connect_cb(nullptr), _connect_cb_arg(nullptr) {} ~AsyncServerImpl() { // end() here would be wrong for the same reason ~AsyncClientImpl does not close: the // last reference can be dropped by a purge on the LwIP thread, and end() is a @@ -582,6 +578,12 @@ static inline lwip_tcp_event_packet_t *_get_async_event() { } static size_t _remove_events_for_client(AsyncClientImpl *client, lwip_tcp_event_packet_t *terminal_event = nullptr) { + if (!_async_queue_mutex) { + // Never started, so nothing can have been queued. A terminal event comes from an + // LwIP callback, and a bound pcb implies a started task. + ASYNCTCP_ASSERT(!terminal_event); + return 0; + } lwip_tcp_event_packet_t *removed_event_chain; { queue_mutex_guard guard; @@ -1162,7 +1164,6 @@ AsyncClientImpl::~AsyncClientImpl() { } AsyncClient::AsyncClient(tcp_pcb *pcb) : _impl(std::make_shared(this)) { - _init_queue_mutex(); if (pcb) { // The pcb's callbacks queue events, so the task must be there to take them. if (_start_async_task()) { From 3326e58926a23bbbfeb791fa4d8b85703bc2839f Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 12:14:54 -0400 Subject: [PATCH 22/35] refactor: Let queue_mutex_guard tolerate a mutex never created Co-Authored-By: Claude Opus 5.5 --- src/AsyncTCP.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index bf4ab529..0fba975d 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -267,7 +267,14 @@ class queue_mutex_guard { bool holds_mutex; public: + // For paths that can only run once _start_async_task() has succeeded. inline queue_mutex_guard() : holds_mutex(xSemaphoreTake(_async_queue_mutex, portMAX_DELAY)){}; + // For paths that can run before it ever has: takes nothing if the mutex was never created, + // in which case nothing can have been queued and no task is reading. + enum if_created_t { + if_created + }; + inline explicit queue_mutex_guard(if_created_t) : holds_mutex(_async_queue_mutex && xSemaphoreTake(_async_queue_mutex, portMAX_DELAY)){}; inline ~queue_mutex_guard() { if (holds_mutex) { xSemaphoreGive(_async_queue_mutex); @@ -578,15 +585,15 @@ static inline lwip_tcp_event_packet_t *_get_async_event() { } static size_t _remove_events_for_client(AsyncClientImpl *client, lwip_tcp_event_packet_t *terminal_event = nullptr) { - if (!_async_queue_mutex) { - // Never started, so nothing can have been queued. A terminal event comes from an - // LwIP callback, and a bound pcb implies a started task. - ASYNCTCP_ASSERT(!terminal_event); - return 0; - } lwip_tcp_event_packet_t *removed_event_chain; { - queue_mutex_guard guard; + queue_mutex_guard guard(queue_mutex_guard::if_created); + if (!guard) { + // Never started, so nothing can have been queued. A terminal event comes from an + // LwIP callback, and a bound pcb implies a started task. + ASYNCTCP_ASSERT(!terminal_event); + return 0; + } removed_event_chain = _async_queue.remove_if([=](lwip_tcp_event_packet_t &pkt) { return pkt.impl.get() == client; }); From d233104edf3983f54e2254df5aea3e578e758fb8 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 12:14:54 -0400 Subject: [PATCH 23/35] fix: Decide whether to deliver an accept under the queue mutex Co-Authored-By: Claude Opus 5.5 --- src/AsyncTCP.cpp | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 0fba975d..ab03252c 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -439,7 +439,9 @@ class AsyncClientImpl : public std::enable_shared_from_this { _facade is the orphan indicator. ~AsyncServer() clears it, and a queued connection that finds it null is dropped as though it never arrived. A connection accepted before end() - is dropped too, on the epoch rather than on _pcb - see below. + is dropped too, on the epoch rather than on _pcb - see below. Both are checked when the + connection is dequeued; one already dequeued is delivered, as a client's event is after + close(). One consequence of outliving the facade: the last reference can be dropped by a purge on the LwIP thread, so this object's destructor - and with it the application's onClient @@ -468,12 +470,12 @@ class AsyncServerImpl : public std::enable_shared_from_this { /* Which listening session we are on. Bumped in LwIP context in end() to ensure any pending connections do not get mishandled if a client should call end() then begin() before LwIP - can dispatch them. + can dispatch them. Written under both the LwIP lock and the queue mutex, so tcp_accept() + and the dequeue can each read it under the one they hold. */ uint32_t _epoch; - // Cleared by ~AsyncServer(). Read without a lock on the async task, on the same terms - // as AsyncClientImpl::_facade - see the note there. + // Cleared by ~AsyncServer() under the queue mutex, and read under it at dequeue. AsyncServer *_facade; // Not synchronized: set up before begin() and read afterwards. @@ -524,8 +526,8 @@ static inline void _prepend_async_event(lwip_tcp_event_packet_t *e) { static inline lwip_tcp_event_packet_t *_get_async_event() { lwip_tcp_event_packet_t *result = nullptr; - // Discarded events are freed once the mutex is released, to keep pbuf_free() out of - // the critical section. + // Discarded events are disposed of once the mutex is released, to keep pbuf_free() and + // close() out of the critical section. lwip_tcp_event_packet_t *discarded = nullptr; { @@ -533,6 +535,18 @@ static inline lwip_tcp_event_packet_t *_get_async_event() { while (1) { lwip_tcp_event_packet_t *e = _async_queue.pop_front(); + if (e && (e->event == LWIP_TCP_ACCEPT)) { + // Decided here, under the lock end() and ~AsyncServer() take to change these. + AsyncServerImpl *server = e->server.get(); + const bool wanted = server && server->_facade // the application still has this server + && (server->_epoch == e->accept.epoch); // ...and has not stopped listening since + if (!wanted) { + e->next = discarded; + discarded = e; + continue; + } + } + if ((!e) || (e->event != LWIP_TCP_POLL)) { result = e; break; @@ -579,6 +593,10 @@ static inline lwip_tcp_event_packet_t *_get_async_event() { while (discarded) { auto t = discarded; discarded = t->next; + if (t->event == LWIP_TCP_ACCEPT) { + // Never delivered, so no callbacks; this also purges anything queued behind it. + t->impl->close(); + } _free_event(t); } return result; @@ -618,15 +636,13 @@ void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { if (e->event == LWIP_TCP_ACCEPT) { // Accept is checked first because we need to handle it before the client facade check, // since we haven't constructed a facade for it yet. - AsyncServerImpl *server = e->server.get(); - const bool wanted = server && server->_facade // the application still has this server - && (server->_epoch == e->accept.epoch) // ...and has not stopped listening since - && e->impl->_pcb; // ...and the connection is still up - AsyncClient *c = wanted ? new (std::nothrow) AsyncClient(e->impl) : nullptr; + // Unwanted connections were discarded at dequeue. One that errored before then had this + // event purged; one that errors from there on is delivered, and its error event follows. + AsyncClient *c = new (std::nothrow) AsyncClient(e->impl); if (c) { - server->_accepted(c); + e->server->_accepted(c); } else { - e->impl->close(); // Not wanted or failed to create a client facade + e->impl->close(); // Failed to create a client facade } } else if ((e->impl->_facade == NULL)) { // A detached implementation has no facade to hand to the user's callbacks, so its @@ -2200,6 +2216,7 @@ AsyncServer::~AsyncServer() { _impl->end(); // Detach: a connection accepted but not yet delivered still points at the implementation, // and this is how it learns there is no longer a server to hand it to. + queue_mutex_guard guard(queue_mutex_guard::if_created); _impl->_facade = nullptr; } @@ -2295,7 +2312,9 @@ void AsyncServerImpl::end() { _pcb = NULL; // PCB is now the property of LwIP } // Disown anything already accepted on the session we just closed. Bumped inside the - // transaction, so it is ordered against tcp_accept() rather than racing it. + // transaction, so it is ordered against tcp_accept() rather than racing it, and under + // the queue mutex, so it is ordered against the dequeue. + queue_mutex_guard guard(queue_mutex_guard::if_created); ++_epoch; return ERR_OK; }); From 2cca21551f8eaa0e42f2514fc090ee44da9f088d Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 15:21:23 -0400 Subject: [PATCH 24/35] fix: Locking over set calls Ensure `setKeepAlive()` and `setNodelay()` are processed in LwIP context. --- src/AsyncTCP.cpp | 54 +++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index ab03252c..f45f3ba3 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1646,14 +1646,17 @@ void AsyncClientImpl::setAckTimeout(uint32_t timeout) { } void AsyncClientImpl::setNoDelay(bool nodelay) const { - if (!_pcb) { - return; - } - if (nodelay) { - tcp_nagle_disable(_pcb); - } else { - tcp_nagle_enable(_pcb); - } + with_tcp_core_lock([&]() { + if (!_pcb) { + return ERR_CONN; + } + if (nodelay) { + tcp_nagle_disable(_pcb); + } else { + tcp_nagle_enable(_pcb); + } + return ERR_OK; + }); } bool AsyncClientImpl::getNoDelay() { @@ -1664,18 +1667,21 @@ bool AsyncClientImpl::getNoDelay() { } void AsyncClientImpl::setKeepAlive(uint32_t ms, uint8_t cnt) { - if (!_pcb) { - return; - } - if (ms != 0) { - _pcb->so_options |= SOF_KEEPALIVE; // Turn on TCP Keepalive for the given pcb - // Set the time between keepalive messages in milli-seconds - _pcb->keep_idle = ms; - _pcb->keep_intvl = ms; - _pcb->keep_cnt = cnt; // The number of unanswered probes required to force closure of the socket - } else { - _pcb->so_options &= ~SOF_KEEPALIVE; // Turn off TCP Keepalive for the given pcb - } + with_tcp_core_lock([&]() { + if (!_pcb) { + return ERR_CONN; + } + if (ms != 0) { + _pcb->so_options |= SOF_KEEPALIVE; // Turn on TCP Keepalive for the given pcb + // Set the time between keepalive messages in milli-seconds + _pcb->keep_idle = ms; + _pcb->keep_intvl = ms; + _pcb->keep_cnt = cnt; // The number of unanswered probes required to force closure of the socket + } else { + _pcb->so_options &= ~SOF_KEEPALIVE; // Turn off TCP Keepalive for the given pcb + } + return ERR_OK; + }); } uint16_t AsyncClientImpl::getMss() const { @@ -2347,13 +2353,19 @@ int8_t AsyncTCP_detail::tcp_accept(void *arg, tcp_pcb *pcb, int8_t err) { return ERR_ABRT; } + // Apply our nodelay setting + if (server->_noDelay) { + tcp_nagle_disable(pcb); + } else { + tcp_nagle_enable(pcb); + } + // Allocate and initialize the AsyncClient implementation for this accepted connection. // The facade will be constructed by the async task when it processes this event -- if the connection // errors out prior to accept running, the event will be discarded and the implementation destroyed // as there's no API to inform the server of the error. e->impl = std::make_shared(nullptr); e->impl->_adopt(pcb); - e->impl->setNoDelay(server->_noDelay); e->server = server->shared_from_this(); // Lock server impl in scope e->accept.epoch = server->_epoch; // And remember in case it's end()ed From 4a059827756d58ed6f148c80c75848c33586223b Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 15:40:44 -0400 Subject: [PATCH 25/35] fix: recv early state check --- src/AsyncTCP.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index f45f3ba3..61384ab0 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1521,6 +1521,13 @@ int8_t AsyncClientImpl::_sent(tcp_pcb *pcb, uint16_t len) { int8_t AsyncClientImpl::_recv(tcp_pcb *pcb, pbuf *pb, int8_t err) { while (pb != NULL) { + // Ensure the object is still valid and the pcb we're processing for is still the one held. + // The callback (or other user code) could close the connection, destroy the client, or + // replace the connection outright while we're in the loop. + if (_pcb != pcb || !_facade) { + break; + } + _rx_last_packet = millis(); // we should not ack before we assimilate the data _ack_pcb = true; @@ -1551,13 +1558,6 @@ int8_t AsyncClientImpl::_recv(tcp_pcb *pcb, pbuf *pb, int8_t err) { _in_callback_ack_len = 0; } } - // Stop if the callback closed the connection, destroyed the client, or replaced the - // connection outright - a callback that reconnects leaves _pcb non-null but pointing - // at a different peer, and the rest of this chain belongs to the old one. Comparing - // against the pcb the data arrived on covers all three. - if (_pcb != pcb || !_facade) { - break; - } } if (pb) { pbuf_free(pb); // release whatever we did not get to From 0cf7a4d51e4d31b39efe9f970f30388f3cdaff3d Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 15:41:54 -0400 Subject: [PATCH 26/35] fix: Do poll state check in LwIP context --- src/AsyncTCP.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 61384ab0..de660c88 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -791,6 +791,11 @@ static int8_t _tcp_connected(void *arg, tcp_pcb *pcb, int8_t err) { } int8_t AsyncTCP_detail::tcp_poll(void *arg, struct tcp_pcb *pcb) { + if (pcb->state < ESTABLISHED) { + // Still connecting; LwIP handles the SYN timeout itself + return ERR_OK; + } + // throttle polling events queueing when event queue is getting filled up, let it handle _onack's { queue_mutex_guard guard; @@ -1570,16 +1575,12 @@ int8_t AsyncClientImpl::_poll(tcp_pcb *pcb) { // async_tcp_log_d("pcb is NULL"); return ERR_OK; } + if (pcb != _pcb) { async_tcp_log_d("0x%08" PRIx32 " != 0x%08" PRIx32, (uint32_t)pcb, (uint32_t)_pcb); return ERR_OK; } - if (_pcb->state < ESTABLISHED) { - // Still connecting; LwIP handles the SYN timeout itself - return ERR_OK; - } - uint32_t now = millis(); // ACK Timeout From fe4b798ab27f75c13b667618ed6137d36ff26bd1 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 15:43:45 -0400 Subject: [PATCH 27/35] fix: poll error check order The ack timeout was exiting the call, so the rx timeout couldn't be checked. Reverse the order so the rx timeout (which closes the connection and is thus terminal) is checked first. --- src/AsyncTCP.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index de660c88..a5fcca89 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1583,24 +1583,26 @@ int8_t AsyncClientImpl::_poll(tcp_pcb *pcb) { uint32_t now = millis(); + // RX Timeout + if (_rx_timeout && (now - _rx_last_packet) >= (_rx_timeout * 1000)) { + async_tcp_log_d("rx timeout %d", state()); + close(); + return ERR_OK; // connection is now closed; don't call poll callback + } + // ACK Timeout if (_ack_timeout) { const uint32_t one_day = 86400000; bool last_tx_is_after_last_ack = (_rx_last_ack - _tx_last_packet + one_day) < one_day; if (last_tx_is_after_last_ack && (now - _tx_last_packet) >= _ack_timeout) { - async_tcp_log_d("ack timeout %d", pcb->state); + async_tcp_log_d("ack timeout %d", state()); if (_timeout_cb) { async_tcp_log_elapsed("onTimeout", _timeout_cb(_timeout_cb_arg, _facade, (now - _tx_last_packet))); } - return ERR_OK; + return ERR_OK; // do not call poll callback } } - // RX Timeout - if (_rx_timeout && (now - _rx_last_packet) >= (_rx_timeout * 1000)) { - async_tcp_log_d("rx timeout %d", pcb->state); - close(); - return ERR_OK; - } + // Everything is fine if (_poll_cb) { async_tcp_log_elapsed("onPoll", _poll_cb(_poll_cb_arg, _facade)); From d4bbb0ce4635bd053c9e757408518a3750042bc2 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 15:46:01 -0400 Subject: [PATCH 28/35] fix: Add entry guards to _fin and _sent --- src/AsyncTCP.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index a5fcca89..98fbb614 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1512,11 +1512,17 @@ void AsyncClientImpl::_error(int8_t err) { } int8_t AsyncClientImpl::_fin(tcp_pcb *pcb, int8_t err) { + if (_pcb != pcb) { + return ERR_CONN; + } close(); return ERR_OK; } int8_t AsyncClientImpl::_sent(tcp_pcb *pcb, uint16_t len) { + if (_pcb != pcb) { + return ERR_CONN; + } _rx_last_ack = _rx_last_packet = millis(); if (_sent_cb) { async_tcp_log_elapsed("onAck", _sent_cb(_sent_cb_arg, _facade, len, (_rx_last_packet - _tx_last_packet))); From 87e6c268b1a6db71eade29e81dc925b4acf2b23d Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 15:57:02 -0400 Subject: [PATCH 29/35] fix: free/freeable exclude pending DNS lookup --- src/AsyncTCP.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 98fbb614..cba6bbb1 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1621,13 +1621,7 @@ int8_t AsyncClientImpl::_poll(tcp_pcb *pcb) { * */ bool AsyncClientImpl::free() { - if (!_pcb) { - return true; - } - if (_pcb->state == CLOSED || _pcb->state > ESTABLISHED) { - return true; - } - return false; + return this->freeable(); // these are synonymous } size_t AsyncClientImpl::write(const char *data, size_t size, uint8_t apiflags) { @@ -1891,10 +1885,7 @@ bool AsyncClientImpl::disconnected() const { } bool AsyncClientImpl::freeable() const { - if (!_pcb) { - return true; - } - return _pcb->state == CLOSED || _pcb->state > ESTABLISHED; + return !connecting() && !connected(); } bool AsyncClientImpl::canSend() const { From 5c1b5cfe847187899a2c0c525f72c5be9558f09a Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 16:12:26 -0400 Subject: [PATCH 30/35] fix: ack timeout handling --- src/AsyncTCP.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index cba6bbb1..9be5b6dd 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1604,8 +1604,9 @@ int8_t AsyncClientImpl::_poll(tcp_pcb *pcb) { async_tcp_log_d("ack timeout %d", state()); if (_timeout_cb) { async_tcp_log_elapsed("onTimeout", _timeout_cb(_timeout_cb_arg, _facade, (now - _tx_last_packet))); + return ERR_OK; // do not call poll callback } - return ERR_OK; // do not call poll callback + // Otherwise fall through to poll - not sure why you'd configure a timeout without a callback } } From 49ca0da4aa63a83c4db527ac95a393384ca4ef22 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 16:17:34 -0400 Subject: [PATCH 31/35] fix: get*Address when IPv6 Return 0 instead of unpredictable garbage. --- src/AsyncTCP.cpp | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 9be5b6dd..3d245948 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1696,14 +1696,7 @@ uint16_t AsyncClientImpl::getMss() const { } uint32_t AsyncClientImpl::getRemoteAddress() const { - if (!_pcb) { - return 0; - } -#if LWIP_IPV4 && LWIP_IPV6 - return _pcb->remote_ip.u_addr.ip4.addr; -#else - return _pcb->remote_ip.addr; -#endif + return getRemoteAddress4().addr; } #if LWIP_IPV6 @@ -1765,14 +1758,7 @@ uint16_t AsyncClientImpl::getRemotePort() const { } uint32_t AsyncClientImpl::getLocalAddress() const { - if (!_pcb) { - return 0; - } -#if LWIP_IPV4 && LWIP_IPV6 - return _pcb->local_ip.u_addr.ip4.addr; -#else - return _pcb->local_ip.addr; -#endif + return getLocalAddress4().addr; } uint16_t AsyncClientImpl::getLocalPort() const { From ce728a4f1b7ac852213146adc657a122450ea9da Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 3 Oct 2026 16:25:08 -0400 Subject: [PATCH 32/35] fix: Correct pbuf tot_len Since we're de-chaining the pbuf, ensure the other length field is correct such that `pbuf_x()` calls can be used in onPacket callbacks. --- src/AsyncTCP.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 3d245948..e37f7b3b 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1542,9 +1542,11 @@ int8_t AsyncClientImpl::_recv(tcp_pcb *pcb, pbuf *pb, int8_t err) { _rx_last_packet = millis(); // we should not ack before we assimilate the data _ack_pcb = true; + // Inline pbuf pop, similar to pbuf_dechain pbuf *b = pb; pb = b->next; b->next = NULL; + b->tot_len = b->len; if (_pb_cb) { // The callback owns b now, and must ackPacket() or free it async_tcp_log_elapsed("onPacket", _pb_cb(_pb_cb_arg, _facade, b)); From af1c67fb49ce39138d0cbcb47da38d7b7e86238f Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 4 Oct 2026 16:05:03 -0400 Subject: [PATCH 33/35] fix: space() available in CLOSE_WAIT It's legal to send bytes in CLOSE_WAIT state. While we still lack full half-close support, this will ensure `space()` accurately reports what `add()` can accept. --- src/AsyncTCP.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index e37f7b3b..9d51291a 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1398,7 +1398,8 @@ int8_t AsyncClientImpl::abort() { } size_t AsyncClientImpl::space() const { - if ((_pcb != NULL) && (_pcb->state == ESTABLISHED)) { + auto st = state(); + if (st == ESTABLISHED || st == CLOSE_WAIT) { return tcp_sndbuf(_pcb); } return 0; From 2c0d3616c06818ce396bd51f821382d0cf4cb208 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 4 Oct 2026 16:14:15 -0400 Subject: [PATCH 34/35] fix: Refactor write path Streamline locking for write calls, and ensure that write() always indicates accepted bytes even if sending fails. This allows clients to track queued data, and LwIP will retry sends on its own timers. This includes a refactor of the ack timeout handling to clarify the semantics given that packets may be sent by LwIP on its own time. --- src/AsyncTCP.cpp | 135 ++++++++++++++++++++++++++--------------------- src/AsyncTCP.h | 8 ++- 2 files changed, 82 insertions(+), 61 deletions(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index 9d51291a..aa910643 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -210,6 +210,7 @@ struct lwip_tcp_event_packet_t { struct { tcp_pcb *pcb; uint16_t len; + uint32_t time; } sent; struct { tcp_pcb *pcb; @@ -354,7 +355,7 @@ class AsyncClientImpl : public std::enable_shared_from_this { uint32_t _rx_ack_len; uint32_t _rx_last_packet; uint32_t _rx_timeout; - uint32_t _rx_last_ack; + uint32_t _ack_since; // ack timer; 0 = idle uint32_t _ack_timeout; uint32_t _in_callback_ack_len; // bytes handed to onData but not yet acked @@ -425,9 +426,15 @@ class AsyncClientImpl : public std::enable_shared_from_this { int8_t _connected(tcp_pcb *pcb, int8_t err); void _error(int8_t err); int8_t _poll(tcp_pcb *pcb); - int8_t _sent(tcp_pcb *pcb, uint16_t len); + int8_t _sent(tcp_pcb *pcb, uint16_t len, uint32_t time); int8_t _fin(tcp_pcb *pcb, int8_t err); int8_t _recv(tcp_pcb *pcb, pbuf *pb, int8_t err); + + // Add and send internal implementations; must be called in LwIP context + // These are called out to share code with write(), but allow the whole + // sequence to go in a single LwIP lock transaction. + size_t _add_locked(const char *data, size_t size, uint8_t apiflags); + err_t _send_locked(); }; /* @@ -657,7 +664,7 @@ void AsyncTCP_detail::handle_async_event(lwip_tcp_event_packet_t *e) { e->impl->_fin(e->fin.pcb, e->fin.err); } else if (e->event == LWIP_TCP_SENT) { // ets_printf("-S: 0x%08x\n", e->sent.pcb); - e->impl->_sent(e->sent.pcb, e->sent.len); + e->impl->_sent(e->sent.pcb, e->sent.len, e->sent.time); } else if (e->event == LWIP_TCP_POLL) { // ets_printf("-P: 0x%08x\n", e->poll.pcb); e->impl->_poll(e->poll.pcb); @@ -785,6 +792,7 @@ static int8_t _tcp_connected(void *arg, tcp_pcb *pcb, int8_t err) { } e->connected.pcb = pcb; e->connected.err = err; + client->_rx_last_packet = millis(); queue_mutex_guard guard; _send_async_event(e); return ERR_OK; @@ -827,6 +835,7 @@ int8_t AsyncTCP_detail::tcp_recv(void *arg, struct tcp_pcb *pcb, struct pbuf *pb async_tcp_log_e("Failed to allocate event packet"); return ERR_MEM; } + client->_rx_last_packet = millis(); if (pb) { // ets_printf("+R: 0x%08x\n", pcb); e->recv.pcb = pcb; @@ -854,6 +863,14 @@ int8_t AsyncTCP_detail::tcp_sent(void *arg, struct tcp_pcb *pcb, uint16_t len) { } e->sent.pcb = pcb; e->sent.len = len; + uint32_t now = millis(); + e->sent.time = now - client->_tx_last_packet; + // Update timeout tracking + if (client->_ack_since) { + // Re-arm the timer if we've still got outstanding data; otherwise cancel it + client->_ack_since = (tcp_sndqueuelen(pcb) ? (now - (now == 0)) : 0); + } + client->_rx_last_packet = now; queue_mutex_guard guard; _send_async_event(e); @@ -959,25 +976,6 @@ typedef struct { }; } tcp_api_call_t; -static err_t _tcp_output_api(struct tcpip_api_call_data *api_call_msg) { - tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; - msg->err = ERR_CONN; - if (msg->client->_pcb) { - msg->err = tcp_output(msg->client->_pcb); - } - return msg->err; -} - -static esp_err_t _tcp_output(AsyncClientImpl *client) { - if (!client->_pcb) { - return ERR_CONN; - } - tcp_api_call_t msg; - msg.client = client; - tcpip_api_call(_tcp_output_api, (struct tcpip_api_call_data *)&msg); - return msg.err; -} - static err_t _tcp_recved_api(struct tcpip_api_call_data *api_call_msg) { tcp_api_call_t *msg = (tcp_api_call_t *)api_call_msg; msg->err = ERR_CONN; @@ -1182,7 +1180,7 @@ static err_t _tcp_dns_start(AsyncClientImpl *client, const char *host, ip_addr_t AsyncClientImpl::AsyncClientImpl(AsyncClient *facade) : _pcb(nullptr), _connect_port(0), _dns_token(nullptr), _dispose_owed(false), _facade(facade), _connect_cb(0), _connect_cb_arg(0), _discard_cb(0), _discard_cb_arg(0), _sent_cb(0), _sent_cb_arg(0), _error_cb(0), _error_cb_arg(0), _recv_cb(0), _recv_cb_arg(0), _pb_cb(0), _pb_cb_arg(0), _timeout_cb(0), - _timeout_cb_arg(0), _poll_cb(0), _poll_cb_arg(0), _ack_pcb(true), _tx_last_packet(0), _rx_ack_len(0), _rx_last_packet(0), _rx_timeout(0), _rx_last_ack(0), + _timeout_cb_arg(0), _poll_cb(0), _poll_cb_arg(0), _ack_pcb(true), _tx_last_packet(0), _rx_ack_len(0), _rx_last_packet(0), _rx_timeout(0), _ack_since(0), _ack_timeout(CONFIG_ASYNC_TCP_MAX_ACK_TIME), _in_callback_ack_len(0) {} AsyncClientImpl::~AsyncClientImpl() { @@ -1405,35 +1403,65 @@ size_t AsyncClientImpl::space() const { return 0; } -size_t AsyncClientImpl::add(const char *data, size_t size, uint8_t apiflags) { - if (!_pcb || size == 0 || data == NULL) { +// Callers must hold the TCP core lock +size_t AsyncClientImpl::_add_locked(const char *data, size_t size, uint8_t apiflags) { + size = std::min(size, space()); + if (size == 0 || tcp_write(_pcb, data, size, apiflags) != ERR_OK) { return 0; } - // Early check outside of lock - if (space() == 0) { + return size; +} + +err_t AsyncClientImpl::_send_locked() { + uint32_t now = millis(); + _tx_last_packet = now; // Save timestamp for `sent` callback + // Re-arm the timer only if it's not already active and there's something to send + if (!_ack_since && tcp_sndqueuelen(_pcb)) { + _ack_since = now - (now == 0); // ensure is nonzero + } + return tcp_output(_pcb); +} + +size_t AsyncClientImpl::add(const char *data, size_t size, uint8_t apiflags) { + if (size == 0 || data == NULL) { return 0; } - auto err = with_tcp_core_lock([&]() -> err_t { + with_tcp_core_lock([&]() -> err_t { if (!_pcb) { + size = 0; return ERR_CONN; - } - size = std::min(size, space()); - return tcp_write(_pcb, data, size, apiflags); + }; + size = _add_locked(data, size, apiflags); + return ERR_OK; }); - if (err != ERR_OK) { - return 0; - } return size; } bool AsyncClientImpl::send() { - auto backup = _tx_last_packet; - _tx_last_packet = millis(); - if (_tcp_output(this) == ERR_OK) { - return true; + auto err = with_tcp_core_lock([&]() -> err_t { + if (!_pcb) { + return ERR_CONN; + }; + return _send_locked(); + }); + return (err == ERR_OK); +} + +size_t AsyncClientImpl::write(const char *data, size_t size, uint8_t apiflags) { + if (size == 0 || data == NULL) { + return 0; } - _tx_last_packet = backup; - return false; + size_t added = 0; + with_tcp_core_lock([&]() -> err_t { + if (_pcb) { + added = _add_locked(data, size, apiflags); + if (added) { + _send_locked(); // on failure lwIP keeps the data queued and retries + } + } + return ERR_OK; + }); + return added; } size_t AsyncClientImpl::ack(size_t len) { @@ -1474,7 +1502,7 @@ void AsyncClientImpl::_adopt(tcp_pcb *pcb) { _pcb = pcb; _rx_ack_len = 0; _tx_last_packet = 0; - _rx_last_ack = 0; + _ack_since = 0; _rx_last_packet = millis(); _bind_tcp_callbacks(pcb, this); } @@ -1485,7 +1513,6 @@ int8_t AsyncClientImpl::_connected(tcp_pcb *pcb, int8_t err) { async_tcp_log_d("%p != %p", (const void *)pcb, (const void *)_pcb); return ERR_OK; } - _rx_last_packet = millis(); if (_connect_cb) { async_tcp_log_elapsed("onConnect", _connect_cb(_connect_cb_arg, _facade)); } @@ -1520,13 +1547,12 @@ int8_t AsyncClientImpl::_fin(tcp_pcb *pcb, int8_t err) { return ERR_OK; } -int8_t AsyncClientImpl::_sent(tcp_pcb *pcb, uint16_t len) { +int8_t AsyncClientImpl::_sent(tcp_pcb *pcb, uint16_t len, uint32_t time) { if (_pcb != pcb) { return ERR_CONN; } - _rx_last_ack = _rx_last_packet = millis(); if (_sent_cb) { - async_tcp_log_elapsed("onAck", _sent_cb(_sent_cb_arg, _facade, len, (_rx_last_packet - _tx_last_packet))); + async_tcp_log_elapsed("onAck", _sent_cb(_sent_cb_arg, _facade, len, time)); } return ERR_OK; } @@ -1540,7 +1566,6 @@ int8_t AsyncClientImpl::_recv(tcp_pcb *pcb, pbuf *pb, int8_t err) { break; } - _rx_last_packet = millis(); // we should not ack before we assimilate the data _ack_pcb = true; // Inline pbuf pop, similar to pbuf_dechain @@ -1601,15 +1626,15 @@ int8_t AsyncClientImpl::_poll(tcp_pcb *pcb) { // ACK Timeout if (_ack_timeout) { - const uint32_t one_day = 86400000; - bool last_tx_is_after_last_ack = (_rx_last_ack - _tx_last_packet + one_day) < one_day; - if (last_tx_is_after_last_ack && (now - _tx_last_packet) >= _ack_timeout) { + uint32_t since = _ack_since; // Not quite interlocked with LwIP - safe enough + // If timeout is armed and timeout has expired + if (since && (now - since) >= _ack_timeout) { async_tcp_log_d("ack timeout %d", state()); if (_timeout_cb) { - async_tcp_log_elapsed("onTimeout", _timeout_cb(_timeout_cb_arg, _facade, (now - _tx_last_packet))); + async_tcp_log_elapsed("onTimeout", _timeout_cb(_timeout_cb_arg, _facade, (now - since))); return ERR_OK; // do not call poll callback } - // Otherwise fall through to poll - not sure why you'd configure a timeout without a callback + // Otherwise fall through to poll } } @@ -1628,14 +1653,6 @@ bool AsyncClientImpl::free() { return this->freeable(); // these are synonymous } -size_t AsyncClientImpl::write(const char *data, size_t size, uint8_t apiflags) { - size_t will_send = add(data, size, apiflags); - if (!will_send || !send()) { - return 0; - } - return will_send; -} - void AsyncClientImpl::setRxTimeout(uint32_t timeout) { _rx_timeout = timeout; } diff --git a/src/AsyncTCP.h b/src/AsyncTCP.h index 21cd0f0e..d074ee03 100644 --- a/src/AsyncTCP.h +++ b/src/AsyncTCP.h @@ -149,6 +149,10 @@ class AsyncClient { /** * @brief send data previously add()'ed + * @note Data queued with add() will be sent on LwIP's internal timer as well; calling send() triggers + * an immediate flush. It is possible that queued data may have already been sent by the time send() + * is executed. send() failures do not clear previously add()ed data; it remains queued and the send + * will be retried on the next timer cycle. * * @return true on success * @return false on error @@ -163,7 +167,7 @@ class AsyncClient { * @param data * @param size * @param apiflags - * @return size_t + * @return size_t amount of data that has been copied */ size_t write(const char *data, size_t size, uint8_t apiflags = ASYNC_WRITE_FLAG_COPY); @@ -194,7 +198,7 @@ class AsyncClient { void setRxTimeout(uint32_t timeout); uint32_t getAckTimeout() const; - // no ACK timeout for the last sent packet in milliseconds + // no ACK timeout for the send() call in milliseconds void setAckTimeout(uint32_t timeout); void setNoDelay(bool nodelay) const; From 4ec9cf416f51a64d73d642bcd5e499335ac37daf Mon Sep 17 00:00:00 2001 From: Will Miles Date: Tue, 6 Oct 2026 23:13:22 -0400 Subject: [PATCH 35/35] fix: Align errorToString with LwIP strings Co-Authored-By: Claude Opus 5.5 --- src/AsyncTCP.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AsyncTCP.cpp b/src/AsyncTCP.cpp index aa910643..2c9cfd45 100644 --- a/src/AsyncTCP.cpp +++ b/src/AsyncTCP.cpp @@ -1910,7 +1910,8 @@ const char *AsyncClient::errorToString(int8_t error) { case ERR_VAL: return "Illegal value"; case ERR_WOULDBLOCK: return "Operation would block"; case ERR_USE: return "Address in use"; - case ERR_ALREADY: return "Already connected"; + case ERR_ALREADY: return "Already connecting"; + case ERR_ISCONN: return "Already connected"; case ERR_CONN: return "Not connected"; case ERR_IF: return "Low-level netif error"; case ERR_ABRT: return "Connection aborted";