From c34eb326ab74017965dd19a64283b33fe098c7d8 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 5 Nov 2026 18:08:44 -0800 Subject: [PATCH 1/3] ZOOKEEPER-5121 (C client) DNS lookup failures can disrupt the client Previously, the ZooKeeper client resolved addresses synchronously inside its main I/O loop. When a DNS lookup failed, the loop exited without sending heartbeats or attempting to reconnect. It also left output parameters uninitialized, feeding invalid state into poll(). Brief DNS blips could silently expire healthy sessions and prevent disconnected clients from falling back to known addresses. Address resolution is now decoupled into a rate-limited periodic refresh. If a lookup fails, the client retains and uses its cached server addresses, reserving hard resolver errors for initial setup calls (zookeeper_init and zoo_set_servers). The event loop now guarantees safe fallback outputs on any early return, preventing poll corruption. New tests verify that heartbeats, timeouts, and reconnections continue uninterrupted throughout simulated DNS outages. Co-Authored-By: Claude Opus 5.5 --- .../zookeeper-client-c/CMakeLists.txt | 1 + .../zookeeper-client-c/Makefile.am | 1 + .../zookeeper-client-c/src/zookeeper.c | 21 ++ .../zookeeper-client-c/tests/LibCMocks.cc | 18 ++ .../zookeeper-client-c/tests/LibCMocks.h | 19 ++ .../zookeeper-client-c/tests/LibCSymTable.cc | 1 + .../zookeeper-client-c/tests/LibCSymTable.h | 2 + .../tests/TestResolverFailure.cc | 240 ++++++++++++++++++ 8 files changed, 303 insertions(+) create mode 100644 zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc diff --git a/zookeeper-client/zookeeper-client-c/CMakeLists.txt b/zookeeper-client/zookeeper-client-c/CMakeLists.txt index 317059a8471..7cc60869ba1 100644 --- a/zookeeper-client/zookeeper-client-c/CMakeLists.txt +++ b/zookeeper-client/zookeeper-client-c/CMakeLists.txt @@ -252,6 +252,7 @@ set(test_sources tests/TestSASLAuth.cc tests/TestReconfig.cc tests/TestReconfigServer.cc + tests/TestResolverFailure.cc tests/TestClientRetry.cc tests/TestOperations.cc tests/TestMulti.cc diff --git a/zookeeper-client/zookeeper-client-c/Makefile.am b/zookeeper-client/zookeeper-client-c/Makefile.am index 7372291d79f..f53fb948ab7 100644 --- a/zookeeper-client/zookeeper-client-c/Makefile.am +++ b/zookeeper-client/zookeeper-client-c/Makefile.am @@ -111,6 +111,7 @@ TEST_SOURCES = \ tests/TestZookeeperClose.cc \ tests/TestReconfig.cc \ tests/TestReconfigServer.cc \ + tests/TestResolverFailure.cc \ tests/TestClientRetry.cc \ tests/TestOperations.cc \ tests/TestMulti.cc \ diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index c8429003b29..c21784abc25 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -1073,6 +1073,10 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a * \param ref_time an optional "reference time," used to determine if * resolution can be skipped in accordance to the delay set by \ref * zoo_set_servers_resolution_delay. Passing NULL prevents skipping. + * A non-NULL ref_time also marks a periodic refresh from the driver. If + * resolution fails during a refresh, the client keeps its last good address + * list, and the function returns ZOK. A failed lookup is not a topology + * change, so the connection and the reconnect logic continue. * * See {@link https://issues.apache.org/jira/browse/ZOOKEEPER-1355} for the * protocol and its evaluation, @@ -1132,6 +1136,17 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) rc = resolve_hosts(zh, hosts, &resolved); if (rc != ZOK) { + // A resolver outage must not stop pings, timeouts, or reconnects. + // Thus a periodic refresh keeps the cached addresses. It records the + // attempt time, so the resolution delay also paces the retries. + // Callers that set a new server list pass NULL and get the error. + if (ref_time && zh->addrs.count > 0) { + LOG_WARN(LOGCALLBACK(zh), + "Failed to resolve server list [%s], keeping the last " + "good addresses (zk retcode=%d)", zh->hostname, rc); + zh->last_resolve = *ref_time; + rc = ZOK; + } goto finish; } @@ -2631,6 +2646,12 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, if(zh==0 || fd==0 ||interest==0 || tv==0) return ZBADARGUMENTS; + // Set safe outputs before any early return. Some event loops ignore the + // return code and pass the outputs to poll(). Thus an error return must + // not leave them uninitialized, and it must not cause a busy loop. + *fd = -1; + *interest = 0; + *tv = get_timeval(zh->recv_timeout/60); if (is_unrecoverable(zh)) return ZINVALIDSTATE; get_system_time(&now); diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc index 8a46ee54dcc..aaa83b1d2a6 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc +++ b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc @@ -119,6 +119,24 @@ Mock_gethostbyname::HostEntry& Mock_gethostbyname::HostEntry::addAddress( } +// ***************************************************************************** +// getaddrinfo + +int getaddrinfo(const char *node, const char *service, + const struct addrinfo *hints, struct addrinfo **res) { + if(!Mock_getaddrinfo::mock_) + return LIBC_SYMBOLS.getaddrinfo(node,service,hints,res); + return Mock_getaddrinfo::mock_->call(node,service,hints,res); +} + +Mock_getaddrinfo* Mock_getaddrinfo::mock_=0; + +int Mock_getaddrinfo::call(const char* node, const char* service, + const struct addrinfo* hints, struct addrinfo** res) { + callCounter++; + return failWith; +} + // ***************************************************************************** // calloc #ifndef USING_DUMA diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h index 5b07cdae92d..048e4ab58f1 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h +++ b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h @@ -66,6 +66,25 @@ class MockFailed_gethostbyname: public Mock_gethostbyname } }; +// ***************************************************************************** +// getaddrinfo + +// Simulates a resolver outage. While an instance exists, every lookup fails +// with failWith. Without an instance, lookups go to the C library. +class Mock_getaddrinfo: public Mock +{ +public: + Mock_getaddrinfo():failWith(EAI_AGAIN),callCounter(0){mock_=this;} + virtual ~Mock_getaddrinfo(){mock_=0;} + + int failWith; + int callCounter; + virtual int call(const char* node, const char* service, + const struct addrinfo* hints, struct addrinfo** res); + + static Mock_getaddrinfo* mock_; +}; + // ***************************************************************************** // calloc diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.cc b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.cc index 23862f3c01d..fa910239244 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.cc +++ b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.cc @@ -35,6 +35,7 @@ LibCSymTable::LibCSymTable() { void* handle=getHandle(); LOAD_SYM(gethostbyname); + LOAD_SYM(getaddrinfo); LOAD_SYM(calloc); LOAD_SYM(realloc); LOAD_SYM(free); diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h index 08028abca8c..fb0959b701a 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h +++ b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h @@ -65,6 +65,8 @@ struct LibCSymTable { DECLARE_SYM(hostent*,gethostbyname,(const char*)); + DECLARE_SYM(int,getaddrinfo,(const char*,const char*, + const struct addrinfo*,struct addrinfo**)); DECLARE_SYM(void*,calloc,(size_t, size_t)); DECLARE_SYM(void*,realloc,(void*, size_t)); DECLARE_SYM(void,free,(void*)); diff --git a/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc b/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc new file mode 100644 index 00000000000..fe8c2f24715 --- /dev/null +++ b/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +extern "C" { +#include +} + +#include "LibCMocks.h" +#include "ZKMocks.h" + +using namespace std; + +/* + * These tests simulate a resolver outage after the client has resolved its + * server list. The client must keep its last good addresses. It must send + * pings, enforce the receive timeout, and reconnect to the cached addresses. + * The tests use the single-threaded library, because they drive the event + * loop directly. + */ +class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE(Zookeeper_resolverFailure); +#ifndef THREADED + CPPUNIT_TEST(testResolverFailureKeepsPinging); + CPPUNIT_TEST(testResolverFailureKeepsReceiveTimeout); + CPPUNIT_TEST(testResolverFailureReconnectsToCachedAddress); + CPPUNIT_TEST(testRejectedServerListKeepsReconnecting); + CPPUNIT_TEST(testFailedInterestSetsSafeOutputs); +#endif + CPPUNIT_TEST_SUITE_END(); + + static const int TIMEOUT = 9; // session timeout in seconds + + zhandle_t *zh; + FILE *logfile; + + static void watcher(zhandle_t *, int, int, const char *, void *) {} + + class PingCountingServer: public ZookeeperServer { + public: + PingCountingServer(): pingCount_(0) {} + virtual void onMessageReceived(const RequestHeader& rh, iarchive* ia) { + if (rh.type == ZOO_PING_OP) { + pingCount_++; + } + } + int pingCount_; + }; + +public: + Zookeeper_resolverFailure() { + logfile = openlogfile("Zookeeper_resolverFailure"); + } + + ~Zookeeper_resolverFailure() { + if (logfile) { + fflush(logfile); + fclose(logfile); + logfile = 0; + } + } + + void setUp() + { + zoo_set_log_stream(logfile); + zoo_deterministic_conn_order(1); + zh = 0; + } + + void tearDown() + { + zookeeper_close(zh); + zh = 0; + } + +#ifndef THREADED + // A resolver outage on a connected session must not stop the pings. + void testResolverFailureKeepsPinging() + { + Mock_gettimeofday timeMock; + PingCountingServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh, &timeMock.tv); + Mock_getaddrinfo resolverOutage; + + // A ping is due after one third of the session timeout. + timeMock.tick(TIMEOUT/3 + 1); + int fd = -2; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT(resolverOutage.callCounter > 0); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD, fd); + CPPUNIT_ASSERT(interest & ZOOKEEPER_READ); + + rc = zookeeper_process(zh, interest); + CPPUNIT_ASSERT_EQUAL((int)ZNOTHING, rc); + CPPUNIT_ASSERT_EQUAL(1, zkServer.pingCount_); + CPPUNIT_ASSERT_EQUAL((int)ZOO_CONNECTED_STATE, zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(1, (int)zh->addrs.count); + } + + // A resolver outage must not hide a dead connection. + void testResolverFailureKeepsReceiveTimeout() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh, &timeMock.tv); + Mock_getaddrinfo resolverOutage; + + // No data arrives for longer than two thirds of the session timeout. + timeMock.tick(TIMEOUT); + int fd = -2; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOPERATIONTIMEOUT, rc); + CPPUNIT_ASSERT_EQUAL(-1, (int)zh->fd->sock); + } + + // A disconnected client must reconnect to the cached addresses. + void testResolverFailureReconnectsToCachedAddress() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT_EQUAL(-1, (int)zh->fd->sock); + Mock_getaddrinfo resolverOutage; + + int fd = -2; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT(resolverOutage.callCounter > 0); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD, fd); + CPPUNIT_ASSERT_EQUAL((int)ZOO_CONNECTING_STATE, zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(0, memcmp(&zh->addr_cur, &zh->addrs.data[0], + sizeof(zh->addr_cur))); + } + + // zoo_set_servers reports a resolver failure, and the client still + // reconnects to the last good addresses. + void testRejectedServerListKeepsReconnecting() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + Mock_getaddrinfo resolverOutage; + CPPUNIT_ASSERT_EQUAL((int)ZSYSTEMERROR, + zoo_set_servers(zh, "zk1.example.com:2121")); + + int fd = -2; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD, fd); + CPPUNIT_ASSERT_EQUAL((int)ZOO_CONNECTING_STATE, zoo_state(zh)); + } + + // Event loops such as do_io() ignore the return code. Thus an error + // return must give values that are safe to pass to poll(). + void testFailedInterestSetsSafeOutputs() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh, &timeMock.tv); + + // With no cached addresses, a resolver failure is still an error. + { + Mock_getaddrinfo resolverOutage; + addrvec_free(&zh->addrs); + int fd = 12345; + int interest = ZOOKEEPER_READ | ZOOKEEPER_WRITE; + timeval tv = { 0, 0 }; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZSYSTEMERROR, rc); + CPPUNIT_ASSERT_EQUAL(-1, fd); + CPPUNIT_ASSERT_EQUAL(0, interest); + CPPUNIT_ASSERT_EQUAL(TIMEOUT*1000/60, + (int)(tv.tv_sec*1000 + tv.tv_usec/1000)); + } + + // A terminal state also gives safe values. + zh->state = ZOO_EXPIRED_SESSION_STATE; + int fd = 12345; + int interest = ZOOKEEPER_READ | ZOOKEEPER_WRITE; + timeval tv = { 0, 0 }; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZINVALIDSTATE, rc); + CPPUNIT_ASSERT_EQUAL(-1, fd); + CPPUNIT_ASSERT_EQUAL(0, interest); + CPPUNIT_ASSERT(tv.tv_sec != 0 || tv.tv_usec != 0); + } +#endif +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(Zookeeper_resolverFailure); From 49e61b8919751ba3ad926258c5eae06690cfd87f Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Sat, 7 Nov 2026 10:24:51 -0800 Subject: [PATCH 2/3] ZOOKEEPER-5121: (C client) Addendum: failover must try every server in turn After a connect failure, both handle_error and zoo_cycle_next_server advanced the server list cursor, but the client skipped every other server. Outside a reconfig pass, handle_error now only records the next server, and zoo_cycle_next_server alone advances the cursor. A reconfig pass selects from addrs_new and addrs_old, so handle_error still advances the cursor there to pace the end-of-list delay. A new test verifies that connect attempts alternate between two refusing servers during a resolver outage. Co-Authored-By: Claude Opus 5.5 --- .../zookeeper-client-c/src/zookeeper.c | 10 +++- .../tests/TestResolverFailure.cc | 51 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index c21784abc25..3d96e4bc4d2 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -2142,9 +2142,15 @@ static void handle_error(zhandle_t *zh,int rc) cleanup(zh, rc); // NOTE: If we're at the end of the list of addresses to connect to, then // we want to delay the next connection attempt to avoid spinning. - // Then increment what host we'll connect to since we failed to connect to current zh->delay = addrvec_atend(&zh->addrs); - addrvec_next(&zh->addrs, &zh->addr_cur); + // Outside a reconfig pass, zoo_cycle_next_server moves the cursor before + // each connect. Here the client only records the next server, so that a + // failure does not skip it. + if (zh->reconfig) { + addrvec_next(&zh->addrs, &zh->addr_cur); + } else { + addrvec_peek(&zh->addrs, &zh->addr_cur); + } } static int handle_socket_error_msg(zhandle_t *zh, int line, const char *func, int rc, diff --git a/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc b/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc index fe8c2f24715..da70a6645a4 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc @@ -18,6 +18,8 @@ #include #include +#include +#include extern "C" { #include @@ -43,6 +45,7 @@ class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture CPPUNIT_TEST(testResolverFailureKeepsReceiveTimeout); CPPUNIT_TEST(testResolverFailureReconnectsToCachedAddress); CPPUNIT_TEST(testRejectedServerListKeepsReconnecting); + CPPUNIT_TEST(testFailoverTriesEveryCachedAddress); CPPUNIT_TEST(testFailedInterestSetsSafeOutputs); #endif CPPUNIT_TEST_SUITE_END(); @@ -65,6 +68,22 @@ class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture int pingCount_; }; + // Refuses each connect, and records the port of each attempt. + class RefusingServer: public Mock_socket { + public: + RefusingServer() { + connectReturns = -1; + connectErrno = ECONNREFUSED; + } + virtual int callConnect(int s, const struct sockaddr *addr, + socklen_t len) { + ports.push_back( + ntohs(((const struct sockaddr_in *)addr)->sin_port)); + return Mock_socket::callConnect(s, addr, len); + } + vector ports; + }; + public: Zookeeper_resolverFailure() { logfile = openlogfile("Zookeeper_resolverFailure"); @@ -195,6 +214,38 @@ class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZOO_CONNECTING_STATE, zoo_state(zh)); } + // Failover must try each cached address in turn. Each connect failure + // must move the cursor only one time, or a two-server list never + // reaches the second server. The first pass after zookeeper_init is a + // reconfig pass, because all addresses are new. Later passes are normal + // passes. The test covers both kinds of pass. + void testFailoverTriesEveryCachedAddress() + { + Mock_gettimeofday timeMock; + RefusingServer refusingServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121,127.0.0.1:2122", watcher, + TIMEOUT*1000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT_EQUAL(2, (int)zh->addrs.count); + Mock_getaddrinfo resolverOutage; + + // Some calls do not connect, because they apply the delay at the + // end of the server list. + for (int i = 0; i < 8; i++) { + int fd = -2; + int interest = 0; + timeval tv; + zookeeper_interest(zh, &fd, &interest, &tv); + } + CPPUNIT_ASSERT(refusingServer.ports.size() >= 5); + for (size_t i = 0; i < refusingServer.ports.size(); i++) { + CPPUNIT_ASSERT_EQUAL(i % 2 == 0 ? 2121 : 2122, + refusingServer.ports[i]); + } + } + // Event loops such as do_io() ignore the return code. Thus an error // return must give values that are safe to pass to poll(). void testFailedInterestSetsSafeOutputs() From 6e747cf8f9910ece78323386db0c9a475cf5c544 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 19 Nov 2026 17:48:18 -0800 Subject: [PATCH 3/3] ZOOKEEPER-5121: (C client) Addendum: pace failed lookups, keep the cached list on a partial refresh - After a periodic refresh in which one or more hosts do not resolve, the next refresh waits. The wait starts at one second, doubles after each failed refresh up to one minute, and counts from the end of the lookup. It applies also while the client is not connected. A lookup in which all hosts resolve clears it. A single lookup can still block the I/O loop for as long as the resolver takes; the wait limits how often this occurs. - A periodic refresh in which some hosts do not resolve keeps all cached addresses. Thus the client does not drop its server only because a name did not resolve. - zoo_set_servers() stages the new list. The list becomes current only if one or more of its hosts resolve. A failed call keeps the current list and addresses, and it does not start the wait. If only some hosts resolve, the client uses the addresses that resolved. - The zoo_set_servers() and zoo_set_servers_resolution_delay() docs describe this behavior. The delay window counts from the last resolution, and a failed routine resolution also counts. Co-Authored-By: Claude Opus 5.5 --- .../zookeeper-client-c/include/zookeeper.h | 19 ++- .../zookeeper-client-c/src/zk_adaptor.h | 2 + .../zookeeper-client-c/src/zookeeper.c | 121 +++++++++++--- .../zookeeper-client-c/tests/LibCMocks.cc | 4 + .../zookeeper-client-c/tests/LibCMocks.h | 15 +- .../tests/TestResolverFailure.cc | 151 +++++++++++++++++- 6 files changed, 285 insertions(+), 27 deletions(-) diff --git a/zookeeper-client/zookeeper-client-c/include/zookeeper.h b/zookeeper-client/zookeeper-client-c/include/zookeeper.h index 1a3f5d1f92f..23f8d84e672 100644 --- a/zookeeper-client/zookeeper-client-c/include/zookeeper.h +++ b/zookeeper-client/zookeeper-client-c/include/zookeeper.h @@ -820,7 +820,11 @@ ZOOAPI const char* zoo_version_str(); * \return ZOK on success or one of the following errcodes on failure: * ZBADARGUMENTS - invalid input parameters * ZINVALIDSTATE - zhandle state is either ZOO_SESSION_EXPIRED_STATE or ZOO_AUTH_FAILED_STATE - * ZSYSTEMERROR -- a system (OS) error occurred; it's worth checking errno to get details + * ZSYSTEMERROR -- a system (OS) error occurred; it's worth checking errno to get details. + * This includes the case where no host in the new list resolves. The client + * then keeps its current server list and addresses. If only some hosts + * in the new list resolve, the call succeeds. The client then uses the + * addresses that resolved. Later routine resolutions retry the other hosts. */ ZOOAPI int zoo_set_servers(zhandle_t *zh, const char *hosts); @@ -839,12 +843,23 @@ ZOOAPI int zoo_set_servers(zhandle_t *zh, const char *hosts); * This method allows an application to influence the rate of polling. * When delay_ms is set to a value greater than zero, the client skips * most "routine" resolutions which would have happened in a window of - * that many milliseconds since the last successful one. + * that many milliseconds since the last resolution. A failed routine + * resolution also counts. * * Setting delay_ms to 0 disables this logic, reverting to the default * behavior. Setting it to -1 disables network resolutions during * normal operation (but not, e.g., on connection loss). * + * If a host does not resolve during a routine resolution, the client + * keeps all of its cached addresses. Until all hosts resolve, routine + * resolutions do not change these addresses. zoo_set_servers() replaces + * them. The client then waits before the next routine resolution, also + * while it is not connected. The wait starts at one second and doubles + * after each failed resolution, up to one minute. A resolution in which + * all hosts resolve clears the wait. This wait applies in addition to + * delay_ms. A single resolution still blocks the I/O loop for as long as + * the resolver takes. The wait only limits how often this occurs. + * * \param delay_ms 0, -1, or the window size in milliseconds * \return ZOK on success or ZBADARGUMENTS for invalid input parameters */ diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index 25049bb335e..2dfb65aee7c 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -195,6 +195,7 @@ struct _zhandle { // Hostlist and list of addresses char *hostname; // hostname contains list of zookeeper servers to connect to + char *hostname_pending; // list from zoo_set_servers() that is not yet resolved struct sockaddr_storage addr_cur; // address of server we're currently connecting/connected to struct sockaddr_storage addr_rw_server; // address of last known read/write server found. @@ -204,6 +205,7 @@ struct _zhandle { struct timeval last_resolve; // time of last hostname resolution int resolve_delay_ms; // see zoo_set_servers_resolution_delay + int resolve_backoff_ms; // wait after a failed refresh, or 0 after a complete lookup int reconfig; // Are we in the process of reconfiguring cluster's ensemble double pOld, pNew; // Probability for selecting between 'addrs_old' and 'addrs_new' diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index 3d96e4bc4d2..8ce18124b55 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -716,6 +716,8 @@ static void destroy(zhandle_t *zh) free(zh->hostname); zh->hostname = NULL; } + free(zh->hostname_pending); + zh->hostname_pending = NULL; if (zh->fd->sock != -1) { close_zsock(zh->fd); memset(&zh->addr_cur, 0, sizeof(zh->addr_cur)); @@ -824,12 +826,41 @@ static int count_hosts(char *hosts) return count+1; } +/* + * Limits for the wait after a failed refresh. The wait starts at the minimum + * and doubles after each failed refresh, up to the maximum. + */ +#define RESOLVE_BACKOFF_MIN_MS 1000 +#define RESOLVE_BACKOFF_MAX_MS 60000 + +/* + * Record a periodic refresh in which one or more hosts did not resolve. A + * slow resolver can block the caller for a long time. Thus the wait starts + * when the lookup ends, not when it starts. The caller holds the reconfig + * lock. + */ +static void note_resolve_failure(zhandle_t *zh) +{ + get_system_time(&zh->last_resolve); + if (zh->resolve_backoff_ms < RESOLVE_BACKOFF_MIN_MS) { + zh->resolve_backoff_ms = RESOLVE_BACKOFF_MIN_MS; + } else if (zh->resolve_backoff_ms < RESOLVE_BACKOFF_MAX_MS / 2) { + zh->resolve_backoff_ms *= 2; + } else { + zh->resolve_backoff_ms = RESOLVE_BACKOFF_MAX_MS; + } +} + /** * Resolve hosts and populate provided address vector with shuffled results. * The contents of the provided address vector will be initialized to an * empty state. + * + * The function skips each host that does not resolve, and it counts these + * hosts in *unresolved. It returns an error if no host resolves. */ -static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *avec) +static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *avec, + int *unresolved) { int rc = ZOK; char *host = NULL; @@ -841,6 +872,8 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a return ZBADARGUMENTS; } + *unresolved = 0; + // initialize address vector addrvec_init(avec); @@ -987,6 +1020,7 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a LOG_ERROR(LOGCALLBACK(zh), "getaddrinfo: %s\n", strerror(errno)); #endif rc=ZSYSTEMERROR; + (*unresolved)++; goto next; } } @@ -1073,10 +1107,14 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a * \param ref_time an optional "reference time," used to determine if * resolution can be skipped in accordance to the delay set by \ref * zoo_set_servers_resolution_delay. Passing NULL prevents skipping. - * A non-NULL ref_time also marks a periodic refresh from the driver. If - * resolution fails during a refresh, the client keeps its last good address - * list, and the function returns ZOK. A failed lookup is not a topology - * change, so the connection and the reconnect logic continue. + * A non-NULL ref_time also marks a periodic refresh from the driver. If a + * host does not resolve during a refresh, the client keeps its cached + * addresses and the function returns ZOK. A failed lookup is not a topology + * change. See zoo_set_servers_resolution_delay for the wait. + * + * A NULL ref_time resolves the list that zoo_set_servers() staged, if there + * is one. The staged list becomes current if one or more of its hosts + * resolve. The client then uses only the addresses that resolved. * * See {@link https://issues.apache.org/jira/browse/ZOOKEEPER-1355} for the * protocol and its evaluation, @@ -1089,6 +1127,7 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) uint32_t num_new = 0; uint32_t i = 0; int found_current = 0; + int unresolved = 0; addrvec_t resolved = { 0 }; // Verify we have a valid handle @@ -1105,6 +1144,17 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) // NOTE: guard access to {hostname, addr_cur, addrs, addrs_old, addrs_new, last_resolve, resolve_delay_ms} lock_reconfig(zh); + // After a failed refresh, the next refresh waits before it tries again. + // Thus a slow or failing resolver cannot block each pass of the event + // loop. The wait applies also while the client is not connected, because + // the client then reconnects to the cached addresses. + if (ref_time && zh->resolve_backoff_ms > 0 && zh->addrs.count > 0) { + int elapsed_ms = calculate_interval(&zh->last_resolve, ref_time); + if (elapsed_ms >= 0 && elapsed_ms < zh->resolve_backoff_ms) { + goto finish; + } + } + // Check if we are due for a host name resolution. (See // zoo_set_servers_resolution_delay. The answer is always "yes" // if no reference is provided or the file descriptor is invalid.) @@ -1126,30 +1176,50 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) } } - // Copy zh->hostname for local use - hosts = strdup(zh->hostname); + // zoo_set_servers() stages a new list in hostname_pending. The list + // becomes current only if one or more of its hosts resolve. A periodic + // refresh always uses the current list. + if (!ref_time && zh->hostname_pending) { + hosts = strdup(zh->hostname_pending); + } else { + hosts = strdup(zh->hostname); + } if (hosts == NULL) { rc = ZSYSTEMERROR; goto finish; } - rc = resolve_hosts(zh, hosts, &resolved); - if (rc != ZOK) + rc = resolve_hosts(zh, hosts, &resolved, &unresolved); + if (rc != ZOK || (ref_time && unresolved > 0 && zh->addrs.count > 0)) { // A resolver outage must not stop pings, timeouts, or reconnects. - // Thus a periodic refresh keeps the cached addresses. It records the - // attempt time, so the resolution delay also paces the retries. - // Callers that set a new server list pass NULL and get the error. + // Thus a periodic refresh keeps all cached addresses if a host does + // not resolve. The client does not drop or change its server only + // because a lookup failed. + // A failed zoo_set_servers() call keeps the current list, and the + // caller gets the error. + if (ref_time) { + note_resolve_failure(zh); + } if (ref_time && zh->addrs.count > 0) { LOG_WARN(LOGCALLBACK(zh), - "Failed to resolve server list [%s], keeping the last " - "good addresses (zk retcode=%d)", zh->hostname, rc); - zh->last_resolve = *ref_time; + "%d host(s) in server list [%s] did not resolve, keeping " + "the last good addresses", unresolved, zh->hostname); rc = ZOK; + } else if (!ref_time && zh->hostname_pending) { + free(zh->hostname_pending); + zh->hostname_pending = NULL; } goto finish; } + // One or more hosts of the new list resolved. It becomes the current list. + if (!ref_time && zh->hostname_pending) { + free(zh->hostname); + zh->hostname = zh->hostname_pending; + zh->hostname_pending = NULL; + } + // Unconditionally note last resolution time. if (ref_time) { zh->last_resolve = *ref_time; @@ -1157,6 +1227,15 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) get_system_time(&zh->last_resolve); } + // A complete lookup clears the wait. A partial periodic refresh gets to + // this point only if no addresses are cached. It uses the addresses that + // resolved, and it starts the wait. + if (unresolved == 0) { + zh->resolve_backoff_ms = 0; + } else if (ref_time) { + note_resolve_failure(zh); + } + // If the addrvec list is identical to last time we ran don't do anything if (addrvec_eq(&zh->addrs, &resolved)) { @@ -1575,13 +1654,15 @@ int zoo_set_servers(zhandle_t *zh, const char *hosts) // NOTE: guard access to {hostname, addr_cur, addrs, addrs_old, addrs_new, last_resolve, resolve_delay_ms} lock_reconfig(zh); - // Reset hostname to new set of hosts to connect to - if (zh->hostname) { - free(zh->hostname); + // Stage the new list. update_addrs() makes it current only if one or + // more of its hosts resolve. Thus a failed call keeps the current list. + free(zh->hostname_pending); + zh->hostname_pending = strdup(hosts); + if (zh->hostname_pending == NULL) { + unlock_reconfig(zh); + return ZSYSTEMERROR; } - zh->hostname = strdup(hosts); - unlock_reconfig(zh); return update_addrs(zh, NULL); diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc index aaa83b1d2a6..d64af2c0ef5 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc +++ b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc @@ -134,6 +134,10 @@ Mock_getaddrinfo* Mock_getaddrinfo::mock_=0; int Mock_getaddrinfo::call(const char* node, const char* service, const struct addrinfo* hints, struct addrinfo** res) { callCounter++; + if (clock) + clock->millitick(lookupMillis); + if (failNode && (!node || strcmp(node, failNode) != 0)) + return LIBC_SYMBOLS.getaddrinfo(node,service,hints,res); return failWith; } diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h index 048e4ab58f1..49c37135961 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h +++ b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h @@ -69,15 +69,24 @@ class MockFailed_gethostbyname: public Mock_gethostbyname // ***************************************************************************** // getaddrinfo -// Simulates a resolver outage. While an instance exists, every lookup fails -// with failWith. Without an instance, lookups go to the C library. +class Mock_gettimeofday; + +// Simulates a resolver outage. While an instance exists, lookups fail with +// failWith. If failNode is set, only lookups of that node fail, and the +// other lookups go to the C library. If clock is set, each lookup advances +// it by lookupMillis, to simulate a slow resolver. Without an instance, +// lookups go to the C library. class Mock_getaddrinfo: public Mock { public: - Mock_getaddrinfo():failWith(EAI_AGAIN),callCounter(0){mock_=this;} + Mock_getaddrinfo():failWith(EAI_AGAIN),failNode(0),clock(0), + lookupMillis(0),callCounter(0){mock_=this;} virtual ~Mock_getaddrinfo(){mock_=0;} int failWith; + const char* failNode; + Mock_gettimeofday* clock; + int lookupMillis; int callCounter; virtual int call(const char* node, const char* service, const struct addrinfo* hints, struct addrinfo** res); diff --git a/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc b/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc index da70a6645a4..f444fddb8b0 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc @@ -47,6 +47,10 @@ class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture CPPUNIT_TEST(testRejectedServerListKeepsReconnecting); CPPUNIT_TEST(testFailoverTriesEveryCachedAddress); CPPUNIT_TEST(testFailedInterestSetsSafeOutputs); + CPPUNIT_TEST(testResolverFailureBacksOff); + CPPUNIT_TEST(testPartialResolutionKeepsConnection); + CPPUNIT_TEST(testPartialResolutionWhileDisconnected); + CPPUNIT_TEST(testAcceptedServerListBecomesCurrent); #endif CPPUNIT_TEST_SUITE_END(); @@ -84,6 +88,19 @@ class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture vector ports; }; + static int portOf(const struct sockaddr_storage *addr) { + return ntohs(((const struct sockaddr_in *)addr)->sin_port); + } + +#ifndef THREADED + static int interestOnce(zhandle_t *zh) { + int fd = -2; + int interest = 0; + timeval tv; + return zookeeper_interest(zh, &fd, &interest, &tv); + } +#endif + public: Zookeeper_resolverFailure() { logfile = openlogfile("Zookeeper_resolverFailure"); @@ -190,8 +207,8 @@ class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture sizeof(zh->addr_cur))); } - // zoo_set_servers reports a resolver failure, and the client still - // reconnects to the last good addresses. + // zoo_set_servers reports a resolver failure. The client keeps its + // current server list, and it reconnects to the last good addresses. void testRejectedServerListKeepsReconnecting() { Mock_gettimeofday timeMock; @@ -204,6 +221,10 @@ class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture Mock_getaddrinfo resolverOutage; CPPUNIT_ASSERT_EQUAL((int)ZSYSTEMERROR, zoo_set_servers(zh, "zk1.example.com:2121")); + CPPUNIT_ASSERT_EQUAL(string("127.0.0.1:2121"), string(zh->hostname)); + CPPUNIT_ASSERT(zh->hostname_pending == 0); + // A rejected list does not delay refreshes of the current list. + CPPUNIT_ASSERT_EQUAL(0, zh->resolve_backoff_ms); int fd = -2; int interest = 0; @@ -272,6 +293,9 @@ class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL(0, interest); CPPUNIT_ASSERT_EQUAL(TIMEOUT*1000/60, (int)(tv.tv_sec*1000 + tv.tv_usec/1000)); + rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZSYSTEMERROR, rc); + CPPUNIT_ASSERT_EQUAL(2, resolverOutage.callCounter); } // A terminal state also gives safe values. @@ -285,6 +309,129 @@ class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL(0, interest); CPPUNIT_ASSERT(tv.tv_sec != 0 || tv.tv_usec != 0); } + + // A failed lookup delays the next periodic lookup, also while the client + // is not connected. The delay starts when the slow lookup ends, and it + // doubles after each failed lookup. A complete lookup clears it. + void testResolverFailureBacksOff() + { + Mock_gettimeofday timeMock; + RefusingServer refusingServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + Mock_getaddrinfo resolverOutage; + resolverOutage.clock = &timeMock; + resolverOutage.lookupMillis = 5000; + + // Each lookup takes 5 seconds and fails. + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(1, resolverOutage.callCounter); + CPPUNIT_ASSERT_EQUAL(1000, zh->resolve_backoff_ms); + CPPUNIT_ASSERT_EQUAL(-1, (int)zh->fd->sock); + + // The wait counts from the end of the lookup. + interestOnce(zh); + timeMock.millitick(900); + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(1, resolverOutage.callCounter); + + timeMock.millitick(200); + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(2, resolverOutage.callCounter); + CPPUNIT_ASSERT_EQUAL(2000, zh->resolve_backoff_ms); + + timeMock.millitick(1500); + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(2, resolverOutage.callCounter); + timeMock.millitick(600); + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(3, resolverOutage.callCounter); + CPPUNIT_ASSERT_EQUAL(4000, zh->resolve_backoff_ms); + CPPUNIT_ASSERT_EQUAL(1, (int)zh->addrs.count); + + // The resolver recovers. A complete lookup clears the wait. + resolverOutage.failNode = "zk1.example.com"; + resolverOutage.lookupMillis = 0; + timeMock.millitick(4100); + int before = resolverOutage.callCounter; + interestOnce(zh); + CPPUNIT_ASSERT(resolverOutage.callCounter > before); + CPPUNIT_ASSERT_EQUAL(0, zh->resolve_backoff_ms); + } + + // While the client is connected, a partial periodic refresh keeps the + // whole cached list. Thus the client keeps its connection to a host + // that does not resolve. + void testPartialResolutionKeepsConnection() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121,127.0.0.2:2122", watcher, + TIMEOUT*1000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT_EQUAL(2, (int)zh->addrs.count); + forceConnected(zh, &timeMock.tv); + for (uint32_t i = 0; i < zh->addrs.count; i++) { + if (portOf(&zh->addrs.data[i]) == 2122) { + zh->addr_cur = zh->addrs.data[i]; + } + } + CPPUNIT_ASSERT_EQUAL(2122, portOf(&zh->addr_cur)); + + Mock_getaddrinfo partialOutage; + partialOutage.failNode = "127.0.0.2"; + int rc = interestOnce(zh); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD, (int)zh->fd->sock); + CPPUNIT_ASSERT_EQUAL((int)ZOO_CONNECTED_STATE, zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(2, (int)zh->addrs.count); + CPPUNIT_ASSERT_EQUAL(1000, zh->resolve_backoff_ms); + } + + // While the client is not connected, a partial periodic refresh also + // keeps the whole cached list, and it starts the wait. + void testPartialResolutionWhileDisconnected() + { + Mock_gettimeofday timeMock; + RefusingServer refusingServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121,127.0.0.2:2122", watcher, + TIMEOUT*1000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT_EQUAL(-1, (int)zh->fd->sock); + Mock_getaddrinfo partialOutage; + partialOutage.failNode = "127.0.0.2"; + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(2, (int)zh->addrs.count); + CPPUNIT_ASSERT_EQUAL(1000, zh->resolve_backoff_ms); + } + + // A server list becomes current if only some of its hosts resolve. + // The client then uses the addresses that resolved. + void testAcceptedServerListBecomesCurrent() + { + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + Mock_getaddrinfo partialOutage; + partialOutage.failNode = "127.0.0.3"; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_set_servers(zh, "127.0.0.2:2122,127.0.0.3:2123")); + CPPUNIT_ASSERT_EQUAL(string("127.0.0.2:2122,127.0.0.3:2123"), + string(zh->hostname)); + CPPUNIT_ASSERT(zh->hostname_pending == 0); + CPPUNIT_ASSERT_EQUAL(1, (int)zh->addrs.count); + CPPUNIT_ASSERT_EQUAL(2122, portOf(&zh->addrs.data[0])); + CPPUNIT_ASSERT_EQUAL(0, zh->resolve_backoff_ms); + } #endif };