diff --git a/zookeeper-client/zookeeper-client-c/CMakeLists.txt b/zookeeper-client/zookeeper-client-c/CMakeLists.txt index 317059a8471..7cc60869ba1 100644 --- a/zookeeper-client/zookeeper-client-c/CMakeLists.txt +++ b/zookeeper-client/zookeeper-client-c/CMakeLists.txt @@ -252,6 +252,7 @@ set(test_sources tests/TestSASLAuth.cc tests/TestReconfig.cc tests/TestReconfigServer.cc + tests/TestResolverFailure.cc tests/TestClientRetry.cc tests/TestOperations.cc tests/TestMulti.cc diff --git a/zookeeper-client/zookeeper-client-c/Makefile.am b/zookeeper-client/zookeeper-client-c/Makefile.am index 7372291d79f..f53fb948ab7 100644 --- a/zookeeper-client/zookeeper-client-c/Makefile.am +++ b/zookeeper-client/zookeeper-client-c/Makefile.am @@ -111,6 +111,7 @@ TEST_SOURCES = \ tests/TestZookeeperClose.cc \ tests/TestReconfig.cc \ tests/TestReconfigServer.cc \ + tests/TestResolverFailure.cc \ tests/TestClientRetry.cc \ tests/TestOperations.cc \ tests/TestMulti.cc \ diff --git a/zookeeper-client/zookeeper-client-c/include/zookeeper.h b/zookeeper-client/zookeeper-client-c/include/zookeeper.h index 1a3f5d1f92f..23f8d84e672 100644 --- a/zookeeper-client/zookeeper-client-c/include/zookeeper.h +++ b/zookeeper-client/zookeeper-client-c/include/zookeeper.h @@ -820,7 +820,11 @@ ZOOAPI const char* zoo_version_str(); * \return ZOK on success or one of the following errcodes on failure: * ZBADARGUMENTS - invalid input parameters * ZINVALIDSTATE - zhandle state is either ZOO_SESSION_EXPIRED_STATE or ZOO_AUTH_FAILED_STATE - * ZSYSTEMERROR -- a system (OS) error occurred; it's worth checking errno to get details + * ZSYSTEMERROR -- a system (OS) error occurred; it's worth checking errno to get details. + * This includes the case where no host in the new list resolves. The client + * then keeps its current server list and addresses. If only some hosts + * in the new list resolve, the call succeeds. The client then uses the + * addresses that resolved. Later routine resolutions retry the other hosts. */ ZOOAPI int zoo_set_servers(zhandle_t *zh, const char *hosts); @@ -839,12 +843,23 @@ ZOOAPI int zoo_set_servers(zhandle_t *zh, const char *hosts); * This method allows an application to influence the rate of polling. * When delay_ms is set to a value greater than zero, the client skips * most "routine" resolutions which would have happened in a window of - * that many milliseconds since the last successful one. + * that many milliseconds since the last resolution. A failed routine + * resolution also counts. * * Setting delay_ms to 0 disables this logic, reverting to the default * behavior. Setting it to -1 disables network resolutions during * normal operation (but not, e.g., on connection loss). * + * If a host does not resolve during a routine resolution, the client + * keeps all of its cached addresses. Until all hosts resolve, routine + * resolutions do not change these addresses. zoo_set_servers() replaces + * them. The client then waits before the next routine resolution, also + * while it is not connected. The wait starts at one second and doubles + * after each failed resolution, up to one minute. A resolution in which + * all hosts resolve clears the wait. This wait applies in addition to + * delay_ms. A single resolution still blocks the I/O loop for as long as + * the resolver takes. The wait only limits how often this occurs. + * * \param delay_ms 0, -1, or the window size in milliseconds * \return ZOK on success or ZBADARGUMENTS for invalid input parameters */ diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index 25049bb335e..2dfb65aee7c 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -195,6 +195,7 @@ struct _zhandle { // Hostlist and list of addresses char *hostname; // hostname contains list of zookeeper servers to connect to + char *hostname_pending; // list from zoo_set_servers() that is not yet resolved struct sockaddr_storage addr_cur; // address of server we're currently connecting/connected to struct sockaddr_storage addr_rw_server; // address of last known read/write server found. @@ -204,6 +205,7 @@ struct _zhandle { struct timeval last_resolve; // time of last hostname resolution int resolve_delay_ms; // see zoo_set_servers_resolution_delay + int resolve_backoff_ms; // wait after a failed refresh, or 0 after a complete lookup int reconfig; // Are we in the process of reconfiguring cluster's ensemble double pOld, pNew; // Probability for selecting between 'addrs_old' and 'addrs_new' diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index c8429003b29..8ce18124b55 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -716,6 +716,8 @@ static void destroy(zhandle_t *zh) free(zh->hostname); zh->hostname = NULL; } + free(zh->hostname_pending); + zh->hostname_pending = NULL; if (zh->fd->sock != -1) { close_zsock(zh->fd); memset(&zh->addr_cur, 0, sizeof(zh->addr_cur)); @@ -824,12 +826,41 @@ static int count_hosts(char *hosts) return count+1; } +/* + * Limits for the wait after a failed refresh. The wait starts at the minimum + * and doubles after each failed refresh, up to the maximum. + */ +#define RESOLVE_BACKOFF_MIN_MS 1000 +#define RESOLVE_BACKOFF_MAX_MS 60000 + +/* + * Record a periodic refresh in which one or more hosts did not resolve. A + * slow resolver can block the caller for a long time. Thus the wait starts + * when the lookup ends, not when it starts. The caller holds the reconfig + * lock. + */ +static void note_resolve_failure(zhandle_t *zh) +{ + get_system_time(&zh->last_resolve); + if (zh->resolve_backoff_ms < RESOLVE_BACKOFF_MIN_MS) { + zh->resolve_backoff_ms = RESOLVE_BACKOFF_MIN_MS; + } else if (zh->resolve_backoff_ms < RESOLVE_BACKOFF_MAX_MS / 2) { + zh->resolve_backoff_ms *= 2; + } else { + zh->resolve_backoff_ms = RESOLVE_BACKOFF_MAX_MS; + } +} + /** * Resolve hosts and populate provided address vector with shuffled results. * The contents of the provided address vector will be initialized to an * empty state. + * + * The function skips each host that does not resolve, and it counts these + * hosts in *unresolved. It returns an error if no host resolves. */ -static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *avec) +static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *avec, + int *unresolved) { int rc = ZOK; char *host = NULL; @@ -841,6 +872,8 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a return ZBADARGUMENTS; } + *unresolved = 0; + // initialize address vector addrvec_init(avec); @@ -987,6 +1020,7 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a LOG_ERROR(LOGCALLBACK(zh), "getaddrinfo: %s\n", strerror(errno)); #endif rc=ZSYSTEMERROR; + (*unresolved)++; goto next; } } @@ -1073,6 +1107,14 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a * \param ref_time an optional "reference time," used to determine if * resolution can be skipped in accordance to the delay set by \ref * zoo_set_servers_resolution_delay. Passing NULL prevents skipping. + * A non-NULL ref_time also marks a periodic refresh from the driver. If a + * host does not resolve during a refresh, the client keeps its cached + * addresses and the function returns ZOK. A failed lookup is not a topology + * change. See zoo_set_servers_resolution_delay for the wait. + * + * A NULL ref_time resolves the list that zoo_set_servers() staged, if there + * is one. The staged list becomes current if one or more of its hosts + * resolve. The client then uses only the addresses that resolved. * * See {@link https://issues.apache.org/jira/browse/ZOOKEEPER-1355} for the * protocol and its evaluation, @@ -1085,6 +1127,7 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) uint32_t num_new = 0; uint32_t i = 0; int found_current = 0; + int unresolved = 0; addrvec_t resolved = { 0 }; // Verify we have a valid handle @@ -1101,6 +1144,17 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) // NOTE: guard access to {hostname, addr_cur, addrs, addrs_old, addrs_new, last_resolve, resolve_delay_ms} lock_reconfig(zh); + // After a failed refresh, the next refresh waits before it tries again. + // Thus a slow or failing resolver cannot block each pass of the event + // loop. The wait applies also while the client is not connected, because + // the client then reconnects to the cached addresses. + if (ref_time && zh->resolve_backoff_ms > 0 && zh->addrs.count > 0) { + int elapsed_ms = calculate_interval(&zh->last_resolve, ref_time); + if (elapsed_ms >= 0 && elapsed_ms < zh->resolve_backoff_ms) { + goto finish; + } + } + // Check if we are due for a host name resolution. (See // zoo_set_servers_resolution_delay. The answer is always "yes" // if no reference is provided or the file descriptor is invalid.) @@ -1122,19 +1176,50 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) } } - // Copy zh->hostname for local use - hosts = strdup(zh->hostname); + // zoo_set_servers() stages a new list in hostname_pending. The list + // becomes current only if one or more of its hosts resolve. A periodic + // refresh always uses the current list. + if (!ref_time && zh->hostname_pending) { + hosts = strdup(zh->hostname_pending); + } else { + hosts = strdup(zh->hostname); + } if (hosts == NULL) { rc = ZSYSTEMERROR; goto finish; } - rc = resolve_hosts(zh, hosts, &resolved); - if (rc != ZOK) + rc = resolve_hosts(zh, hosts, &resolved, &unresolved); + if (rc != ZOK || (ref_time && unresolved > 0 && zh->addrs.count > 0)) { + // A resolver outage must not stop pings, timeouts, or reconnects. + // Thus a periodic refresh keeps all cached addresses if a host does + // not resolve. The client does not drop or change its server only + // because a lookup failed. + // A failed zoo_set_servers() call keeps the current list, and the + // caller gets the error. + if (ref_time) { + note_resolve_failure(zh); + } + if (ref_time && zh->addrs.count > 0) { + LOG_WARN(LOGCALLBACK(zh), + "%d host(s) in server list [%s] did not resolve, keeping " + "the last good addresses", unresolved, zh->hostname); + rc = ZOK; + } else if (!ref_time && zh->hostname_pending) { + free(zh->hostname_pending); + zh->hostname_pending = NULL; + } goto finish; } + // One or more hosts of the new list resolved. It becomes the current list. + if (!ref_time && zh->hostname_pending) { + free(zh->hostname); + zh->hostname = zh->hostname_pending; + zh->hostname_pending = NULL; + } + // Unconditionally note last resolution time. if (ref_time) { zh->last_resolve = *ref_time; @@ -1142,6 +1227,15 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) get_system_time(&zh->last_resolve); } + // A complete lookup clears the wait. A partial periodic refresh gets to + // this point only if no addresses are cached. It uses the addresses that + // resolved, and it starts the wait. + if (unresolved == 0) { + zh->resolve_backoff_ms = 0; + } else if (ref_time) { + note_resolve_failure(zh); + } + // If the addrvec list is identical to last time we ran don't do anything if (addrvec_eq(&zh->addrs, &resolved)) { @@ -1560,13 +1654,15 @@ int zoo_set_servers(zhandle_t *zh, const char *hosts) // NOTE: guard access to {hostname, addr_cur, addrs, addrs_old, addrs_new, last_resolve, resolve_delay_ms} lock_reconfig(zh); - // Reset hostname to new set of hosts to connect to - if (zh->hostname) { - free(zh->hostname); + // Stage the new list. update_addrs() makes it current only if one or + // more of its hosts resolve. Thus a failed call keeps the current list. + free(zh->hostname_pending); + zh->hostname_pending = strdup(hosts); + if (zh->hostname_pending == NULL) { + unlock_reconfig(zh); + return ZSYSTEMERROR; } - zh->hostname = strdup(hosts); - unlock_reconfig(zh); return update_addrs(zh, NULL); @@ -2127,9 +2223,15 @@ static void handle_error(zhandle_t *zh,int rc) cleanup(zh, rc); // NOTE: If we're at the end of the list of addresses to connect to, then // we want to delay the next connection attempt to avoid spinning. - // Then increment what host we'll connect to since we failed to connect to current zh->delay = addrvec_atend(&zh->addrs); - addrvec_next(&zh->addrs, &zh->addr_cur); + // Outside a reconfig pass, zoo_cycle_next_server moves the cursor before + // each connect. Here the client only records the next server, so that a + // failure does not skip it. + if (zh->reconfig) { + addrvec_next(&zh->addrs, &zh->addr_cur); + } else { + addrvec_peek(&zh->addrs, &zh->addr_cur); + } } static int handle_socket_error_msg(zhandle_t *zh, int line, const char *func, int rc, @@ -2631,6 +2733,12 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, if(zh==0 || fd==0 ||interest==0 || tv==0) return ZBADARGUMENTS; + // Set safe outputs before any early return. Some event loops ignore the + // return code and pass the outputs to poll(). Thus an error return must + // not leave them uninitialized, and it must not cause a busy loop. + *fd = -1; + *interest = 0; + *tv = get_timeval(zh->recv_timeout/60); if (is_unrecoverable(zh)) return ZINVALIDSTATE; get_system_time(&now); diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc index 8a46ee54dcc..d64af2c0ef5 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc +++ b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc @@ -119,6 +119,28 @@ Mock_gethostbyname::HostEntry& Mock_gethostbyname::HostEntry::addAddress( } +// ***************************************************************************** +// getaddrinfo + +int getaddrinfo(const char *node, const char *service, + const struct addrinfo *hints, struct addrinfo **res) { + if(!Mock_getaddrinfo::mock_) + return LIBC_SYMBOLS.getaddrinfo(node,service,hints,res); + return Mock_getaddrinfo::mock_->call(node,service,hints,res); +} + +Mock_getaddrinfo* Mock_getaddrinfo::mock_=0; + +int Mock_getaddrinfo::call(const char* node, const char* service, + const struct addrinfo* hints, struct addrinfo** res) { + callCounter++; + if (clock) + clock->millitick(lookupMillis); + if (failNode && (!node || strcmp(node, failNode) != 0)) + return LIBC_SYMBOLS.getaddrinfo(node,service,hints,res); + return failWith; +} + // ***************************************************************************** // calloc #ifndef USING_DUMA diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h index 5b07cdae92d..49c37135961 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h +++ b/zookeeper-client/zookeeper-client-c/tests/LibCMocks.h @@ -66,6 +66,34 @@ class MockFailed_gethostbyname: public Mock_gethostbyname } }; +// ***************************************************************************** +// getaddrinfo + +class Mock_gettimeofday; + +// Simulates a resolver outage. While an instance exists, lookups fail with +// failWith. If failNode is set, only lookups of that node fail, and the +// other lookups go to the C library. If clock is set, each lookup advances +// it by lookupMillis, to simulate a slow resolver. Without an instance, +// lookups go to the C library. +class Mock_getaddrinfo: public Mock +{ +public: + Mock_getaddrinfo():failWith(EAI_AGAIN),failNode(0),clock(0), + lookupMillis(0),callCounter(0){mock_=this;} + virtual ~Mock_getaddrinfo(){mock_=0;} + + int failWith; + const char* failNode; + Mock_gettimeofday* clock; + int lookupMillis; + int callCounter; + virtual int call(const char* node, const char* service, + const struct addrinfo* hints, struct addrinfo** res); + + static Mock_getaddrinfo* mock_; +}; + // ***************************************************************************** // calloc diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.cc b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.cc index 23862f3c01d..fa910239244 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.cc +++ b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.cc @@ -35,6 +35,7 @@ LibCSymTable::LibCSymTable() { void* handle=getHandle(); LOAD_SYM(gethostbyname); + LOAD_SYM(getaddrinfo); LOAD_SYM(calloc); LOAD_SYM(realloc); LOAD_SYM(free); diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h index 08028abca8c..fb0959b701a 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h +++ b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h @@ -65,6 +65,8 @@ struct LibCSymTable { DECLARE_SYM(hostent*,gethostbyname,(const char*)); + DECLARE_SYM(int,getaddrinfo,(const char*,const char*, + const struct addrinfo*,struct addrinfo**)); DECLARE_SYM(void*,calloc,(size_t, size_t)); DECLARE_SYM(void*,realloc,(void*, size_t)); DECLARE_SYM(void,free,(void*)); diff --git a/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc b/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc new file mode 100644 index 00000000000..f444fddb8b0 --- /dev/null +++ b/zookeeper-client/zookeeper-client-c/tests/TestResolverFailure.cc @@ -0,0 +1,438 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +extern "C" { +#include +} + +#include "LibCMocks.h" +#include "ZKMocks.h" + +using namespace std; + +/* + * These tests simulate a resolver outage after the client has resolved its + * server list. The client must keep its last good addresses. It must send + * pings, enforce the receive timeout, and reconnect to the cached addresses. + * The tests use the single-threaded library, because they drive the event + * loop directly. + */ +class Zookeeper_resolverFailure : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE(Zookeeper_resolverFailure); +#ifndef THREADED + CPPUNIT_TEST(testResolverFailureKeepsPinging); + CPPUNIT_TEST(testResolverFailureKeepsReceiveTimeout); + CPPUNIT_TEST(testResolverFailureReconnectsToCachedAddress); + CPPUNIT_TEST(testRejectedServerListKeepsReconnecting); + CPPUNIT_TEST(testFailoverTriesEveryCachedAddress); + CPPUNIT_TEST(testFailedInterestSetsSafeOutputs); + CPPUNIT_TEST(testResolverFailureBacksOff); + CPPUNIT_TEST(testPartialResolutionKeepsConnection); + CPPUNIT_TEST(testPartialResolutionWhileDisconnected); + CPPUNIT_TEST(testAcceptedServerListBecomesCurrent); +#endif + CPPUNIT_TEST_SUITE_END(); + + static const int TIMEOUT = 9; // session timeout in seconds + + zhandle_t *zh; + FILE *logfile; + + static void watcher(zhandle_t *, int, int, const char *, void *) {} + + class PingCountingServer: public ZookeeperServer { + public: + PingCountingServer(): pingCount_(0) {} + virtual void onMessageReceived(const RequestHeader& rh, iarchive* ia) { + if (rh.type == ZOO_PING_OP) { + pingCount_++; + } + } + int pingCount_; + }; + + // Refuses each connect, and records the port of each attempt. + class RefusingServer: public Mock_socket { + public: + RefusingServer() { + connectReturns = -1; + connectErrno = ECONNREFUSED; + } + virtual int callConnect(int s, const struct sockaddr *addr, + socklen_t len) { + ports.push_back( + ntohs(((const struct sockaddr_in *)addr)->sin_port)); + return Mock_socket::callConnect(s, addr, len); + } + vector ports; + }; + + static int portOf(const struct sockaddr_storage *addr) { + return ntohs(((const struct sockaddr_in *)addr)->sin_port); + } + +#ifndef THREADED + static int interestOnce(zhandle_t *zh) { + int fd = -2; + int interest = 0; + timeval tv; + return zookeeper_interest(zh, &fd, &interest, &tv); + } +#endif + +public: + Zookeeper_resolverFailure() { + logfile = openlogfile("Zookeeper_resolverFailure"); + } + + ~Zookeeper_resolverFailure() { + if (logfile) { + fflush(logfile); + fclose(logfile); + logfile = 0; + } + } + + void setUp() + { + zoo_set_log_stream(logfile); + zoo_deterministic_conn_order(1); + zh = 0; + } + + void tearDown() + { + zookeeper_close(zh); + zh = 0; + } + +#ifndef THREADED + // A resolver outage on a connected session must not stop the pings. + void testResolverFailureKeepsPinging() + { + Mock_gettimeofday timeMock; + PingCountingServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh, &timeMock.tv); + Mock_getaddrinfo resolverOutage; + + // A ping is due after one third of the session timeout. + timeMock.tick(TIMEOUT/3 + 1); + int fd = -2; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT(resolverOutage.callCounter > 0); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD, fd); + CPPUNIT_ASSERT(interest & ZOOKEEPER_READ); + + rc = zookeeper_process(zh, interest); + CPPUNIT_ASSERT_EQUAL((int)ZNOTHING, rc); + CPPUNIT_ASSERT_EQUAL(1, zkServer.pingCount_); + CPPUNIT_ASSERT_EQUAL((int)ZOO_CONNECTED_STATE, zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(1, (int)zh->addrs.count); + } + + // A resolver outage must not hide a dead connection. + void testResolverFailureKeepsReceiveTimeout() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh, &timeMock.tv); + Mock_getaddrinfo resolverOutage; + + // No data arrives for longer than two thirds of the session timeout. + timeMock.tick(TIMEOUT); + int fd = -2; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOPERATIONTIMEOUT, rc); + CPPUNIT_ASSERT_EQUAL(-1, (int)zh->fd->sock); + } + + // A disconnected client must reconnect to the cached addresses. + void testResolverFailureReconnectsToCachedAddress() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT_EQUAL(-1, (int)zh->fd->sock); + Mock_getaddrinfo resolverOutage; + + int fd = -2; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT(resolverOutage.callCounter > 0); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD, fd); + CPPUNIT_ASSERT_EQUAL((int)ZOO_CONNECTING_STATE, zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(0, memcmp(&zh->addr_cur, &zh->addrs.data[0], + sizeof(zh->addr_cur))); + } + + // zoo_set_servers reports a resolver failure. The client keeps its + // current server list, and it reconnects to the last good addresses. + void testRejectedServerListKeepsReconnecting() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + Mock_getaddrinfo resolverOutage; + CPPUNIT_ASSERT_EQUAL((int)ZSYSTEMERROR, + zoo_set_servers(zh, "zk1.example.com:2121")); + CPPUNIT_ASSERT_EQUAL(string("127.0.0.1:2121"), string(zh->hostname)); + CPPUNIT_ASSERT(zh->hostname_pending == 0); + // A rejected list does not delay refreshes of the current list. + CPPUNIT_ASSERT_EQUAL(0, zh->resolve_backoff_ms); + + int fd = -2; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD, fd); + CPPUNIT_ASSERT_EQUAL((int)ZOO_CONNECTING_STATE, zoo_state(zh)); + } + + // Failover must try each cached address in turn. Each connect failure + // must move the cursor only one time, or a two-server list never + // reaches the second server. The first pass after zookeeper_init is a + // reconfig pass, because all addresses are new. Later passes are normal + // passes. The test covers both kinds of pass. + void testFailoverTriesEveryCachedAddress() + { + Mock_gettimeofday timeMock; + RefusingServer refusingServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121,127.0.0.1:2122", watcher, + TIMEOUT*1000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT_EQUAL(2, (int)zh->addrs.count); + Mock_getaddrinfo resolverOutage; + + // Some calls do not connect, because they apply the delay at the + // end of the server list. + for (int i = 0; i < 8; i++) { + int fd = -2; + int interest = 0; + timeval tv; + zookeeper_interest(zh, &fd, &interest, &tv); + } + CPPUNIT_ASSERT(refusingServer.ports.size() >= 5); + for (size_t i = 0; i < refusingServer.ports.size(); i++) { + CPPUNIT_ASSERT_EQUAL(i % 2 == 0 ? 2121 : 2122, + refusingServer.ports[i]); + } + } + + // Event loops such as do_io() ignore the return code. Thus an error + // return must give values that are safe to pass to poll(). + void testFailedInterestSetsSafeOutputs() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh, &timeMock.tv); + + // With no cached addresses, a resolver failure is still an error. + { + Mock_getaddrinfo resolverOutage; + addrvec_free(&zh->addrs); + int fd = 12345; + int interest = ZOOKEEPER_READ | ZOOKEEPER_WRITE; + timeval tv = { 0, 0 }; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZSYSTEMERROR, rc); + CPPUNIT_ASSERT_EQUAL(-1, fd); + CPPUNIT_ASSERT_EQUAL(0, interest); + CPPUNIT_ASSERT_EQUAL(TIMEOUT*1000/60, + (int)(tv.tv_sec*1000 + tv.tv_usec/1000)); + rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZSYSTEMERROR, rc); + CPPUNIT_ASSERT_EQUAL(2, resolverOutage.callCounter); + } + + // A terminal state also gives safe values. + zh->state = ZOO_EXPIRED_SESSION_STATE; + int fd = 12345; + int interest = ZOOKEEPER_READ | ZOOKEEPER_WRITE; + timeval tv = { 0, 0 }; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZINVALIDSTATE, rc); + CPPUNIT_ASSERT_EQUAL(-1, fd); + CPPUNIT_ASSERT_EQUAL(0, interest); + CPPUNIT_ASSERT(tv.tv_sec != 0 || tv.tv_usec != 0); + } + + // A failed lookup delays the next periodic lookup, also while the client + // is not connected. The delay starts when the slow lookup ends, and it + // doubles after each failed lookup. A complete lookup clears it. + void testResolverFailureBacksOff() + { + Mock_gettimeofday timeMock; + RefusingServer refusingServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + Mock_getaddrinfo resolverOutage; + resolverOutage.clock = &timeMock; + resolverOutage.lookupMillis = 5000; + + // Each lookup takes 5 seconds and fails. + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(1, resolverOutage.callCounter); + CPPUNIT_ASSERT_EQUAL(1000, zh->resolve_backoff_ms); + CPPUNIT_ASSERT_EQUAL(-1, (int)zh->fd->sock); + + // The wait counts from the end of the lookup. + interestOnce(zh); + timeMock.millitick(900); + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(1, resolverOutage.callCounter); + + timeMock.millitick(200); + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(2, resolverOutage.callCounter); + CPPUNIT_ASSERT_EQUAL(2000, zh->resolve_backoff_ms); + + timeMock.millitick(1500); + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(2, resolverOutage.callCounter); + timeMock.millitick(600); + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(3, resolverOutage.callCounter); + CPPUNIT_ASSERT_EQUAL(4000, zh->resolve_backoff_ms); + CPPUNIT_ASSERT_EQUAL(1, (int)zh->addrs.count); + + // The resolver recovers. A complete lookup clears the wait. + resolverOutage.failNode = "zk1.example.com"; + resolverOutage.lookupMillis = 0; + timeMock.millitick(4100); + int before = resolverOutage.callCounter; + interestOnce(zh); + CPPUNIT_ASSERT(resolverOutage.callCounter > before); + CPPUNIT_ASSERT_EQUAL(0, zh->resolve_backoff_ms); + } + + // While the client is connected, a partial periodic refresh keeps the + // whole cached list. Thus the client keeps its connection to a host + // that does not resolve. + void testPartialResolutionKeepsConnection() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121,127.0.0.2:2122", watcher, + TIMEOUT*1000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT_EQUAL(2, (int)zh->addrs.count); + forceConnected(zh, &timeMock.tv); + for (uint32_t i = 0; i < zh->addrs.count; i++) { + if (portOf(&zh->addrs.data[i]) == 2122) { + zh->addr_cur = zh->addrs.data[i]; + } + } + CPPUNIT_ASSERT_EQUAL(2122, portOf(&zh->addr_cur)); + + Mock_getaddrinfo partialOutage; + partialOutage.failNode = "127.0.0.2"; + int rc = interestOnce(zh); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD, (int)zh->fd->sock); + CPPUNIT_ASSERT_EQUAL((int)ZOO_CONNECTED_STATE, zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(2, (int)zh->addrs.count); + CPPUNIT_ASSERT_EQUAL(1000, zh->resolve_backoff_ms); + } + + // While the client is not connected, a partial periodic refresh also + // keeps the whole cached list, and it starts the wait. + void testPartialResolutionWhileDisconnected() + { + Mock_gettimeofday timeMock; + RefusingServer refusingServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121,127.0.0.2:2122", watcher, + TIMEOUT*1000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT_EQUAL(-1, (int)zh->fd->sock); + Mock_getaddrinfo partialOutage; + partialOutage.failNode = "127.0.0.2"; + interestOnce(zh); + CPPUNIT_ASSERT_EQUAL(2, (int)zh->addrs.count); + CPPUNIT_ASSERT_EQUAL(1000, zh->resolve_backoff_ms); + } + + // A server list becomes current if only some of its hosts resolve. + // The client then uses the addresses that resolved. + void testAcceptedServerListBecomesCurrent() + { + CloseFinally guard(&zh); + + zh = zookeeper_init("127.0.0.1:2121", watcher, TIMEOUT*1000, + TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + Mock_getaddrinfo partialOutage; + partialOutage.failNode = "127.0.0.3"; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_set_servers(zh, "127.0.0.2:2122,127.0.0.3:2123")); + CPPUNIT_ASSERT_EQUAL(string("127.0.0.2:2122,127.0.0.3:2123"), + string(zh->hostname)); + CPPUNIT_ASSERT(zh->hostname_pending == 0); + CPPUNIT_ASSERT_EQUAL(1, (int)zh->addrs.count); + CPPUNIT_ASSERT_EQUAL(2122, portOf(&zh->addrs.data[0])); + CPPUNIT_ASSERT_EQUAL(0, zh->resolve_backoff_ms); + } +#endif +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(Zookeeper_resolverFailure);