Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions zookeeper-client/zookeeper-client-c/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ set(test_sources
tests/TestSASLAuth.cc
tests/TestReconfig.cc
tests/TestReconfigServer.cc
tests/TestResolverFailure.cc
tests/TestClientRetry.cc
tests/TestOperations.cc
tests/TestMulti.cc
Expand Down
1 change: 1 addition & 0 deletions zookeeper-client/zookeeper-client-c/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ TEST_SOURCES = \
tests/TestZookeeperClose.cc \
tests/TestReconfig.cc \
tests/TestReconfigServer.cc \
tests/TestResolverFailure.cc \
tests/TestClientRetry.cc \
tests/TestOperations.cc \
tests/TestMulti.cc \
Expand Down
19 changes: 17 additions & 2 deletions zookeeper-client/zookeeper-client-c/include/zookeeper.h
Original file line number Diff line number Diff line change
Expand Up @@ -820,7 +820,11 @@ ZOOAPI const char* zoo_version_str();
* \return ZOK on success or one of the following errcodes on failure:
* ZBADARGUMENTS - invalid input parameters
* ZINVALIDSTATE - zhandle state is either ZOO_SESSION_EXPIRED_STATE or ZOO_AUTH_FAILED_STATE
* ZSYSTEMERROR -- a system (OS) error occurred; it's worth checking errno to get details
* ZSYSTEMERROR -- a system (OS) error occurred; it's worth checking errno to get details.
* This includes the case where no host in the new list resolves. The client
* then keeps its current server list and addresses. If only some hosts
* in the new list resolve, the call succeeds. The client then uses the
* addresses that resolved. Later routine resolutions retry the other hosts.
*/
ZOOAPI int zoo_set_servers(zhandle_t *zh, const char *hosts);

Expand All @@ -839,12 +843,23 @@ ZOOAPI int zoo_set_servers(zhandle_t *zh, const char *hosts);
* This method allows an application to influence the rate of polling.
* When delay_ms is set to a value greater than zero, the client skips
* most "routine" resolutions which would have happened in a window of
* that many milliseconds since the last successful one.
* that many milliseconds since the last resolution. A failed routine
* resolution also counts.
*
* Setting delay_ms to 0 disables this logic, reverting to the default
* behavior. Setting it to -1 disables network resolutions during
* normal operation (but not, e.g., on connection loss).
*
* If a host does not resolve during a routine resolution, the client
* keeps all of its cached addresses. Until all hosts resolve, routine
* resolutions do not change these addresses. zoo_set_servers() replaces
* them. The client then waits before the next routine resolution, also
* while it is not connected. The wait starts at one second and doubles
* after each failed resolution, up to one minute. A resolution in which
* all hosts resolve clears the wait. This wait applies in addition to
* delay_ms. A single resolution still blocks the I/O loop for as long as
* the resolver takes. The wait only limits how often this occurs.
*
* \param delay_ms 0, -1, or the window size in milliseconds
* \return ZOK on success or ZBADARGUMENTS for invalid input parameters
*/
Expand Down
2 changes: 2 additions & 0 deletions zookeeper-client/zookeeper-client-c/src/zk_adaptor.h
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ struct _zhandle {

// Hostlist and list of addresses
char *hostname; // hostname contains list of zookeeper servers to connect to
char *hostname_pending; // list from zoo_set_servers() that is not yet resolved
struct sockaddr_storage addr_cur; // address of server we're currently connecting/connected to
struct sockaddr_storage addr_rw_server; // address of last known read/write server found.

Expand All @@ -204,6 +205,7 @@ struct _zhandle {

struct timeval last_resolve; // time of last hostname resolution
int resolve_delay_ms; // see zoo_set_servers_resolution_delay
int resolve_backoff_ms; // wait after a failed refresh, or 0 after a complete lookup

int reconfig; // Are we in the process of reconfiguring cluster's ensemble
double pOld, pNew; // Probability for selecting between 'addrs_old' and 'addrs_new'
Expand Down
132 changes: 120 additions & 12 deletions zookeeper-client/zookeeper-client-c/src/zookeeper.c
Original file line number Diff line number Diff line change
Expand Up @@ -716,6 +716,8 @@ static void destroy(zhandle_t *zh)
free(zh->hostname);
zh->hostname = NULL;
}
free(zh->hostname_pending);
zh->hostname_pending = NULL;
if (zh->fd->sock != -1) {
close_zsock(zh->fd);
memset(&zh->addr_cur, 0, sizeof(zh->addr_cur));
Expand Down Expand Up @@ -824,12 +826,41 @@ static int count_hosts(char *hosts)
return count+1;
}

/*
* Limits for the wait after a failed refresh. The wait starts at the minimum
* and doubles after each failed refresh, up to the maximum.
*/
#define RESOLVE_BACKOFF_MIN_MS 1000
#define RESOLVE_BACKOFF_MAX_MS 60000

/*
* Record a periodic refresh in which one or more hosts did not resolve. A
* slow resolver can block the caller for a long time. Thus the wait starts
* when the lookup ends, not when it starts. The caller holds the reconfig
* lock.
*/
static void note_resolve_failure(zhandle_t *zh)
{
get_system_time(&zh->last_resolve);
if (zh->resolve_backoff_ms < RESOLVE_BACKOFF_MIN_MS) {
zh->resolve_backoff_ms = RESOLVE_BACKOFF_MIN_MS;
} else if (zh->resolve_backoff_ms < RESOLVE_BACKOFF_MAX_MS / 2) {
zh->resolve_backoff_ms *= 2;
} else {
zh->resolve_backoff_ms = RESOLVE_BACKOFF_MAX_MS;
}
}

/**
* Resolve hosts and populate provided address vector with shuffled results.
* The contents of the provided address vector will be initialized to an
* empty state.
*
* The function skips each host that does not resolve, and it counts these
* hosts in *unresolved. It returns an error if no host resolves.
*/
static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *avec)
static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *avec,
int *unresolved)
{
int rc = ZOK;
char *host = NULL;
Expand All @@ -841,6 +872,8 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a
return ZBADARGUMENTS;
}

*unresolved = 0;

// initialize address vector
addrvec_init(avec);

Expand Down Expand Up @@ -987,6 +1020,7 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a
LOG_ERROR(LOGCALLBACK(zh), "getaddrinfo: %s\n", strerror(errno));
#endif
rc=ZSYSTEMERROR;
(*unresolved)++;
goto next;
}
}
Expand Down Expand Up @@ -1073,6 +1107,14 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a
* \param ref_time an optional "reference time," used to determine if
* resolution can be skipped in accordance to the delay set by \ref
* zoo_set_servers_resolution_delay. Passing NULL prevents skipping.
* A non-NULL ref_time also marks a periodic refresh from the driver. If a
* host does not resolve during a refresh, the client keeps its cached
* addresses and the function returns ZOK. A failed lookup is not a topology
* change. See zoo_set_servers_resolution_delay for the wait.
*
* A NULL ref_time resolves the list that zoo_set_servers() staged, if there
* is one. The staged list becomes current if one or more of its hosts
* resolve. The client then uses only the addresses that resolved.
*
* See {@link https://issues.apache.org/jira/browse/ZOOKEEPER-1355} for the
* protocol and its evaluation,
Expand All @@ -1085,6 +1127,7 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time)
uint32_t num_new = 0;
uint32_t i = 0;
int found_current = 0;
int unresolved = 0;
addrvec_t resolved = { 0 };

// Verify we have a valid handle
Expand All @@ -1101,6 +1144,17 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time)
// NOTE: guard access to {hostname, addr_cur, addrs, addrs_old, addrs_new, last_resolve, resolve_delay_ms}
lock_reconfig(zh);

// After a failed refresh, the next refresh waits before it tries again.
// Thus a slow or failing resolver cannot block each pass of the event
// loop. The wait applies also while the client is not connected, because
// the client then reconnects to the cached addresses.
if (ref_time && zh->resolve_backoff_ms > 0 && zh->addrs.count > 0) {
int elapsed_ms = calculate_interval(&zh->last_resolve, ref_time);
if (elapsed_ms >= 0 && elapsed_ms < zh->resolve_backoff_ms) {
goto finish;
}
}

// Check if we are due for a host name resolution. (See
// zoo_set_servers_resolution_delay. The answer is always "yes"
// if no reference is provided or the file descriptor is invalid.)
Expand All @@ -1122,26 +1176,66 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time)
}
}

// Copy zh->hostname for local use
hosts = strdup(zh->hostname);
// zoo_set_servers() stages a new list in hostname_pending. The list
// becomes current only if one or more of its hosts resolve. A periodic
// refresh always uses the current list.
if (!ref_time && zh->hostname_pending) {
hosts = strdup(zh->hostname_pending);
} else {
hosts = strdup(zh->hostname);
}
if (hosts == NULL) {
rc = ZSYSTEMERROR;
goto finish;
}

rc = resolve_hosts(zh, hosts, &resolved);
if (rc != ZOK)
rc = resolve_hosts(zh, hosts, &resolved, &unresolved);
if (rc != ZOK || (ref_time && unresolved > 0 && zh->addrs.count > 0))
{
// A resolver outage must not stop pings, timeouts, or reconnects.
// Thus a periodic refresh keeps all cached addresses if a host does
// not resolve. The client does not drop or change its server only
// because a lookup failed.
// A failed zoo_set_servers() call keeps the current list, and the
// caller gets the error.
if (ref_time) {
note_resolve_failure(zh);
}
if (ref_time && zh->addrs.count > 0) {
LOG_WARN(LOGCALLBACK(zh),
"%d host(s) in server list [%s] did not resolve, keeping "
"the last good addresses", unresolved, zh->hostname);
rc = ZOK;
} else if (!ref_time && zh->hostname_pending) {
free(zh->hostname_pending);
zh->hostname_pending = NULL;
}
goto finish;
}

// One or more hosts of the new list resolved. It becomes the current list.
if (!ref_time && zh->hostname_pending) {
free(zh->hostname);
zh->hostname = zh->hostname_pending;
zh->hostname_pending = NULL;
}

// Unconditionally note last resolution time.
if (ref_time) {
zh->last_resolve = *ref_time;
} else {
get_system_time(&zh->last_resolve);
}

// A complete lookup clears the wait. A partial periodic refresh gets to
// this point only if no addresses are cached. It uses the addresses that
// resolved, and it starts the wait.
if (unresolved == 0) {
zh->resolve_backoff_ms = 0;
} else if (ref_time) {
note_resolve_failure(zh);
}

// If the addrvec list is identical to last time we ran don't do anything
if (addrvec_eq(&zh->addrs, &resolved))
{
Expand Down Expand Up @@ -1560,13 +1654,15 @@ int zoo_set_servers(zhandle_t *zh, const char *hosts)
// NOTE: guard access to {hostname, addr_cur, addrs, addrs_old, addrs_new, last_resolve, resolve_delay_ms}
lock_reconfig(zh);

// Reset hostname to new set of hosts to connect to
if (zh->hostname) {
free(zh->hostname);
// Stage the new list. update_addrs() makes it current only if one or
// more of its hosts resolve. Thus a failed call keeps the current list.
free(zh->hostname_pending);
zh->hostname_pending = strdup(hosts);
if (zh->hostname_pending == NULL) {
unlock_reconfig(zh);
return ZSYSTEMERROR;
}

zh->hostname = strdup(hosts);

unlock_reconfig(zh);

return update_addrs(zh, NULL);
Expand Down Expand Up @@ -2127,9 +2223,15 @@ static void handle_error(zhandle_t *zh,int rc)
cleanup(zh, rc);
// NOTE: If we're at the end of the list of addresses to connect to, then
// we want to delay the next connection attempt to avoid spinning.
// Then increment what host we'll connect to since we failed to connect to current
zh->delay = addrvec_atend(&zh->addrs);
addrvec_next(&zh->addrs, &zh->addr_cur);
// Outside a reconfig pass, zoo_cycle_next_server moves the cursor before
// each connect. Here the client only records the next server, so that a
// failure does not skip it.
if (zh->reconfig) {
addrvec_next(&zh->addrs, &zh->addr_cur);
} else {
addrvec_peek(&zh->addrs, &zh->addr_cur);
}
}

static int handle_socket_error_msg(zhandle_t *zh, int line, const char *func, int rc,
Expand Down Expand Up @@ -2631,6 +2733,12 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest,

if(zh==0 || fd==0 ||interest==0 || tv==0)
return ZBADARGUMENTS;
// Set safe outputs before any early return. Some event loops ignore the
// return code and pass the outputs to poll(). Thus an error return must
// not leave them uninitialized, and it must not cause a busy loop.
*fd = -1;
*interest = 0;
*tv = get_timeval(zh->recv_timeout/60);
if (is_unrecoverable(zh))
return ZINVALIDSTATE;
get_system_time(&now);
Expand Down
22 changes: 22 additions & 0 deletions zookeeper-client/zookeeper-client-c/tests/LibCMocks.cc
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,28 @@ Mock_gethostbyname::HostEntry& Mock_gethostbyname::HostEntry::addAddress(
}


// *****************************************************************************
// getaddrinfo

int getaddrinfo(const char *node, const char *service,
const struct addrinfo *hints, struct addrinfo **res) {
if(!Mock_getaddrinfo::mock_)
return LIBC_SYMBOLS.getaddrinfo(node,service,hints,res);
return Mock_getaddrinfo::mock_->call(node,service,hints,res);
}

Mock_getaddrinfo* Mock_getaddrinfo::mock_=0;

int Mock_getaddrinfo::call(const char* node, const char* service,
const struct addrinfo* hints, struct addrinfo** res) {
callCounter++;
if (clock)
clock->millitick(lookupMillis);
if (failNode && (!node || strcmp(node, failNode) != 0))
return LIBC_SYMBOLS.getaddrinfo(node,service,hints,res);
return failWith;
}

// *****************************************************************************
// calloc
#ifndef USING_DUMA
Expand Down
28 changes: 28 additions & 0 deletions zookeeper-client/zookeeper-client-c/tests/LibCMocks.h
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,34 @@ class MockFailed_gethostbyname: public Mock_gethostbyname
}
};

// *****************************************************************************
// getaddrinfo

class Mock_gettimeofday;

// Simulates a resolver outage. While an instance exists, lookups fail with
// failWith. If failNode is set, only lookups of that node fail, and the
// other lookups go to the C library. If clock is set, each lookup advances
// it by lookupMillis, to simulate a slow resolver. Without an instance,
// lookups go to the C library.
class Mock_getaddrinfo: public Mock
{
public:
Mock_getaddrinfo():failWith(EAI_AGAIN),failNode(0),clock(0),
lookupMillis(0),callCounter(0){mock_=this;}
virtual ~Mock_getaddrinfo(){mock_=0;}

int failWith;
const char* failNode;
Mock_gettimeofday* clock;
int lookupMillis;
int callCounter;
virtual int call(const char* node, const char* service,
const struct addrinfo* hints, struct addrinfo** res);

static Mock_getaddrinfo* mock_;
};

// *****************************************************************************
// calloc

Expand Down
1 change: 1 addition & 0 deletions zookeeper-client/zookeeper-client-c/tests/LibCSymTable.cc
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ LibCSymTable::LibCSymTable()
{
void* handle=getHandle();
LOAD_SYM(gethostbyname);
LOAD_SYM(getaddrinfo);
LOAD_SYM(calloc);
LOAD_SYM(realloc);
LOAD_SYM(free);
Expand Down
2 changes: 2 additions & 0 deletions zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@
struct LibCSymTable
{
DECLARE_SYM(hostent*,gethostbyname,(const char*));
DECLARE_SYM(int,getaddrinfo,(const char*,const char*,
const struct addrinfo*,struct addrinfo**));
DECLARE_SYM(void*,calloc,(size_t, size_t));
DECLARE_SYM(void*,realloc,(void*, size_t));
DECLARE_SYM(void,free,(void*));
Expand Down
Loading