Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions zookeeper-client/zookeeper-client-c/include/zookeeper.h
Original file line number Diff line number Diff line change
Expand Up @@ -812,6 +812,14 @@ ZOOAPI const char* zoo_version_str();
* the client moves back to the normal mode of operation where it will pick an
* arbitrary server from the 'host' string.
*
* This call does not drop the connection itself. It asks the driver to drop
* it. The driver is the IO thread in the multithreaded library. In the
* single-threaded library, the driver is the next zookeeper_interest() call.
* Until the driver acts, zoo_state() continues to return the current state,
* for example ZOO_CONNECTED_STATE or ZOO_READONLY_STATE. If the session was
* connected, the watcher then receives a ZOO_CONNECTING_STATE session event.
* All outstanding requests complete with ZCONNECTIONLOSS.
*
* See {@link https://issues.apache.org/jira/browse/ZOOKEEPER-1355} for the
* protocol and its evaluation,
*
Expand Down
22 changes: 16 additions & 6 deletions zookeeper-client/zookeeper-client-c/src/mt_adaptor.c
Original file line number Diff line number Diff line change
Expand Up @@ -167,9 +167,6 @@ void *do_io(void *);
void *do_completion(void *);
#endif


int wakeup_io_thread(zhandle_t *zh);

#ifdef WIN32
static int set_nonblock(SOCKET fd){
ULONG nonblocking_flag = 1;
Expand Down Expand Up @@ -336,11 +333,24 @@ int wakeup_io_thread(zhandle_t *zh)
{
struct adaptor_threads *adaptor_threads = zh->adaptor_priv;
char c=0;
/*
* A full pipe already holds a wakeup for the IO thread, so a full pipe
* is not an error. Retry the write if a signal interrupts it.
*/
#ifndef WIN32
return write(adaptor_threads->self_pipe[1],&c,1)==1? ZOK: ZSYSTEMERROR;
for (;;) {
if (write(adaptor_threads->self_pipe[1],&c,1)==1)
return ZOK;
if (errno == EAGAIN || errno == EWOULDBLOCK)
return ZOK;
if (errno != EINTR)
return ZSYSTEMERROR;
}
#else
return send(adaptor_threads->self_pipe[1], &c, 1, 0)==1? ZOK: ZSYSTEMERROR;
#endif
if (send(adaptor_threads->self_pipe[1], &c, 1, 0)==1)
return ZOK;
return WSAGetLastError() == WSAEWOULDBLOCK ? ZOK : ZSYSTEMERROR;
#endif
}

int adaptor_send_queue(zhandle_t *zh, int timeout)
Expand Down
3 changes: 3 additions & 0 deletions zookeeper-client/zookeeper-client-c/src/zk_adaptor.h
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,8 @@ struct _zhandle {
int reconfig; // Are we in the process of reconfiguring cluster's ensemble
double pOld, pNew; // Probability for selecting between 'addrs_old' and 'addrs_new'
int delay;
int disconnect_requested; // When set, the next zookeeper_interest() call drops the
// connection. The reconfig lock guards this flag.
int disable_reconnection_attempt; // When set, client will not try reconnect to a different server in
// server list. This makes a sticky server for client, and is useful
// for testing if a sticky server is required, or if client wants to
Expand Down Expand Up @@ -291,6 +293,7 @@ struct sync_completion *alloc_sync_completion(void);
int wait_sync_completion(struct sync_completion *sc);
void free_sync_completion(struct sync_completion *sc);
void notify_sync_completion(struct sync_completion *sc);
int wakeup_io_thread(zhandle_t *zh);
#endif
int adaptor_send_queue(zhandle_t *zh, int timeout);
int process_async(int outstanding_sync);
Expand Down
124 changes: 108 additions & 16 deletions zookeeper-client/zookeeper-client-c/src/zookeeper.c
Original file line number Diff line number Diff line change
Expand Up @@ -653,9 +653,10 @@ void close_zsock(zsock_t *fd)
* Release the SSL_CTX only together with its SSL object. A context
* without an SSL object exists only while init_ssl_for_socket()
* configures it on the IO thread. That function releases it on
* every failure path (ZOOKEEPER-5004). An application thread can
* call this function (for example through zoo_set_servers()), and
* a release here could free the context during its configuration.
* every failure path (ZOOKEEPER-5004). Only the driver closes the
* session socket. destroy() closes it after the IO thread stops.
* Thus no other thread can run init_ssl_for_socket() at the same
* time. Keep this rule as a defensive measure.
*/
if (fd->ssl_sock) {
SSL_free(fd->ssl_sock);
Expand Down Expand Up @@ -1050,6 +1051,51 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a
return rc;
}

/*
* Only the driver closes the connection. The driver is the IO thread, or the
* caller of zookeeper_interest() in the single-threaded library. Other code
* sets a flag to ask for a disconnect. The next zookeeper_interest() call
* does the full teardown. The teardown sends the session event and fails the
* pending requests. Thus no stale request or partial frame goes to the next
* server.
*
* The reconfig lock guards the flag. The caller of this function holds it.
*/
static void request_disconnect_locked(zhandle_t *zh)
{
zh->disconnect_requested = 1;
}

static void wakeup_driver(zhandle_t *zh)
{
#ifdef THREADED
/* The IO thread can sleep in poll(). Wake it so that it acts at once. */
if (zh->adaptor_priv) {
wakeup_io_thread(zh);
}
#endif
}

static void request_disconnect(zhandle_t *zh)
{
lock_reconfig(zh);
request_disconnect_locked(zh);
unlock_reconfig(zh);
wakeup_driver(zh);
}

/* Return and clear the disconnect request. */
static int take_disconnect_request(zhandle_t *zh)
{
int requested;

lock_reconfig(zh);
requested = zh->disconnect_requested;
zh->disconnect_requested = 0;
unlock_reconfig(zh);
return requested;
}

/**
* Updates the list of servers and determine if changing connections is necessary.
* Permutes server list for proper load balancing.
Expand Down Expand Up @@ -1216,13 +1262,11 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time)
addrvec_free(&zh->addrs);
zh->addrs = resolved;

// If we need to do a reconfig and we're currently connected to a server,
// then force close that connection so on next interest() call we'll make a
// new connection
// If the client must move to another server, ask the driver to drop the
// current connection. Its next connect uses the new list.
if (zh->reconfig == 1 && zh->fd->sock != -1)
{
close_zsock(zh->fd);
zh->state = ZOO_NOTCONNECTED_STATE;
request_disconnect_locked(zh);
}

finish:
Expand Down Expand Up @@ -1551,6 +1595,9 @@ zhandle_t *zookeeper_init_sasl(const char *host, watcher_fn watcher,
*/
int zoo_set_servers(zhandle_t *zh, const char *hosts)
{
int rc;
int disconnect;

if (hosts == NULL)
{
LOG_ERROR(LOGCALLBACK(zh), "New server list cannot be empty");
Expand All @@ -1569,7 +1616,19 @@ int zoo_set_servers(zhandle_t *zh, const char *hosts)

unlock_reconfig(zh);

return update_addrs(zh, NULL);
rc = update_addrs(zh, NULL);

// This call can come from any thread or callback. If a disconnect is
// pending, wake the IO thread so that it acts at once. The update_addrs()
// call in zookeeper_interest() needs no wakeup, because the driver
// already runs.
lock_reconfig(zh);
disconnect = zh->disconnect_requested;
unlock_reconfig(zh);
if (disconnect) {
wakeup_driver(zh);
}
return rc;
}

/*
Expand Down Expand Up @@ -2452,7 +2511,15 @@ static struct timeval get_timeval(int interval)
get_buffer_len(oa));
leave_critical(zh);
close_buffer_oarchive(&oa, rc < 0);
#ifdef THREADED
/*
* Only the IO thread sends pings, from zookeeper_interest(). That call
* then asks for write readiness, so the IO thread needs no wakeup.
*/
return rc < 0 ? rc : ZOK;
#else
return rc<0 ? rc : adaptor_send_queue(zh, 0);
#endif
}

/* upper bound of a timeout for seeking for r/w server when in read-only mode */
Expand Down Expand Up @@ -2631,6 +2698,12 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest,

if(zh==0 || fd==0 ||interest==0 || tv==0)
return ZBADARGUMENTS;
// Set safe outputs before any early return. Some event loops ignore the
// return code and pass the outputs to poll(). Thus an error return must
// not leave them uninitialized, and it must not cause a busy loop.
*fd = -1;
*interest = 0;
*tv = get_timeval(zh->recv_timeout/60);
if (is_unrecoverable(zh))
return ZINVALIDSTATE;
get_system_time(&now);
Expand All @@ -2644,6 +2717,16 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest,
api_prolog(zh);

rc = update_addrs(zh, &now);

// Do the teardown that a server list change or a failed send asked for.
// If an error closed the socket already, do nothing. The connect code
// below then selects the next server.
if (take_disconnect_request(zh) && zh->fd->sock != -1) {
LOG_INFO(LOGCALLBACK(zh), "Closing connection to %s on request",
format_endpoint_info(&zh->addr_cur));
cleanup(zh, ZCONNECTIONLOSS);
}

if (rc != ZOK) {
return api_epilog(zh, rc);
}
Expand Down Expand Up @@ -2773,7 +2856,14 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest,
rc = send_ping(zh);
if (rc < 0) {
LOG_ERROR(LOGCALLBACK(zh), "failed to send PING request (zk retcode=%d)",rc);
return api_epilog(zh,rc);
// zookeeper_interest() runs in the driver, so it
// does the teardown now. It must not return the
// closed socket.
handle_error(zh, ZCONNECTIONLOSS);
*fd = -1;
*interest = 0;
*tv = get_timeval(0);
return api_epilog(zh, ZCONNECTIONLOSS);
}
}
send_to = zh->recv_timeout/3;
Expand Down Expand Up @@ -4150,14 +4240,16 @@ static int Request_path_watch_init(zhandle_t *zh, int mode,
* ASYNC API
*---------------------------------------------------------------------------*/

/* make an attempt to send queued requests immediately without blocking */
/*
* Try to send the queued requests without blocking. The caller can be an
* application thread or a callback, so a failed send does not close the
* socket here. The driver does the teardown on its next pass. The pending
* requests then fail with ZCONNECTIONLOSS.
*/
static int nonblocking_send(zhandle_t *zh, int rc)
{
if (adaptor_send_queue(zh, 0) < 0) {
if (zh->fd->sock != -1) {
close_zsock(zh->fd);
zh->state = ZOO_NOTCONNECTED_STATE;
}
if (adaptor_send_queue(zh, 0) < 0 && zh->fd->sock != -1) {
request_disconnect(zh);
}
return (rc < 0) ? ZMARSHALLINGERROR : ZOK;
}
Expand Down
Loading