From eccc461d7f527118c4cc36723d50093a353c94a2 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 5 Nov 2026 17:36:13 -0800 Subject: [PATCH 1/3] ZOOKEEPER-5120: (C client) Forced closes leaks request queues and corrupts session state Calling threads close the client socket directly during server reconfigurations or failed async writes, bypassing proper cleanup and producing invalid internal states. This triggers several race conditions: healthy connections are dropped when the wake-up pipe backs up, pending requests are routed to new servers prematurely, watchers are exposed to inconsistent states, and authentication failures are masked. The fix is to manage the lifecycle strictly in the I/O driver. Calling threads no longer close sockets or mutate client state themselves. Instead they set a disconnect request flag and signal the I/O thread. The driver handles the teardown deterministically, notifying watchers of a reconnecting state, cleanly failing in-flight requests with connection loss, and hardening the wake up path against pipe congestion and signal interruptions. Add Zookeeper_operations tests testServerListChangeFailsPendingRequests, testSendFailureDefersTeardown, testPingSendFailureTearsDown, testServerListChangeKeepsAuthFailedState (ST), testServerListChangeDuringSessionEstablishment and testFullWakeupPipeKeepsConnection (MT). Co-Authored-By: Claude Opus 5.5 --- .../zookeeper-client-c/src/mt_adaptor.c | 19 +- .../zookeeper-client-c/src/zk_adaptor.h | 3 + .../zookeeper-client-c/src/zookeeper.c | 102 ++++- .../tests/TestOperations.cc | 371 +++++++++++++++++- 4 files changed, 478 insertions(+), 17 deletions(-) diff --git a/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c b/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c index fdef1b4e4da..0a945f4a474 100644 --- a/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c +++ b/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c @@ -336,11 +336,24 @@ int wakeup_io_thread(zhandle_t *zh) { struct adaptor_threads *adaptor_threads = zh->adaptor_priv; char c=0; + /* + * A full pipe already holds a wakeup for the IO thread, so a full pipe + * is not an error. Retry the write if a signal interrupts it. + */ #ifndef WIN32 - return write(adaptor_threads->self_pipe[1],&c,1)==1? ZOK: ZSYSTEMERROR; + for (;;) { + if (write(adaptor_threads->self_pipe[1],&c,1)==1) + return ZOK; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return ZOK; + if (errno != EINTR) + return ZSYSTEMERROR; + } #else - return send(adaptor_threads->self_pipe[1], &c, 1, 0)==1? ZOK: ZSYSTEMERROR; -#endif + if (send(adaptor_threads->self_pipe[1], &c, 1, 0)==1) + return ZOK; + return WSAGetLastError() == WSAEWOULDBLOCK ? ZOK : ZSYSTEMERROR; +#endif } int adaptor_send_queue(zhandle_t *zh, int timeout) diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index 25049bb335e..8dfe1312d13 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -208,6 +208,8 @@ struct _zhandle { int reconfig; // Are we in the process of reconfiguring cluster's ensemble double pOld, pNew; // Probability for selecting between 'addrs_old' and 'addrs_new' int delay; + int disconnect_requested; // When set, the next zookeeper_interest() call drops the + // connection. The reconfig lock guards this flag. int disable_reconnection_attempt; // When set, client will not try reconnect to a different server in // server list. This makes a sticky server for client, and is useful // for testing if a sticky server is required, or if client wants to @@ -291,6 +293,7 @@ struct sync_completion *alloc_sync_completion(void); int wait_sync_completion(struct sync_completion *sc); void free_sync_completion(struct sync_completion *sc); void notify_sync_completion(struct sync_completion *sc); +int wakeup_io_thread(zhandle_t *zh); #endif int adaptor_send_queue(zhandle_t *zh, int timeout); int process_async(int outstanding_sync); diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index c8429003b29..ea4b7695416 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -1050,6 +1050,51 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a return rc; } +/* + * Only the driver closes the connection. The driver is the IO thread, or the + * caller of zookeeper_interest() in the single-threaded library. Other code + * sets a flag to ask for a disconnect. The next zookeeper_interest() call + * does the full teardown. The teardown sends the session event and fails the + * pending requests. Thus no stale request or partial frame goes to the next + * server. + * + * The reconfig lock guards the flag. The caller of this function holds it. + */ +static void request_disconnect_locked(zhandle_t *zh) +{ + zh->disconnect_requested = 1; +} + +static void wakeup_driver(zhandle_t *zh) +{ +#ifdef THREADED + /* The IO thread can sleep in poll(). Wake it so that it acts at once. */ + if (zh->adaptor_priv) { + wakeup_io_thread(zh); + } +#endif +} + +static void request_disconnect(zhandle_t *zh) +{ + lock_reconfig(zh); + request_disconnect_locked(zh); + unlock_reconfig(zh); + wakeup_driver(zh); +} + +/* Return and clear the disconnect request. */ +static int take_disconnect_request(zhandle_t *zh) +{ + int requested; + + lock_reconfig(zh); + requested = zh->disconnect_requested; + zh->disconnect_requested = 0; + unlock_reconfig(zh); + return requested; +} + /** * Updates the list of servers and determine if changing connections is necessary. * Permutes server list for proper load balancing. @@ -1079,6 +1124,7 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a */ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) { + int disconnect = 0; int rc = ZOK; char *hosts = NULL; uint32_t num_old = 0; @@ -1216,19 +1262,22 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) addrvec_free(&zh->addrs); zh->addrs = resolved; - // If we need to do a reconfig and we're currently connected to a server, - // then force close that connection so on next interest() call we'll make a - // new connection + // If the client must move to another server, ask the driver to drop the + // current connection. Its next connect uses the new list. if (zh->reconfig == 1 && zh->fd->sock != -1) { - close_zsock(zh->fd); - zh->state = ZOO_NOTCONNECTED_STATE; + request_disconnect_locked(zh); + disconnect = 1; } finish: unlock_reconfig(zh); + if (disconnect) { + wakeup_driver(zh); + } + // If we short-circuited out and never assigned resolved to zh->addrs then we // need to free resolved to avoid a memleak. if (resolved.data && zh->addrs.data != resolved.data) @@ -2452,7 +2501,15 @@ static struct timeval get_timeval(int interval) get_buffer_len(oa)); leave_critical(zh); close_buffer_oarchive(&oa, rc < 0); +#ifdef THREADED + /* + * Only the IO thread sends pings, from zookeeper_interest(). That call + * then asks for write readiness, so the IO thread needs no wakeup. + */ + return rc < 0 ? rc : ZOK; +#else return rc<0 ? rc : adaptor_send_queue(zh, 0); +#endif } /* upper bound of a timeout for seeking for r/w server when in read-only mode */ @@ -2644,6 +2701,16 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, api_prolog(zh); rc = update_addrs(zh, &now); + + // Do the teardown that a server list change or a failed send asked for. + // If an error closed the socket already, do nothing. The connect code + // below then selects the next server. + if (take_disconnect_request(zh) && zh->fd->sock != -1) { + LOG_INFO(LOGCALLBACK(zh), "Closing connection to %s on request", + format_endpoint_info(&zh->addr_cur)); + cleanup(zh, ZCONNECTIONLOSS); + } + if (rc != ZOK) { return api_epilog(zh, rc); } @@ -2772,8 +2839,15 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, if (zh->sent_requests.head == 0) { rc = send_ping(zh); if (rc < 0) { - LOG_ERROR(LOGCALLBACK(zh), "failed to send PING request (zk retcode=%d)",rc); - return api_epilog(zh,rc); + // This is the driver thread, so do the teardown + // now. Do not return the closed socket. + *fd = -1; + *interest = 0; + *tv = get_timeval(0); + return api_epilog(zh, handle_socket_error_msg(zh, + __LINE__, __func__, ZCONNECTIONLOSS, + "failed to send PING request (zk retcode=%d)", + rc)); } } send_to = zh->recv_timeout/3; @@ -4150,14 +4224,16 @@ static int Request_path_watch_init(zhandle_t *zh, int mode, * ASYNC API *---------------------------------------------------------------------------*/ -/* make an attempt to send queued requests immediately without blocking */ +/* + * Try to send the queued requests without blocking. The caller can be an + * application thread or a callback, so a failed send does not close the + * socket here. The driver does the teardown on its next pass. The pending + * requests then fail with ZCONNECTIONLOSS. + */ static int nonblocking_send(zhandle_t *zh, int rc) { - if (adaptor_send_queue(zh, 0) < 0) { - if (zh->fd->sock != -1) { - close_zsock(zh->fd); - zh->state = ZOO_NOTCONNECTED_STATE; - } + if (adaptor_send_queue(zh, 0) < 0 && zh->fd->sock != -1) { + request_disconnect(zh); } return (rc < 0) ? ZMARSHALLINGERROR : ZOK; } diff --git a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc index 534c7599c1f..10039e58c9f 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc @@ -21,6 +21,10 @@ #include "ZKMocks.h" #include +#include +#include +#include +#include using namespace std; @@ -36,6 +40,10 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_TEST(testCloseWhileInProgressFromCompletion); CPPUNIT_TEST(testCloseWhileMultiInProgressFromMain); CPPUNIT_TEST(testCloseWhileMultiInProgressFromCompletion); + CPPUNIT_TEST(testServerListChangeFailsPendingRequests); + CPPUNIT_TEST(testSendFailureDefersTeardown); + CPPUNIT_TEST(testPingSendFailureTearsDown); + CPPUNIT_TEST(testServerListChangeKeepsAuthFailedState); CPPUNIT_TEST(testConnectResponseFull); CPPUNIT_TEST(testConnectResponseNoReadOnlyFlag); CPPUNIT_TEST(testConnectResponseSplitAtReadOnlyFlag); @@ -43,6 +51,8 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture #else CPPUNIT_TEST(testAsyncWatcher1); CPPUNIT_TEST(testAsyncGetOperation); + CPPUNIT_TEST(testServerListChangeDuringSessionEstablishment); + CPPUNIT_TEST(testFullWakeupPipeKeepsConnection); #endif CPPUNIT_TEST(testOperationsAndDisconnectConcurrently1); CPPUNIT_TEST(testOperationsAndDisconnectConcurrently2); @@ -118,6 +128,35 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture bool called_; int rc_; }; + + // Records the session states that the watcher receives, in order. + class SessionEvents{ + public: + void add(int state){ + synchronized(mx_); + states_.push_back(state); + } + vector get() const{ + synchronized(mx_); + return states_; + } + bool contains(int state) const{ + vector s=get(); + return find(s.begin(),s.end(),state)!=s.end(); + } + mutable Mutex mx_; + vector states_; + }; + + static void sessionWatcher(zhandle_t *, int type, int state, + const char *, void *ctx){ + if(type==ZOO_SESSION_EVENT && ctx!=0) + ((SessionEvents*)ctx)->add(state); + } + + static int portOf(const struct sockaddr_storage& addr){ + return ntohs(((const struct sockaddr_in*)&addr)->sin_port); + } #ifndef THREADED // send two get data requests; verify that the corresponding completions called void testConcurrentOperations1() @@ -714,6 +753,189 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZCLOSING,res2.rc_); } + // Records the status of any completion type. + class StatusCompletion: public AsyncCompletion{ + public: + StatusCompletion():called_(false),rc_(ZAPIERROR){} + virtual void dataCompl(int rc, const char *value, int len, const Stat *stat){ + record(rc); + } + virtual void statCompl(int rc, const Stat *stat){ record(rc); } + virtual void stringCompl(int rc, const char *value){ record(rc); } + virtual void voidCompl(int rc){ record(rc); } + void record(int rc){ called_=true; rc_=rc; } + bool called_; + int rc_; + }; + + // A server list change that drops the current server must do a full + // teardown. The pending requests fail with ZCONNECTIONLOSS, and no + // request or partial frame goes to the next server. + void testServerListChangeFailsPendingRequests() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + SessionEvents events; + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + + // The server receives this request but does not reply. + StatusCompletion sentRes; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/a",0,asyncCompletion,&sentRes)); + CPPUNIT_ASSERT(zh->sent_requests.head!=0); + // This request stays in the send queue, partly sent. + zkServer.sendErrno=EAGAIN; + StatusCompletion queuedRes; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/b",0,asyncCompletion,&queuedRes)); + CPPUNIT_ASSERT(zh->to_send.head!=0); + zh->to_send.head->curr_offset=2; + zkServer.sendErrno=0; + + // The new list does not contain the current server. The call only + // asks for a disconnect. It does not close the socket. + CPPUNIT_ASSERT_EQUAL((int)ZOK,zoo_set_servers(zh,"127.0.0.2:2122")); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,zh->fd->sock); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,zoo_state(zh)); + CPPUNIT_ASSERT(!sentRes.called_); + CPPUNIT_ASSERT(!queuedRes.called_); + + // The next driver call does the teardown and connects to the new + // server with empty queues. + int fd=0; + int interest=0; + timeval tv; + CPPUNIT_ASSERT_EQUAL((int)ZOK,zookeeper_interest(zh,&fd,&interest,&tv)); + CPPUNIT_ASSERT(sentRes.called_); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,sentRes.rc_); + CPPUNIT_ASSERT(queuedRes.called_); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,queuedRes.rc_); + CPPUNIT_ASSERT(events.contains(ZOO_CONNECTING_STATE)); + CPPUNIT_ASSERT(zh->sent_requests.head==0); + CPPUNIT_ASSERT(zh->to_send.head==0); + CPPUNIT_ASSERT_EQUAL(2122,portOf(zh->addr_cur)); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTING_STATE,zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(0,zh->disconnect_requested); + } + + // A failed send in an asynchronous call must not close the socket in + // the caller. The next driver call does the full teardown. + void testSendFailureDefersTeardown() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + SessionEvents events; + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + + zkServer.sendErrno=ECONNRESET; + StatusCompletion res; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/a",0,asyncCompletion,&res)); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,zh->fd->sock); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(1,zh->disconnect_requested); + CPPUNIT_ASSERT(!res.called_); + zkServer.sendErrno=0; + + int fd=0; + int interest=0; + timeval tv; + CPPUNIT_ASSERT_EQUAL((int)ZOK,zookeeper_interest(zh,&fd,&interest,&tv)); + CPPUNIT_ASSERT(res.called_); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,res.rc_); + CPPUNIT_ASSERT(events.contains(ZOO_CONNECTING_STATE)); + CPPUNIT_ASSERT(zh->sent_requests.head==0); + CPPUNIT_ASSERT(zh->to_send.head==0); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTING_STATE,zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(0,zh->disconnect_requested); + } + + // A failed ping send must close the connection and must not return the + // closed socket. + void testPingSendFailureTearsDown() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + SessionEvents events; + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + + // A ping is due after one third of the session timeout. + timeMock.tick(4); + zkServer.sendErrno=ECONNRESET; + int fd=0; + int interest=0; + timeval tv; + int rc=zookeeper_interest(zh,&fd,&interest,&tv); + zkServer.sendErrno=0; + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,rc); + CPPUNIT_ASSERT_EQUAL(-1,fd); + CPPUNIT_ASSERT_EQUAL(0,interest); + CPPUNIT_ASSERT_EQUAL(-1,(int)zh->fd->sock); + CPPUNIT_ASSERT(events.contains(ZOO_CONNECTING_STATE)); + CPPUNIT_ASSERT(zh->to_send.head==0); + CPPUNIT_ASSERT_EQUAL(0,zoo_state(zh)); + } + + // Changes the server list from an authentication completion. + struct AuthFailureContext{ + zhandle_t *zh; + int rc; + int setServersRc; + }; + static void setServersOnAuthCompletion(int rc, const void *data){ + AuthFailureContext *ctx=(AuthFailureContext*)data; + ctx->rc=rc; + ctx->setServersRc=zoo_set_servers(ctx->zh,"127.0.0.2:2122"); + } + + // A server list change from a failed authentication callback must not + // hide the failure. The handle stays in ZOO_AUTH_FAILED_STATE. + void testServerListChangeKeepsAuthFailedState() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + SessionEvents events; + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + + // The server rejects the credential. + zkServer.addOperationResponse(new ZooStatResponse(0,ZAUTHFAILED)); + AuthFailureContext ctx={zh,ZAPIERROR,ZAPIERROR}; + CPPUNIT_ASSERT_EQUAL((int)ZOK,zoo_add_auth(zh,"digest","u:p",3, + setServersOnAuthCompletion,&ctx)); + + int fd=0; + int interest=0; + timeval tv; + CPPUNIT_ASSERT_EQUAL((int)ZOK,zookeeper_interest(zh,&fd,&interest,&tv)); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED,zookeeper_process(zh,interest)); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED,ctx.rc); + CPPUNIT_ASSERT_EQUAL((int)ZOK,ctx.setServersRc); + CPPUNIT_ASSERT_EQUAL(ZOO_AUTH_FAILED_STATE,zoo_state(zh)); + CPPUNIT_ASSERT(events.contains(ZOO_AUTH_FAILED_STATE)); + CPPUNIT_ASSERT_EQUAL((int)ZINVALIDSTATE, + zookeeper_interest(zh,&fd,&interest,&tv)); + } + void testConnectResponseFull() { CloseFinally guard(&zh); @@ -1092,7 +1314,154 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); CPPUNIT_ASSERT(ensureCondition(action.isNodeChangedTriggered(),1000)<1000); - CPPUNIT_ASSERT_EQUAL(string("/x/y/z"),action.path_); + CPPUNIT_ASSERT_EQUAL(string("/x/y/z"),action.path_); + } + + // A mock server that holds the first handshake reply until the test + // releases it. + class HeldHandshakeServer: public ZookeeperServer{ + public: + HeldHandshakeServer():held_(false),released_(false){} + virtual void notifyBufferSent(const std::string& buffer){ + { + synchronized(mx_); + if(!released_ && HandshakeRequest::isValid(buffer)){ + handshake_=buffer; + held_=true; + return; + } + } + ZookeeperServer::notifyBufferSent(buffer); + } + void release(){ + string buffer; + { + synchronized(mx_); + released_=true; + buffer=handshake_; + } + ZookeeperServer::notifyBufferSent(buffer); + } + SyncedBoolCondition isHeld() const{ + return SyncedBoolCondition(held_,mx_); + } + mutable Mutex mx_; + bool held_; + bool released_; + string handshake_; + }; + + // Releases the authentication lock if an assertion fails. + struct AuthLockGuard{ + AuthLockGuard(zhandle_t* zh):zh_(zh),locked_(true){zoo_lock_auth(zh_);} + ~AuthLockGuard(){unlock();} + void unlock(){ + if(locked_){ + locked_=false; + zoo_unlock_auth(zh_); + } + } + zhandle_t* zh_; + bool locked_; + }; + + // A server list change while the IO thread completes the session setup + // must not give the watcher an invalid state. The watcher gets + // ZOO_CONNECTED_STATE, then ZOO_CONNECTING_STATE, then the session + // connects to the new server. + void testServerListChangeDuringSessionEstablishment() + { + Mock_gettimeofday timeMock; + HeldHandshakeServer zkServer; + Mock_poll pollMock(&zkServer,ZookeeperServer::FD); + CloseFinally guard(&zh); + SessionEvents events; + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + CPPUNIT_ASSERT(ensureCondition(zkServer.isHeld(),1000)<1000); + + { + // The IO thread sets the connected state, then waits for this + // lock in send_auth_info() before it sends the session event. + AuthLockGuard authLock(zh); + zkServer.release(); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh),1000)<1000); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_set_servers(zh,"127.0.0.2:2122")); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,zoo_state(zh)); + } + + // Wait for the reconnect to the new server. + struct SecondConnect{ + SecondConnect(const SessionEvents& e):e_(e){} + bool operator()() const{ + vector s=e_.get(); + return count(s.begin(),s.end(),ZOO_CONNECTED_STATE)>=2; + } + const SessionEvents& e_; + }; + CPPUNIT_ASSERT(ensureCondition(SecondConnect(events),2000)<2000); + + vector states=events.get(); + CPPUNIT_ASSERT(states.size()>=3); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,states[0]); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTING_STATE,states[1]); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,states[2]); + CPPUNIT_ASSERT(!events.contains(ZOO_NOTCONNECTED_STATE)); + CPPUNIT_ASSERT_EQUAL(2122,portOf(zh->addr_cur)); + } + // Holds the reconfig lock, and so stops the IO thread in its next + // zookeeper_interest() call. Releases the lock if an assertion fails. + struct ReconfigLockGuard{ + ReconfigLockGuard(zhandle_t* zh):zh_(zh),locked_(true){lock_reconfig(zh_);} + ~ReconfigLockGuard(){unlock();} + void unlock(){ + if(locked_){ + locked_=false; + unlock_reconfig(zh_); + } + } + zhandle_t* zh_; + bool locked_; + }; + + // A full wakeup pipe already holds a wakeup for the IO thread. An + // asynchronous call must then succeed and keep the connection. + void testFullWakeupPipeKeepsConnection() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + Mock_poll pollMock(&zkServer,ZookeeperServer::FD); + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121",watcher,10000,TEST_CLIENT_ID,0,0); + CPPUNIT_ASSERT(zh!=0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh),1000)<1000); + + AsyncGetOperationCompletion res; + { + // Stop the IO thread so that it does not empty the pipe. + ReconfigLockGuard reconfigLock(zh); + millisleep(200); + struct adaptor_threads* adaptor= + (struct adaptor_threads*)zh->adaptor_priv; + char buf[4096]={0}; + while(write(adaptor->self_pipe[1],buf,sizeof(buf))>0){} + CPPUNIT_ASSERT_EQUAL(EAGAIN,errno); + CPPUNIT_ASSERT_EQUAL((int)ZOK,wakeup_io_thread(zh)); + + zkServer.addOperationResponse(new ZooGetResponse("1",1)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/x/y/1",0,asyncCompletion,&res)); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,zh->fd->sock); + CPPUNIT_ASSERT_EQUAL(0,zh->disconnect_requested); + } + + CPPUNIT_ASSERT(ensureCondition(res,1000)<1000); + CPPUNIT_ASSERT_EQUAL((int)ZOK,res.rc_); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,zoo_state(zh)); } #endif }; From 234d1aacc705311c4989aba0d45df0d7ab7db147 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 19 Nov 2026 16:57:46 -0800 Subject: [PATCH 2/3] ZOOKEEPER-5120: Arrange test helpers to merge with ZOOKEEPER-5119 Both changes add StatusCompletion and portOf() to TestOperations.cc. Put one identical copy after AsyncVoidOperationCompletion in both changes, so that they merge in either order without a conflict or a duplicate definition. Move the other new includes and helpers away from the lines that ZOOKEEPER-5119 changes. Co-Authored-By: Claude Opus 5.5 --- .../tests/TestOperations.cc | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc index 10039e58c9f..ddea2f38417 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc @@ -18,13 +18,13 @@ #include #include "CppAssertHelper.h" +#include +#include #include "ZKMocks.h" #include #include #include -#include -#include using namespace std; @@ -112,23 +112,6 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture NodeStat stat_; }; - class AsyncVoidOperationCompletion: public AsyncCompletion{ - public: - AsyncVoidOperationCompletion():called_(false),rc_(ZAPIERROR){} - virtual void voidCompl(int rc){ - synchronized(mx_); - called_=true; - rc_=rc; - } - bool operator()()const{ - synchronized(mx_); - return called_; - } - mutable Mutex mx_; - bool called_; - int rc_; - }; - // Records the session states that the watcher receives, in order. class SessionEvents{ public: @@ -154,6 +137,38 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture ((SessionEvents*)ctx)->add(state); } + class AsyncVoidOperationCompletion: public AsyncCompletion{ + public: + AsyncVoidOperationCompletion():called_(false),rc_(ZAPIERROR){} + virtual void voidCompl(int rc){ + synchronized(mx_); + called_=true; + rc_=rc; + } + bool operator()()const{ + synchronized(mx_); + return called_; + } + mutable Mutex mx_; + bool called_; + int rc_; + }; + + // Records the status of any completion type. + class StatusCompletion: public AsyncCompletion{ + public: + StatusCompletion():called_(false),rc_(ZAPIERROR){} + virtual void dataCompl(int rc, const char *value, int len, const Stat *stat){ + record(rc); + } + virtual void statCompl(int rc, const Stat *stat){ record(rc); } + virtual void stringCompl(int rc, const char *value){ record(rc); } + virtual void voidCompl(int rc){ record(rc); } + void record(int rc){ called_=true; rc_=rc; } + bool called_; + int rc_; + }; + static int portOf(const struct sockaddr_storage& addr){ return ntohs(((const struct sockaddr_in*)&addr)->sin_port); } @@ -753,21 +768,6 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZCLOSING,res2.rc_); } - // Records the status of any completion type. - class StatusCompletion: public AsyncCompletion{ - public: - StatusCompletion():called_(false),rc_(ZAPIERROR){} - virtual void dataCompl(int rc, const char *value, int len, const Stat *stat){ - record(rc); - } - virtual void statCompl(int rc, const Stat *stat){ record(rc); } - virtual void stringCompl(int rc, const char *value){ record(rc); } - virtual void voidCompl(int rc){ record(rc); } - void record(int rc){ called_=true; rc_=rc; } - bool called_; - int rc_; - }; - // A server list change that drops the current server must do a full // teardown. The pending requests fail with ZCONNECTIONLOSS, and no // request or partial frame goes to the next server. From 1c0aa0879a7988fc12fe2de127d800b67a072a5a Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Fri, 20 Nov 2026 00:41:43 -0800 Subject: [PATCH 3/3] ZOOKEEPER-5120: Set safe interest outputs and wake the driver from zoo_set_servers - Set safe outputs at the start of zookeeper_interest(): fd -1, no interest, and a timeout of recv_timeout/60. An early error return then does not leave them uninitialized, and it does not cause a busy loop. - Document in zoo_set_servers() that, when the call must drop the connection, it asks the driver to do it. Until the driver acts, zoo_state() returns the current state. If the session was connected, the watcher then receives ZOO_CONNECTING_STATE. Outstanding requests complete with ZCONNECTIONLOSS. - Keep the "failed to send PING request" log line on a failed ping send, then do the teardown with handle_error(). - When a disconnect is pending, wake the IO thread from zoo_set_servers(), not from update_addrs(). The IO thread also calls update_addrs(), and it does not need to wake itself. - Remove the duplicate wakeup_io_thread() declaration from mt_adaptor.c. - Update the close_zsock() comment, because only the driver closes the session socket now. - testFullWakeupPipeKeepsConnection parks the IO thread in a poll() mock. It no longer holds the reconfig lock and sleeps. At DEBUG log level, zoo_aget() takes the reconfig lock, so the old test could deadlock. - Add testServerListChangeFailsBlockedSyncCall (MT): a synchronous zoo_get() that waits when zoo_set_servers() drops the server returns ZCONNECTIONLOSS. A poll() mock with a 60 second idle timeout makes the test fail if zoo_set_servers() does not wake the IO thread. The mock does not report the session socket as writable, so the close at the end of the test does not send a close request. - Declare SessionEvents before CloseFinally, so that the close sends its events to a live object. - Close the handle before the join in the MT test, so that a failed assertion cannot hang the test. TestJob::join() is now idempotent. This change and ZOOKEEPER-5119 add the same lines, so they merge in either order. Co-Authored-By: Claude Opus 5.5 --- .../zookeeper-client-c/include/zookeeper.h | 8 + .../zookeeper-client-c/src/mt_adaptor.c | 3 - .../zookeeper-client-c/src/zookeeper.c | 48 ++-- .../tests/TestOperations.cc | 247 +++++++++++++++--- .../zookeeper-client-c/tests/ThreadingUtil.h | 2 + 5 files changed, 254 insertions(+), 54 deletions(-) diff --git a/zookeeper-client/zookeeper-client-c/include/zookeeper.h b/zookeeper-client/zookeeper-client-c/include/zookeeper.h index 1a3f5d1f92f..df897552f38 100644 --- a/zookeeper-client/zookeeper-client-c/include/zookeeper.h +++ b/zookeeper-client/zookeeper-client-c/include/zookeeper.h @@ -812,6 +812,14 @@ ZOOAPI const char* zoo_version_str(); * the client moves back to the normal mode of operation where it will pick an * arbitrary server from the 'host' string. * + * This call does not drop the connection itself. It asks the driver to drop + * it. The driver is the IO thread in the multithreaded library. In the + * single-threaded library, the driver is the next zookeeper_interest() call. + * Until the driver acts, zoo_state() continues to return the current state, + * for example ZOO_CONNECTED_STATE or ZOO_READONLY_STATE. If the session was + * connected, the watcher then receives a ZOO_CONNECTING_STATE session event. + * All outstanding requests complete with ZCONNECTIONLOSS. + * * See {@link https://issues.apache.org/jira/browse/ZOOKEEPER-1355} for the * protocol and its evaluation, * diff --git a/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c b/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c index 0a945f4a474..33c6751af34 100644 --- a/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c +++ b/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c @@ -167,9 +167,6 @@ void *do_io(void *); void *do_completion(void *); #endif - -int wakeup_io_thread(zhandle_t *zh); - #ifdef WIN32 static int set_nonblock(SOCKET fd){ ULONG nonblocking_flag = 1; diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index ea4b7695416..1899617e811 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -653,9 +653,10 @@ void close_zsock(zsock_t *fd) * Release the SSL_CTX only together with its SSL object. A context * without an SSL object exists only while init_ssl_for_socket() * configures it on the IO thread. That function releases it on - * every failure path (ZOOKEEPER-5004). An application thread can - * call this function (for example through zoo_set_servers()), and - * a release here could free the context during its configuration. + * every failure path (ZOOKEEPER-5004). Only the driver closes the + * session socket. destroy() closes it after the IO thread stops. + * Thus no other thread can run init_ssl_for_socket() at the same + * time. Keep this rule as a defensive measure. */ if (fd->ssl_sock) { SSL_free(fd->ssl_sock); @@ -1124,7 +1125,6 @@ static int take_disconnect_request(zhandle_t *zh) */ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) { - int disconnect = 0; int rc = ZOK; char *hosts = NULL; uint32_t num_old = 0; @@ -1267,17 +1267,12 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) if (zh->reconfig == 1 && zh->fd->sock != -1) { request_disconnect_locked(zh); - disconnect = 1; } finish: unlock_reconfig(zh); - if (disconnect) { - wakeup_driver(zh); - } - // If we short-circuited out and never assigned resolved to zh->addrs then we // need to free resolved to avoid a memleak. if (resolved.data && zh->addrs.data != resolved.data) @@ -1600,6 +1595,9 @@ zhandle_t *zookeeper_init_sasl(const char *host, watcher_fn watcher, */ int zoo_set_servers(zhandle_t *zh, const char *hosts) { + int rc; + int disconnect; + if (hosts == NULL) { LOG_ERROR(LOGCALLBACK(zh), "New server list cannot be empty"); @@ -1618,7 +1616,19 @@ int zoo_set_servers(zhandle_t *zh, const char *hosts) unlock_reconfig(zh); - return update_addrs(zh, NULL); + rc = update_addrs(zh, NULL); + + // This call can come from any thread or callback. If a disconnect is + // pending, wake the IO thread so that it acts at once. The update_addrs() + // call in zookeeper_interest() needs no wakeup, because the driver + // already runs. + lock_reconfig(zh); + disconnect = zh->disconnect_requested; + unlock_reconfig(zh); + if (disconnect) { + wakeup_driver(zh); + } + return rc; } /* @@ -2688,6 +2698,12 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, if(zh==0 || fd==0 ||interest==0 || tv==0) return ZBADARGUMENTS; + // Set safe outputs before any early return. Some event loops ignore the + // return code and pass the outputs to poll(). Thus an error return must + // not leave them uninitialized, and it must not cause a busy loop. + *fd = -1; + *interest = 0; + *tv = get_timeval(zh->recv_timeout/60); if (is_unrecoverable(zh)) return ZINVALIDSTATE; get_system_time(&now); @@ -2839,15 +2855,15 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, if (zh->sent_requests.head == 0) { rc = send_ping(zh); if (rc < 0) { - // This is the driver thread, so do the teardown - // now. Do not return the closed socket. + LOG_ERROR(LOGCALLBACK(zh), "failed to send PING request (zk retcode=%d)",rc); + // zookeeper_interest() runs in the driver, so it + // does the teardown now. It must not return the + // closed socket. + handle_error(zh, ZCONNECTIONLOSS); *fd = -1; *interest = 0; *tv = get_timeval(0); - return api_epilog(zh, handle_socket_error_msg(zh, - __LINE__, __func__, ZCONNECTIONLOSS, - "failed to send PING request (zk retcode=%d)", - rc)); + return api_epilog(zh, ZCONNECTIONLOSS); } } send_to = zh->recv_timeout/3; diff --git a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc index ddea2f38417..5d1b2af2c4f 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc @@ -53,6 +53,7 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_TEST(testAsyncGetOperation); CPPUNIT_TEST(testServerListChangeDuringSessionEstablishment); CPPUNIT_TEST(testFullWakeupPipeKeepsConnection); + CPPUNIT_TEST(testServerListChangeFailsBlockedSyncCall); #endif CPPUNIT_TEST(testOperationsAndDisconnectConcurrently1); CPPUNIT_TEST(testOperationsAndDisconnectConcurrently2); @@ -172,6 +173,23 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture static int portOf(const struct sockaddr_storage& addr){ return ntohs(((const struct sockaddr_in*)&addr)->sin_port); } + +#ifdef THREADED + // Closes the handle and then joins the job, also when an assertion + // fails. The close makes a waiting synchronous call return ZCLOSING, + // so the join cannot hang. The join also occurs before the job destroys + // its own members. Declare the guard after the job. + struct CloseThenJoin{ + CloseThenJoin(CloseFinally& close,TestJob& job): + close_(close),job_(job){} + ~CloseThenJoin(){ + close_.execute(); + job_.join(); + } + CloseFinally& close_; + TestJob& job_; + }; +#endif #ifndef THREADED // send two get data requests; verify that the corresponding completions called void testConcurrentOperations1() @@ -775,8 +793,8 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture { Mock_gettimeofday timeMock; ZookeeperServer zkServer; - CloseFinally guard(&zh); SessionEvents events; + CloseFinally guard(&zh); zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, TEST_CLIENT_ID,&events,0); @@ -829,8 +847,8 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture { Mock_gettimeofday timeMock; ZookeeperServer zkServer; - CloseFinally guard(&zh); SessionEvents events; + CloseFinally guard(&zh); zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, TEST_CLIENT_ID,&events,0); @@ -866,8 +884,8 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture { Mock_gettimeofday timeMock; ZookeeperServer zkServer; - CloseFinally guard(&zh); SessionEvents events; + CloseFinally guard(&zh); zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, TEST_CLIENT_ID,&events,0); @@ -909,8 +927,8 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture { Mock_gettimeofday timeMock; ZookeeperServer zkServer; - CloseFinally guard(&zh); SessionEvents events; + CloseFinally guard(&zh); zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, TEST_CLIENT_ID,&events,0); @@ -1374,8 +1392,8 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture Mock_gettimeofday timeMock; HeldHandshakeServer zkServer; Mock_poll pollMock(&zkServer,ZookeeperServer::FD); - CloseFinally guard(&zh); SessionEvents events; + CloseFinally guard(&zh); zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, TEST_CLIENT_ID,&events,0); @@ -1412,19 +1430,48 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT(!events.contains(ZOO_NOTCONNECTED_STATE)); CPPUNIT_ASSERT_EQUAL(2122,portOf(zh->addr_cur)); } - // Holds the reconfig lock, and so stops the IO thread in its next - // zookeeper_interest() call. Releases the lock if an assertion fails. - struct ReconfigLockGuard{ - ReconfigLockGuard(zhandle_t* zh):zh_(zh),locked_(true){lock_reconfig(zh_);} - ~ReconfigLockGuard(){unlock();} - void unlock(){ - if(locked_){ - locked_=false; - unlock_reconfig(zh_); + // A poll() mock that can park the IO thread. A parked IO thread does not + // read the wakeup pipe, so the test controls the pipe contents. + class ParkingPoll: public Mock_poll{ + public: + ParkingPoll(Mock_socket* s,int fd): + Mock_poll(s,fd),park_(false),parked_(false){} + virtual int call(struct pollfd *fds, POLL_NFDS_TYPE nfds, int to){ + for(;;){ + { + synchronized(mx_); + if(!park_){ + parked_=false; + break; + } + parked_=true; + } + millisleep(1); } + return Mock_poll::call(fds,nfds,to); } - zhandle_t* zh_; - bool locked_; + void park(){ + synchronized(mx_); + park_=true; + } + void release(){ + synchronized(mx_); + park_=false; + } + SyncedBoolCondition isParked() const{ + return SyncedBoolCondition(parked_,mx_); + } + mutable Mutex mx_; + bool park_; + bool parked_; + }; + + // Releases a parked IO thread, also when an assertion fails. Declare it + // after CloseFinally, so that the release occurs before the close. + struct ParkRelease{ + ParkRelease(ParkingPoll& poll):poll_(poll){} + ~ParkRelease(){ poll_.release(); } + ParkingPoll& poll_; }; // A full wakeup pipe already holds a wakeup for the IO thread. An @@ -1433,36 +1480,166 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture { Mock_gettimeofday timeMock; ZookeeperServer zkServer; - Mock_poll pollMock(&zkServer,ZookeeperServer::FD); + ParkingPoll pollMock(&zkServer,ZookeeperServer::FD); + // The completion must outlive the close and the park release, + // because both can complete the request. + AsyncGetOperationCompletion res; CloseFinally guard(&zh); + ParkRelease parkRelease(pollMock); zh=zookeeper_init("127.0.0.1:2121",watcher,10000,TEST_CLIENT_ID,0,0); CPPUNIT_ASSERT(zh!=0); CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh),1000)<1000); - AsyncGetOperationCompletion res; - { - // Stop the IO thread so that it does not empty the pipe. - ReconfigLockGuard reconfigLock(zh); - millisleep(200); - struct adaptor_threads* adaptor= - (struct adaptor_threads*)zh->adaptor_priv; - char buf[4096]={0}; - while(write(adaptor->self_pipe[1],buf,sizeof(buf))>0){} - CPPUNIT_ASSERT_EQUAL(EAGAIN,errno); - CPPUNIT_ASSERT_EQUAL((int)ZOK,wakeup_io_thread(zh)); - - zkServer.addOperationResponse(new ZooGetResponse("1",1)); - CPPUNIT_ASSERT_EQUAL((int)ZOK, - zoo_aget(zh,"/x/y/1",0,asyncCompletion,&res)); - CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,zh->fd->sock); - CPPUNIT_ASSERT_EQUAL(0,zh->disconnect_requested); - } + // Park the IO thread, then fill the pipe. The pipe stays full until + // the release. + pollMock.park(); + CPPUNIT_ASSERT(ensureCondition(pollMock.isParked(),1000)<1000); + struct adaptor_threads* adaptor= + (struct adaptor_threads*)zh->adaptor_priv; + char buf[4096]={0}; + while(write(adaptor->self_pipe[1],buf,sizeof(buf))>0){} + CPPUNIT_ASSERT_EQUAL(EAGAIN,errno); + CPPUNIT_ASSERT_EQUAL((int)ZOK,wakeup_io_thread(zh)); + + zkServer.addOperationResponse(new ZooGetResponse("1",1)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/x/y/1",0,asyncCompletion,&res)); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,zh->fd->sock); + lock_reconfig(zh); + int requested=zh->disconnect_requested; + unlock_reconfig(zh); + CPPUNIT_ASSERT_EQUAL(0,requested); + pollMock.release(); CPPUNIT_ASSERT(ensureCondition(res,1000)<1000); CPPUNIT_ASSERT_EQUAL((int)ZOK,res.rc_); CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,zoo_state(zh)); } + + // Counts the get requests that reach the server. The server does not + // reply to them. + class GetRequestServer: public ZookeeperServer{ + public: + virtual void onMessageReceived(const RequestHeader& rh, iarchive* ia){ + if(rh.type==ZOO_GETDATA_OP) + ++getRequests; + } + AtomicInt getRequests; + }; + + // Checks that a get request reached the server. + struct GetRequestReceived{ + GetRequestReceived(const AtomicInt& n):n_(n){} + bool operator()() const{ return n_.get()>0; } + const AtomicInt& n_; + }; + + // Runs one synchronous get request and records its result. + class BlockedGetJob: public TestJob{ + public: + BlockedGetJob(zhandle_t* zh):zh_(zh),done_(false),rc_(ZAPIERROR){} + virtual TestJob* clone() const { return new BlockedGetJob(zh_); } + virtual void run(){ + char buf[1]; + int size=sizeof(buf); + int rc=zoo_get(zh_,"/a",0,buf,&size,0); + synchronized(mx_); + rc_=rc; + done_=true; + } + virtual void validate(const char* file, int line) const{} + bool operator()() const{ + synchronized(mx_); + return done_; + } + int rc() const{ + synchronized(mx_); + return rc_; + } + zhandle_t* zh_; + mutable Mutex mx_; + bool done_; + int rc_; + }; + + // Checks for a connected session with the given server. + struct ConnectedTo{ + ConnectedTo(zhandle_t* zh,const char* server):zh_(zh),server_(server){} + bool operator()() const{ + return zoo_state(zh_)==ZOO_CONNECTED_STATE && + server_==zoo_get_current_server(zh_); + } + zhandle_t* zh_; + string server_; + }; + + // Checks that the watcher received a session event with the given state. + struct EventSeen{ + EventSeen(const SessionEvents& e,int state):e_(e),state_(state){} + bool operator()() const{ return e_.contains(state_); } + const SessionEvents& e_; + int state_; + }; + + // A poll() mock with a long timeout, so that an idle IO thread wakes + // only for the wakeup pipe. zookeeper_close() and the debug check for + // a readable socket poll only the session socket. The mock answers a + // poll of only the session socket at once, so these polls do not wait + // for the long timeout. The mock does not report the socket as + // writable, so zookeeper_close() does not send the close request. The + // test does not need it. + class LongIdlePoll: public Mock_poll{ + public: + LongIdlePoll(Mock_socket* s,int fd):Mock_poll(s,fd){ + timeout=60000; + } + virtual int call(struct pollfd *fds, POLL_NFDS_TYPE nfds, int to){ + if(nfds==1 && fds[0].fd==myFD){ + bool readable=isFDReadable(); + fds[0].revents=readable?POLLIN:0; + return readable?1:0; + } + return Mock_poll::call(fds,nfds,to); + } + }; + + // A synchronous call that waits when zoo_set_servers() drops the current + // server must return ZCONNECTIONLOSS. The client then connects to the + // new server. + void testServerListChangeFailsBlockedSyncCall() + { + Mock_gettimeofday timeMock; + GetRequestServer zkServer; + // The test fails if zoo_set_servers() does not wake the IO thread. + LongIdlePoll pollMock(&zkServer,ZookeeperServer::FD); + SessionEvents events; + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + CPPUNIT_ASSERT(ensureCondition( + ConnectedTo(zh,"127.0.0.1:2121"),1000)<1000); + + BlockedGetJob job(zh); + CloseThenJoin closeThenJoin(guard,job); + job.start(); + CPPUNIT_ASSERT(ensureCondition( + GetRequestReceived(zkServer.getRequests),1000)<1000); + CPPUNIT_ASSERT(!job()); + + CPPUNIT_ASSERT_EQUAL((int)ZOK,zoo_set_servers(zh,"127.0.0.2:2122")); + CPPUNIT_ASSERT(ensureCondition(job,1000)<1000); + job.join(); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,job.rc()); + // The completion thread sends the session event, so it can arrive + // after the synchronous call returns. + CPPUNIT_ASSERT(ensureCondition( + EventSeen(events,ZOO_CONNECTING_STATE),1000)<1000); + CPPUNIT_ASSERT(ensureCondition( + ConnectedTo(zh,"127.0.0.2:2122"),1000)<1000); + } #endif }; diff --git a/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h b/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h index 45791fd2f59..2ed01a7faf4 100644 --- a/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h +++ b/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h @@ -189,6 +189,8 @@ class TestJob { pthread_join(thread_,0); else pthread_detach(thread_); + // A thread can be joined only once, so later calls do nothing. + hasRun_=false; } private: void awaitStart() {