diff --git a/zookeeper-client/zookeeper-client-c/include/zookeeper.h b/zookeeper-client/zookeeper-client-c/include/zookeeper.h index 1a3f5d1f92f..df897552f38 100644 --- a/zookeeper-client/zookeeper-client-c/include/zookeeper.h +++ b/zookeeper-client/zookeeper-client-c/include/zookeeper.h @@ -812,6 +812,14 @@ ZOOAPI const char* zoo_version_str(); * the client moves back to the normal mode of operation where it will pick an * arbitrary server from the 'host' string. * + * This call does not drop the connection itself. It asks the driver to drop + * it. The driver is the IO thread in the multithreaded library. In the + * single-threaded library, the driver is the next zookeeper_interest() call. + * Until the driver acts, zoo_state() continues to return the current state, + * for example ZOO_CONNECTED_STATE or ZOO_READONLY_STATE. If the session was + * connected, the watcher then receives a ZOO_CONNECTING_STATE session event. + * All outstanding requests complete with ZCONNECTIONLOSS. + * * See {@link https://issues.apache.org/jira/browse/ZOOKEEPER-1355} for the * protocol and its evaluation, * diff --git a/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c b/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c index fdef1b4e4da..33c6751af34 100644 --- a/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c +++ b/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c @@ -167,9 +167,6 @@ void *do_io(void *); void *do_completion(void *); #endif - -int wakeup_io_thread(zhandle_t *zh); - #ifdef WIN32 static int set_nonblock(SOCKET fd){ ULONG nonblocking_flag = 1; @@ -336,11 +333,24 @@ int wakeup_io_thread(zhandle_t *zh) { struct adaptor_threads *adaptor_threads = zh->adaptor_priv; char c=0; + /* + * A full pipe already holds a wakeup for the IO thread, so a full pipe + * is not an error. Retry the write if a signal interrupts it. + */ #ifndef WIN32 - return write(adaptor_threads->self_pipe[1],&c,1)==1? ZOK: ZSYSTEMERROR; + for (;;) { + if (write(adaptor_threads->self_pipe[1],&c,1)==1) + return ZOK; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return ZOK; + if (errno != EINTR) + return ZSYSTEMERROR; + } #else - return send(adaptor_threads->self_pipe[1], &c, 1, 0)==1? ZOK: ZSYSTEMERROR; -#endif + if (send(adaptor_threads->self_pipe[1], &c, 1, 0)==1) + return ZOK; + return WSAGetLastError() == WSAEWOULDBLOCK ? ZOK : ZSYSTEMERROR; +#endif } int adaptor_send_queue(zhandle_t *zh, int timeout) diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index 25049bb335e..8dfe1312d13 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -208,6 +208,8 @@ struct _zhandle { int reconfig; // Are we in the process of reconfiguring cluster's ensemble double pOld, pNew; // Probability for selecting between 'addrs_old' and 'addrs_new' int delay; + int disconnect_requested; // When set, the next zookeeper_interest() call drops the + // connection. The reconfig lock guards this flag. int disable_reconnection_attempt; // When set, client will not try reconnect to a different server in // server list. This makes a sticky server for client, and is useful // for testing if a sticky server is required, or if client wants to @@ -291,6 +293,7 @@ struct sync_completion *alloc_sync_completion(void); int wait_sync_completion(struct sync_completion *sc); void free_sync_completion(struct sync_completion *sc); void notify_sync_completion(struct sync_completion *sc); +int wakeup_io_thread(zhandle_t *zh); #endif int adaptor_send_queue(zhandle_t *zh, int timeout); int process_async(int outstanding_sync); diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index c8429003b29..1899617e811 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -653,9 +653,10 @@ void close_zsock(zsock_t *fd) * Release the SSL_CTX only together with its SSL object. A context * without an SSL object exists only while init_ssl_for_socket() * configures it on the IO thread. That function releases it on - * every failure path (ZOOKEEPER-5004). An application thread can - * call this function (for example through zoo_set_servers()), and - * a release here could free the context during its configuration. + * every failure path (ZOOKEEPER-5004). Only the driver closes the + * session socket. destroy() closes it after the IO thread stops. + * Thus no other thread can run init_ssl_for_socket() at the same + * time. Keep this rule as a defensive measure. */ if (fd->ssl_sock) { SSL_free(fd->ssl_sock); @@ -1050,6 +1051,51 @@ static int resolve_hosts(const zhandle_t *zh, const char *hosts_in, addrvec_t *a return rc; } +/* + * Only the driver closes the connection. The driver is the IO thread, or the + * caller of zookeeper_interest() in the single-threaded library. Other code + * sets a flag to ask for a disconnect. The next zookeeper_interest() call + * does the full teardown. The teardown sends the session event and fails the + * pending requests. Thus no stale request or partial frame goes to the next + * server. + * + * The reconfig lock guards the flag. The caller of this function holds it. + */ +static void request_disconnect_locked(zhandle_t *zh) +{ + zh->disconnect_requested = 1; +} + +static void wakeup_driver(zhandle_t *zh) +{ +#ifdef THREADED + /* The IO thread can sleep in poll(). Wake it so that it acts at once. */ + if (zh->adaptor_priv) { + wakeup_io_thread(zh); + } +#endif +} + +static void request_disconnect(zhandle_t *zh) +{ + lock_reconfig(zh); + request_disconnect_locked(zh); + unlock_reconfig(zh); + wakeup_driver(zh); +} + +/* Return and clear the disconnect request. */ +static int take_disconnect_request(zhandle_t *zh) +{ + int requested; + + lock_reconfig(zh); + requested = zh->disconnect_requested; + zh->disconnect_requested = 0; + unlock_reconfig(zh); + return requested; +} + /** * Updates the list of servers and determine if changing connections is necessary. * Permutes server list for proper load balancing. @@ -1216,13 +1262,11 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) addrvec_free(&zh->addrs); zh->addrs = resolved; - // If we need to do a reconfig and we're currently connected to a server, - // then force close that connection so on next interest() call we'll make a - // new connection + // If the client must move to another server, ask the driver to drop the + // current connection. Its next connect uses the new list. if (zh->reconfig == 1 && zh->fd->sock != -1) { - close_zsock(zh->fd); - zh->state = ZOO_NOTCONNECTED_STATE; + request_disconnect_locked(zh); } finish: @@ -1551,6 +1595,9 @@ zhandle_t *zookeeper_init_sasl(const char *host, watcher_fn watcher, */ int zoo_set_servers(zhandle_t *zh, const char *hosts) { + int rc; + int disconnect; + if (hosts == NULL) { LOG_ERROR(LOGCALLBACK(zh), "New server list cannot be empty"); @@ -1569,7 +1616,19 @@ int zoo_set_servers(zhandle_t *zh, const char *hosts) unlock_reconfig(zh); - return update_addrs(zh, NULL); + rc = update_addrs(zh, NULL); + + // This call can come from any thread or callback. If a disconnect is + // pending, wake the IO thread so that it acts at once. The update_addrs() + // call in zookeeper_interest() needs no wakeup, because the driver + // already runs. + lock_reconfig(zh); + disconnect = zh->disconnect_requested; + unlock_reconfig(zh); + if (disconnect) { + wakeup_driver(zh); + } + return rc; } /* @@ -2452,7 +2511,15 @@ static struct timeval get_timeval(int interval) get_buffer_len(oa)); leave_critical(zh); close_buffer_oarchive(&oa, rc < 0); +#ifdef THREADED + /* + * Only the IO thread sends pings, from zookeeper_interest(). That call + * then asks for write readiness, so the IO thread needs no wakeup. + */ + return rc < 0 ? rc : ZOK; +#else return rc<0 ? rc : adaptor_send_queue(zh, 0); +#endif } /* upper bound of a timeout for seeking for r/w server when in read-only mode */ @@ -2631,6 +2698,12 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, if(zh==0 || fd==0 ||interest==0 || tv==0) return ZBADARGUMENTS; + // Set safe outputs before any early return. Some event loops ignore the + // return code and pass the outputs to poll(). Thus an error return must + // not leave them uninitialized, and it must not cause a busy loop. + *fd = -1; + *interest = 0; + *tv = get_timeval(zh->recv_timeout/60); if (is_unrecoverable(zh)) return ZINVALIDSTATE; get_system_time(&now); @@ -2644,6 +2717,16 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, api_prolog(zh); rc = update_addrs(zh, &now); + + // Do the teardown that a server list change or a failed send asked for. + // If an error closed the socket already, do nothing. The connect code + // below then selects the next server. + if (take_disconnect_request(zh) && zh->fd->sock != -1) { + LOG_INFO(LOGCALLBACK(zh), "Closing connection to %s on request", + format_endpoint_info(&zh->addr_cur)); + cleanup(zh, ZCONNECTIONLOSS); + } + if (rc != ZOK) { return api_epilog(zh, rc); } @@ -2773,7 +2856,14 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, rc = send_ping(zh); if (rc < 0) { LOG_ERROR(LOGCALLBACK(zh), "failed to send PING request (zk retcode=%d)",rc); - return api_epilog(zh,rc); + // zookeeper_interest() runs in the driver, so it + // does the teardown now. It must not return the + // closed socket. + handle_error(zh, ZCONNECTIONLOSS); + *fd = -1; + *interest = 0; + *tv = get_timeval(0); + return api_epilog(zh, ZCONNECTIONLOSS); } } send_to = zh->recv_timeout/3; @@ -4150,14 +4240,16 @@ static int Request_path_watch_init(zhandle_t *zh, int mode, * ASYNC API *---------------------------------------------------------------------------*/ -/* make an attempt to send queued requests immediately without blocking */ +/* + * Try to send the queued requests without blocking. The caller can be an + * application thread or a callback, so a failed send does not close the + * socket here. The driver does the teardown on its next pass. The pending + * requests then fail with ZCONNECTIONLOSS. + */ static int nonblocking_send(zhandle_t *zh, int rc) { - if (adaptor_send_queue(zh, 0) < 0) { - if (zh->fd->sock != -1) { - close_zsock(zh->fd); - zh->state = ZOO_NOTCONNECTED_STATE; - } + if (adaptor_send_queue(zh, 0) < 0 && zh->fd->sock != -1) { + request_disconnect(zh); } return (rc < 0) ? ZMARSHALLINGERROR : ZOK; } diff --git a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc index 534c7599c1f..5d1b2af2c4f 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc @@ -18,9 +18,13 @@ #include #include "CppAssertHelper.h" +#include +#include #include "ZKMocks.h" #include +#include +#include using namespace std; @@ -36,6 +40,10 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_TEST(testCloseWhileInProgressFromCompletion); CPPUNIT_TEST(testCloseWhileMultiInProgressFromMain); CPPUNIT_TEST(testCloseWhileMultiInProgressFromCompletion); + CPPUNIT_TEST(testServerListChangeFailsPendingRequests); + CPPUNIT_TEST(testSendFailureDefersTeardown); + CPPUNIT_TEST(testPingSendFailureTearsDown); + CPPUNIT_TEST(testServerListChangeKeepsAuthFailedState); CPPUNIT_TEST(testConnectResponseFull); CPPUNIT_TEST(testConnectResponseNoReadOnlyFlag); CPPUNIT_TEST(testConnectResponseSplitAtReadOnlyFlag); @@ -43,6 +51,9 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture #else CPPUNIT_TEST(testAsyncWatcher1); CPPUNIT_TEST(testAsyncGetOperation); + CPPUNIT_TEST(testServerListChangeDuringSessionEstablishment); + CPPUNIT_TEST(testFullWakeupPipeKeepsConnection); + CPPUNIT_TEST(testServerListChangeFailsBlockedSyncCall); #endif CPPUNIT_TEST(testOperationsAndDisconnectConcurrently1); CPPUNIT_TEST(testOperationsAndDisconnectConcurrently2); @@ -102,6 +113,31 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture NodeStat stat_; }; + // Records the session states that the watcher receives, in order. + class SessionEvents{ + public: + void add(int state){ + synchronized(mx_); + states_.push_back(state); + } + vector get() const{ + synchronized(mx_); + return states_; + } + bool contains(int state) const{ + vector s=get(); + return find(s.begin(),s.end(),state)!=s.end(); + } + mutable Mutex mx_; + vector states_; + }; + + static void sessionWatcher(zhandle_t *, int type, int state, + const char *, void *ctx){ + if(type==ZOO_SESSION_EVENT && ctx!=0) + ((SessionEvents*)ctx)->add(state); + } + class AsyncVoidOperationCompletion: public AsyncCompletion{ public: AsyncVoidOperationCompletion():called_(false),rc_(ZAPIERROR){} @@ -118,6 +154,42 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture bool called_; int rc_; }; + + // Records the status of any completion type. + class StatusCompletion: public AsyncCompletion{ + public: + StatusCompletion():called_(false),rc_(ZAPIERROR){} + virtual void dataCompl(int rc, const char *value, int len, const Stat *stat){ + record(rc); + } + virtual void statCompl(int rc, const Stat *stat){ record(rc); } + virtual void stringCompl(int rc, const char *value){ record(rc); } + virtual void voidCompl(int rc){ record(rc); } + void record(int rc){ called_=true; rc_=rc; } + bool called_; + int rc_; + }; + + static int portOf(const struct sockaddr_storage& addr){ + return ntohs(((const struct sockaddr_in*)&addr)->sin_port); + } + +#ifdef THREADED + // Closes the handle and then joins the job, also when an assertion + // fails. The close makes a waiting synchronous call return ZCLOSING, + // so the join cannot hang. The join also occurs before the job destroys + // its own members. Declare the guard after the job. + struct CloseThenJoin{ + CloseThenJoin(CloseFinally& close,TestJob& job): + close_(close),job_(job){} + ~CloseThenJoin(){ + close_.execute(); + job_.join(); + } + CloseFinally& close_; + TestJob& job_; + }; +#endif #ifndef THREADED // send two get data requests; verify that the corresponding completions called void testConcurrentOperations1() @@ -714,6 +786,174 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZCLOSING,res2.rc_); } + // A server list change that drops the current server must do a full + // teardown. The pending requests fail with ZCONNECTIONLOSS, and no + // request or partial frame goes to the next server. + void testServerListChangeFailsPendingRequests() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + SessionEvents events; + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + + // The server receives this request but does not reply. + StatusCompletion sentRes; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/a",0,asyncCompletion,&sentRes)); + CPPUNIT_ASSERT(zh->sent_requests.head!=0); + // This request stays in the send queue, partly sent. + zkServer.sendErrno=EAGAIN; + StatusCompletion queuedRes; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/b",0,asyncCompletion,&queuedRes)); + CPPUNIT_ASSERT(zh->to_send.head!=0); + zh->to_send.head->curr_offset=2; + zkServer.sendErrno=0; + + // The new list does not contain the current server. The call only + // asks for a disconnect. It does not close the socket. + CPPUNIT_ASSERT_EQUAL((int)ZOK,zoo_set_servers(zh,"127.0.0.2:2122")); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,zh->fd->sock); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,zoo_state(zh)); + CPPUNIT_ASSERT(!sentRes.called_); + CPPUNIT_ASSERT(!queuedRes.called_); + + // The next driver call does the teardown and connects to the new + // server with empty queues. + int fd=0; + int interest=0; + timeval tv; + CPPUNIT_ASSERT_EQUAL((int)ZOK,zookeeper_interest(zh,&fd,&interest,&tv)); + CPPUNIT_ASSERT(sentRes.called_); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,sentRes.rc_); + CPPUNIT_ASSERT(queuedRes.called_); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,queuedRes.rc_); + CPPUNIT_ASSERT(events.contains(ZOO_CONNECTING_STATE)); + CPPUNIT_ASSERT(zh->sent_requests.head==0); + CPPUNIT_ASSERT(zh->to_send.head==0); + CPPUNIT_ASSERT_EQUAL(2122,portOf(zh->addr_cur)); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTING_STATE,zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(0,zh->disconnect_requested); + } + + // A failed send in an asynchronous call must not close the socket in + // the caller. The next driver call does the full teardown. + void testSendFailureDefersTeardown() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + SessionEvents events; + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + + zkServer.sendErrno=ECONNRESET; + StatusCompletion res; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/a",0,asyncCompletion,&res)); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,zh->fd->sock); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(1,zh->disconnect_requested); + CPPUNIT_ASSERT(!res.called_); + zkServer.sendErrno=0; + + int fd=0; + int interest=0; + timeval tv; + CPPUNIT_ASSERT_EQUAL((int)ZOK,zookeeper_interest(zh,&fd,&interest,&tv)); + CPPUNIT_ASSERT(res.called_); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,res.rc_); + CPPUNIT_ASSERT(events.contains(ZOO_CONNECTING_STATE)); + CPPUNIT_ASSERT(zh->sent_requests.head==0); + CPPUNIT_ASSERT(zh->to_send.head==0); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTING_STATE,zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL(0,zh->disconnect_requested); + } + + // A failed ping send must close the connection and must not return the + // closed socket. + void testPingSendFailureTearsDown() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + SessionEvents events; + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + + // A ping is due after one third of the session timeout. + timeMock.tick(4); + zkServer.sendErrno=ECONNRESET; + int fd=0; + int interest=0; + timeval tv; + int rc=zookeeper_interest(zh,&fd,&interest,&tv); + zkServer.sendErrno=0; + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,rc); + CPPUNIT_ASSERT_EQUAL(-1,fd); + CPPUNIT_ASSERT_EQUAL(0,interest); + CPPUNIT_ASSERT_EQUAL(-1,(int)zh->fd->sock); + CPPUNIT_ASSERT(events.contains(ZOO_CONNECTING_STATE)); + CPPUNIT_ASSERT(zh->to_send.head==0); + CPPUNIT_ASSERT_EQUAL(0,zoo_state(zh)); + } + + // Changes the server list from an authentication completion. + struct AuthFailureContext{ + zhandle_t *zh; + int rc; + int setServersRc; + }; + static void setServersOnAuthCompletion(int rc, const void *data){ + AuthFailureContext *ctx=(AuthFailureContext*)data; + ctx->rc=rc; + ctx->setServersRc=zoo_set_servers(ctx->zh,"127.0.0.2:2122"); + } + + // A server list change from a failed authentication callback must not + // hide the failure. The handle stays in ZOO_AUTH_FAILED_STATE. + void testServerListChangeKeepsAuthFailedState() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + SessionEvents events; + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + + // The server rejects the credential. + zkServer.addOperationResponse(new ZooStatResponse(0,ZAUTHFAILED)); + AuthFailureContext ctx={zh,ZAPIERROR,ZAPIERROR}; + CPPUNIT_ASSERT_EQUAL((int)ZOK,zoo_add_auth(zh,"digest","u:p",3, + setServersOnAuthCompletion,&ctx)); + + int fd=0; + int interest=0; + timeval tv; + CPPUNIT_ASSERT_EQUAL((int)ZOK,zookeeper_interest(zh,&fd,&interest,&tv)); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED,zookeeper_process(zh,interest)); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED,ctx.rc); + CPPUNIT_ASSERT_EQUAL((int)ZOK,ctx.setServersRc); + CPPUNIT_ASSERT_EQUAL(ZOO_AUTH_FAILED_STATE,zoo_state(zh)); + CPPUNIT_ASSERT(events.contains(ZOO_AUTH_FAILED_STATE)); + CPPUNIT_ASSERT_EQUAL((int)ZINVALIDSTATE, + zookeeper_interest(zh,&fd,&interest,&tv)); + } + void testConnectResponseFull() { CloseFinally guard(&zh); @@ -1092,7 +1332,313 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); CPPUNIT_ASSERT(ensureCondition(action.isNodeChangedTriggered(),1000)<1000); - CPPUNIT_ASSERT_EQUAL(string("/x/y/z"),action.path_); + CPPUNIT_ASSERT_EQUAL(string("/x/y/z"),action.path_); + } + + // A mock server that holds the first handshake reply until the test + // releases it. + class HeldHandshakeServer: public ZookeeperServer{ + public: + HeldHandshakeServer():held_(false),released_(false){} + virtual void notifyBufferSent(const std::string& buffer){ + { + synchronized(mx_); + if(!released_ && HandshakeRequest::isValid(buffer)){ + handshake_=buffer; + held_=true; + return; + } + } + ZookeeperServer::notifyBufferSent(buffer); + } + void release(){ + string buffer; + { + synchronized(mx_); + released_=true; + buffer=handshake_; + } + ZookeeperServer::notifyBufferSent(buffer); + } + SyncedBoolCondition isHeld() const{ + return SyncedBoolCondition(held_,mx_); + } + mutable Mutex mx_; + bool held_; + bool released_; + string handshake_; + }; + + // Releases the authentication lock if an assertion fails. + struct AuthLockGuard{ + AuthLockGuard(zhandle_t* zh):zh_(zh),locked_(true){zoo_lock_auth(zh_);} + ~AuthLockGuard(){unlock();} + void unlock(){ + if(locked_){ + locked_=false; + zoo_unlock_auth(zh_); + } + } + zhandle_t* zh_; + bool locked_; + }; + + // A server list change while the IO thread completes the session setup + // must not give the watcher an invalid state. The watcher gets + // ZOO_CONNECTED_STATE, then ZOO_CONNECTING_STATE, then the session + // connects to the new server. + void testServerListChangeDuringSessionEstablishment() + { + Mock_gettimeofday timeMock; + HeldHandshakeServer zkServer; + Mock_poll pollMock(&zkServer,ZookeeperServer::FD); + SessionEvents events; + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + CPPUNIT_ASSERT(ensureCondition(zkServer.isHeld(),1000)<1000); + + { + // The IO thread sets the connected state, then waits for this + // lock in send_auth_info() before it sends the session event. + AuthLockGuard authLock(zh); + zkServer.release(); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh),1000)<1000); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_set_servers(zh,"127.0.0.2:2122")); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,zoo_state(zh)); + } + + // Wait for the reconnect to the new server. + struct SecondConnect{ + SecondConnect(const SessionEvents& e):e_(e){} + bool operator()() const{ + vector s=e_.get(); + return count(s.begin(),s.end(),ZOO_CONNECTED_STATE)>=2; + } + const SessionEvents& e_; + }; + CPPUNIT_ASSERT(ensureCondition(SecondConnect(events),2000)<2000); + + vector states=events.get(); + CPPUNIT_ASSERT(states.size()>=3); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,states[0]); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTING_STATE,states[1]); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,states[2]); + CPPUNIT_ASSERT(!events.contains(ZOO_NOTCONNECTED_STATE)); + CPPUNIT_ASSERT_EQUAL(2122,portOf(zh->addr_cur)); + } + // A poll() mock that can park the IO thread. A parked IO thread does not + // read the wakeup pipe, so the test controls the pipe contents. + class ParkingPoll: public Mock_poll{ + public: + ParkingPoll(Mock_socket* s,int fd): + Mock_poll(s,fd),park_(false),parked_(false){} + virtual int call(struct pollfd *fds, POLL_NFDS_TYPE nfds, int to){ + for(;;){ + { + synchronized(mx_); + if(!park_){ + parked_=false; + break; + } + parked_=true; + } + millisleep(1); + } + return Mock_poll::call(fds,nfds,to); + } + void park(){ + synchronized(mx_); + park_=true; + } + void release(){ + synchronized(mx_); + park_=false; + } + SyncedBoolCondition isParked() const{ + return SyncedBoolCondition(parked_,mx_); + } + mutable Mutex mx_; + bool park_; + bool parked_; + }; + + // Releases a parked IO thread, also when an assertion fails. Declare it + // after CloseFinally, so that the release occurs before the close. + struct ParkRelease{ + ParkRelease(ParkingPoll& poll):poll_(poll){} + ~ParkRelease(){ poll_.release(); } + ParkingPoll& poll_; + }; + + // A full wakeup pipe already holds a wakeup for the IO thread. An + // asynchronous call must then succeed and keep the connection. + void testFullWakeupPipeKeepsConnection() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + ParkingPoll pollMock(&zkServer,ZookeeperServer::FD); + // The completion must outlive the close and the park release, + // because both can complete the request. + AsyncGetOperationCompletion res; + CloseFinally guard(&zh); + ParkRelease parkRelease(pollMock); + + zh=zookeeper_init("127.0.0.1:2121",watcher,10000,TEST_CLIENT_ID,0,0); + CPPUNIT_ASSERT(zh!=0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh),1000)<1000); + + // Park the IO thread, then fill the pipe. The pipe stays full until + // the release. + pollMock.park(); + CPPUNIT_ASSERT(ensureCondition(pollMock.isParked(),1000)<1000); + struct adaptor_threads* adaptor= + (struct adaptor_threads*)zh->adaptor_priv; + char buf[4096]={0}; + while(write(adaptor->self_pipe[1],buf,sizeof(buf))>0){} + CPPUNIT_ASSERT_EQUAL(EAGAIN,errno); + CPPUNIT_ASSERT_EQUAL((int)ZOK,wakeup_io_thread(zh)); + + zkServer.addOperationResponse(new ZooGetResponse("1",1)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/x/y/1",0,asyncCompletion,&res)); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,zh->fd->sock); + lock_reconfig(zh); + int requested=zh->disconnect_requested; + unlock_reconfig(zh); + CPPUNIT_ASSERT_EQUAL(0,requested); + + pollMock.release(); + CPPUNIT_ASSERT(ensureCondition(res,1000)<1000); + CPPUNIT_ASSERT_EQUAL((int)ZOK,res.rc_); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE,zoo_state(zh)); + } + + // Counts the get requests that reach the server. The server does not + // reply to them. + class GetRequestServer: public ZookeeperServer{ + public: + virtual void onMessageReceived(const RequestHeader& rh, iarchive* ia){ + if(rh.type==ZOO_GETDATA_OP) + ++getRequests; + } + AtomicInt getRequests; + }; + + // Checks that a get request reached the server. + struct GetRequestReceived{ + GetRequestReceived(const AtomicInt& n):n_(n){} + bool operator()() const{ return n_.get()>0; } + const AtomicInt& n_; + }; + + // Runs one synchronous get request and records its result. + class BlockedGetJob: public TestJob{ + public: + BlockedGetJob(zhandle_t* zh):zh_(zh),done_(false),rc_(ZAPIERROR){} + virtual TestJob* clone() const { return new BlockedGetJob(zh_); } + virtual void run(){ + char buf[1]; + int size=sizeof(buf); + int rc=zoo_get(zh_,"/a",0,buf,&size,0); + synchronized(mx_); + rc_=rc; + done_=true; + } + virtual void validate(const char* file, int line) const{} + bool operator()() const{ + synchronized(mx_); + return done_; + } + int rc() const{ + synchronized(mx_); + return rc_; + } + zhandle_t* zh_; + mutable Mutex mx_; + bool done_; + int rc_; + }; + + // Checks for a connected session with the given server. + struct ConnectedTo{ + ConnectedTo(zhandle_t* zh,const char* server):zh_(zh),server_(server){} + bool operator()() const{ + return zoo_state(zh_)==ZOO_CONNECTED_STATE && + server_==zoo_get_current_server(zh_); + } + zhandle_t* zh_; + string server_; + }; + + // Checks that the watcher received a session event with the given state. + struct EventSeen{ + EventSeen(const SessionEvents& e,int state):e_(e),state_(state){} + bool operator()() const{ return e_.contains(state_); } + const SessionEvents& e_; + int state_; + }; + + // A poll() mock with a long timeout, so that an idle IO thread wakes + // only for the wakeup pipe. zookeeper_close() and the debug check for + // a readable socket poll only the session socket. The mock answers a + // poll of only the session socket at once, so these polls do not wait + // for the long timeout. The mock does not report the socket as + // writable, so zookeeper_close() does not send the close request. The + // test does not need it. + class LongIdlePoll: public Mock_poll{ + public: + LongIdlePoll(Mock_socket* s,int fd):Mock_poll(s,fd){ + timeout=60000; + } + virtual int call(struct pollfd *fds, POLL_NFDS_TYPE nfds, int to){ + if(nfds==1 && fds[0].fd==myFD){ + bool readable=isFDReadable(); + fds[0].revents=readable?POLLIN:0; + return readable?1:0; + } + return Mock_poll::call(fds,nfds,to); + } + }; + + // A synchronous call that waits when zoo_set_servers() drops the current + // server must return ZCONNECTIONLOSS. The client then connects to the + // new server. + void testServerListChangeFailsBlockedSyncCall() + { + Mock_gettimeofday timeMock; + GetRequestServer zkServer; + // The test fails if zoo_set_servers() does not wake the IO thread. + LongIdlePoll pollMock(&zkServer,ZookeeperServer::FD); + SessionEvents events; + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121",sessionWatcher,10000, + TEST_CLIENT_ID,&events,0); + CPPUNIT_ASSERT(zh!=0); + CPPUNIT_ASSERT(ensureCondition( + ConnectedTo(zh,"127.0.0.1:2121"),1000)<1000); + + BlockedGetJob job(zh); + CloseThenJoin closeThenJoin(guard,job); + job.start(); + CPPUNIT_ASSERT(ensureCondition( + GetRequestReceived(zkServer.getRequests),1000)<1000); + CPPUNIT_ASSERT(!job()); + + CPPUNIT_ASSERT_EQUAL((int)ZOK,zoo_set_servers(zh,"127.0.0.2:2122")); + CPPUNIT_ASSERT(ensureCondition(job,1000)<1000); + job.join(); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,job.rc()); + // The completion thread sends the session event, so it can arrive + // after the synchronous call returns. + CPPUNIT_ASSERT(ensureCondition( + EventSeen(events,ZOO_CONNECTING_STATE),1000)<1000); + CPPUNIT_ASSERT(ensureCondition( + ConnectedTo(zh,"127.0.0.2:2122"),1000)<1000); } #endif }; diff --git a/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h b/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h index 45791fd2f59..2ed01a7faf4 100644 --- a/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h +++ b/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h @@ -189,6 +189,8 @@ class TestJob { pthread_join(thread_,0); else pthread_detach(thread_); + // A thread can be joined only once, so later calls do nothing. + hasRun_=false; } private: void awaitStart() {