From 879aee20a120793bcce8b95066a1b524400af15b Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 5 Nov 2026 16:21:46 -0800 Subject: [PATCH 1/2] ZOOKEEPER-5119: (C client) Switching from RO to RW servers corrupts state When a read-only session finds a read/write server, zookeeper_interest() now handles the switch as a normal reconnect: - Fail pending requests with ZCONNECTIONLOSS instead of a faked ZOK reply. - Return fd -1, no interest, and a zero timeout after the switch, so the caller does not poll the closed socket and reconnects at once. Also return fd -1 after a receive timeout, and the current socket after a failed ping send. - Keep the read/write server hint in the handle (has_addr_rw_server) instead of a file-static pointer that any handle could consume. Clear the hint when the server list changes. Add test Zookeeper_operations::testReadOnlyToReadWriteTransition. Co-Authored-By: Claude Opus 5.5 --- .../zookeeper-client-c/src/zk_adaptor.h | 1 + .../zookeeper-client-c/src/zookeeper.c | 34 ++++- .../tests/TestOperations.cc | 128 ++++++++++++++++++ 3 files changed, 157 insertions(+), 6 deletions(-) diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index 25049bb335e..d24080c2254 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -197,6 +197,7 @@ struct _zhandle { char *hostname; // hostname contains list of zookeeper servers to connect to struct sockaddr_storage addr_cur; // address of server we're currently connecting/connected to struct sockaddr_storage addr_rw_server; // address of last known read/write server found. + int has_addr_rw_server; // set when addr_rw_server holds a hint for the next connect addrvec_t addrs; // current list of addresses we're connected to addrvec_t addrs_old; // old list of addresses that we are no longer connected to diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index c8429003b29..bad4eae0324 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -285,7 +285,6 @@ static int handle_socket_error_msg(zhandle_t *zh, int line, const char *func, in static void cleanup_bufs(zhandle_t *zh,int callCompletion,int rc); static int disable_conn_permute=0; // permute enabled by default -static struct sockaddr_storage *addr_rw_server = 0; static void *SYNCHRONOUS_MARKER = (void*)&SYNCHRONOUS_MARKER; static int isValidPath(const char* path, const int mode); @@ -1216,6 +1215,10 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) addrvec_free(&zh->addrs); zh->addrs = resolved; + // The read/write server hint comes from the old list. Drop it so that + // the next connect follows the new list. + zh->has_addr_rw_server = 0; + // If we need to do a reconfig and we're currently connected to a server, // then force close that connection so on next interest() call we'll make a // new connection @@ -2518,7 +2521,7 @@ static int ping_rw_server(zhandle_t* zh) out: close_zsock(&fd); - addr_rw_server = rc ? &zh->addr_rw_server : 0; + zh->has_addr_rw_server = rc; return rc; } @@ -2672,9 +2675,9 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, LOG_WARN(LOGCALLBACK(zh), "Delaying connection after exhaustively trying all servers [%s]", zh->hostname); unlock_reconfig(zh); } else { - if (addr_rw_server) { - zh->addr_cur = *addr_rw_server; - addr_rw_server = 0; + if (zh->has_addr_rw_server) { + zh->addr_cur = zh->addr_rw_server; + zh->has_addr_rw_server = 0; } else { // No need to delay -- grab the next server and attempt connection zoo_cycle_next_server(zh); @@ -2754,6 +2757,8 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, #else errno = ETIMEDOUT; #endif + // The error handler closes the socket, so do not return its fd. + *fd = -1; *interest=0; *tv = get_timeval(0); return api_epilog(zh,handle_socket_error_msg(zh, @@ -2773,6 +2778,10 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, rc = send_ping(zh); if (rc < 0) { LOG_ERROR(LOGCALLBACK(zh), "failed to send PING request (zk retcode=%d)",rc); + // A failed send can close the socket. Return the + // current fd so that the caller does not poll a + // closed descriptor. + *fd = zh->fd->sock; return api_epilog(zh,rc); } } @@ -2795,7 +2804,20 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, LOG_INFO(LOGCALLBACK(zh), "r/w server found at %s", format_endpoint_info(&addr)); - cleanup(zh, ZOK); + // The read-only server did not complete the pending + // requests. Fail them with ZCONNECTIONLOSS, as for any + // other disconnect. + cleanup(zh, ZCONNECTIONLOSS); + // The socket is closed. Ask the caller to call again at + // once, so that it connects to the read/write server. + // The probe can block, so clear the deadline to prevent + // a false "exceeded deadline" warning. + *fd = -1; + *interest = 0; + *tv = get_timeval(0); + zh->next_deadline.tv_sec = 0; + zh->next_deadline.tv_usec = 0; + return api_epilog(zh, ZOK); } else { addrvec_next(&zh->addrs, NULL); } diff --git a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc index 534c7599c1f..4e518d86123 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc @@ -21,6 +21,8 @@ #include "ZKMocks.h" #include +#include +#include using namespace std; @@ -36,6 +38,7 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_TEST(testCloseWhileInProgressFromCompletion); CPPUNIT_TEST(testCloseWhileMultiInProgressFromMain); CPPUNIT_TEST(testCloseWhileMultiInProgressFromCompletion); + CPPUNIT_TEST(testReadOnlyToReadWriteTransition); CPPUNIT_TEST(testConnectResponseFull); CPPUNIT_TEST(testConnectResponseNoReadOnlyFlag); CPPUNIT_TEST(testConnectResponseSplitAtReadOnlyFlag); @@ -714,6 +717,131 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZCLOSING,res2.rc_); } + // Records the status of any completion type. + class StatusCompletion: public AsyncCompletion{ + public: + StatusCompletion():called_(false),rc_(ZAPIERROR){} + virtual void dataCompl(int rc, const char *value, int len, const Stat *stat){ + record(rc); + } + virtual void statCompl(int rc, const Stat *stat){ record(rc); } + virtual void stringCompl(int rc, const char *value){ record(rc); } + virtual void voidCompl(int rc){ record(rc); } + void record(int rc){ called_=true; rc_=rc; } + bool called_; + int rc_; + }; + + // A mock server that also answers the read-only probe. While the probe + // is enabled, connect() succeeds and the reply to "isro" is "rw". + class ReadWriteProbeServer: public ZookeeperServer{ + public: + ReadWriteProbeServer():probeEnabled(false),probeCount(0), + rwReplyPending(false){} + virtual int callConnect(int s,const struct sockaddr *addr,socklen_t len){ + if(probeEnabled){ + probeCount++; + return 0; + } + return ZookeeperServer::callConnect(s,addr,len); + } + virtual ssize_t callSend(int s,const void *buf,size_t len,int flags){ + if(probeEnabled && len==4 && memcmp(buf,"isro",4)==0){ + rwReplyPending=true; + return len; + } + return ZookeeperServer::callSend(s,buf,len,flags); + } + virtual ssize_t callRecv(int s,void *buf,size_t len,int flags){ + if(rwReplyPending){ + rwReplyPending=false; + memcpy(buf,"rw",2); + return 2; + } + return ZookeeperServer::callRecv(s,buf,len,flags); + } + bool probeEnabled; + int probeCount; + bool rwReplyPending; + }; + + static int portOf(const struct sockaddr_storage& addr){ + return ntohs(((const struct sockaddr_in*)&addr)->sin_port); + } + + // A read-only session that finds a read/write server must fail all + // pending requests with ZCONNECTIONLOSS. It must not return the closed + // socket, and only this handle can use the read/write server hint. + void testReadOnlyToReadWriteTransition() + { + Mock_gettimeofday timeMock; + ReadWriteProbeServer zkServer; + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121,127.0.0.1:2122",watcher,10000, + TEST_CLIENT_ID,0,ZOO_READONLY); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + zh->state=ZOO_READONLY_STATE; + zh->last_ping_rw=timeMock.tv; + zh->ping_rw_timeout=200; + // The probe uses the next server in the list. + struct sockaddr_storage rwServer=zh->addrs.data[zh->addrs.next]; + + // The server does not reply to these requests. + StatusCompletion delRes, setRes, getRes, createRes, multiRes; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_adelete(zh,"/a",-1,asyncCompletion,&delRes)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aset(zh,"/a","x",1,-1,asyncCompletion,&setRes)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/a",0,asyncCompletion,&getRes)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_acreate(zh,"/a","x",1,&ZOO_OPEN_ACL_UNSAFE,0, + asyncCompletion,&createRes)); + zoo_op_t ops[1]; + zoo_op_result_t results[1]; + zoo_create_op_init(&ops[0],"/b",0,-1,&ZOO_OPEN_ACL_UNSAFE,0,0,0); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_amulti(zh,1,ops,results,asyncCompletion,&multiRes)); + + // The probe is due and finds a read/write server. + timeMock.tick(1); + int fd=0; + int interest=0; + timeval tv; + zkServer.probeEnabled=true; + int rc=zookeeper_interest(zh,&fd,&interest,&tv); + zkServer.probeEnabled=false; + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(1,zkServer.probeCount); + + CPPUNIT_ASSERT_EQUAL(-1,fd); + CPPUNIT_ASSERT_EQUAL(0,interest); + CPPUNIT_ASSERT_EQUAL(0L,(long)tv.tv_sec); + CPPUNIT_ASSERT_EQUAL(0L,(long)tv.tv_usec); + + StatusCompletion* all[]={&delRes,&setRes,&getRes,&createRes,&multiRes}; + for(size_t i=0;icalled_); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,all[i]->rc_); + } + + // Another handle must not use the hint of this handle. + zhandle_t* other=zookeeper_init("127.0.0.1:2181",watcher,10000,0,0,0); + CPPUNIT_ASSERT(other!=0); + CloseFinally otherGuard(&other); + rc=zookeeper_interest(other,&fd,&interest,&tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(2181,portOf(other->addr_cur)); + + // This handle reconnects to the read/write server. + rc=zookeeper_interest(zh,&fd,&interest,&tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(portOf(rwServer),portOf(zh->addr_cur)); + CPPUNIT_ASSERT_EQUAL(0,zh->has_addr_rw_server); + } + void testConnectResponseFull() { CloseFinally guard(&zh); From 265a10ab01e8c5236db44902296ea0940f4e42bf Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 19 Nov 2026 16:56:43 -0800 Subject: [PATCH 2/2] ZOOKEEPER-5119: Lock the read/write server hint and close the probe socket - Drop the ping-failure change that returned zh->fd->sock. The socket is still open at that point, so the change had no effect. The original commit message is wrong on this point. - Close the probe socket when the probe connect fails. Set the SSL fields before the first exit, because close_zsock() reads them. - Read and update the server list cursor and the read/write server hint under the reconfig lock. The probe itself runs without the lock, because it can block. A version counter on the server list makes the probe drop its result when zoo_set_servers() changes the list during the probe. - Store a 0/1 value in has_addr_rw_server. - Add tests: a hint dropped by a server list change, a probe made stale by a server list change, a probe that finds another read-only server, a failed probe connect, a receive timeout, and a synchronous call during the switch (MT). - The read-only test helper ends the reconfig pass, as a real connect does. Thus a reconnect without the hint follows the cursor, and the transition test checks that only the hint selects the read/write server. - Close the handle before the join in the MT test, so that a failed assertion cannot hang the test. TestJob::join() is now idempotent. - Put the shared test helpers in one block after AsyncVoidOperationCompletion, so that this change and ZOOKEEPER-5120 merge in either order. Co-Authored-By: Claude Opus 5.5 --- .../zookeeper-client-c/src/zk_adaptor.h | 5 +- .../zookeeper-client-c/src/zookeeper.c | 88 ++- .../tests/TestOperations.cc | 587 ++++++++++++++---- .../zookeeper-client-c/tests/ThreadingUtil.h | 2 + 4 files changed, 521 insertions(+), 161 deletions(-) diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index d24080c2254..0abdd7939ec 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -197,7 +197,10 @@ struct _zhandle { char *hostname; // hostname contains list of zookeeper servers to connect to struct sockaddr_storage addr_cur; // address of server we're currently connecting/connected to struct sockaddr_storage addr_rw_server; // address of last known read/write server found. - int has_addr_rw_server; // set when addr_rw_server holds a hint for the next connect + int has_addr_rw_server; // set when addr_rw_server holds a hint for the next connect. + // The reconfig lock guards this field. + unsigned int addrs_version; // changes each time update_addrs() replaces addrs. + // The reconfig lock guards this field. addrvec_t addrs; // current list of addresses we're connected to addrvec_t addrs_old; // old list of addresses that we are no longer connected to diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index bad4eae0324..d0d628000b1 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -1212,12 +1212,13 @@ int update_addrs(zhandle_t *zh, const struct timeval *ref_time) } } - addrvec_free(&zh->addrs); - zh->addrs = resolved; - // The read/write server hint comes from the old list. Drop it so that - // the next connect follows the new list. + // the next connect follows the new list. The version change tells a + // probe in progress that its result is stale. zh->has_addr_rw_server = 0; + zh->addrs_version++; + addrvec_free(&zh->addrs); + zh->addrs = resolved; // If we need to do a reconfig and we're currently connected to a server, // then force close that connection so on next interest() call we'll make a @@ -2466,39 +2467,46 @@ static int ping_rw_server(zhandle_t* zh) { char buf[10]; zsock_t fd; - int rc; + struct sockaddr_storage addr; + unsigned int addrs_version; + int rc = 0; sendsize_t ssize; int sock_flags; - addrvec_peek(&zh->addrs, &zh->addr_rw_server); + // An application thread can change the server list at any time. Read + // the list under the reconfig lock, but do not hold the lock during the + // probe, because the probe can block. + lock_reconfig(zh); + addrvec_peek(&zh->addrs, &addr); + addrs_version = zh->addrs_version; + unlock_reconfig(zh); #ifdef SOCK_CLOEXEC_ENABLED sock_flags = SOCK_STREAM | SOCK_CLOEXEC; #else sock_flags = SOCK_STREAM; #endif - fd.sock = socket(zh->addr_rw_server.ss_family, sock_flags, 0); + fd.sock = socket(addr.ss_family, sock_flags, 0); +#ifdef HAVE_OPENSSL_H + // close_zsock() reads these fields, so set them before the first exit. + fd.ssl_sock = NULL; + fd.ssl_ctx = NULL; +#endif if (fd.sock < 0) { - return 0; + goto out; } zookeeper_set_sock_nodelay(zh, fd.sock); zookeeper_set_sock_timeout(zh, fd.sock, 1); - rc = zookeeper_connect(zh, &zh->addr_rw_server, fd.sock); - if (rc < 0) { - return 0; + if (zookeeper_connect(zh, &addr, fd.sock) < 0) { + goto out; } #ifdef HAVE_OPENSSL_H - fd.ssl_sock = NULL; - fd.ssl_ctx = NULL; - if (zh->fd->cert != NULL) { fd.cert = zh->fd->cert; - rc = init_ssl_for_socket(&fd, zh, 0); - if (rc != ZOK) { - rc = 0; + if (init_ssl_for_socket(&fd, zh, 0) != ZOK) { goto out; } } @@ -2506,14 +2514,11 @@ static int ping_rw_server(zhandle_t* zh) ssize = zookeeper_send(&fd, "isro", 4); if (ssize < 0) { - rc = 0; goto out; } memset(buf, 0, sizeof(buf)); - rc = zookeeper_recv(&fd, buf, sizeof(buf), 0); - if (rc < 0) { - rc = 0; + if (zookeeper_recv(&fd, buf, sizeof(buf), 0) < 0) { goto out; } @@ -2521,7 +2526,17 @@ static int ping_rw_server(zhandle_t* zh) out: close_zsock(&fd); - zh->has_addr_rw_server = rc; + // Keep the result only if the server list did not change during the + // probe. Otherwise the probed server can be absent from the new list. + lock_reconfig(zh); + if (zh->addrs_version != addrs_version) { + rc = 0; + } + zh->has_addr_rw_server = rc != 0; + if (rc) { + zh->addr_rw_server = addr; + } + unlock_reconfig(zh); return rc; } @@ -2675,10 +2690,18 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, LOG_WARN(LOGCALLBACK(zh), "Delaying connection after exhaustively trying all servers [%s]", zh->hostname); unlock_reconfig(zh); } else { - if (zh->has_addr_rw_server) { + // A server list change can clear the read/write server hint + // on another thread, so read and clear it under the lock. + int use_rw_server; + + lock_reconfig(zh); + use_rw_server = zh->has_addr_rw_server; + zh->has_addr_rw_server = 0; + if (use_rw_server) { zh->addr_cur = zh->addr_rw_server; - zh->has_addr_rw_server = 0; - } else { + } + unlock_reconfig(zh); + if (!use_rw_server) { // No need to delay -- grab the next server and attempt connection zoo_cycle_next_server(zh); } @@ -2778,10 +2801,6 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, rc = send_ping(zh); if (rc < 0) { LOG_ERROR(LOGCALLBACK(zh), "failed to send PING request (zk retcode=%d)",rc); - // A failed send can close the socket. Return the - // current fd so that the caller does not poll a - // closed descriptor. - *fd = zh->fd->sock; return api_epilog(zh,rc); } } @@ -2798,20 +2817,19 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, zh->ping_rw_timeout = min(zh->ping_rw_timeout * 2, MAX_RW_TIMEOUT); if (ping_rw_server(zh)) { - struct sockaddr_storage addr; - addrvec_peek(&zh->addrs, &addr); zh->ping_rw_timeout = MIN_RW_TIMEOUT; LOG_INFO(LOGCALLBACK(zh), "r/w server found at %s", - format_endpoint_info(&addr)); + format_endpoint_info(&zh->addr_rw_server)); // The read-only server did not complete the pending // requests. Fail them with ZCONNECTIONLOSS, as for any // other disconnect. cleanup(zh, ZCONNECTIONLOSS); // The socket is closed. Ask the caller to call again at // once, so that it connects to the read/write server. - // The probe can block, so clear the deadline to prevent - // a false "exceeded deadline" warning. + // The probe can block. Clear the deadline, so that the + // next call does not log a false "Exceeded deadline" + // warning. *fd = -1; *interest = 0; *tv = get_timeval(0); @@ -2819,7 +2837,9 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, zh->next_deadline.tv_usec = 0; return api_epilog(zh, ZOK); } else { + lock_reconfig(zh); addrvec_next(&zh->addrs, NULL); + unlock_reconfig(zh); } } send_to = min(send_to, zh->ping_rw_timeout - idle_ping_rw); diff --git a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc index 4e518d86123..ab35254188b 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc @@ -26,6 +26,89 @@ using namespace std; +// A mock server that also answers the read-only probe of the client. The +// test arms the probe. The server on rwPort replies with probeReply, which +// is "rw" by default. If rwPort is 0, every server replies with probeReply. +// The other servers reply "ro". A "rw" reply disarms the probe. The test can +// make the probe connect fail, or change the server list during the probe. +// The server can also accept handshakes as a read-only server. +class ReadWriteProbeServer: public ZookeeperServer{ +public: + ReadWriteProbeServer():probeEnabled(false),probeConnectFails(false), + probeReply("rw"),rwPort(0),probeCount(0),probeCloseCount(0), + readOnlyHandshake(false),reconfigHandle(0),reconfigHosts(0), + probeInProgress(false),probePort(0),replyPending(false){} + virtual int callConnect(int s,const struct sockaddr *addr,socklen_t len){ + if(probeEnabled && !probeInProgress){ + probeInProgress=true; + probePort=ntohs(((const struct sockaddr_in*)addr)->sin_port); + ++probeCount; + if(probeConnectFails){ + errno=ECONNREFUSED; + return -1; + } + return 0; + } + return ZookeeperServer::callConnect(s,addr,len); + } + virtual ssize_t callSend(int s,const void *buf,size_t len,int flags){ + if(probeInProgress && len==4 && memcmp(buf,"isro",4)==0){ + // Change the server list while the probe runs. + if(reconfigHandle!=0) + zoo_set_servers(reconfigHandle,reconfigHosts); + replyPending=true; + return len; + } + return ZookeeperServer::callSend(s,buf,len,flags); + } + virtual ssize_t callRecv(int s,void *buf,size_t len,int flags){ + if(replyPending){ + replyPending=false; + std::string reply= + (rwPort==0 || rwPort==probePort) ? probeReply : "ro"; + if(reply=="rw") + probeEnabled=false; + memcpy(buf,reply.data(),reply.size()); + return reply.size(); + } + return ZookeeperServer::callRecv(s,buf,len,flags); + } + virtual int callClose(int fd){ + if(probeInProgress && fd==FD){ + // The probe socket is not the session socket. Keep the queues. + probeInProgress=false; + ++probeCloseCount; + return 0; + } + return ZookeeperServer::callClose(fd); + } + virtual void notifyBufferSent(const std::string& buffer){ + if(readOnlyHandshake && HandshakeRequest::isValid(buffer)){ + unique_ptr req(HandshakeRequest::parse(buffer)); + if(req.get()!=0){ + HandshakeResponse* resp=new HandshakeResponse(req->sessionId); + resp->readOnly=1; + addRecvResponse(resp); + return; + } + } + ZookeeperServer::notifyBufferSent(buffer); + } + volatile bool probeEnabled; + volatile bool probeConnectFails; + std::string probeReply; + volatile int rwPort; + AtomicInt probeCount; + AtomicInt probeCloseCount; + volatile bool readOnlyHandshake; + zhandle_t* reconfigHandle; + const char* reconfigHosts; +private: + volatile bool probeInProgress; + volatile int probePort; + volatile bool replyPending; +}; + class Zookeeper_operations : public CPPUNIT_NS::TestFixture { CPPUNIT_TEST_SUITE(Zookeeper_operations); @@ -38,12 +121,18 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_TEST(testCloseWhileInProgressFromCompletion); CPPUNIT_TEST(testCloseWhileMultiInProgressFromMain); CPPUNIT_TEST(testCloseWhileMultiInProgressFromCompletion); - CPPUNIT_TEST(testReadOnlyToReadWriteTransition); CPPUNIT_TEST(testConnectResponseFull); CPPUNIT_TEST(testConnectResponseNoReadOnlyFlag); CPPUNIT_TEST(testConnectResponseSplitAtReadOnlyFlag); CPPUNIT_TEST(testConnectResponseNoReadOnlyFlagSplit); + CPPUNIT_TEST(testReadOnlyToReadWriteTransition); + CPPUNIT_TEST(testReadOnlyHintDroppedByServerListChange); + CPPUNIT_TEST(testReadOnlyProbeDroppedByServerListChange); + CPPUNIT_TEST(testReadOnlyProbeFindsReadOnlyServer); + CPPUNIT_TEST(testReadOnlyProbeConnectFailureClosesSocket); + CPPUNIT_TEST(testReceiveTimeoutReturnsNoSocket); #else + CPPUNIT_TEST(testReadOnlyToReadWriteFailsSyncCall); CPPUNIT_TEST(testAsyncWatcher1); CPPUNIT_TEST(testAsyncGetOperation); #endif @@ -121,6 +210,42 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture bool called_; int rc_; }; + + // Records the status of any completion type. + class StatusCompletion: public AsyncCompletion{ + public: + StatusCompletion():called_(false),rc_(ZAPIERROR){} + virtual void dataCompl(int rc, const char *value, int len, const Stat *stat){ + record(rc); + } + virtual void statCompl(int rc, const Stat *stat){ record(rc); } + virtual void stringCompl(int rc, const char *value){ record(rc); } + virtual void voidCompl(int rc){ record(rc); } + void record(int rc){ called_=true; rc_=rc; } + bool called_; + int rc_; + }; + + static int portOf(const struct sockaddr_storage& addr){ + return ntohs(((const struct sockaddr_in*)&addr)->sin_port); + } + +#ifdef THREADED + // Closes the handle and then joins the job, also when an assertion + // fails. The close makes a waiting synchronous call return ZCLOSING, + // so the join cannot hang. The join also occurs before the job destroys + // its own members. Declare the guard after the job. + struct CloseThenJoin{ + CloseThenJoin(CloseFinally& close,TestJob& job): + close_(close),job_(job){} + ~CloseThenJoin(){ + close_.execute(); + job_.join(); + } + CloseFinally& close_; + TestJob& job_; + }; +#endif #ifndef THREADED // send two get data requests; verify that the corresponding completions called void testConcurrentOperations1() @@ -717,131 +842,6 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZCLOSING,res2.rc_); } - // Records the status of any completion type. - class StatusCompletion: public AsyncCompletion{ - public: - StatusCompletion():called_(false),rc_(ZAPIERROR){} - virtual void dataCompl(int rc, const char *value, int len, const Stat *stat){ - record(rc); - } - virtual void statCompl(int rc, const Stat *stat){ record(rc); } - virtual void stringCompl(int rc, const char *value){ record(rc); } - virtual void voidCompl(int rc){ record(rc); } - void record(int rc){ called_=true; rc_=rc; } - bool called_; - int rc_; - }; - - // A mock server that also answers the read-only probe. While the probe - // is enabled, connect() succeeds and the reply to "isro" is "rw". - class ReadWriteProbeServer: public ZookeeperServer{ - public: - ReadWriteProbeServer():probeEnabled(false),probeCount(0), - rwReplyPending(false){} - virtual int callConnect(int s,const struct sockaddr *addr,socklen_t len){ - if(probeEnabled){ - probeCount++; - return 0; - } - return ZookeeperServer::callConnect(s,addr,len); - } - virtual ssize_t callSend(int s,const void *buf,size_t len,int flags){ - if(probeEnabled && len==4 && memcmp(buf,"isro",4)==0){ - rwReplyPending=true; - return len; - } - return ZookeeperServer::callSend(s,buf,len,flags); - } - virtual ssize_t callRecv(int s,void *buf,size_t len,int flags){ - if(rwReplyPending){ - rwReplyPending=false; - memcpy(buf,"rw",2); - return 2; - } - return ZookeeperServer::callRecv(s,buf,len,flags); - } - bool probeEnabled; - int probeCount; - bool rwReplyPending; - }; - - static int portOf(const struct sockaddr_storage& addr){ - return ntohs(((const struct sockaddr_in*)&addr)->sin_port); - } - - // A read-only session that finds a read/write server must fail all - // pending requests with ZCONNECTIONLOSS. It must not return the closed - // socket, and only this handle can use the read/write server hint. - void testReadOnlyToReadWriteTransition() - { - Mock_gettimeofday timeMock; - ReadWriteProbeServer zkServer; - CloseFinally guard(&zh); - - zh=zookeeper_init("127.0.0.1:2121,127.0.0.1:2122",watcher,10000, - TEST_CLIENT_ID,0,ZOO_READONLY); - CPPUNIT_ASSERT(zh!=0); - forceConnected(zh, &timeMock.tv); - zh->state=ZOO_READONLY_STATE; - zh->last_ping_rw=timeMock.tv; - zh->ping_rw_timeout=200; - // The probe uses the next server in the list. - struct sockaddr_storage rwServer=zh->addrs.data[zh->addrs.next]; - - // The server does not reply to these requests. - StatusCompletion delRes, setRes, getRes, createRes, multiRes; - CPPUNIT_ASSERT_EQUAL((int)ZOK, - zoo_adelete(zh,"/a",-1,asyncCompletion,&delRes)); - CPPUNIT_ASSERT_EQUAL((int)ZOK, - zoo_aset(zh,"/a","x",1,-1,asyncCompletion,&setRes)); - CPPUNIT_ASSERT_EQUAL((int)ZOK, - zoo_aget(zh,"/a",0,asyncCompletion,&getRes)); - CPPUNIT_ASSERT_EQUAL((int)ZOK, - zoo_acreate(zh,"/a","x",1,&ZOO_OPEN_ACL_UNSAFE,0, - asyncCompletion,&createRes)); - zoo_op_t ops[1]; - zoo_op_result_t results[1]; - zoo_create_op_init(&ops[0],"/b",0,-1,&ZOO_OPEN_ACL_UNSAFE,0,0,0); - CPPUNIT_ASSERT_EQUAL((int)ZOK, - zoo_amulti(zh,1,ops,results,asyncCompletion,&multiRes)); - - // The probe is due and finds a read/write server. - timeMock.tick(1); - int fd=0; - int interest=0; - timeval tv; - zkServer.probeEnabled=true; - int rc=zookeeper_interest(zh,&fd,&interest,&tv); - zkServer.probeEnabled=false; - CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); - CPPUNIT_ASSERT_EQUAL(1,zkServer.probeCount); - - CPPUNIT_ASSERT_EQUAL(-1,fd); - CPPUNIT_ASSERT_EQUAL(0,interest); - CPPUNIT_ASSERT_EQUAL(0L,(long)tv.tv_sec); - CPPUNIT_ASSERT_EQUAL(0L,(long)tv.tv_usec); - - StatusCompletion* all[]={&delRes,&setRes,&getRes,&createRes,&multiRes}; - for(size_t i=0;icalled_); - CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,all[i]->rc_); - } - - // Another handle must not use the hint of this handle. - zhandle_t* other=zookeeper_init("127.0.0.1:2181",watcher,10000,0,0,0); - CPPUNIT_ASSERT(other!=0); - CloseFinally otherGuard(&other); - rc=zookeeper_interest(other,&fd,&interest,&tv); - CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); - CPPUNIT_ASSERT_EQUAL(2181,portOf(other->addr_cur)); - - // This handle reconnects to the read/write server. - rc=zookeeper_interest(zh,&fd,&interest,&tv); - CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); - CPPUNIT_ASSERT_EQUAL(portOf(rwServer),portOf(zh->addr_cur)); - CPPUNIT_ASSERT_EQUAL(0,zh->has_addr_rw_server); - } - void testConnectResponseFull() { CloseFinally guard(&zh); @@ -962,7 +962,342 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, static_cast(zh->state)); } + // Starts a read-only session on 127.0.0.1:2121. The next server in the + // list is 127.0.0.1:2122, so the probe goes to that server. + void startReadOnlySession(Mock_gettimeofday& timeMock, const char* hosts) + { + zoo_deterministic_conn_order(1); + zh=zookeeper_init(hosts,watcher,10000,TEST_CLIENT_ID,0,ZOO_READONLY); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + // A real connect ends the reconfig pass. Thus a reconnect follows + // the cursor, not the first new address. + zh->reconfig=0; + zh->state=ZOO_READONLY_STATE; + zh->last_ping_rw=timeMock.tv; + zh->ping_rw_timeout=200; + CPPUNIT_ASSERT_EQUAL(2121,portOf(zh->addr_cur)); + CPPUNIT_ASSERT_EQUAL(2122,portOf(zh->addrs.data[zh->addrs.next])); + } + + // Makes the read-only probe due and runs one zookeeper_interest() call. + int runProbe(Mock_gettimeofday& timeMock, ReadWriteProbeServer& zkServer, + int& fd, int& interest, timeval& tv) + { + timeMock.tick(1); + zkServer.probeEnabled=true; + int rc=zookeeper_interest(zh,&fd,&interest,&tv); + zkServer.probeEnabled=false; + return rc; + } + + // A read-only session that finds a read/write server must fail all + // pending requests with ZCONNECTIONLOSS. zookeeper_interest() must not + // return the closed socket. Only this handle can use the read/write + // server hint. + void testReadOnlyToReadWriteTransition() + { + Mock_gettimeofday timeMock; + ReadWriteProbeServer zkServer; + CloseFinally guard(&zh); + startReadOnlySession(timeMock,"127.0.0.1:2121,127.0.0.1:2122"); + + // The server does not reply to these requests. + StatusCompletion delRes, setRes, getRes, createRes, multiRes; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_adelete(zh,"/a",-1,asyncCompletion,&delRes)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aset(zh,"/a","x",1,-1,asyncCompletion,&setRes)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/a",0,asyncCompletion,&getRes)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_acreate(zh,"/a","x",1,&ZOO_OPEN_ACL_UNSAFE,0, + asyncCompletion,&createRes)); + zoo_op_t ops[1]; + zoo_op_result_t results[1]; + zoo_create_op_init(&ops[0],"/b",0,-1,&ZOO_OPEN_ACL_UNSAFE,0,0,0); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_amulti(zh,1,ops,results,asyncCompletion,&multiRes)); + + // The probe finds a read/write server. + int fd=0; + int interest=0; + timeval tv; + int rc=runProbe(timeMock,zkServer,fd,interest,tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(1,(int)zkServer.probeCount); + CPPUNIT_ASSERT_EQUAL(1,(int)zkServer.probeCloseCount); + CPPUNIT_ASSERT_EQUAL(1,zh->has_addr_rw_server); + + CPPUNIT_ASSERT_EQUAL(-1,fd); + CPPUNIT_ASSERT_EQUAL(0,interest); + CPPUNIT_ASSERT_EQUAL(0L,(long)tv.tv_sec); + CPPUNIT_ASSERT_EQUAL(0L,(long)tv.tv_usec); + + StatusCompletion* all[]={&delRes,&setRes,&getRes,&createRes,&multiRes}; + for(size_t i=0;icalled_); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,all[i]->rc_); + } + + // Another handle must not use the hint of this handle. + zhandle_t* other=zookeeper_init("127.0.0.1:2181",watcher,10000,0,0,0); + CPPUNIT_ASSERT(other!=0); + CloseFinally otherGuard(&other); + rc=zookeeper_interest(other,&fd,&interest,&tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(2181,portOf(other->addr_cur)); + + // Move the cursor past the probed server. Only the hint can then + // select it for the reconnect. + zh->addrs.next++; + + // This handle reconnects to the read/write server. + rc=zookeeper_interest(zh,&fd,&interest,&tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(2122,portOf(zh->addr_cur)); + CPPUNIT_ASSERT_EQUAL(0,zh->has_addr_rw_server); + } + + // A server list change after the probe drops the read/write server + // hint. The reconnect then uses the new list. + void testReadOnlyHintDroppedByServerListChange() + { + Mock_gettimeofday timeMock; + ReadWriteProbeServer zkServer; + CloseFinally guard(&zh); + startReadOnlySession(timeMock,"127.0.0.1:2121,127.0.0.1:2122"); + + int fd=0; + int interest=0; + timeval tv; + int rc=runProbe(timeMock,zkServer,fd,interest,tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(-1,fd); + CPPUNIT_ASSERT_EQUAL(1,zh->has_addr_rw_server); + + // The address comparison ignores ports, so change the number of + // servers to make the new list differ from the old list. + CPPUNIT_ASSERT_EQUAL((int)ZOK,zoo_set_servers(zh,"127.0.0.1:2123")); + CPPUNIT_ASSERT_EQUAL(0,zh->has_addr_rw_server); + + rc=zookeeper_interest(zh,&fd,&interest,&tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(2123,portOf(zh->addr_cur)); + } + + // A server list change while the probe runs makes the probe result + // stale. The client must not switch to the probed server. + void testReadOnlyProbeDroppedByServerListChange() + { + Mock_gettimeofday timeMock; + ReadWriteProbeServer zkServer; + CloseFinally guard(&zh); + startReadOnlySession(timeMock,"127.0.0.1:2121,127.0.0.1:2122"); + + StatusCompletion getRes; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/a",0,asyncCompletion,&getRes)); + + // The new list keeps the current server, so the client stays + // connected. The probed server 127.0.0.1:2122 is not in the list. + zkServer.reconfigHandle=zh; + zkServer.reconfigHosts="127.0.0.1:2121"; + int fd=0; + int interest=0; + timeval tv; + int rc=runProbe(timeMock,zkServer,fd,interest,tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(1,(int)zkServer.probeCount); + CPPUNIT_ASSERT_EQUAL(1,(int)zh->addrs.count); + + CPPUNIT_ASSERT_EQUAL(0,zh->has_addr_rw_server); + CPPUNIT_ASSERT_EQUAL((int)ZOO_READONLY_STATE,zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,fd); + CPPUNIT_ASSERT(!getRes.called_); + } + + // A probe that finds another read-only server does not switch. The + // next probe goes to the next server in the list. + void testReadOnlyProbeFindsReadOnlyServer() + { + Mock_gettimeofday timeMock; + ReadWriteProbeServer zkServer; + CloseFinally guard(&zh); + startReadOnlySession(timeMock, + "127.0.0.1:2121,127.0.0.1:2122,127.0.0.1:2123"); + + StatusCompletion getRes; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_aget(zh,"/a",0,asyncCompletion,&getRes)); + + zkServer.probeReply="ro"; + int fd=0; + int interest=0; + timeval tv; + int rc=runProbe(timeMock,zkServer,fd,interest,tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(1,(int)zkServer.probeCount); + CPPUNIT_ASSERT_EQUAL(1,(int)zkServer.probeCloseCount); + + CPPUNIT_ASSERT_EQUAL(0,zh->has_addr_rw_server); + CPPUNIT_ASSERT_EQUAL((int)ZOO_READONLY_STATE,zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,fd); + CPPUNIT_ASSERT(!getRes.called_); + CPPUNIT_ASSERT_EQUAL(2123,portOf(zh->addrs.data[zh->addrs.next])); + } + + // A probe that cannot connect must close its socket. + void testReadOnlyProbeConnectFailureClosesSocket() + { + Mock_gettimeofday timeMock; + ReadWriteProbeServer zkServer; + CloseFinally guard(&zh); + startReadOnlySession(timeMock, + "127.0.0.1:2121,127.0.0.1:2122,127.0.0.1:2123"); + + zkServer.probeConnectFails=true; + int fd=0; + int interest=0; + timeval tv; + int rc=runProbe(timeMock,zkServer,fd,interest,tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK,rc); + CPPUNIT_ASSERT_EQUAL(1,(int)zkServer.probeCount); + CPPUNIT_ASSERT_EQUAL(1,(int)zkServer.probeCloseCount); + + CPPUNIT_ASSERT_EQUAL(0,zh->has_addr_rw_server); + CPPUNIT_ASSERT_EQUAL((int)ZOO_READONLY_STATE,zoo_state(zh)); + CPPUNIT_ASSERT_EQUAL((int)ZookeeperServer::FD,fd); + CPPUNIT_ASSERT_EQUAL(2123,portOf(zh->addrs.data[zh->addrs.next])); + } + + // A receive timeout closes the socket, so zookeeper_interest() must + // not return it. + void testReceiveTimeoutReturnsNoSocket() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + CloseFinally guard(&zh); + + zh=zookeeper_init("127.0.0.1:2121",watcher,10000,TEST_CLIENT_ID,0,0); + CPPUNIT_ASSERT(zh!=0); + forceConnected(zh, &timeMock.tv); + + // Exceed two thirds of the session timeout with no reply. + timeMock.tick(7); + int fd=0; + int interest=0; + timeval tv; + int rc=zookeeper_interest(zh,&fd,&interest,&tv); + CPPUNIT_ASSERT_EQUAL((int)ZOPERATIONTIMEOUT,rc); + CPPUNIT_ASSERT_EQUAL(-1,fd); + CPPUNIT_ASSERT_EQUAL(0,interest); + CPPUNIT_ASSERT_EQUAL(-1,zh->fd->sock); + } #else + // Checks for a read-only session with the given server. + struct ReadOnlySessionWith{ + ReadOnlySessionWith(zhandle_t* zh,const char* server): + zh_(zh),server_(server){} + bool operator()() const{ + return zoo_state(zh_)==ZOO_READONLY_STATE && + server_==zoo_get_current_server(zh_); + } + zhandle_t* zh_; + string server_; + }; + + // Runs one synchronous get request. + class SyncGetJob: public TestJob{ + public: + SyncGetJob(zhandle_t* zh):zh_(zh),done_(false),rc_(ZAPIERROR){} + virtual TestJob* clone() const { return new SyncGetJob(zh_); } + virtual void run(){ + char buf[1]; + int size=sizeof(buf); + int rc=zoo_get(zh_,"/a",0,buf,&size,0); + synchronized(mx_); + rc_=rc; + done_=true; + } + virtual void validate(const char* file, int line) const{} + bool operator()() const{ + synchronized(mx_); + return done_; + } + int rc() const{ + synchronized(mx_); + return rc_; + } + zhandle_t* zh_; + mutable Mutex mx_; + bool done_; + int rc_; + }; + + // Counts the get requests that reach the server. + class CountingProbeServer: public ReadWriteProbeServer{ + public: + virtual void onMessageReceived(const RequestHeader& rh, iarchive* ia){ + if(rh.type==ZOO_GETDATA_OP) + ++getRequests; + } + AtomicInt getRequests; + }; + + // Checks that a counter reaches a value. + struct CountReaches{ + CountReaches(const AtomicInt& n,int value):n_(n),value_(value){} + bool operator()() const{ return n_.get()>=value_; } + const AtomicInt& n_; + int value_; + }; + + // A synchronous call that waits during the switch to a read/write + // server must return ZCONNECTIONLOSS. The client then connects to the + // read/write server. + void testReadOnlyToReadWriteFailsSyncCall() + { + // The clock is frozen, so the probe runs only when the test moves it. + Mock_gettimeofday timeMock; + CountingProbeServer zkServer; + zkServer.readOnlyHandshake=true; + Mock_poll pollMock(&zkServer,ZookeeperServer::FD); + CloseFinally guard(&zh); + + zoo_deterministic_conn_order(1); + zh=zookeeper_init("127.0.0.1:2121,127.0.0.1:2122",watcher,10000, + TEST_CLIENT_ID,0,ZOO_READONLY); + CPPUNIT_ASSERT(zh!=0); + CPPUNIT_ASSERT(ensureCondition( + ReadOnlySessionWith(zh,"127.0.0.1:2121"),1000)<1000); + + // The server does not reply to the request. + SyncGetJob job(zh); + CloseThenJoin closeThenJoin(guard,job); + job.start(); + CPPUNIT_ASSERT(ensureCondition( + CountReaches(zkServer.getRequests,1),1000)<1000); + CPPUNIT_ASSERT(!job()); + + // The first probe of a new session goes to the current server, + // which is read-only. The next probe finds the read/write server. + zkServer.rwPort=2122; + zkServer.probeEnabled=true; + timeMock.tick(1); + CPPUNIT_ASSERT(ensureCondition( + CountReaches(zkServer.probeCloseCount,1),1000)<1000); + CPPUNIT_ASSERT(!job()); + timeMock.tick(1); + CPPUNIT_ASSERT(ensureCondition(job,1000)<1000); + job.join(); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS,job.rc()); + CPPUNIT_ASSERT_EQUAL(2,(int)zkServer.probeCount); + + CPPUNIT_ASSERT(ensureCondition( + ReadOnlySessionWith(zh,"127.0.0.1:2122"),1000)<1000); + } + class TestGetDataJob: public TestJob{ public: TestGetDataJob(ZookeeperServer* svr,zhandle_t* zh, int reps=500) diff --git a/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h b/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h index 45791fd2f59..2ed01a7faf4 100644 --- a/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h +++ b/zookeeper-client/zookeeper-client-c/tests/ThreadingUtil.h @@ -189,6 +189,8 @@ class TestJob { pthread_join(thread_,0); else pthread_detach(thread_); + // A thread can be joined only once, so later calls do nothing. + hasRun_=false; } private: void awaitStart() {