diff --git a/zookeeper-client/zookeeper-client-c/CMakeLists.txt b/zookeeper-client/zookeeper-client-c/CMakeLists.txt index 317059a8471..96b8ed3a99a 100644 --- a/zookeeper-client/zookeeper-client-c/CMakeLists.txt +++ b/zookeeper-client/zookeeper-client-c/CMakeLists.txt @@ -255,6 +255,7 @@ set(test_sources tests/TestClientRetry.cc tests/TestOperations.cc tests/TestMulti.cc + tests/TestMultiFailure.cc tests/TestWatchers.cc tests/TestClient.cc tests/ZooKeeperQuorumServer.cc diff --git a/zookeeper-client/zookeeper-client-c/Makefile.am b/zookeeper-client/zookeeper-client-c/Makefile.am index 7372291d79f..1a0051dcf05 100644 --- a/zookeeper-client/zookeeper-client-c/Makefile.am +++ b/zookeeper-client/zookeeper-client-c/Makefile.am @@ -114,6 +114,7 @@ TEST_SOURCES = \ tests/TestClientRetry.cc \ tests/TestOperations.cc \ tests/TestMulti.cc \ + tests/TestMultiFailure.cc \ tests/TestWatchers.cc \ tests/TestClient.cc \ tests/ZooKeeperQuorumServer.cc \ diff --git a/zookeeper-client/zookeeper-client-c/include/zookeeper.h b/zookeeper-client/zookeeper-client-c/include/zookeeper.h index 1a3f5d1f92f..b704dadb704 100644 --- a/zookeeper-client/zookeeper-client-c/include/zookeeper.h +++ b/zookeeper-client/zookeeper-client-c/include/zookeeper.h @@ -1815,9 +1815,14 @@ ZOOAPI int zoo_aset_acl(zhandle_t *zh, const char *path, int version, * ops supported by a multi op (see \ref zoo_acreate, \ref zoo_adelete, \ref zoo_aset). * \param data the data that will be passed to the completion routine when * the function completes. - * \return the return code for the function call. This can be any of the - * values that can be returned by the ops supported by a multi op (see - * \ref zoo_acreate, \ref zoo_adelete, \ref zoo_aset). + * \return ZOK on success or one of the following errcodes on failure: + * ZBADARGUMENTS - invalid input parameters + * ZINVALIDSTATE - zhandle state is either ZOO_SESSION_EXPIRED_STATE or ZOO_AUTH_FAILED_STATE + * ZMARSHALLINGERROR - failed to marshall a request; possibly, out of memory + * ZUNIMPLEMENTED - an operation type is not supported in a multi + * ZSYSTEMERROR - out of memory + * If the call fails, every result slot holds the return code and the + * completion is not called. */ ZOOAPI int zoo_amulti(zhandle_t *zh, int count, const zoo_op_t *ops, zoo_op_result_t *results, void_completion_t, const void *data); @@ -2583,7 +2588,9 @@ ZOOAPI int zoo_set_acl(zhandle_t *zh, const char *path, int version, * \param results an array to hold the results of the operations * \return the return code for the function call. This can be any of the * values that can be returned by the ops supported by a multi op (see - * \ref zoo_acreate, \ref zoo_adelete, \ref zoo_aset). + * \ref zoo_acreate, \ref zoo_adelete, \ref zoo_aset), or one of the + * call errors of \ref zoo_amulti. If the multi fails as a whole, for + * example on connection loss, every result slot holds an error code. */ ZOOAPI int zoo_multi(zhandle_t *zh, int count, const zoo_op_t *ops, zoo_op_result_t *results); diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index c8429003b29..ca29e753ffa 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -252,6 +252,7 @@ static const char* format_endpoint_info(const struct sockaddr_storage* ep); /* deserialize forward declarations */ static void deserialize_response(zhandle_t *zh, int type, int xid, int failed, int rc, completion_list_t *cptr, struct iarchive *ia); static int deserialize_multi(zhandle_t *zh, int xid, completion_list_t *cptr, struct iarchive *ia); +static void cleanup_failed_multi(zhandle_t *zh, int xid, int rc, completion_list_t *cptr); /* completion routine forward declarations */ static int add_completion(zhandle_t *zh, int xid, int completion_type, @@ -2035,6 +2036,13 @@ void free_completions(zhandle_t *zh,int callCompletion,int reason) struct sync_completion *sc = (struct sync_completion*)cptr->data; sc->rc = reason; + /* + * The caller reads the result array of a multi after the + * wait ends. Fill every slot with the failure code first. + */ + if (cptr->c.type == COMPLETION_MULTI) { + cleanup_failed_multi(zh, cptr->xid, reason, cptr); + } notify_sync_completion(sc); zh->outstanding_sync--; destroy_completion_entry(cptr); @@ -3306,16 +3314,47 @@ static void cleanup_failed_multi(zhandle_t *zh, int xid, int rc, completion_list } } +/* + * Deliver a Create2Response to a string completion. The server answers + * container creates in a multi with a path and a stat. The sub-operation + * result has no place for the stat, so this function discards it. + */ +static void deserialize_create2_to_string(zhandle_t *zh, int rc, + completion_list_t *cptr, struct iarchive *ia) +{ + struct Create2Response res; + const char *client_path; + memset(&res, 0, sizeof(res)); + deserialize_Create2Response(ia, "reply", &res); + client_path = sub_string(zh, res.path); + cptr->c.string_result(rc, client_path, cptr->data); + free_duplicate_path(client_path, res.path); + deallocate_Create2Response(&res); +} + +/* + * Decode the sub-results of a multi reply. The completion type of each + * sub-operation selects its record format. Two wire types are exceptions. + * An error result has an ErrorResponse, and a container create has a + * Create2Response. Every sub-operation gets a result. This is also true + * when the reply does not match the request. + */ static int deserialize_multi(zhandle_t *zh, int xid, completion_list_t *cptr, struct iarchive *ia) { int rc = 0; completion_head_t *clist = &cptr->c.clist; struct MultiHeader mhdr = {0, 0, 0}; assert(clist); - deserialize_MultiHeader(ia, "multiheader", &mhdr); + if (deserialize_MultiHeader(ia, "multiheader", &mhdr) < 0) { + mhdr.done = 1; + } while (!mhdr.done) { completion_list_t *entry = dequeue_completion_nolock(clist); - assert(entry); + if (entry == NULL) { + LOG_ERROR(LOGCALLBACK(zh), + "Multi reply xid=%#x has more results than operations", xid); + return ZMARSHALLINGERROR; + } if (mhdr.type == -1) { struct ErrorResponse er; @@ -3326,14 +3365,27 @@ static int deserialize_multi(zhandle_t *zh, int xid, completion_list_t *cptr, st } } - deserialize_response(zh, entry->c.type, xid, mhdr.type == -1, mhdr.err, entry, ia); - deserialize_MultiHeader(ia, "multiheader", &mhdr); + if (mhdr.type == ZOO_CREATE2_OP && entry->c.type == COMPLETION_STRING) { + deserialize_create2_to_string(zh, mhdr.err, entry, ia); + } else { + deserialize_response(zh, entry->c.type, xid, mhdr.type == -1, mhdr.err, entry, ia); + } + if (deserialize_MultiHeader(ia, "multiheader", &mhdr) < 0) { + mhdr.done = 1; + } //While deserializing the response we must destroy completion entry for each operation in //the zoo_multi transaction. Otherwise this results in memory leak when client invokes zoo_multi //operation. destroy_completion_entry(entry); } + if (clist->head != NULL) { + LOG_ERROR(LOGCALLBACK(zh), + "Multi reply xid=%#x has fewer results than operations", xid); + cleanup_failed_multi(zh, xid, ZMARSHALLINGERROR, cptr); + rc = ZMARSHALLINGERROR; + } + return rc; } @@ -3880,7 +3932,11 @@ static int do_add_completion(zhandle_t *zh, const void *dc, if (!c) return ZSYSTEMERROR; lock_completion_list(&zh->sent_requests); - if (zh->close_requested != 1) { + /* + * Refuse new requests on a closing or unrecoverable handle. No later + * teardown drains the queue, so a synchronous caller waits forever. + */ + if (zh->close_requested != 1 && !is_unrecoverable(zh)) { queue_completion_nolock(&zh->sent_requests, c, add_to_front); if (dc == SYNCHRONOUS_MARKER) { zh->outstanding_sync++; @@ -4830,6 +4886,23 @@ static void op_result_stat_completion(int err, const struct Stat *stat, const vo } } +/* + * Fail every result slot of a multi that zoo_amulti() does not submit. + * No reply arrives for such a multi, so the slots get the return code. + */ +static void fail_multi_results(int count, zoo_op_result_t *results, int rc) +{ + int i; + if (results == NULL) { + return; + } + for (i = 0; i < count; i++) { + results[i].err = rc; + results[i].value = NULL; + results[i].stat = NULL; + } +} + static int CheckVersionRequest_init(zhandle_t *zh, struct CheckVersionRequest *req, const char *path, int version) { @@ -4856,6 +4929,11 @@ int zoo_amulti(zhandle_t *zh, int count, const zoo_op_t *ops, int rc = serialize_RequestHeader(oa, "header", &h); int index = 0; + if (count > 0 && (ops == NULL || results == NULL)) { + close_buffer_oarchive(&oa, 1); + fail_multi_results(count, results, ZBADARGUMENTS); + return ZBADARGUMENTS; + } for (index=0; index < count; index++) { const zoo_op_t *op = ops+index; zoo_op_result_t *result = results+index; @@ -4973,10 +5051,18 @@ int zoo_amulti(zhandle_t *zh, int count, const zoo_op_t *ops, LOG_DEBUG(LOGCALLBACK(zh), "Sending multi request xid=%#x with %d subrequests to %s", h.xid, index, zoo_get_current_server(zh)); - if (rc == ZUNIMPLEMENTED) { - return rc; + /* + * A failed call-time check returns its own code. A serializer error + * becomes ZMARSHALLINGERROR. + */ + if (rc != ZUNIMPLEMENTED && rc != ZBADARGUMENTS && rc != ZINVALIDSTATE + && rc != ZSYSTEMERROR) { + rc = nonblocking_send(zh, rc); } - return nonblocking_send(zh, rc); + if (rc != ZOK) { + fail_multi_results(count, results, rc); + } + return rc; } typedef union WatchesRequest WatchesRequest; @@ -5528,7 +5614,15 @@ static void process_sync_completion(zhandle_t *zh, case COMPLETION_VOID: break; case COMPLETION_MULTI: - sc->rc = deserialize_multi(zh, cptr->xid, cptr, ia); + /* + * A reply with an error in its header has no multi body. Fail + * every sub-operation with the header error. + */ + if (sc->rc == 0) { + sc->rc = deserialize_multi(zh, cptr->xid, cptr, ia); + } else { + cleanup_failed_multi(zh, cptr->xid, sc->rc, cptr); + } break; default: LOG_DEBUG(LOGCALLBACK(zh), "Unsupported completion type=%d", cptr->c.type); diff --git a/zookeeper-client/zookeeper-client-c/tests/TestMulti.cc b/zookeeper-client/zookeeper-client-c/tests/TestMulti.cc index 6dfeb96e0c0..025b16b887a 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestMulti.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestMulti.cc @@ -167,6 +167,7 @@ class Zookeeper_multi : public CPPUNIT_NS::TestFixture //flaw in the test suite setup. CPPUNIT_TEST(testCreate); CPPUNIT_TEST(testCreateDelete); + CPPUNIT_TEST(testCreateContainer); CPPUNIT_TEST(testInvalidVersion); CPPUNIT_TEST(testNestedCreate); CPPUNIT_TEST(testSetData); @@ -329,6 +330,48 @@ class Zookeeper_multi : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZNONODE, rc); } + /** + * Test container create followed by other operations. The server + * answers a container create with a path and a stat. The client must + * decode the sub-results that follow it correctly. + */ + void testCreateContainer() { + int rc; + watchctx_t ctx; + zhandle_t *zk = createClient(&ctx); + int sz = 512; + char p1[sz]; + char p2[sz]; + p1[0] = p2[0] = '\0'; + struct Stat s1; + memset(&s1, 0, sizeof(s1)); + int nops = 4; + zoo_op_t ops[nops]; + zoo_op_result_t results[nops]; + + zoo_create_op_init(&ops[0], "/multiContainer", "", 0, + &ZOO_OPEN_ACL_UNSAFE, ZOO_CONTAINER, p1, sz); + zoo_create_op_init(&ops[1], "/multiContainer/a", "", 0, + &ZOO_OPEN_ACL_UNSAFE, 0, p2, sz); + zoo_set_op_init(&ops[2], "/multiContainer/a", "x", 1, 0, &s1); + zoo_check_op_init(&ops[3], "/multiContainer", 0); + + rc = zoo_multi(zk, nops, ops, results); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + for (int i = 0; i < nops; i++) { + CPPUNIT_ASSERT_EQUAL((int)ZOK, results[i].err); + } + CPPUNIT_ASSERT(strcmp(p1, "/multiContainer") == 0); + CPPUNIT_ASSERT(strcmp(p2, "/multiContainer/a") == 0); + CPPUNIT_ASSERT_EQUAL(1, s1.version); + CPPUNIT_ASSERT_EQUAL(1, s1.dataLength); + + zoo_delete_op_init(&ops[0], "/multiContainer/a", -1); + zoo_delete_op_init(&ops[1], "/multiContainer", -1); + rc = zoo_multi(zk, 2, ops, results); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + } + /** * Test invalid versions */ diff --git a/zookeeper-client/zookeeper-client-c/tests/TestMultiFailure.cc b/zookeeper-client/zookeeper-client-c/tests/TestMultiFailure.cc new file mode 100644 index 00000000000..c2eaa8544ea --- /dev/null +++ b/zookeeper-client/zookeeper-client-c/tests/TestMultiFailure.cc @@ -0,0 +1,538 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "CppAssertHelper.h" + +#include +#include + +#include "ZKMocks.h" +#include + +using namespace std; + +/* A sentinel error code. No result slot holds it after a multi completes. */ +static const int SENTINEL_ERR = 0x5a5a5a5a; + +static Stat makeStat(int32_t version) +{ + Stat stat; + memset(&stat, 0, sizeof(stat)); + stat.czxid = 0x1111; + stat.mzxid = 0x2222; + stat.version = version; + stat.dataLength = 1; + stat.pzxid = 0x3333; + return stat; +} + +static void fillSentinel(zoo_op_result_t *results, int count) +{ + for (int i = 0; i < count; i++) { + results[i].err = SENTINEL_ERR; + } +} + +/* A reply that has only a ReplyHeader. Servers send such a reply when they + * reject a request as a whole, for example ZNOTREADONLY. */ +class HeaderOnlyResponse: public Response +{ +public: + HeaderOnlyResponse(int rc): xid_(0), rc_(rc) {} + virtual void setXID(int32_t xid) { xid_ = xid; } + virtual string toString() const { + oarchive *oa = create_buffer_oarchive(); + ReplyHeader h = {xid_, 1, rc_}; + serialize_ReplyHeader(oa, "hdr", &h); + int32_t len = htonl(get_buffer_len(oa)); + string res((char*)&len, sizeof(len)); + res.append(get_buffer(oa), get_buffer_len(oa)); + close_buffer_oarchive(&oa, 1); + return res; + } +private: + int32_t xid_; + int rc_; +}; + +/* The reply that a server sends for a multi with a container create and a + * set data. The server answers the container create with a Create2Response, + * which has a path and a stat. */ +class ContainerMultiResponse: public Response +{ +public: + ContainerMultiResponse(const char *path, const Stat& createStat, + const Stat& setStat) + : xid_(0), path_(path), createStat_(createStat), setStat_(setStat) {} + virtual void setXID(int32_t xid) { xid_ = xid; } + virtual string toString() const { + oarchive *oa = create_buffer_oarchive(); + ReplyHeader h = {xid_, 1, ZOK}; + serialize_ReplyHeader(oa, "hdr", &h); + + MultiHeader create = {ZOO_CREATE2_OP, 0, 0}; + serialize_MultiHeader(oa, "multiheader", &create); + Create2Response cr; + cr.path = (char*)path_.c_str(); + cr.stat = createStat_; + serialize_Create2Response(oa, "reply", &cr); + + MultiHeader set = {ZOO_SETDATA_OP, 0, 0}; + serialize_MultiHeader(oa, "multiheader", &set); + SetDataResponse sr; + sr.stat = setStat_; + serialize_SetDataResponse(oa, "reply", &sr); + + MultiHeader done = {-1, 1, -1}; + serialize_MultiHeader(oa, "multiheader", &done); + + int32_t len = htonl(get_buffer_len(oa)); + string res((char*)&len, sizeof(len)); + res.append(get_buffer(oa), get_buffer_len(oa)); + close_buffer_oarchive(&oa, 1); + return res; + } +private: + int32_t xid_; + string path_; + Stat createStat_; + Stat setStat_; +}; + +/* A multi reply with a number of successful check results. A check result + * has a MultiHeader and no body. */ +class CheckMultiResponse: public Response +{ +public: + CheckMultiResponse(int count): xid_(0), count_(count) {} + virtual void setXID(int32_t xid) { xid_ = xid; } + virtual string toString() const { + oarchive *oa = create_buffer_oarchive(); + ReplyHeader h = {xid_, 1, ZOK}; + serialize_ReplyHeader(oa, "hdr", &h); + for (int i = 0; i < count_; i++) { + MultiHeader check = {ZOO_CHECK_OP, 0, 0}; + serialize_MultiHeader(oa, "multiheader", &check); + } + MultiHeader done = {-1, 1, -1}; + serialize_MultiHeader(oa, "multiheader", &done); + + int32_t len = htonl(get_buffer_len(oa)); + string res((char*)&len, sizeof(len)); + res.append(get_buffer(oa), get_buffer_len(oa)); + close_buffer_oarchive(&oa, 1); + return res; + } +private: + int32_t xid_; + int count_; +}; + +#ifdef THREADED +/* A mock server that drops the connection when a multi request arrives. The + * multi is then in flight when the connection fails. */ +class DropOnMultiServer: public ZookeeperServer +{ +public: + virtual void onMessageReceived(const RequestHeader& rh, iarchive*) { + if (rh.type == ZOO_MULTI_OP) { + setServerDown(0); + setConnectionLost(); + } + } +}; + +/* Runs zoo_multi() on its own thread, so that a hang fails the test and does + * not block the test run. */ +struct MultiCall { + zhandle_t *zh; + int count; + const zoo_op_t *ops; + zoo_op_result_t *results; + int rc; + AtomicInt done; + pthread_t thread; + + static void *run(void *arg) { + MultiCall *call = (MultiCall*)arg; + call->rc = zoo_multi(call->zh, call->count, call->ops, call->results); + call->done = 1; + return 0; + } + void start() { + rc = ZAPIERROR; + done = 0; + pthread_create(&thread, 0, run, this); + } + bool operator()() const { return done.get() != 0; } +}; + +/* Records the result of an asynchronous multi. */ +struct AsyncMulti { + int rc; + AtomicInt done; + + AsyncMulti(): rc(ZAPIERROR) { done = 0; } + static void completion(int rc, const void *data) { + AsyncMulti *m = (AsyncMulti*)data; + m->rc = rc; + m->done = 1; + } + bool operator()() const { return done.get() != 0; } +}; +#endif + +class Zookeeper_multiFailure : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE(Zookeeper_multiFailure); + CPPUNIT_TEST(testValidationFailureFillsResults); + CPPUNIT_TEST(testNullArguments); +#ifdef THREADED + CPPUNIT_TEST(testConnectionLossFillsResults); + CPPUNIT_TEST(testHeaderOnlyErrorReply); + CPPUNIT_TEST(testContainerCreateKeepsAlignment); + CPPUNIT_TEST(testAsyncContainerCreate); + CPPUNIT_TEST(testMoreResultsThanOperations); + CPPUNIT_TEST(testFewerResultsThanOperations); + CPPUNIT_TEST(testZeroCountOnExpiredHandle); + CPPUNIT_TEST(testMultiOnExpiredHandle); +#endif + CPPUNIT_TEST_SUITE_END(); + + zhandle_t *zh; + FILE *logfile; + + static void watcher(zhandle_t *, int, int, const char *, void *) {} + +public: + Zookeeper_multiFailure() { + logfile = openlogfile("Zookeeper_multiFailure"); + } + + ~Zookeeper_multiFailure() { + if (logfile) { + fflush(logfile); + fclose(logfile); + logfile = 0; + } + } + + void setUp() + { + zoo_set_log_stream(logfile); + zoo_deterministic_conn_order(0); + zh = 0; + } + + void tearDown() + { + zookeeper_close(zh); + } + + /* zoo_amulti() rejects a multi with an invalid path with ZBADARGUMENTS + * before it queues the request. Every result slot then holds that code. + * This includes the slots after the invalid operation. */ + void testValidationFailureFillsResults() + { + zh = zookeeper_init("localhost:2121", watcher, 10000, 0, 0, 0); + CPPUNIT_ASSERT(zh != 0); + + char buf[64]; + zoo_op_t ops[3]; + zoo_op_result_t results[3]; + zoo_create_op_init(&ops[0], "/a", "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, + buf, sizeof(buf)); + zoo_delete_op_init(&ops[1], "no-leading-slash", -1); + zoo_check_op_init(&ops[2], "/c", 0); + fillSentinel(results, 3); + + int rc = zoo_amulti(zh, 3, ops, results, 0, 0); + CPPUNIT_ASSERT_EQUAL((int)ZBADARGUMENTS, rc); + for (int i = 0; i < 3; i++) { + CPPUNIT_ASSERT_EQUAL(rc, results[i].err); + } + CPPUNIT_ASSERT(results[0].value == NULL); + } + + /* NULL operation or result arrays give ZBADARGUMENTS and do not crash. */ + void testNullArguments() + { + zh = zookeeper_init("localhost:2121", watcher, 10000, 0, 0, 0); + CPPUNIT_ASSERT(zh != 0); + + zoo_op_t ops[1]; + zoo_op_result_t results[1]; + zoo_check_op_init(&ops[0], "/c", 0); + + fillSentinel(results, 1); + CPPUNIT_ASSERT_EQUAL((int)ZBADARGUMENTS, + zoo_amulti(zh, 1, NULL, results, 0, 0)); + CPPUNIT_ASSERT_EQUAL((int)ZBADARGUMENTS, results[0].err); + + CPPUNIT_ASSERT_EQUAL((int)ZBADARGUMENTS, + zoo_amulti(zh, 1, ops, NULL, 0, 0)); + } + +#ifdef THREADED + /* A synchronous multi in flight when the connection fails returns + * ZCONNECTIONLOSS. Every result slot also holds ZCONNECTIONLOSS. */ + void testConnectionLossFillsResults() + { + Mock_gettimeofday timeMock; + DropOnMultiServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + + char buf[64]; + Stat stat; + zoo_op_t ops[3]; + zoo_op_result_t results[3]; + zoo_create_op_init(&ops[0], "/a", "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, + buf, sizeof(buf)); + zoo_set_op_init(&ops[1], "/b", "x", 1, -1, &stat); + zoo_delete_op_init(&ops[2], "/c", -1); + fillSentinel(results, 3); + + int rc = zoo_multi(zh, 3, ops, results); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS, rc); + for (int i = 0; i < 3; i++) { + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS, results[i].err); + } + CPPUNIT_ASSERT(results[0].value == NULL); + CPPUNIT_ASSERT(results[1].stat == NULL); + } + + /* A read-only server answers a write multi with a ZNOTREADONLY header + * and no body. The synchronous multi returns that error in every slot. */ + void testHeaderOnlyErrorReply() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + + zoo_op_t ops[2]; + zoo_op_result_t results[2]; + zoo_delete_op_init(&ops[0], "/a", -1); + zoo_check_op_init(&ops[1], "/b", 0); + fillSentinel(results, 2); + + zkServer.addOperationResponse(new HeaderOnlyResponse(ZNOTREADONLY)); + int rc = zoo_multi(zh, 2, ops, results); + CPPUNIT_ASSERT_EQUAL((int)ZNOTREADONLY, rc); + for (int i = 0; i < 2; i++) { + CPPUNIT_ASSERT_EQUAL((int)ZNOTREADONLY, results[i].err); + } + CPPUNIT_ASSERT(ClientConnected(zh)()); + } + + /* The Create2Response of a container create has a stat. The client must + * consume the stat, so that it decodes the next sub-result correctly. */ + void testContainerCreateKeepsAlignment() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + + char buf[64]; + Stat setStat; + memset(&setStat, 0, sizeof(setStat)); + zoo_op_t ops[2]; + zoo_op_result_t results[2]; + zoo_create_op_init(&ops[0], "/container", "", 0, &ZOO_OPEN_ACL_UNSAFE, + ZOO_CONTAINER, buf, sizeof(buf)); + zoo_set_op_init(&ops[1], "/container", "x", 1, -1, &setStat); + CPPUNIT_ASSERT_EQUAL((int)ZOO_CREATE_CONTAINER_OP, ops[0].type); + fillSentinel(results, 2); + + zkServer.addOperationResponse(new ContainerMultiResponse("/container", + makeStat(0), makeStat(7))); + int rc = zoo_multi(zh, 2, ops, results); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL((int)ZOK, results[0].err); + CPPUNIT_ASSERT_EQUAL(string("/container"), string(buf)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, results[1].err); + CPPUNIT_ASSERT_EQUAL(7, setStat.version); + CPPUNIT_ASSERT_EQUAL((int64_t)0x2222, setStat.mzxid); + } + + /* The asynchronous multi decodes a container create in the same way. */ + void testAsyncContainerCreate() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + + char buf[64]; + Stat setStat; + memset(&setStat, 0, sizeof(setStat)); + zoo_op_t ops[2]; + zoo_op_result_t results[2]; + zoo_create_op_init(&ops[0], "/container", "", 0, &ZOO_OPEN_ACL_UNSAFE, + ZOO_CONTAINER, buf, sizeof(buf)); + zoo_set_op_init(&ops[1], "/container", "x", 1, -1, &setStat); + fillSentinel(results, 2); + + AsyncMulti multi; + zkServer.addOperationResponse(new ContainerMultiResponse("/container", + makeStat(0), makeStat(7))); + CPPUNIT_ASSERT_EQUAL((int)ZOK, zoo_amulti(zh, 2, ops, results, + AsyncMulti::completion, &multi)); + CPPUNIT_ASSERT(ensureCondition(multi, 1000) < 1000); + CPPUNIT_ASSERT_EQUAL((int)ZOK, multi.rc); + CPPUNIT_ASSERT_EQUAL((int)ZOK, results[0].err); + CPPUNIT_ASSERT_EQUAL(string("/container"), string(buf)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, results[1].err); + CPPUNIT_ASSERT_EQUAL(7, setStat.version); + } + + /* A reply with more sub-results than operations fails the multi with + * ZMARSHALLINGERROR. The operations keep the results that they got. */ + void testMoreResultsThanOperations() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + + zoo_op_t ops[2]; + zoo_op_result_t results[2]; + zoo_check_op_init(&ops[0], "/a", 0); + zoo_check_op_init(&ops[1], "/b", 0); + fillSentinel(results, 2); + + zkServer.addOperationResponse(new CheckMultiResponse(3)); + int rc = zoo_multi(zh, 2, ops, results); + CPPUNIT_ASSERT_EQUAL((int)ZMARSHALLINGERROR, rc); + for (int i = 0; i < 2; i++) { + CPPUNIT_ASSERT_EQUAL((int)ZOK, results[i].err); + } + } + + /* A reply with fewer sub-results than operations fails the multi with + * ZMARSHALLINGERROR. The operations without a result get that code. */ + void testFewerResultsThanOperations() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + + zoo_op_t ops[3]; + zoo_op_result_t results[3]; + zoo_check_op_init(&ops[0], "/a", 0); + zoo_check_op_init(&ops[1], "/b", 0); + zoo_check_op_init(&ops[2], "/c", 0); + fillSentinel(results, 3); + + zkServer.addOperationResponse(new CheckMultiResponse(1)); + int rc = zoo_multi(zh, 3, ops, results); + CPPUNIT_ASSERT_EQUAL((int)ZMARSHALLINGERROR, rc); + CPPUNIT_ASSERT_EQUAL((int)ZOK, results[0].err); + CPPUNIT_ASSERT_EQUAL((int)ZMARSHALLINGERROR, results[1].err); + CPPUNIT_ASSERT_EQUAL((int)ZMARSHALLINGERROR, results[2].err); + } + + void expireSession(ZookeeperServer& zkServer) + { + zkServer.returnSessionExpired(); + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(SessionExpired(zh), 1000) < 1000); + } + + /* A zero-count multi on an expired handle returns an error at once. It + * must not wait for a reply that no I/O thread delivers. */ + void testZeroCountOnExpiredHandle() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + expireSession(zkServer); + + MultiCall call; + call.zh = zh; + call.count = 0; + call.ops = NULL; + call.results = NULL; + call.start(); + bool returned = ensureCondition(call, 1000) < 1000; + if (!returned) { + /* Close the handle to wake the waiting thread. */ + zookeeper_close(zh); + zh = 0; + } + pthread_join(call.thread, 0); + CPPUNIT_ASSERT_MESSAGE("zoo_multi() hangs on an expired handle", returned); + CPPUNIT_ASSERT_EQUAL((int)ZINVALIDSTATE, call.rc); + } + + /* A multi with operations on an expired handle also fails at once with + * ZINVALIDSTATE, and every result slot holds that code. */ + void testMultiOnExpiredHandle() + { + Mock_gettimeofday timeMock; + ZookeeperServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + expireSession(zkServer); + + zoo_op_t ops[2]; + zoo_op_result_t results[2]; + zoo_delete_op_init(&ops[0], "/a", -1); + zoo_check_op_init(&ops[1], "/b", 0); + fillSentinel(results, 2); + + int rc = zoo_multi(zh, 2, ops, results); + CPPUNIT_ASSERT_EQUAL((int)ZINVALIDSTATE, rc); + for (int i = 0; i < 2; i++) { + CPPUNIT_ASSERT_EQUAL(rc, results[i].err); + } + } +#endif +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(Zookeeper_multiFailure);