diff --git a/zookeeper-client/zookeeper-client-c/CMakeLists.txt b/zookeeper-client/zookeeper-client-c/CMakeLists.txt index 317059a8471..2e3afeac231 100644 --- a/zookeeper-client/zookeeper-client-c/CMakeLists.txt +++ b/zookeeper-client/zookeeper-client-c/CMakeLists.txt @@ -249,6 +249,7 @@ set(test_sources tests/ThreadingUtil.cc tests/TestZookeeperInit.cc tests/TestZookeeperClose.cc + tests/TestAuthCompletion.cc tests/TestSASLAuth.cc tests/TestReconfig.cc tests/TestReconfigServer.cc diff --git a/zookeeper-client/zookeeper-client-c/Makefile.am b/zookeeper-client/zookeeper-client-c/Makefile.am index 7372291d79f..4b86905f221 100644 --- a/zookeeper-client/zookeeper-client-c/Makefile.am +++ b/zookeeper-client/zookeeper-client-c/Makefile.am @@ -109,6 +109,7 @@ TEST_SOURCES = \ tests/ThreadingUtil.cc \ tests/TestZookeeperInit.cc \ tests/TestZookeeperClose.cc \ + tests/TestAuthCompletion.cc \ tests/TestReconfig.cc \ tests/TestReconfigServer.cc \ tests/TestClientRetry.cc \ diff --git a/zookeeper-client/zookeeper-client-c/include/zookeeper.h b/zookeeper-client/zookeeper-client-c/include/zookeeper.h index 1a3f5d1f92f..1130ecf79cd 100644 --- a/zookeeper-client/zookeeper-client-c/include/zookeeper.h +++ b/zookeeper-client/zookeeper-client-c/include/zookeeper.h @@ -1849,6 +1849,11 @@ ZOOAPI const char* zerror(int c); * the following result codes may be passed into the completion callback: * ZOK operation completed successfully * ZAUTHFAILED authentication failed + * If the server rejects an earlier credential, a request that is still + * pending also receives ZAUTHFAILED. If the connection closes before the + * reply, the completion receives the error that closed it, for example + * ZCONNECTIONLOSS. The client keeps the credential and sends it again on the + * next connection. That resend has no completion. * \param data the data that will be passed to the completion routine when the * function completes. * \return ZOK on success or one of the following errcodes on failure: diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index 25049bb335e..cd79847541c 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -115,7 +115,6 @@ struct sync_completion { }; typedef struct _auth_info { - int state; /* 0=>inactive, >0 => active */ char* scheme; struct buffer auth; void_completion_t completion; @@ -177,9 +176,21 @@ struct adaptor_threads { }; #endif +/** + * Outstanding SetAuth request. The list is a FIFO that matches the order of + * frames on the wire. Each reply completes the head entry. + */ +typedef struct _auth_request { + auth_info *auth; /* credential carried by this frame */ + void_completion_t completion; /* NULL on a reconnect resend */ + const char *data; + struct _auth_request *next; +} auth_request_t; + /** the auth list for adding auth */ typedef struct _auth_list_head { auth_info *auth; + auth_request_t *pending; /* FIFO of SetAuth frames on the wire */ #ifdef THREADED pthread_mutex_t lock; #endif diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index c8429003b29..194291e8101 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -361,7 +361,7 @@ static int is_sasl_auth_in_progress(zhandle_t* zh) * Extract the type field (ZOO_*_OP) of a serialized RequestHeader. * * (This is not the most efficient way of fetching 4 bytes, but it is - * currently only used during SASL negotiation.) + * only used during SASL negotiation and session establishment.) * * \param buffer the buffer to extract the request type from. Must * start with a serialized RequestHeader; @@ -502,18 +502,6 @@ int zoo_recv_timeout(zhandle_t *zh) /** these functions are thread unsafe, so make sure that zoo_lock_auth is called before you access them **/ -static auth_info* get_last_auth(auth_list_head_t *auth_list) { - auth_info *element; - element = auth_list->auth; - if (element == NULL) { - return NULL; - } - while (element->next != NULL) { - element = element->next; - } - return element; -} - static void free_auth_completion(auth_completion_list_t *a_list) { auth_completion_list_t *tmp, *ftmp; if (a_list == NULL) { @@ -556,12 +544,22 @@ static void add_auth_completion(auth_completion_list_t* a_list, void_completion_ return; } +/* + * Collect completions that still wait for an auth result. Then clear the + * pending FIFO. The caller has dropped the connection that carried those + * frames. + */ static void get_auth_completions(auth_list_head_t *auth_list, auth_completion_list_t *a_list) { + auth_request_t *req; auth_info *element; - element = auth_list->auth; - if (element == NULL) { - return; + while ((req = auth_list->pending) != NULL) { + auth_list->pending = req->next; + if (req->completion) { + add_auth_completion(a_list, &req->completion, req->data); + } + free(req); } + element = auth_list->auth; while (element) { if (element->completion) { add_auth_completion(a_list, &element->completion, element->data); @@ -590,24 +588,17 @@ static void add_last_auth(auth_list_head_t *auth_list, auth_info *add_el) { static void init_auth_info(auth_list_head_t *auth_list) { auth_list->auth = NULL; -} - -static void mark_active_auth(zhandle_t *zh) { - auth_list_head_t auth_h = zh->auth_h; - auth_info *element; - if (auth_h.auth == NULL) { - return; - } - element = auth_h.auth; - while (element != NULL) { - element->state = 1; - element = element->next; - } + auth_list->pending = NULL; } static void free_auth_info(auth_list_head_t *auth_list) { auth_info *auth = auth_list->auth; + auth_request_t *req; + while ((req = auth_list->pending) != NULL) { + auth_list->pending = req->next; + free(req); + } while (auth != NULL) { auth_info* old_auth = NULL; if(auth->scheme!=NULL) @@ -2149,79 +2140,162 @@ static int handle_socket_error_msg(zhandle_t *zh, int line, const char *func, in return rc; } +/* + * Apply one SetAuth reply to the head of the pending FIFO. The server + * replies to SetAuth in receive order, so the head is the matching request. + * An empty FIFO means the reply is not for this handle. The client ignores + * it. The session stays connected. + */ static void auth_completion_func(int rc, zhandle_t* zh) { - void_completion_t auth_completion = NULL; - auth_completion_list_t a_list; - auth_completion_list_t *a_tmp; + auth_request_t *req; if(zh==NULL) return; zoo_lock_auth(zh); - + req = zh->auth_h.pending; + if (req == NULL) { + zoo_unlock_auth(zh); + LOG_WARN(LOGCALLBACK(zh), + "Ignoring an authentication reply (rc=%d) that has no pending request", + rc); + return; + } + zh->auth_h.pending = req->next; if(rc!=0){ zh->state=ZOO_AUTH_FAILED_STATE; - }else{ - //change state for all auths - mark_active_auth(zh); } - a_list.completion = NULL; - a_list.next = NULL; - get_auth_completions(&zh->auth_h, &a_list); zoo_unlock_auth(zh); if (rc) { LOG_ERROR(LOGCALLBACK(zh), "Authentication scheme %s failed. Connection closed.", - zh->auth_h.auth->scheme); + req->auth->scheme); } else { - LOG_INFO(LOGCALLBACK(zh), "Authentication scheme %s succeeded", zh->auth_h.auth->scheme); + LOG_INFO(LOGCALLBACK(zh), "Authentication scheme %s succeeded", req->auth->scheme); } - a_tmp = &a_list; - // chain call user's completion function - while (a_tmp->completion != NULL) { - auth_completion = a_tmp->completion; - auth_completion(rc, a_tmp->auth_data); - a_tmp = a_tmp->next; - if (a_tmp == NULL) - break; + if (req->completion) { + req->completion(rc, req->data); } - free_auth_completion(&a_list); + free(req); } -static int send_info_packet(zhandle_t *zh, auth_info* auth) { +/* + * Enqueue one SetAuth frame and its pending-FIFO entry as one locked + * operation. add_to_front must match the send-queue position. Then the FIFO + * stays in wire order. If the credential has a completion, that completion + * moves to the pending entry. + */ +static int send_info_packet(zhandle_t *zh, auth_info* auth, int add_to_front) { struct oarchive *oa; struct RequestHeader h = {AUTH_XID, ZOO_SETAUTH_OP}; struct AuthPacket req; + auth_request_t *ar; + auth_request_t **link; + buffer_list_t *b = NULL; int rc; + + ar = calloc(1, sizeof(*ar)); + if (ar == NULL) { + return ZSYSTEMERROR; + } oa = create_buffer_oarchive(); rc = serialize_RequestHeader(oa, "header", &h); req.type=0; // ignored by the server req.scheme = auth->scheme; req.auth = auth->auth; rc = rc < 0 ? rc : serialize_AuthPacket(oa, "req", &req); - /* add this buffer to the head of the send queue */ - rc = rc < 0 ? rc : queue_front_buffer_bytes(&zh->to_send, get_buffer(oa), - get_buffer_len(oa)); + if (rc >= 0) { + b = allocate_buffer(get_buffer(oa), get_buffer_len(oa)); + if (b == NULL) { + rc = ZSYSTEMERROR; + } + } /* Free serialized buffer if not enqueued */ close_buffer_oarchive(&oa, rc < 0); + if (rc < 0) { + free(ar); + return rc; + } - return rc; + ar->auth = auth; + ar->completion = auth->completion; + ar->data = auth->data; + auth->completion = NULL; + queue_buffer(&zh->to_send, b, add_to_front); + link = &zh->auth_h.pending; + if (!add_to_front) { + while (*link != NULL) { + link = &(*link)->next; + } + } + ar->next = *link; + *link = ar; + return ZOK; } -/** send all auths, not just the last one **/ +/* + * Drop unsent SetAuth frames from the send queue. This runs before a session + * resends credentials. A leftover frame would send the same credential twice, + * and its reply would have no FIFO entry. + */ +static void remove_auth_frames(buffer_head_t *list) +{ + buffer_list_t *b; + buffer_list_t *next; + buffer_list_t *last = NULL; + int32_t type; + + lock_buffer_list(list); + for (b = list->head; b != NULL; b = next) { + next = b->next; + if (extract_request_type(b->buffer, b->len, &type) == ZOK && + type == ZOO_SETAUTH_OP) { + if (last == NULL) { + list->head = next; + } else { + last->next = next; + } + free_buffer(b); + } else { + last = b; + } + } + list->last = last; + unlock_buffer_list(list); +} + +/* + * Resend every stored credential on a newly established session. First + * return pending completions to their credentials and drop leftover SetAuth + * frames. Then enqueue one new frame per credential. Any unsent completion + * goes with that frame. + */ static int send_auth_info(zhandle_t *zh) { - int rc = 0; + int rc = ZOK; auth_info *auth = NULL; + auth_request_t *req; zoo_lock_auth(zh); + while ((req = zh->auth_h.pending) != NULL) { + zh->auth_h.pending = req->next; + if (req->completion) { + req->auth->completion = req->completion; + req->auth->data = req->data; + } + free(req); + } + remove_auth_frames(&zh->to_send); auth = zh->auth_h.auth; if (auth == NULL) { zoo_unlock_auth(zh); return ZOK; } while (auth != NULL) { - rc = send_info_packet(zh, auth); + int auth_rc = send_info_packet(zh, auth, 1); + if (auth_rc < 0) { + rc = auth_rc; + } auth = auth->next; } zoo_unlock_auth(zh); @@ -2229,23 +2303,6 @@ static int send_auth_info(zhandle_t *zh) { return (rc <0) ? ZMARSHALLINGERROR:ZOK; } -static int send_last_auth_info(zhandle_t *zh) -{ - int rc = 0; - auth_info *auth = NULL; - - zoo_lock_auth(zh); - auth = get_last_auth(&zh->auth_h); - if(auth==NULL) { - zoo_unlock_auth(zh); - return ZOK; // there is nothing to send - } - rc = send_info_packet(zh, auth); - zoo_unlock_auth(zh); - LOG_DEBUG(LOGCALLBACK(zh), "Sending auth info request to %s",zoo_get_current_server(zh)); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; -} - static void free_key_list(char **list, int count) { int i; @@ -5304,6 +5361,8 @@ int zoo_add_auth(zhandle_t *zh,const char* scheme,const char* cert, { struct buffer auth; auth_info *authinfo; + int rc = ZOK; + int sent; if(scheme==NULL || zh==NULL) return ZBADARGUMENTS; @@ -5328,25 +5387,49 @@ int zoo_add_auth(zhandle_t *zh,const char* scheme,const char* cert, auth.len = 0; } - zoo_lock_auth(zh); - authinfo = (auth_info*) malloc(sizeof(auth_info)); + authinfo = (auth_info*) calloc(1, sizeof(auth_info)); + if (authinfo == NULL) { + deallocate_Buffer(&auth); + return ZSYSTEMERROR; + } authinfo->scheme=strdup(scheme); + if (authinfo->scheme == NULL) { + deallocate_Buffer(&auth); + free(authinfo); + return ZSYSTEMERROR; + } authinfo->auth=auth; authinfo->completion=completion; authinfo->data=data; authinfo->next = NULL; - add_last_auth(&zh->auth_h, authinfo); - zoo_unlock_auth(zh); - if (is_connected(zh) || - // When associating, only send info packets if no SASL - // negotiation is planned. (Such packets would be queued in - // front of SASL packets, which is forbidden, and SASL - // completion is followed by a 'send_auth_info' anyway.) - (zh->state == ZOO_ASSOCIATING_STATE && !has_sasl_client(zh))) { - return send_last_auth_info(zh); + // Hold the auth lock across enqueue. This pairs the credential with its + // FIFO entry. Concurrent callers cannot send the same tail credential. + // Append the frame after earlier application requests. If the session + // is not connected, store the credential only. send_auth_info() sends + // it when the session is established. + zoo_lock_auth(zh); + sent = is_connected(zh); + if (sent) { + rc = send_info_packet(zh, authinfo, 0); } + if (rc == ZOK) { + add_last_auth(&zh->auth_h, authinfo); + } + zoo_unlock_auth(zh); + if (rc != ZOK) { + // The credential is not in the list. The completion does not run. + free(authinfo->scheme); + deallocate_Buffer(&authinfo->auth); + free(authinfo); + return rc == ZSYSTEMERROR ? ZSYSTEMERROR : ZMARSHALLINGERROR; + } + if (sent) { + LOG_DEBUG(LOGCALLBACK(zh), "Sending auth info request to %s", + zoo_get_current_server(zh)); + return nonblocking_send(zh, ZOK); + } return ZOK; } diff --git a/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc b/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc new file mode 100644 index 00000000000..386ac7b4a93 --- /dev/null +++ b/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc @@ -0,0 +1,655 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "CppAssertHelper.h" + +#include +#include +#include +#include +#include +#include + +#include "ZKMocks.h" +#include + +using namespace std; + +// SetAuth reply. The server sends only a reply header. +class AuthResponse: public Response +{ +public: + AuthResponse(int rc): rc_(rc) {} + virtual string toString() const { + oarchive* oa = create_buffer_oarchive(); + ReplyHeader h = {AUTH_XID, 0, rc_}; + serialize_ReplyHeader(oa, "hdr", &h); + int32_t len = htonl(get_buffer_len(oa)); + string res((char*)&len, sizeof(len)); + res.append(get_buffer(oa), get_buffer_len(oa)); + close_buffer_oarchive(&oa, 1); + return res; + } +private: + int rc_; +}; + +// Mock server that records SetAuth frames in arrival order. Accepts the +// digest scheme and rejects all others. Tests can hold replies and release +// them in the same order. +class AuthServer: public ZookeeperServer +{ +public: + AuthServer(): holdReplies_(false) {} + + virtual void onMessageReceived(const RequestHeader& rh, iarchive* ia) { + if (rh.type != ZOO_SETAUTH_OP) { + return; + } + AuthPacket pkt; + deserialize_AuthPacket(ia, "req", &pkt); + string cred; + if (pkt.auth.buff != 0 && pkt.auth.len > 0) { + cred.assign(pkt.auth.buff, pkt.auth.len); + } + int rc = strcmp(pkt.scheme, "digest") == 0 ? ZOK : ZAUTHFAILED; + bool reply; + { + synchronized(mx_); + schemes_.push_back(pkt.scheme); + creds_.push_back(cred); + reply = !holdReplies_; + if (!reply) { + held_.push_back(rc); + } + } + deallocate_AuthPacket(&pkt); + if (reply) { + addRecvResponse(new AuthResponse(rc)); + } + } + + void holdReplies() { + synchronized(mx_); + holdReplies_ = true; + } + + void releaseReply() { + int rc; + { + synchronized(mx_); + CPPUNIT_ASSERT(!held_.empty()); + rc = held_.front(); + held_.pop_front(); + } + addRecvResponse(new AuthResponse(rc)); + } + + // Discard held replies. They belong to a connection that is gone. + void dropHeldReplies() { + synchronized(mx_); + held_.clear(); + } + + vector schemes() const { + synchronized(mx_); + return schemes_; + } + + vector creds() const { + synchronized(mx_); + return creds_; + } + + size_t frameCount() const { + synchronized(mx_); + return schemes_.size(); + } + +private: + mutable Mutex mx_; + bool holdReplies_; + deque held_; + vector schemes_; + vector creds_; +}; + +// Result of one zoo_add_auth completion. +struct AuthResult +{ + AuthResult(): calls(0), rc(ZAPIERROR) {} + int calls; + int rc; +}; + +static void authCompletion(int rc, const void *data) +{ + AuthResult *res = (AuthResult *)data; + res->calls++; + res->rc = rc; +} + +#ifdef THREADED +// The I/O thread runs completions and the test thread reads the results. +// A mutex guards the results, and a counter tells the test when to read. +static Mutex authResultsMx; +static AtomicInt authCompletions; + +static void lockedAuthCompletion(int rc, const void *data) +{ + synchronized(authResultsMx); + authCompletion(rc, data); + ++authCompletions; +} + +struct CompletionsAtLeast { + CompletionsAtLeast(int count): count_(count) {} + bool operator()() const { + return authCompletions.get() >= count_; + } + int count_; +}; + +// The log callback has no context argument. The race test uses these +// globals. raceMx keeps the callback from reading raceHandle before +// zookeeper_init2 returns. +static Mutex raceMx; +static zhandle_t *raceHandle; +static bool raceFired; +static int raceRc; +static AuthResult raceResult; + +// Call zoo_add_auth on the I/O thread in the window where the session is +// connected but send_auth_info has not run. +static void addAuthDuringEstablishment(const char *message) +{ + zhandle_t *zh; + if (strstr(message, "session establishment complete") == NULL) { + return; + } + { + synchronized(raceMx); + if (raceHandle == NULL || raceFired) { + return; + } + raceFired = true; + zh = raceHandle; + } + raceRc = zoo_add_auth(zh, "digest", "race:pass", 9, + lockedAuthCompletion, &raceResult); +} +#endif + +class Zookeeper_authCompletion : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE(Zookeeper_authCompletion); +#ifndef THREADED + CPPUNIT_TEST(testEachReplyCompletesItsOwnRequest); + CPPUNIT_TEST(testRejectedCredentialIsNotReportedOk); + CPPUNIT_TEST(testResentCredentialDoesNotCompleteNewRequest); + CPPUNIT_TEST(testUnsolicitedAuthReplyIsIgnored); + CPPUNIT_TEST(testStoredCredentialsCompleteSeparately); + CPPUNIT_TEST(testReconnectResendDoesNotCompleteNewRequest); +#else + CPPUNIT_TEST(testConcurrentAddAuthSendsEachCredential); + CPPUNIT_TEST(testAddAuthDuringSessionEstablishment); +#endif + CPPUNIT_TEST_SUITE_END(); + + zhandle_t *zh; + FILE *logfile; + + static void watcher(zhandle_t *, int, int, const char *, void *) {} + +public: + Zookeeper_authCompletion() { + logfile = openlogfile("Zookeeper_authCompletion"); + } + + ~Zookeeper_authCompletion() { + if (logfile) { + fflush(logfile); + fclose(logfile); + logfile = 0; + } + } + + void setUp() { + zoo_set_log_stream(logfile); + zoo_deterministic_conn_order(0); + zh = 0; + } + + void tearDown() { + zookeeper_close(zh); + } + +#ifndef THREADED + // Drive the event loop. Each pass sends queued frames and reads at most + // one reply. Stop if the handle becomes unrecoverable. + void drive(int passes) { + for (int i = 0; i < passes; i++) { + int fd = 0; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + if (rc != ZOK) { + return; + } + rc = zookeeper_process(zh, interest); + if (rc != ZOK && rc != ZNOTHING) { + return; + } + } + } + + // Complete a real handshake so session establishment sends stored + // credentials. + void connect() { + int fd = 0; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + rc = zookeeper_process(zh, interest); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL(ZOO_ASSOCIATING_STATE, zoo_state(zh)); + rc = zookeeper_process(zh, interest); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } + + // Drive the event loop after a connection loss until the session is + // established again. The first pass can only clear the reconnect delay. + void reconnect() { + for (int i = 0; i < 10 && zoo_state(zh) != ZOO_CONNECTED_STATE; i++) { + int fd = 0; + int interest = 0; + timeval tv; + if (zookeeper_interest(zh, &fd, &interest, &tv) == ZOK) { + zookeeper_process(zh, interest); + } + } + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } + + // Two credentials go out in the order of the zoo_add_auth calls. Each + // completion gets the result of its own frame. + void testEachReplyCompletesItsOwnRequest() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh); + zkServer.holdReplies(); + + AuthResult good, bad; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user:pass", 9, authCompletion, &good)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &bad)); + drive(1); + + vector schemes = zkServer.schemes(); + CPPUNIT_ASSERT_EQUAL((size_t)2, schemes.size()); + CPPUNIT_ASSERT_EQUAL(string("digest"), schemes[0]); + CPPUNIT_ASSERT_EQUAL(string("nosuchscheme"), schemes[1]); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, good.rc); + CPPUNIT_ASSERT_EQUAL(0, bad.calls); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL(1, bad.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, bad.rc); + CPPUNIT_ASSERT_EQUAL(ZOO_AUTH_FAILED_STATE, zoo_state(zh)); + } + + // Rejected credential first, then a valid credential. The rejected + // completion never receives ZOK. + void testRejectedCredentialIsNotReportedOk() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh); + + AuthResult bad, good; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &bad)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user:pass", 9, authCompletion, &good)); + drive(5); + + CPPUNIT_ASSERT_EQUAL(1, bad.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, bad.rc); + vector schemes = zkServer.schemes(); + CPPUNIT_ASSERT(schemes.size() >= 1); + CPPUNIT_ASSERT_EQUAL(string("nosuchscheme"), schemes[0]); + // The session failed on the first rejection, so the second + // completion also receives ZAUTHFAILED. + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, good.rc); + CPPUNIT_ASSERT_EQUAL(ZOO_AUTH_FAILED_STATE, zoo_state(zh)); + } + + // Session establishment sends the stored credential. A later + // zoo_add_auth call does not receive the result of that frame. + void testResentCredentialDoesNotCompleteNewRequest() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + zkServer.holdReplies(); + + // The handle is not connected. The handshake sends this credential + // with its completion. + AuthResult first, second; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user:pass", 9, authCompletion, &first)); + CPPUNIT_ASSERT_EQUAL((size_t)0, zkServer.frameCount()); + connect(); + drive(1); + CPPUNIT_ASSERT_EQUAL((size_t)1, zkServer.frameCount()); + + // Add a second credential while the first reply is still in flight. + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &second)); + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, first.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, first.rc); + CPPUNIT_ASSERT_EQUAL(0, second.calls); + + CPPUNIT_ASSERT_EQUAL((size_t)2, zkServer.frameCount()); + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, first.calls); + CPPUNIT_ASSERT_EQUAL(1, second.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, second.rc); + } + + // A handle with no credentials receives auth replies. The client + // ignores them. The session stays connected. + void testUnsolicitedAuthReplyIsIgnored() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh); + + zkServer.addRecvResponse(new AuthResponse(ZOK)); + zkServer.addRecvResponse(new AuthResponse(ZAUTHFAILED)); + int rc = zookeeper_process(zh, ZOOKEEPER_READ); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + rc = zookeeper_process(zh, ZOOKEEPER_READ); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } + + // Session establishment sends two stored credentials. The server + // accepts one and rejects the other. Each completion gets the result of + // its own frame. + void testStoredCredentialsCompleteSeparately() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + zkServer.holdReplies(); + + AuthResult bad, good; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &bad)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user:pass", 9, authCompletion, &good)); + connect(); + drive(2); + + // Session establishment puts each frame at the front of the send + // queue. The last stored credential goes out first. + vector schemes = zkServer.schemes(); + CPPUNIT_ASSERT_EQUAL((size_t)2, schemes.size()); + CPPUNIT_ASSERT_EQUAL(string("digest"), schemes[0]); + CPPUNIT_ASSERT_EQUAL(string("nosuchscheme"), schemes[1]); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, good.rc); + CPPUNIT_ASSERT_EQUAL(0, bad.calls); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL(1, bad.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, bad.rc); + CPPUNIT_ASSERT_EQUAL(ZOO_AUTH_FAILED_STATE, zoo_state(zh)); + } + + // The connection drops while two stored credentials wait for replies. + // The reconnect sends both credentials again without completions. A + // new zoo_add_auth call does not receive the result of a resent frame. + void testReconnectResendDoesNotCompleteNewRequest() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + zkServer.holdReplies(); + + AuthResult first, second, fresh; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user1:pass", 10, authCompletion, &first)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user2:pass", 10, authCompletion, &second)); + connect(); + drive(2); + CPPUNIT_ASSERT_EQUAL((size_t)2, zkServer.frameCount()); + + // The pending completions receive the connection-loss code. + zkServer.dropHeldReplies(); + zkServer.setConnectionLost(); + drive(1); + zkServer.connectionLost = false; + CPPUNIT_ASSERT_EQUAL(1, first.calls); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS, first.rc); + CPPUNIT_ASSERT_EQUAL(1, second.calls); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS, second.rc); + + reconnect(); + drive(2); + vector creds = zkServer.creds(); + CPPUNIT_ASSERT_EQUAL((size_t)4, creds.size()); + set resent(creds.begin() + 2, creds.end()); + CPPUNIT_ASSERT_EQUAL((size_t)1, resent.count("user1:pass")); + CPPUNIT_ASSERT_EQUAL((size_t)1, resent.count("user2:pass")); + + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &fresh)); + drive(1); + CPPUNIT_ASSERT_EQUAL((size_t)5, zkServer.frameCount()); + + zkServer.releaseReply(); + drive(1); + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(0, fresh.calls); + CPPUNIT_ASSERT_EQUAL(1, first.calls); + CPPUNIT_ASSERT_EQUAL(1, second.calls); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, fresh.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, fresh.rc); + } +#else + struct AddAuthArgs { + zhandle_t *zh; + int thread; + int count; + AuthResult *results; + int failures; + }; + + static void *addAuthThread(void *arg) { + AddAuthArgs *args = (AddAuthArgs *)arg; + for (int i = 0; i < args->count; i++) { + ostringstream cred; + cred << "user" << args->thread << "-" << i << ":pass"; + string s = cred.str(); + if (zoo_add_auth(args->zh, "digest", s.c_str(), (int)s.size(), + lockedAuthCompletion, &args->results[i]) != ZOK) { + args->failures++; + } + } + return NULL; + } + + struct FrameCountAtLeast { + FrameCountAtLeast(const AuthServer& server, size_t count): + server_(server), count_(count) {} + bool operator()() const { + return server_.frameCount() >= count_; + } + const AuthServer& server_; + size_t count_; + }; + + // Concurrent zoo_add_auth calls. Each call sends its own credential + // exactly once, and each completion runs once with ZOK. + void testConcurrentAddAuthSendsEachCredential() { + const int THREADS = 8; + const int PER_THREAD = 50; + Mock_gettimeofday timeMock; + AuthServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + + authCompletions = 0; + vector results(THREADS * PER_THREAD); + pthread_t threads[THREADS]; + AddAuthArgs args[THREADS]; + for (int t = 0; t < THREADS; t++) { + args[t].zh = zh; + args[t].thread = t; + args[t].count = PER_THREAD; + args[t].results = &results[t * PER_THREAD]; + args[t].failures = 0; + CPPUNIT_ASSERT_EQUAL(0, + pthread_create(&threads[t], NULL, addAuthThread, &args[t])); + } + for (int t = 0; t < THREADS; t++) { + pthread_join(threads[t], NULL); + } + for (int t = 0; t < THREADS; t++) { + CPPUNIT_ASSERT_EQUAL(0, args[t].failures); + } + + const size_t total = THREADS * PER_THREAD; + ensureCondition(FrameCountAtLeast(zkServer, total), 5000); + ensureCondition(CompletionsAtLeast((int)total), 5000); + vector creds = zkServer.creds(); + CPPUNIT_ASSERT_EQUAL(total, creds.size()); + set unique(creds.begin(), creds.end()); + CPPUNIT_ASSERT_EQUAL(total, unique.size()); + { + synchronized(authResultsMx); + for (size_t i = 0; i < total; i++) { + CPPUNIT_ASSERT_EQUAL(1, results[i].calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, results[i].rc); + } + } + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } + + // zoo_add_auth runs on the I/O thread after the session becomes + // connected but before send_auth_info runs. This is the race with a + // caller thread. send_auth_info must take back the queued frame and + // send the credential once, with its completion. A second credential + // then acts as a barrier for all earlier frames. + void testAddAuthDuringSessionEstablishment() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + authCompletions = 0; + raceResult = AuthResult(); + raceRc = ZAPIERROR; + { + synchronized(raceMx); + raceFired = false; + zh = zookeeper_init2("localhost:2121", watcher, 10000, + TEST_CLIENT_ID, 0, 0, addAuthDuringEstablishment); + raceHandle = zh; + } + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + ensureCondition(CompletionsAtLeast(1), 5000); + + AuthResult barrier; + CPPUNIT_ASSERT_EQUAL((int)ZOK, zoo_add_auth(zh, "digest", "barrier:pass", + 12, lockedAuthCompletion, &barrier)); + ensureCondition(CompletionsAtLeast(2), 5000); + { + synchronized(raceMx); + raceHandle = NULL; + } + + CPPUNIT_ASSERT(raceFired); + CPPUNIT_ASSERT_EQUAL((int)ZOK, raceRc); + vector creds = zkServer.creds(); + CPPUNIT_ASSERT_EQUAL((size_t)2, creds.size()); + CPPUNIT_ASSERT_EQUAL(string("race:pass"), creds[0]); + CPPUNIT_ASSERT_EQUAL(string("barrier:pass"), creds[1]); + { + synchronized(authResultsMx); + CPPUNIT_ASSERT_EQUAL(1, raceResult.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, raceResult.rc); + CPPUNIT_ASSERT_EQUAL(1, barrier.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, barrier.rc); + } + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } +#endif +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(Zookeeper_authCompletion);