From 2f21695880683dba290ba99585b632af3abb6887 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Fri, 6 Nov 2026 11:09:36 -0800 Subject: [PATCH 1/2] ZOOKEEPER-5123: (C client) Broadcast completion of auth can return ZOK for rejected credentials When the C client received a reply to a SetAuth request, auth_completion_func ran every pending callback with the status of that one reply without checking which credential the reply belongs to. The client also put SetAuth frames at the front of the send queue, so frames from back-to-back calls went out in reverse order. There were additional related problems on the same path: - Credentials that the client sends again during a reconnect could complete newly issued zoo_add_auth calls before the server checked them. - An auth reply on a handle with no credentials dereferenced a NULL list and crashed the I/O thread. - Concurrent zoo_add_auth calls could both send the last credential in the list. - If a frame could not be queued zoo_add_auth would return an error but leave the callback registered. Each handle now keeps a FIFO of pending SetAuth frames in the order of frames on the wire. Each reply completes only the entry at the head. Related fixes: - An auth reply with no pending frame is logged and ignored. Before, it dereferenced a NULL credential list. - zoo_add_auth holds the auth lock while it queues the frame of its own credential. Concurrent calls no longer send the tail credential twice. The frame goes to the end of the send queue. - zoo_add_auth checks its allocations. If the frame cannot be queued, the credential is not added and its completion never runs. - zoo_add_auth no longer sends a frame in ZOO_ASSOCIATING_STATE. Session establishment sends the credential with its completion. Adds the Zookeeper_authCompletion suite in tests/TestAuthCompletion.cc. Co-Authored-By: Claude Opus 5.5 --- .../zookeeper-client-c/CMakeLists.txt | 1 + .../zookeeper-client-c/Makefile.am | 1 + .../zookeeper-client-c/src/zk_adaptor.h | 12 + .../zookeeper-client-c/src/zookeeper.c | 254 +++++++---- .../tests/TestAuthCompletion.cc | 409 ++++++++++++++++++ 5 files changed, 592 insertions(+), 85 deletions(-) create mode 100644 zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc diff --git a/zookeeper-client/zookeeper-client-c/CMakeLists.txt b/zookeeper-client/zookeeper-client-c/CMakeLists.txt index 317059a8471..2e3afeac231 100644 --- a/zookeeper-client/zookeeper-client-c/CMakeLists.txt +++ b/zookeeper-client/zookeeper-client-c/CMakeLists.txt @@ -249,6 +249,7 @@ set(test_sources tests/ThreadingUtil.cc tests/TestZookeeperInit.cc tests/TestZookeeperClose.cc + tests/TestAuthCompletion.cc tests/TestSASLAuth.cc tests/TestReconfig.cc tests/TestReconfigServer.cc diff --git a/zookeeper-client/zookeeper-client-c/Makefile.am b/zookeeper-client/zookeeper-client-c/Makefile.am index 7372291d79f..4b86905f221 100644 --- a/zookeeper-client/zookeeper-client-c/Makefile.am +++ b/zookeeper-client/zookeeper-client-c/Makefile.am @@ -109,6 +109,7 @@ TEST_SOURCES = \ tests/ThreadingUtil.cc \ tests/TestZookeeperInit.cc \ tests/TestZookeeperClose.cc \ + tests/TestAuthCompletion.cc \ tests/TestReconfig.cc \ tests/TestReconfigServer.cc \ tests/TestClientRetry.cc \ diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index 25049bb335e..af7d643d3b2 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -177,9 +177,21 @@ struct adaptor_threads { }; #endif +/** + * Outstanding SetAuth request. The list is a FIFO that matches the order of + * frames on the wire. Each reply completes the head entry. + */ +typedef struct _auth_request { + auth_info *auth; /* credential carried by this frame */ + void_completion_t completion; /* NULL on a reconnect resend */ + const char *data; + struct _auth_request *next; +} auth_request_t; + /** the auth list for adding auth */ typedef struct _auth_list_head { auth_info *auth; + auth_request_t *pending; /* FIFO of SetAuth frames on the wire */ #ifdef THREADED pthread_mutex_t lock; #endif diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index c8429003b29..f60de7a0a20 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -361,7 +361,7 @@ static int is_sasl_auth_in_progress(zhandle_t* zh) * Extract the type field (ZOO_*_OP) of a serialized RequestHeader. * * (This is not the most efficient way of fetching 4 bytes, but it is - * currently only used during SASL negotiation.) + * only used during SASL negotiation and session establishment.) * * \param buffer the buffer to extract the request type from. Must * start with a serialized RequestHeader; @@ -502,18 +502,6 @@ int zoo_recv_timeout(zhandle_t *zh) /** these functions are thread unsafe, so make sure that zoo_lock_auth is called before you access them **/ -static auth_info* get_last_auth(auth_list_head_t *auth_list) { - auth_info *element; - element = auth_list->auth; - if (element == NULL) { - return NULL; - } - while (element->next != NULL) { - element = element->next; - } - return element; -} - static void free_auth_completion(auth_completion_list_t *a_list) { auth_completion_list_t *tmp, *ftmp; if (a_list == NULL) { @@ -556,12 +544,22 @@ static void add_auth_completion(auth_completion_list_t* a_list, void_completion_ return; } +/* + * Collect completions that still wait for an auth result. Then clear the + * pending FIFO. The caller has dropped the connection that carried those + * frames. + */ static void get_auth_completions(auth_list_head_t *auth_list, auth_completion_list_t *a_list) { + auth_request_t *req; auth_info *element; - element = auth_list->auth; - if (element == NULL) { - return; + while ((req = auth_list->pending) != NULL) { + auth_list->pending = req->next; + if (req->completion) { + add_auth_completion(a_list, &req->completion, req->data); + } + free(req); } + element = auth_list->auth; while (element) { if (element->completion) { add_auth_completion(a_list, &element->completion, element->data); @@ -590,24 +588,17 @@ static void add_last_auth(auth_list_head_t *auth_list, auth_info *add_el) { static void init_auth_info(auth_list_head_t *auth_list) { auth_list->auth = NULL; -} - -static void mark_active_auth(zhandle_t *zh) { - auth_list_head_t auth_h = zh->auth_h; - auth_info *element; - if (auth_h.auth == NULL) { - return; - } - element = auth_h.auth; - while (element != NULL) { - element->state = 1; - element = element->next; - } + auth_list->pending = NULL; } static void free_auth_info(auth_list_head_t *auth_list) { auth_info *auth = auth_list->auth; + auth_request_t *req; + while ((req = auth_list->pending) != NULL) { + auth_list->pending = req->next; + free(req); + } while (auth != NULL) { auth_info* old_auth = NULL; if(auth->scheme!=NULL) @@ -2149,79 +2140,164 @@ static int handle_socket_error_msg(zhandle_t *zh, int line, const char *func, in return rc; } +/* + * Apply one SetAuth reply to the head of the pending FIFO. The server + * replies to SetAuth in receive order, so the head is the matching request. + * An empty FIFO means the reply is not for this handle. The client ignores + * it. The session stays connected. + */ static void auth_completion_func(int rc, zhandle_t* zh) { - void_completion_t auth_completion = NULL; - auth_completion_list_t a_list; - auth_completion_list_t *a_tmp; + auth_request_t *req; if(zh==NULL) return; zoo_lock_auth(zh); - + req = zh->auth_h.pending; + if (req == NULL) { + zoo_unlock_auth(zh); + LOG_WARN(LOGCALLBACK(zh), + "Ignoring an authentication reply (rc=%d) that has no pending request", + rc); + return; + } + zh->auth_h.pending = req->next; if(rc!=0){ zh->state=ZOO_AUTH_FAILED_STATE; }else{ - //change state for all auths - mark_active_auth(zh); + req->auth->state = 1; } - a_list.completion = NULL; - a_list.next = NULL; - get_auth_completions(&zh->auth_h, &a_list); zoo_unlock_auth(zh); if (rc) { LOG_ERROR(LOGCALLBACK(zh), "Authentication scheme %s failed. Connection closed.", - zh->auth_h.auth->scheme); + req->auth->scheme); } else { - LOG_INFO(LOGCALLBACK(zh), "Authentication scheme %s succeeded", zh->auth_h.auth->scheme); + LOG_INFO(LOGCALLBACK(zh), "Authentication scheme %s succeeded", req->auth->scheme); } - a_tmp = &a_list; - // chain call user's completion function - while (a_tmp->completion != NULL) { - auth_completion = a_tmp->completion; - auth_completion(rc, a_tmp->auth_data); - a_tmp = a_tmp->next; - if (a_tmp == NULL) - break; + if (req->completion) { + req->completion(rc, req->data); } - free_auth_completion(&a_list); + free(req); } -static int send_info_packet(zhandle_t *zh, auth_info* auth) { +/* + * Enqueue one SetAuth frame and its pending-FIFO entry as one locked + * operation. add_to_front must match the send-queue position. Then the FIFO + * stays in wire order. If the credential has a completion, that completion + * moves to the pending entry. + */ +static int send_info_packet(zhandle_t *zh, auth_info* auth, int add_to_front) { struct oarchive *oa; struct RequestHeader h = {AUTH_XID, ZOO_SETAUTH_OP}; struct AuthPacket req; + auth_request_t *ar; + auth_request_t **link; + buffer_list_t *b = NULL; int rc; + + ar = calloc(1, sizeof(*ar)); + if (ar == NULL) { + return ZSYSTEMERROR; + } oa = create_buffer_oarchive(); rc = serialize_RequestHeader(oa, "header", &h); req.type=0; // ignored by the server req.scheme = auth->scheme; req.auth = auth->auth; rc = rc < 0 ? rc : serialize_AuthPacket(oa, "req", &req); - /* add this buffer to the head of the send queue */ - rc = rc < 0 ? rc : queue_front_buffer_bytes(&zh->to_send, get_buffer(oa), - get_buffer_len(oa)); + if (rc >= 0) { + b = allocate_buffer(get_buffer(oa), get_buffer_len(oa)); + if (b == NULL) { + rc = ZSYSTEMERROR; + } + } /* Free serialized buffer if not enqueued */ close_buffer_oarchive(&oa, rc < 0); + if (rc < 0) { + free(ar); + return rc; + } - return rc; + ar->auth = auth; + ar->completion = auth->completion; + ar->data = auth->data; + auth->completion = NULL; + queue_buffer(&zh->to_send, b, add_to_front); + link = &zh->auth_h.pending; + if (!add_to_front) { + while (*link != NULL) { + link = &(*link)->next; + } + } + ar->next = *link; + *link = ar; + return ZOK; } -/** send all auths, not just the last one **/ +/* + * Drop unsent SetAuth frames from the send queue. This runs before a session + * resends credentials. A leftover frame would send the same credential twice, + * and its reply would have no FIFO entry. + */ +static void remove_auth_frames(buffer_head_t *list) +{ + buffer_list_t *b; + buffer_list_t *next; + buffer_list_t *last = NULL; + int32_t type; + + lock_buffer_list(list); + for (b = list->head; b != NULL; b = next) { + next = b->next; + if (extract_request_type(b->buffer, b->len, &type) == ZOK && + type == ZOO_SETAUTH_OP) { + if (last == NULL) { + list->head = next; + } else { + last->next = next; + } + free_buffer(b); + } else { + last = b; + } + } + list->last = last; + unlock_buffer_list(list); +} + +/* + * Resend every stored credential on a newly established session. First + * return pending completions to their credentials and drop leftover SetAuth + * frames. Then enqueue one new frame per credential. Any unsent completion + * goes with that frame. + */ static int send_auth_info(zhandle_t *zh) { - int rc = 0; + int rc = ZOK; auth_info *auth = NULL; + auth_request_t *req; zoo_lock_auth(zh); + while ((req = zh->auth_h.pending) != NULL) { + zh->auth_h.pending = req->next; + if (req->completion) { + req->auth->completion = req->completion; + req->auth->data = req->data; + } + free(req); + } + remove_auth_frames(&zh->to_send); auth = zh->auth_h.auth; if (auth == NULL) { zoo_unlock_auth(zh); return ZOK; } while (auth != NULL) { - rc = send_info_packet(zh, auth); + int auth_rc = send_info_packet(zh, auth, 1); + if (auth_rc < 0) { + rc = auth_rc; + } auth = auth->next; } zoo_unlock_auth(zh); @@ -2229,23 +2305,6 @@ static int send_auth_info(zhandle_t *zh) { return (rc <0) ? ZMARSHALLINGERROR:ZOK; } -static int send_last_auth_info(zhandle_t *zh) -{ - int rc = 0; - auth_info *auth = NULL; - - zoo_lock_auth(zh); - auth = get_last_auth(&zh->auth_h); - if(auth==NULL) { - zoo_unlock_auth(zh); - return ZOK; // there is nothing to send - } - rc = send_info_packet(zh, auth); - zoo_unlock_auth(zh); - LOG_DEBUG(LOGCALLBACK(zh), "Sending auth info request to %s",zoo_get_current_server(zh)); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; -} - static void free_key_list(char **list, int count) { int i; @@ -5304,6 +5363,8 @@ int zoo_add_auth(zhandle_t *zh,const char* scheme,const char* cert, { struct buffer auth; auth_info *authinfo; + int rc = ZOK; + int sent; if(scheme==NULL || zh==NULL) return ZBADARGUMENTS; @@ -5328,25 +5389,48 @@ int zoo_add_auth(zhandle_t *zh,const char* scheme,const char* cert, auth.len = 0; } - zoo_lock_auth(zh); - authinfo = (auth_info*) malloc(sizeof(auth_info)); + authinfo = (auth_info*) calloc(1, sizeof(auth_info)); + if (authinfo == NULL) { + deallocate_Buffer(&auth); + return ZSYSTEMERROR; + } authinfo->scheme=strdup(scheme); + if (authinfo->scheme == NULL) { + deallocate_Buffer(&auth); + free(authinfo); + return ZSYSTEMERROR; + } authinfo->auth=auth; authinfo->completion=completion; authinfo->data=data; authinfo->next = NULL; - add_last_auth(&zh->auth_h, authinfo); - zoo_unlock_auth(zh); - if (is_connected(zh) || - // When associating, only send info packets if no SASL - // negotiation is planned. (Such packets would be queued in - // front of SASL packets, which is forbidden, and SASL - // completion is followed by a 'send_auth_info' anyway.) - (zh->state == ZOO_ASSOCIATING_STATE && !has_sasl_client(zh))) { - return send_last_auth_info(zh); + // Hold the auth lock across enqueue. This pairs the credential with its + // FIFO entry. Concurrent callers cannot send the same tail credential. + // Append the frame after earlier application requests. If the session + // is not connected, store the credential only. send_auth_info() sends + // it when the session is established. + zoo_lock_auth(zh); + sent = is_connected(zh); + if (sent) { + rc = send_info_packet(zh, authinfo, 0); } + if (rc == ZOK) { + add_last_auth(&zh->auth_h, authinfo); + } + zoo_unlock_auth(zh); + if (rc != ZOK) { + // The credential is not in the list. The completion does not run. + free(authinfo->scheme); + deallocate_Buffer(&authinfo->auth); + free(authinfo); + return ZMARSHALLINGERROR; + } + if (sent) { + LOG_DEBUG(LOGCALLBACK(zh), "Sending auth info request to %s", + zoo_get_current_server(zh)); + } return ZOK; } diff --git a/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc b/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc new file mode 100644 index 00000000000..61523d44236 --- /dev/null +++ b/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc @@ -0,0 +1,409 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "CppAssertHelper.h" + +#include +#include +#include +#include +#include + +#include "ZKMocks.h" +#include + +using namespace std; + +// SetAuth reply. The server sends only a reply header. +class AuthResponse: public Response +{ +public: + AuthResponse(int rc): rc_(rc) {} + virtual string toString() const { + oarchive* oa = create_buffer_oarchive(); + ReplyHeader h = {AUTH_XID, 0, rc_}; + serialize_ReplyHeader(oa, "hdr", &h); + int32_t len = htonl(get_buffer_len(oa)); + string res((char*)&len, sizeof(len)); + res.append(get_buffer(oa), get_buffer_len(oa)); + close_buffer_oarchive(&oa, 1); + return res; + } +private: + int rc_; +}; + +// Mock server that records SetAuth frames in arrival order. Accepts the +// digest scheme and rejects all others. Tests can hold replies and release +// them in the same order. +class AuthServer: public ZookeeperServer +{ +public: + AuthServer(): holdReplies_(false) {} + + virtual void onMessageReceived(const RequestHeader& rh, iarchive* ia) { + if (rh.type != ZOO_SETAUTH_OP) { + return; + } + AuthPacket pkt; + deserialize_AuthPacket(ia, "req", &pkt); + string cred; + if (pkt.auth.buff != 0 && pkt.auth.len > 0) { + cred.assign(pkt.auth.buff, pkt.auth.len); + } + int rc = strcmp(pkt.scheme, "digest") == 0 ? ZOK : ZAUTHFAILED; + bool reply; + { + synchronized(mx_); + schemes_.push_back(pkt.scheme); + creds_.push_back(cred); + reply = !holdReplies_; + if (!reply) { + held_.push_back(rc); + } + } + deallocate_AuthPacket(&pkt); + if (reply) { + addRecvResponse(new AuthResponse(rc)); + } + } + + void holdReplies() { + synchronized(mx_); + holdReplies_ = true; + } + + void releaseReply() { + int rc; + { + synchronized(mx_); + CPPUNIT_ASSERT(!held_.empty()); + rc = held_.front(); + held_.pop_front(); + } + addRecvResponse(new AuthResponse(rc)); + } + + vector schemes() const { + synchronized(mx_); + return schemes_; + } + + vector creds() const { + synchronized(mx_); + return creds_; + } + + size_t frameCount() const { + synchronized(mx_); + return schemes_.size(); + } + +private: + mutable Mutex mx_; + bool holdReplies_; + deque held_; + vector schemes_; + vector creds_; +}; + +// Result of one zoo_add_auth completion. +struct AuthResult +{ + AuthResult(): calls(0), rc(ZAPIERROR) {} + int calls; + int rc; +}; + +static void authCompletion(int rc, const void *data) +{ + AuthResult *res = (AuthResult *)data; + res->calls++; + res->rc = rc; +} + +class Zookeeper_authCompletion : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE(Zookeeper_authCompletion); +#ifndef THREADED + CPPUNIT_TEST(testEachReplyCompletesItsOwnRequest); + CPPUNIT_TEST(testRejectedCredentialIsNotReportedOk); + CPPUNIT_TEST(testResentCredentialDoesNotCompleteNewRequest); + CPPUNIT_TEST(testUnsolicitedAuthReplyIsIgnored); +#else + CPPUNIT_TEST(testConcurrentAddAuthSendsEachCredential); +#endif + CPPUNIT_TEST_SUITE_END(); + + zhandle_t *zh; + FILE *logfile; + + static void watcher(zhandle_t *, int, int, const char *, void *) {} + +public: + Zookeeper_authCompletion() { + logfile = openlogfile("Zookeeper_authCompletion"); + } + + ~Zookeeper_authCompletion() { + if (logfile) { + fflush(logfile); + fclose(logfile); + logfile = 0; + } + } + + void setUp() { + zoo_set_log_stream(logfile); + zoo_deterministic_conn_order(0); + zh = 0; + } + + void tearDown() { + zookeeper_close(zh); + } + +#ifndef THREADED + // Drive the event loop. Each pass sends queued frames and reads at most + // one reply. Stop if the handle becomes unrecoverable. + void drive(int passes) { + for (int i = 0; i < passes; i++) { + int fd = 0; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + if (rc != ZOK) { + return; + } + rc = zookeeper_process(zh, interest); + if (rc != ZOK && rc != ZNOTHING) { + return; + } + } + } + + // Complete a real handshake so session establishment sends stored + // credentials. + void connect() { + int fd = 0; + int interest = 0; + timeval tv; + int rc = zookeeper_interest(zh, &fd, &interest, &tv); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + rc = zookeeper_process(zh, interest); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL(ZOO_ASSOCIATING_STATE, zoo_state(zh)); + rc = zookeeper_process(zh, interest); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } + + // Two credentials go out in the order of the zoo_add_auth calls. Each + // completion gets the result of its own frame. + void testEachReplyCompletesItsOwnRequest() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh); + zkServer.holdReplies(); + + AuthResult good, bad; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user:pass", 9, authCompletion, &good)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &bad)); + drive(1); + + vector schemes = zkServer.schemes(); + CPPUNIT_ASSERT_EQUAL((size_t)2, schemes.size()); + CPPUNIT_ASSERT_EQUAL(string("digest"), schemes[0]); + CPPUNIT_ASSERT_EQUAL(string("nosuchscheme"), schemes[1]); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, good.rc); + CPPUNIT_ASSERT_EQUAL(0, bad.calls); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL(1, bad.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, bad.rc); + CPPUNIT_ASSERT_EQUAL(ZOO_AUTH_FAILED_STATE, zoo_state(zh)); + } + + // Rejected credential first, then a valid credential. The rejected + // completion never receives ZOK. + void testRejectedCredentialIsNotReportedOk() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh); + + AuthResult bad, good; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &bad)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user:pass", 9, authCompletion, &good)); + drive(5); + + CPPUNIT_ASSERT_EQUAL(1, bad.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, bad.rc); + vector schemes = zkServer.schemes(); + CPPUNIT_ASSERT(schemes.size() >= 1); + CPPUNIT_ASSERT_EQUAL(string("nosuchscheme"), schemes[0]); + // The session failed on the first rejection, so the second + // completion also receives ZAUTHFAILED. + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, good.rc); + CPPUNIT_ASSERT_EQUAL(ZOO_AUTH_FAILED_STATE, zoo_state(zh)); + } + + // Session establishment sends the stored credential. A later + // zoo_add_auth call does not receive the result of that frame. + void testResentCredentialDoesNotCompleteNewRequest() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + zkServer.holdReplies(); + + // The handle is not connected. The handshake sends this credential + // with its completion. + AuthResult first, second; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user:pass", 9, authCompletion, &first)); + CPPUNIT_ASSERT_EQUAL((size_t)0, zkServer.frameCount()); + connect(); + drive(1); + CPPUNIT_ASSERT_EQUAL((size_t)1, zkServer.frameCount()); + + // Add a second credential while the first reply is still in flight. + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &second)); + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, first.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, first.rc); + CPPUNIT_ASSERT_EQUAL(0, second.calls); + + CPPUNIT_ASSERT_EQUAL((size_t)2, zkServer.frameCount()); + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, first.calls); + CPPUNIT_ASSERT_EQUAL(1, second.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, second.rc); + } + + // A handle with no credentials receives auth replies. The client + // ignores them. The session stays connected. + void testUnsolicitedAuthReplyIsIgnored() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + forceConnected(zh); + + zkServer.addRecvResponse(new AuthResponse(ZOK)); + zkServer.addRecvResponse(new AuthResponse(ZAUTHFAILED)); + int rc = zookeeper_process(zh, ZOOKEEPER_READ); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + rc = zookeeper_process(zh, ZOOKEEPER_READ); + CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } +#else + struct AddAuthArgs { + zhandle_t *zh; + int thread; + int count; + }; + + static void *addAuthThread(void *arg) { + AddAuthArgs *args = (AddAuthArgs *)arg; + for (int i = 0; i < args->count; i++) { + ostringstream cred; + cred << "user" << args->thread << "-" << i << ":pass"; + string s = cred.str(); + zoo_add_auth(args->zh, "digest", s.c_str(), (int)s.size(), NULL, NULL); + } + return NULL; + } + + struct FrameCountAtLeast { + FrameCountAtLeast(const AuthServer& server, size_t count): + server_(server), count_(count) {} + bool operator()() const { + return server_.frameCount() >= count_; + } + const AuthServer& server_; + size_t count_; + }; + + // Concurrent zoo_add_auth calls. Each call sends its own credential + // exactly once. + void testConcurrentAddAuthSendsEachCredential() { + const int THREADS = 8; + const int PER_THREAD = 50; + Mock_gettimeofday timeMock; + AuthServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + + pthread_t threads[THREADS]; + AddAuthArgs args[THREADS]; + for (int t = 0; t < THREADS; t++) { + args[t].zh = zh; + args[t].thread = t; + args[t].count = PER_THREAD; + CPPUNIT_ASSERT_EQUAL(0, + pthread_create(&threads[t], NULL, addAuthThread, &args[t])); + } + for (int t = 0; t < THREADS; t++) { + pthread_join(threads[t], NULL); + } + + const size_t total = THREADS * PER_THREAD; + ensureCondition(FrameCountAtLeast(zkServer, total), 5000); + vector creds = zkServer.creds(); + CPPUNIT_ASSERT_EQUAL(total, creds.size()); + set unique(creds.begin(), creds.end()); + CPPUNIT_ASSERT_EQUAL(total, unique.size()); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } +#endif +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(Zookeeper_authCompletion); From 86f63df2608483b1dbd5b8c42cdcdf3784c87339 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Fri, 20 Nov 2026 13:35:57 -0800 Subject: [PATCH 2/2] ZOOKEEPER-5123: Cover auth reconnect paths and clarify the completion contract Address review feedback: - Add tests for two stored credentials with separate results, for a reconnect resend that does not complete a new request, and for zoo_add_auth on the I/O thread during session establishment. The last test drives the code that takes back pending completions and drops queued SetAuth frames. - Give each call in the concurrency test a completion, and check that it runs once with ZOK and that zoo_add_auth returns ZOK. - Document in zookeeper.h which result a pending completion receives after an earlier rejection or a disconnect. - Send the frame at once when connected, as the other async calls do. - Return ZSYSTEMERROR from zoo_add_auth as itself. - Remove the write-only auth_info.state field. Co-Authored-By: Claude Opus 5.5 --- .../zookeeper-client-c/include/zookeeper.h | 5 + .../zookeeper-client-c/src/zk_adaptor.h | 1 - .../zookeeper-client-c/src/zookeeper.c | 5 +- .../tests/TestAuthCompletion.cc | 250 +++++++++++++++++- 4 files changed, 255 insertions(+), 6 deletions(-) diff --git a/zookeeper-client/zookeeper-client-c/include/zookeeper.h b/zookeeper-client/zookeeper-client-c/include/zookeeper.h index 1a3f5d1f92f..1130ecf79cd 100644 --- a/zookeeper-client/zookeeper-client-c/include/zookeeper.h +++ b/zookeeper-client/zookeeper-client-c/include/zookeeper.h @@ -1849,6 +1849,11 @@ ZOOAPI const char* zerror(int c); * the following result codes may be passed into the completion callback: * ZOK operation completed successfully * ZAUTHFAILED authentication failed + * If the server rejects an earlier credential, a request that is still + * pending also receives ZAUTHFAILED. If the connection closes before the + * reply, the completion receives the error that closed it, for example + * ZCONNECTIONLOSS. The client keeps the credential and sends it again on the + * next connection. That resend has no completion. * \param data the data that will be passed to the completion routine when the * function completes. * \return ZOK on success or one of the following errcodes on failure: diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index af7d643d3b2..cd79847541c 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -115,7 +115,6 @@ struct sync_completion { }; typedef struct _auth_info { - int state; /* 0=>inactive, >0 => active */ char* scheme; struct buffer auth; void_completion_t completion; diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index f60de7a0a20..194291e8101 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -2165,8 +2165,6 @@ static void auth_completion_func(int rc, zhandle_t* zh) zh->auth_h.pending = req->next; if(rc!=0){ zh->state=ZOO_AUTH_FAILED_STATE; - }else{ - req->auth->state = 1; } zoo_unlock_auth(zh); if (rc) { @@ -5425,11 +5423,12 @@ int zoo_add_auth(zhandle_t *zh,const char* scheme,const char* cert, free(authinfo->scheme); deallocate_Buffer(&authinfo->auth); free(authinfo); - return ZMARSHALLINGERROR; + return rc == ZSYSTEMERROR ? ZSYSTEMERROR : ZMARSHALLINGERROR; } if (sent) { LOG_DEBUG(LOGCALLBACK(zh), "Sending auth info request to %s", zoo_get_current_server(zh)); + return nonblocking_send(zh, ZOK); } return ZOK; } diff --git a/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc b/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc index 61523d44236..386ac7b4a93 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestAuthCompletion.cc @@ -19,6 +19,7 @@ #include #include "CppAssertHelper.h" +#include #include #include #include @@ -100,6 +101,12 @@ class AuthServer: public ZookeeperServer addRecvResponse(new AuthResponse(rc)); } + // Discard held replies. They belong to a connection that is gone. + void dropHeldReplies() { + synchronized(mx_); + held_.clear(); + } + vector schemes() const { synchronized(mx_); return schemes_; @@ -138,6 +145,57 @@ static void authCompletion(int rc, const void *data) res->rc = rc; } +#ifdef THREADED +// The I/O thread runs completions and the test thread reads the results. +// A mutex guards the results, and a counter tells the test when to read. +static Mutex authResultsMx; +static AtomicInt authCompletions; + +static void lockedAuthCompletion(int rc, const void *data) +{ + synchronized(authResultsMx); + authCompletion(rc, data); + ++authCompletions; +} + +struct CompletionsAtLeast { + CompletionsAtLeast(int count): count_(count) {} + bool operator()() const { + return authCompletions.get() >= count_; + } + int count_; +}; + +// The log callback has no context argument. The race test uses these +// globals. raceMx keeps the callback from reading raceHandle before +// zookeeper_init2 returns. +static Mutex raceMx; +static zhandle_t *raceHandle; +static bool raceFired; +static int raceRc; +static AuthResult raceResult; + +// Call zoo_add_auth on the I/O thread in the window where the session is +// connected but send_auth_info has not run. +static void addAuthDuringEstablishment(const char *message) +{ + zhandle_t *zh; + if (strstr(message, "session establishment complete") == NULL) { + return; + } + { + synchronized(raceMx); + if (raceHandle == NULL || raceFired) { + return; + } + raceFired = true; + zh = raceHandle; + } + raceRc = zoo_add_auth(zh, "digest", "race:pass", 9, + lockedAuthCompletion, &raceResult); +} +#endif + class Zookeeper_authCompletion : public CPPUNIT_NS::TestFixture { CPPUNIT_TEST_SUITE(Zookeeper_authCompletion); @@ -146,8 +204,11 @@ class Zookeeper_authCompletion : public CPPUNIT_NS::TestFixture CPPUNIT_TEST(testRejectedCredentialIsNotReportedOk); CPPUNIT_TEST(testResentCredentialDoesNotCompleteNewRequest); CPPUNIT_TEST(testUnsolicitedAuthReplyIsIgnored); + CPPUNIT_TEST(testStoredCredentialsCompleteSeparately); + CPPUNIT_TEST(testReconnectResendDoesNotCompleteNewRequest); #else CPPUNIT_TEST(testConcurrentAddAuthSendsEachCredential); + CPPUNIT_TEST(testAddAuthDuringSessionEstablishment); #endif CPPUNIT_TEST_SUITE_END(); @@ -214,6 +275,20 @@ class Zookeeper_authCompletion : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); } + // Drive the event loop after a connection loss until the session is + // established again. The first pass can only clear the reconnect delay. + void reconnect() { + for (int i = 0; i < 10 && zoo_state(zh) != ZOO_CONNECTED_STATE; i++) { + int fd = 0; + int interest = 0; + timeval tv; + if (zookeeper_interest(zh, &fd, &interest, &tv) == ZOK) { + zookeeper_process(zh, interest); + } + } + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } + // Two credentials go out in the order of the zoo_add_auth calls. Each // completion gets the result of its own frame. void testEachReplyCompletesItsOwnRequest() { @@ -340,11 +415,114 @@ class Zookeeper_authCompletion : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT_EQUAL((int)ZOK, rc); CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); } + + // Session establishment sends two stored credentials. The server + // accepts one and rejects the other. Each completion gets the result of + // its own frame. + void testStoredCredentialsCompleteSeparately() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + zkServer.holdReplies(); + + AuthResult bad, good; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &bad)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user:pass", 9, authCompletion, &good)); + connect(); + drive(2); + + // Session establishment puts each frame at the front of the send + // queue. The last stored credential goes out first. + vector schemes = zkServer.schemes(); + CPPUNIT_ASSERT_EQUAL((size_t)2, schemes.size()); + CPPUNIT_ASSERT_EQUAL(string("digest"), schemes[0]); + CPPUNIT_ASSERT_EQUAL(string("nosuchscheme"), schemes[1]); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, good.rc); + CPPUNIT_ASSERT_EQUAL(0, bad.calls); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, good.calls); + CPPUNIT_ASSERT_EQUAL(1, bad.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, bad.rc); + CPPUNIT_ASSERT_EQUAL(ZOO_AUTH_FAILED_STATE, zoo_state(zh)); + } + + // The connection drops while two stored credentials wait for replies. + // The reconnect sends both credentials again without completions. A + // new zoo_add_auth call does not receive the result of a resent frame. + void testReconnectResendDoesNotCompleteNewRequest() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + CloseFinally guard(&zh); + + zh = zookeeper_init("localhost:2121", watcher, 10000, TEST_CLIENT_ID, 0, 0); + CPPUNIT_ASSERT(zh != 0); + zkServer.holdReplies(); + + AuthResult first, second, fresh; + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user1:pass", 10, authCompletion, &first)); + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "digest", "user2:pass", 10, authCompletion, &second)); + connect(); + drive(2); + CPPUNIT_ASSERT_EQUAL((size_t)2, zkServer.frameCount()); + + // The pending completions receive the connection-loss code. + zkServer.dropHeldReplies(); + zkServer.setConnectionLost(); + drive(1); + zkServer.connectionLost = false; + CPPUNIT_ASSERT_EQUAL(1, first.calls); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS, first.rc); + CPPUNIT_ASSERT_EQUAL(1, second.calls); + CPPUNIT_ASSERT_EQUAL((int)ZCONNECTIONLOSS, second.rc); + + reconnect(); + drive(2); + vector creds = zkServer.creds(); + CPPUNIT_ASSERT_EQUAL((size_t)4, creds.size()); + set resent(creds.begin() + 2, creds.end()); + CPPUNIT_ASSERT_EQUAL((size_t)1, resent.count("user1:pass")); + CPPUNIT_ASSERT_EQUAL((size_t)1, resent.count("user2:pass")); + + CPPUNIT_ASSERT_EQUAL((int)ZOK, + zoo_add_auth(zh, "nosuchscheme", "x", 1, authCompletion, &fresh)); + drive(1); + CPPUNIT_ASSERT_EQUAL((size_t)5, zkServer.frameCount()); + + zkServer.releaseReply(); + drive(1); + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(0, fresh.calls); + CPPUNIT_ASSERT_EQUAL(1, first.calls); + CPPUNIT_ASSERT_EQUAL(1, second.calls); + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + + zkServer.releaseReply(); + drive(1); + CPPUNIT_ASSERT_EQUAL(1, fresh.calls); + CPPUNIT_ASSERT_EQUAL((int)ZAUTHFAILED, fresh.rc); + } #else struct AddAuthArgs { zhandle_t *zh; int thread; int count; + AuthResult *results; + int failures; }; static void *addAuthThread(void *arg) { @@ -353,7 +531,10 @@ class Zookeeper_authCompletion : public CPPUNIT_NS::TestFixture ostringstream cred; cred << "user" << args->thread << "-" << i << ":pass"; string s = cred.str(); - zoo_add_auth(args->zh, "digest", s.c_str(), (int)s.size(), NULL, NULL); + if (zoo_add_auth(args->zh, "digest", s.c_str(), (int)s.size(), + lockedAuthCompletion, &args->results[i]) != ZOK) { + args->failures++; + } } return NULL; } @@ -369,7 +550,7 @@ class Zookeeper_authCompletion : public CPPUNIT_NS::TestFixture }; // Concurrent zoo_add_auth calls. Each call sends its own credential - // exactly once. + // exactly once, and each completion runs once with ZOK. void testConcurrentAddAuthSendsEachCredential() { const int THREADS = 8; const int PER_THREAD = 50; @@ -382,25 +563,90 @@ class Zookeeper_authCompletion : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT(zh != 0); CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + authCompletions = 0; + vector results(THREADS * PER_THREAD); pthread_t threads[THREADS]; AddAuthArgs args[THREADS]; for (int t = 0; t < THREADS; t++) { args[t].zh = zh; args[t].thread = t; args[t].count = PER_THREAD; + args[t].results = &results[t * PER_THREAD]; + args[t].failures = 0; CPPUNIT_ASSERT_EQUAL(0, pthread_create(&threads[t], NULL, addAuthThread, &args[t])); } for (int t = 0; t < THREADS; t++) { pthread_join(threads[t], NULL); } + for (int t = 0; t < THREADS; t++) { + CPPUNIT_ASSERT_EQUAL(0, args[t].failures); + } const size_t total = THREADS * PER_THREAD; ensureCondition(FrameCountAtLeast(zkServer, total), 5000); + ensureCondition(CompletionsAtLeast((int)total), 5000); vector creds = zkServer.creds(); CPPUNIT_ASSERT_EQUAL(total, creds.size()); set unique(creds.begin(), creds.end()); CPPUNIT_ASSERT_EQUAL(total, unique.size()); + { + synchronized(authResultsMx); + for (size_t i = 0; i < total; i++) { + CPPUNIT_ASSERT_EQUAL(1, results[i].calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, results[i].rc); + } + } + CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); + } + + // zoo_add_auth runs on the I/O thread after the session becomes + // connected but before send_auth_info runs. This is the race with a + // caller thread. send_auth_info must take back the queued frame and + // send the credential once, with its completion. A second credential + // then acts as a barrier for all earlier frames. + void testAddAuthDuringSessionEstablishment() { + Mock_gettimeofday timeMock; + AuthServer zkServer; + Mock_poll pollMock(&zkServer, ZookeeperServer::FD); + CloseFinally guard(&zh); + + authCompletions = 0; + raceResult = AuthResult(); + raceRc = ZAPIERROR; + { + synchronized(raceMx); + raceFired = false; + zh = zookeeper_init2("localhost:2121", watcher, 10000, + TEST_CLIENT_ID, 0, 0, addAuthDuringEstablishment); + raceHandle = zh; + } + CPPUNIT_ASSERT(zh != 0); + CPPUNIT_ASSERT(ensureCondition(ClientConnected(zh), 1000) < 1000); + ensureCondition(CompletionsAtLeast(1), 5000); + + AuthResult barrier; + CPPUNIT_ASSERT_EQUAL((int)ZOK, zoo_add_auth(zh, "digest", "barrier:pass", + 12, lockedAuthCompletion, &barrier)); + ensureCondition(CompletionsAtLeast(2), 5000); + { + synchronized(raceMx); + raceHandle = NULL; + } + + CPPUNIT_ASSERT(raceFired); + CPPUNIT_ASSERT_EQUAL((int)ZOK, raceRc); + vector creds = zkServer.creds(); + CPPUNIT_ASSERT_EQUAL((size_t)2, creds.size()); + CPPUNIT_ASSERT_EQUAL(string("race:pass"), creds[0]); + CPPUNIT_ASSERT_EQUAL(string("barrier:pass"), creds[1]); + { + synchronized(authResultsMx); + CPPUNIT_ASSERT_EQUAL(1, raceResult.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, raceResult.rc); + CPPUNIT_ASSERT_EQUAL(1, barrier.calls); + CPPUNIT_ASSERT_EQUAL((int)ZOK, barrier.rc); + } CPPUNIT_ASSERT_EQUAL(ZOO_CONNECTED_STATE, zoo_state(zh)); } #endif