diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 4ec70dbd..27805d33 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -38,6 +38,10 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; import io.questdb.client.impl.ConfigView; @@ -48,6 +52,7 @@ import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.IntList; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -790,12 +795,16 @@ default Sender uuidColumn(CharSequence name, long lo, long hi) { * unconnected sender; the I/O thread runs the same retry loop in * the background. The user thread can call {@code at()} / * {@code flush()} immediately; rows accumulate in the cursor SF - * engine until the wire is up. Connect failures are retried - * indefinitely in the background; a terminal upgrade failure - * (auth reject, capability mismatch) is delivered to the async - * error inbox as a {@link io.questdb.client.SenderError} (no - * synchronous throw on the user call site). Wire - * {@code error_handler=...} to observe these. + * engine until the wire is up. Transport failures (unreachable or + * dropped server) retry indefinitely and are never surfaced -- the + * buffered rows are safe in SF and the server may still appear. A + * terminal auth, upgrade or capability rejection on the initial + * connect (before the wire has ever come up) has no caller left to + * throw at, so it is delivered to the async error inbox as a + * {@link io.questdb.client.SenderError}; wire {@code error_handler=...} + * to observe it. Once the wire has come up even once, SF owns the + * buffered data and the same rejections (e.g. a credential rotation) + * become transients retried indefinitely. * *

* Default resolution when the caller does not pick a value: @@ -988,6 +997,12 @@ final class LineSenderBuilder { // build() time. 0 or negative is a documented "disable" value, so // a Long.MIN_VALUE sentinel keeps it distinguishable from "unset". private static final long DURABLE_ACK_KEEPALIVE_NOT_SET = Long.MIN_VALUE; + private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(LineSenderBuilder.class); + // How many quarantined copies of one slot may pile up under sf_dir before build() + // refuses to set aside another. Each is an unreplayable slot a human still has to + // look at; accumulating them without bound would turn a disk-space problem into a + // second incident. + private static final int MAX_QUARANTINE_SLOT_ATTEMPTS = 64; private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1; // sf-client.md section 4.4: the inbox capacity must accommodate the // distinct error categories in a bursty error stream so that drop-oldest @@ -1003,6 +1018,16 @@ final class LineSenderBuilder { private static final int PROTOCOL_TCP = 0; private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; + @TestOnly + private static volatile Runnable quarantineAfterCloseHook; + @TestOnly + private static volatile FilesFacade quarantineFilesFacade = FilesFacade.INSTANCE; + // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the + // sender's own slot name, so a restarted sender does not re-adopt it as its own; + // quarantineTornSlot then marks it .failed, so the orphan drainer skips it too and + // the bytes stay put for a human to inspect and resend. + private static final String QUARANTINE_SLOT_SUFFIX = + OrphanScanner.QUARANTINE_SLOT_INFIX; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); private long authTimeoutMillis = QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS; @@ -1051,6 +1076,7 @@ final class LineSenderBuilder { private int errorInboxCapacity = PARAMETER_NOT_SET_EXPLICITLY; private int maxFrameRejections = PARAMETER_NOT_SET_EXPLICITLY; private long poisonMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; + private long catchUpCapGapMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; private String httpPath; private String httpSettingsPath; private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY; @@ -1059,8 +1085,8 @@ final class LineSenderBuilder { // explicitly", which build() resolves to SYNC when any reconnect_* // knob was tuned by the user, otherwise OFF. SYNC retries on the // user thread up to the reconnect cap. ASYNC returns immediately - // and lets the I/O thread retry in the background, surfacing - // terminal failures via the error inbox. + // and lets the I/O thread retry every endpoint/transport failure in + // the background without surfacing it to the producer. private InitialConnectMode initialConnectMode = null; private String keyId; private int maxBackgroundDrainers = DEFAULT_MAX_BACKGROUND_DRAINERS; @@ -1505,6 +1531,10 @@ public Sender build() { long actualPoisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY ? poisonMinEscalationWindowMillis : CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + long actualCatchUpCapGapMinEscalationWindowMillis = + catchUpCapGapMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY + ? catchUpCapGapMinEscalationWindowMillis + : CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; // sfDir is the parent (group root); the actual slot lives // under sfDir/senderId. This is what the engine sees — the @@ -1520,7 +1550,11 @@ public Sender build() { slotPath = null; } else { if (!Files.exists(sfDir)) { - int rc = Files.mkdir(sfDir, Files.DIR_MODE_DEFAULT); + // DIR_MODE_SHARED, matching .slot-locks: a second uid has to create + // its own slot directory in here, so restricting this to 0755 makes + // the shared lock directory pointless (see Files.DIR_MODE_SHARED). + // Under the usual umask 022 this is still 0755, plus sticky. + int rc = Files.mkdir(sfDir, Files.DIR_MODE_SHARED); // mkdir is non-zero on failure, but "already exists" // is one such failure. Multiple SF senders sharing one // sf_dir can be built concurrently (the pool calls @@ -1540,55 +1574,105 @@ public Sender build() { sfAppendDeadlineMillis == PARAMETER_NOT_SET_EXPLICITLY ? CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS : sfAppendDeadlineMillis * 1_000_000L; - CursorSendEngine cursorEngine = new CursorSendEngine( - slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? errorInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; - int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? connectionListenerInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; - List wsEndpoints = - new ArrayList<>(hosts.size()); - for (int i = 0, n = hosts.size(); i < n; i++) { - wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); - } - QwpWebSocketSender connected; - try { - connected = QwpWebSocketSender.connect( - wsEndpoints, - wsTlsConfig, - actualAutoFlushRows, - actualAutoFlushBytes, - actualAutoFlushIntervalNanos, - wsAuthHeader, - requestDurableAck, - cursorEngine, - actualCloseFlushTimeoutMillis, - actualReconnectMaxDurationMillis, - actualReconnectInitialBackoffMillis, - actualReconnectMaxBackoffMillis, - actualInitialConnectMode, - errorHandler, - actualErrorInboxCapacity, - actualDurableAckKeepaliveIntervalMillis, - authTimeoutMillis, - connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, - connectionListener, - actualConnectionListenerInboxCapacity, - actualMaxFrameRejections, - actualPoisonMinEscalationWindowMillis - ); - } catch (Throwable t) { - // connect() failed before ownership of cursorEngine - // transferred — close it ourselves. - try { - cursorEngine.close(); - } catch (Throwable ignored) { - // best-effort + QwpWebSocketSender connected = null; + // The parent-anchored logical lock is stable across a slot rename. Keep it + // from before the directory-local lock is acquired until connect() has either + // adopted that engine or quarantine has closed, renamed and recreated it. + // This closes the inode-swap window in which an already-queued orphan drainer + // could otherwise acquire the renamed directory's old .lock and later operate + // on the fresh slot through the original pathname. + try (SlotLock logicalSlotLock = slotPath == null + ? null + : SlotLock.acquireLogical(slotPath)) { + CursorSendEngine cursorEngine = new CursorSendEngine( + slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? errorInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; + int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? connectionListenerInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; + List wsEndpoints = + new ArrayList<>(hosts.size()); + for (int i = 0, n = hosts.size(); i < n; i++) { + wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); + } + // The recovery seed inside connect() is the authority on whether a recovered + // slot can be replayed: it rebuilds the dictionary from its intact prefix and + // then from the surviving frames' own delta sections, and throws + // UnreplayableSlotException only once neither source holds the missing ids. + // Quarantining on anything weaker would set aside slots that recovery can + // still rescue, so build() waits for that verdict rather than pre-judging it. + boolean quarantined = false; + while (connected == null) { + try { + connected = QwpWebSocketSender.connect( + wsEndpoints, + wsTlsConfig, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + wsAuthHeader, + requestDurableAck, + cursorEngine, + actualCloseFlushTimeoutMillis, + actualReconnectMaxDurationMillis, + actualReconnectInitialBackoffMillis, + actualReconnectMaxBackoffMillis, + actualInitialConnectMode, + errorHandler, + actualErrorInboxCapacity, + actualDurableAckKeepaliveIntervalMillis, + authTimeoutMillis, + connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, + connectionListener, + actualConnectionListenerInboxCapacity, + actualMaxFrameRejections, + actualPoisonMinEscalationWindowMillis, + actualCatchUpCapGapMinEscalationWindowMillis + ); + } catch (UnreplayableSlotException e) { + // The one failure build() recovers from. The slot's frames reference ids + // that nothing still holds, so they can never go on the wire -- but that is + // no reason to take the producer down with them. Before this, the throw + // escaped build() and, because senderId is stable and a not-fully-drained + // slot is retained on close, every retry re-recovered the same slot and + // threw again: the application could not construct a Sender at all, so it + // could not even BUFFER new rows. An already-lost batch became an unbounded + // outage of everything after it. + // + // Set the slot aside instead, keep its bytes for forensics and resend, and + // start the producer on a clean one. Once only: a second such failure would + // mean the FRESH slot is unreplayable, which cannot happen, so let it out + // rather than loop. + if (quarantined || slotPath == null) { + try { + // close(false): we still hold the logical slot lock. + cursorEngine.close(false); + } catch (Throwable ignored) { + // best-effort + } + throw e; + } + quarantined = true; + cursorEngine = quarantineTornSlot( + cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos, errorHandler); + } catch (Throwable t) { + // connect() failed before ownership of cursorEngine + // transferred — close it ourselves. close(false) + // because logicalSlotLock is still held here: a fresh + // slot is fully drained, so the default close would + // unlink the very lock file this scope holds. + try { + cursorEngine.close(false); + } catch (Throwable ignored) { + // best-effort + } + throw t; + } } - throw t; } // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY @@ -1703,6 +1787,35 @@ public Sender build() { return sender; } + /** + * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must + * persist before an orphan store-and-forward drainer quarantines its slot. + *

+ * A cap gap means a symbol already accepted by one node is too large to + * re-register on the node the sender just failed over to, because that node + * advertises a smaller maximum batch size. On a homogeneous cluster this cannot + * happen; it takes a heterogeneous or mid-roll cluster, or an operator lowering + * the cap below existing data. + *

+ * A foreground sender retries such a gap indefinitely because the larger-cap node + * may simply be away. An orphan drainer may quarantine only once the gap has BOTH + * recurred many times AND persisted for this long. Raise it for a cluster whose + * rolling restarts take longer than the 5-minute default; set it to {@code 0} to + * quarantine an orphan slot as soon as the retry count is exhausted. + *

+ * WebSocket transport only. + */ + public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + if (millis < 0) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis must be >= 0: ").put(millis); + } + this.catchUpCapGapMinEscalationWindowMillis = millis; + return this; + } + /** * close() drain timeout in milliseconds. The sender's {@code close()} * method blocks up to this many millis waiting for the server to ACK @@ -2149,11 +2262,10 @@ public LineSenderBuilder initialConnectMode(InitialConnectMode mode) { } /** - * Opt in to retrying the initial connect with the same backoff / - * cap / auth-terminal policy as in-flight reconnect. Set true if - * your deployment expects the server to come up shortly after the - * sender. Auth failures (HTTP 401/403/non-101) stay terminal in - * either mode. + * Opt in to retrying the initial connect with backoff on the calling + * thread, bounded by the configured reconnect cap. Set true if your + * deployment expects the server to come up shortly after the sender. + * Auth failures (HTTP 401/403/non-101) stay terminal in this SYNC mode. *

* When this method is not called, the resolution rule documented * on {@link InitialConnectMode} applies: SYNC implicitly when any @@ -2494,8 +2606,8 @@ public LineSenderBuilder reconnectMaxBackoffMillis(long millis) { * with exponential backoff until connect succeeds or this many * millis elapse, then throws. The background reconnect loop * (mid-stream outages and async initial connect) does NOT consult - * this value: it retries indefinitely and halts only on a terminal - * auth/upgrade error or {@code close()}. + * this value: endpoint and transport failures are retried indefinitely + * until {@code close()}. *

* Default {@code 300_000} (5 minutes). Lower for fail-fast startup; * higher for tolerating a slow server boot. Must be positive: a zero @@ -2916,6 +3028,126 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na } } + /** + * Sets a recovered slot whose symbol dictionary cannot cover its surviving frames + * aside, and returns a fresh engine on an empty slot so the producer can keep + * producing. + *

+ * Such a slot is unreplayable BY THIS PRODUCER: its frames reference symbol ids the + * recovered dictionary lost (a host/power crash tore the unsynced side-file), so a + * producer seeded from the short dictionary would hand those ids to different + * symbols and silently misattribute values. Detecting that is correct and + * load-bearing -- but simply THROWING is not a safe response. {@code senderId} + * defaults to a stable name, so a restarted process re-adopts the same slot; the + * engine's close retains a slot that is not fully drained; and so every subsequent + * {@code build()} would re-recover the same slot and throw again -- forever, until + * an operator deleted the directory by hand. The application could not construct a + * Sender at all, and so could not even BUFFER new rows. That trades a bounded, + * already-lost batch for an unbounded outage of everything after it, which inverts + * the one guarantee store-and-forward exists to give. + *

+ * So: rename the slot aside instead, and mark it {@code .failed}. The verdict is + * authoritative -- the recovery seed already tried every source of truth (the + * persisted prefix AND the surviving frames' own deltas), and the orphan drainer's + * own replay guard uses that same walk, so there is nothing a drainer could rebuild + * that the seed did not. {@code markFailed} (below) therefore quarantines the copy + * for a human rather than leaving a drainer to retry an unreplayable slot forever; + * a full-dictionary-fallback slot never reaches here, because its dictionary is + * discarded at recovery and it never throws. The bytes are preserved on disk for + * forensics and a manual resend, and the new name -- NOT the sender's own slot name + * -- keeps a restarted sender from re-adopting it. The producer, meanwhile, starts + * on a clean empty slot and never notices. + *

+ * If the rename fails (a Windows share lock, a read-only mount) there is no way to + * free the slot name without destroying data, so fall back to the old behaviour and + * throw -- loudly, and never silently dropping bytes. + */ + private static CursorSendEngine quarantineTornSlot( + CursorSendEngine torn, UnreplayableSlotException cause, String sfDir, + String senderId, String slotPath, + long sfMaxBytes, long sfMaxTotalBytes, long sfAppendDeadlineNanos, + io.questdb.client.SenderErrorHandler errorHandler + ) { + // The verdict, and the reason, come from the recovery seed -- the only code that + // has tried every source of truth. Recomputing them here would mean a second, + // independently-drifting notion of "unreplayable". + final String detail = cause.getMessage(); + // Release the slot lock and the dictionary fd before renaming. connect()'s failure + // path already closed the engine; close() is idempotent, so make it explicit rather + // than depend on that. close(false): build() holds the logical slot lock across this + // whole transition -- that is precisely what serialises the rename against a queued + // orphan drainer -- so the engine must not unlink it. + torn.close(false); + Runnable hook = quarantineAfterCloseHook; + if (hook != null) { + hook.run(); + } + + FilesFacade ff = quarantineFilesFacade; + String quarantinePath = null; + for (int i = 0; i < MAX_QUARANTINE_SLOT_ATTEMPTS; i++) { + String candidate = sfDir + "/" + senderId + QUARANTINE_SLOT_SUFFIX + i; + if (!ff.exists(candidate)) { + quarantinePath = candidate; + break; + } + } + if (quarantinePath == null || ff.rename(slotPath, quarantinePath) != 0) { + throw new LineSenderException( + detail + "; the affected data must be resent. The slot could not be set aside " + + "automatically (" + (quarantinePath == null + ? "too many quarantined slots already under " + sfDir + : "rename to " + quarantinePath + " failed") + + "), so this sender cannot start until " + + slotPath + " is moved or removed by hand"); + } + // Mark the quarantined copy so the orphan drainer treats it as a + // human-in-the-loop slot rather than silently retrying it forever. + OrphanScanner.markFailed(quarantinePath, detail); + LOG.error("{} -- the slot has been set aside at {} and the affected data must be resent; " + + "this sender continues on a fresh, empty slot at {}", + detail, quarantinePath, slotPath); + // build() no longer throws for this -- it starts the producer on a fresh slot so + // the outage stays bounded. But abandoning buffered rows is precisely the event a + // caller must be able to act on, and LOG.error alone cannot carry it: this client + // ships slf4j-api with no binding, so an embedding app with no provider gets a NOP + // logger and the loss is announced nowhere. Deliver it programmatically too, so an + // errorHandler can alert / page / record it. Dispatched synchronously here because + // the async SenderErrorDispatcher belongs to the connected sender, which does not + // exist yet at build time. A throwing handler must not turn a contained outage back + // into a failed build, so swallow anything it raises. + if (errorHandler != null) { + try { + errorHandler.onError(new SenderError( + SenderError.Category.PROTOCOL_VIOLATION, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + detail + " [slot set aside at " + quarantinePath + + "; sender continues on a fresh slot at " + slotPath + ']', + SenderError.NO_MESSAGE_SEQUENCE, + SenderError.NO_MESSAGE_SEQUENCE, + SenderError.NO_MESSAGE_SEQUENCE, + null, + System.nanoTime() + )); + } catch (Throwable handlerFailure) { + LOG.error("sender error handler threw while reporting a quarantined slot: {}", + String.valueOf(handlerFailure)); + } + } + return new CursorSendEngine(slotPath, sfMaxBytes, sfMaxTotalBytes, sfAppendDeadlineNanos); + } + + @TestOnly + public static void setQuarantineAfterCloseHookForTest(Runnable hook) { + quarantineAfterCloseHook = hook; + } + + @TestOnly + public static void setQuarantineFilesFacadeForTest(FilesFacade ff) { + quarantineFilesFacade = ff == null ? FilesFacade.INSTANCE : ff; + } + private static int resolveIPv4(String host) { try { byte[] addr = InetAddress.getByName(host).getAddress(); @@ -3413,6 +3645,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "poison_min_escalation_window_millis"); poisonMinEscalationWindowMillis(parseLongValue(sink, "poison_min_escalation_window_millis")); + } else if (Chars.equals("catchup_cap_gap_min_escalation_window_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "catchup_cap_gap_min_escalation_window_millis"); + catchUpCapGapMinEscalationWindowMillis(parseLongValue(sink, "catchup_cap_gap_min_escalation_window_millis")); } else if (Chars.equals("initial_connect_retry", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("initial_connect_retry is only supported for WebSocket transport"); @@ -3686,6 +3924,9 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("poison_min_escalation_window_millis")) { poisonMinEscalationWindowMillis(wsLong(view, v, "poison_min_escalation_window_millis")); } + if (view.has("catchup_cap_gap_min_escalation_window_millis")) { + catchUpCapGapMinEscalationWindowMillis(wsLong(view, v, "catchup_cap_gap_min_escalation_window_millis")); + } if (view.has("sf_append_deadline_millis")) { sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis")); } @@ -3862,6 +4103,10 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("max_background_drainers", maxBackgroundDrainers); m.put("max_frame_rejections", maxFrameRejections); m.put("poison_min_escalation_window_millis", poisonMinEscalationWindowMillis); + m.put("catchup_cap_gap_min_escalation_window_millis", + catchUpCapGapMinEscalationWindowMillis == PARAMETER_NOT_SET_EXPLICITLY + ? CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS + : catchUpCapGapMinEscalationWindowMillis); m.put("error_inbox_capacity", errorInboxCapacity); m.put("connection_listener_inbox_capacity", connectionListenerInboxCapacity); m.put("token", httpToken); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index fd16dca7..ef5a6bf4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -460,6 +460,35 @@ public void sendBinary(long dataPtr, int length, int timeout) { sendBuffer.reset(); } + /** + * Sends two native-memory slices as one WebSocket binary frame. The slices + * are copied directly into the WebSocket send buffer and masked there, so a + * caller assembling a small protocol prefix around a large immutable body + * does not need a second contiguous staging buffer. + * + * @param firstPtr pointer to the first payload slice + * @param firstLength first payload-slice length + * @param secondPtr pointer to the second payload slice + * @param secondLength second payload-slice length + * @param timeout timeout in milliseconds + */ + public void sendBinary( + long firstPtr, + int firstLength, + long secondPtr, + int secondLength, + int timeout + ) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginFrame(); + sendBuffer.putBlockOfBytes(firstPtr, firstLength); + sendBuffer.putBlockOfBytes(secondPtr, secondLength); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + /** * Sends binary data with default timeout. */ @@ -467,6 +496,13 @@ public void sendBinary(long dataPtr, int length) { sendBinary(dataPtr, length, defaultTimeout); } + /** + * Sends two native-memory slices as one binary frame with the default timeout. + */ + public void sendBinary(long firstPtr, int firstLength, long secondPtr, int secondLength) { + sendBinary(firstPtr, firstLength, secondPtr, secondLength, defaultTimeout); + } + /** * Sends a close frame. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 3b4c9a90..d1b0f13e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -50,6 +50,37 @@ public GlobalSymbolDictionary(int initialCapacity) { this.idToSymbol = new ObjList<>(initialCapacity); } + /** + * Appends {@code symbol} at the next sequential id, matching a recovered / + * persisted dictionary's dense id order, WITHOUT de-duplicating. + *

+ * Recovery ({@code QwpWebSocketSender.seedGlobalDictionaryFromPersisted}) + * replays the persisted entries in id order to rebuild this dictionary. It must + * NOT collapse two source strings that decode to the same characters, because + * the persisted {@code .symbol-dict}, the on-wire delta and the I/O-thread + * catch-up mirror all key on the entry POSITION (id), not on the string. The + * only strings that collide this way are malformed lone UTF-16 surrogates, + * which the UTF-8 encoder maps to {@code '?'}: {@link #getOrAddSymbol} would + * de-dup them and leave this dictionary SHORTER than the persisted entry count, + * desyncing the producer's delta baseline from the catch-up mirror (which uses + * {@code pd.size()}) and silently misattributing later symbols. Appending + * unconditionally keeps {@link #size()} equal to that count. The reverse lookup + * keeps the highest id for a colliding string, which is harmless: both ids + * encode to the same bytes, so resolving either is equivalent. + * + * @param symbol the recovered symbol string (must not be null) + * @return the id assigned (the previous {@link #size()}) + */ + public int addRecoveredSymbol(String symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + int newId = idToSymbol.size(); + symbolToId.put(symbol, newId); + idToSymbol.add(symbol); + return newId; + } + /** * Clears all symbols from the dictionary. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index aad95f3c..86343772 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -76,6 +76,27 @@ public static int varintSize(long value) { return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; } + /** + * Writes {@code value} as an unsigned LEB128 varint directly at native address + * {@code addr} and returns the address just past the last byte. The canonical + * raw-address varint writer shared by the SF cursor's persisted dictionary and + * catch-up frame builder. + *

+ * {@code value} must be non-negative: the signed {@code value > 0x7F} loop emits + * a SINGLE truncated byte for a negative long, whereas {@link #varintSize} + * returns 10 for it -- a size/write mismatch that would corrupt the stream. All + * callers pass ids/lengths/counts (non-negative); the assert pins that contract. + */ + public static long writeVarint(long addr, long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + @Override public void close() { if (bufferPtr != 0) { @@ -305,6 +326,7 @@ public void putUtf8(CharSequence value) { */ @Override public void putVarint(long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; ensureCapacity(10); // max varint bytes long addr = bufferPtr + position; while (value > 0x7F) { @@ -336,11 +358,7 @@ public void skip(int bytes) { } private static void writeVarintDirect(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr, (byte) value); + writeVarint(addr, value); } private void encodeUtf8(CharSequence value, int utf8Len) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index ced1a1b5..f38d9e93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; @@ -39,6 +41,16 @@ public class QwpWebSocketEncoder implements QuietCloseable { private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private NativeBufferWriter buffer; + // Byte offsets, within the buffer, of the symbol-dict delta ENTRY region + // ([len][utf8]... only, without the two section varints) that beginMessage + // last wrote. Let the producer persist those bytes straight to the slot's + // .symbol-dict instead of re-encoding the same symbols (see + // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next + // beginMessage; stored as offsets so they survive a buffer realloc. + private int deltaCount; + private int deltaEntriesEnd; + private int deltaEntriesStart; + private int deltaStart; // QWP ingress always advertises Gorilla timestamp encoding. The column // writer still emits a per-column encoding byte and falls back to raw // values when delta-of-delta overflows int32. @@ -65,8 +77,8 @@ public void beginMessage( int batchMaxId ) { buffer.reset(); - int deltaStart = confirmedMaxId + 1; - int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); + deltaStart = confirmedMaxId + 1; + deltaCount = Math.max(0, batchMaxId - confirmedMaxId); byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT); byte origFlags = flags; flags = headerFlags; @@ -75,10 +87,12 @@ public void beginMessage( payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); + deltaEntriesStart = buffer.getPosition(); for (int id = deltaStart; id < deltaStart + deltaCount; id++) { String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } + deltaEntriesEnd = buffer.getPosition(); columnWriter.setBuffer(buffer); } @@ -90,6 +104,64 @@ public void close() { } } + /** + * Copies one single-table split message from the combined message currently + * staged in this encoder. The table body is copied byte-for-byte from its + * recorded offset; columns and rows are not encoded again. + */ + public int copySplitMessage( + MicrobatchBuffer target, + int tableBodyOffset, + int tableBodyLength, + boolean deferCommit, + int confirmedMaxId, + int batchMaxId + ) { + if (target.getBufferPos() != 0) { + throw new IllegalStateException("split message target is not empty"); + } + if (tableBodyOffset < deltaEntriesEnd + || tableBodyLength < 0 + || (long) tableBodyOffset + tableBodyLength > buffer.getPosition()) { + throw new IllegalArgumentException("table body slice is outside the staged message"); + } + + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + int messageSize = splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + target.ensureCapacity(messageSize); + + long source = buffer.getBufferPtr(); + long destination = target.getBufferPtr(); + Vect.memcpy(destination, source, HEADER_SIZE); + + byte splitFlags = Unsafe.getUnsafe().getByte(source + HEADER_OFFSET_FLAGS); + if (deferCommit) { + splitFlags |= FLAG_DEFER_COMMIT; + } else { + splitFlags &= ~FLAG_DEFER_COMMIT; + } + Unsafe.getUnsafe().putByte(destination + HEADER_OFFSET_FLAGS, splitFlags); + Unsafe.getUnsafe().putShort(destination + 6, (short) 1); + Unsafe.getUnsafe().putInt(destination + 8, messageSize - HEADER_SIZE); + + long writeAddress = destination + HEADER_SIZE; + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaStart); + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaCount); + if (deltaEntriesLength > 0) { + Vect.memcpy(writeAddress, source + deltaEntriesStart, deltaEntriesLength); + writeAddress += deltaEntriesLength; + } + Vect.memcpy(writeAddress, source + tableBodyOffset, tableBodyLength); + writeAddress += tableBodyLength; + assert writeAddress == destination + messageSize; + + target.setBufferPos(messageSize); + return messageSize; + } + public int encode(QwpTableBuffer tableBuffer) { buffer.reset(); writeHeader(1, 0); @@ -122,6 +194,33 @@ public QwpBufferWriter getBuffer() { return buffer; } + /** + * Byte length of the symbol-dict delta ENTRY region ({@code [len][utf8]...}, + * excluding the two section varints) that {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesLen() { + return deltaEntriesEnd - deltaEntriesStart; + } + + /** + * Byte offset, within {@link #getBuffer()}, of the symbol-dict delta ENTRY + * region {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesStart() { + return deltaEntriesStart; + } + + public int getSplitMessageSize(int tableBodyLength, int confirmedMaxId, int batchMaxId) { + if (tableBodyLength < 0) { + throw new IllegalArgumentException("tableBodyLength must be non-negative"); + } + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + return splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + } + public void setDeferCommit(boolean defer) { if (defer) { flags |= FLAG_DEFER_COMMIT; @@ -144,4 +243,35 @@ public void writeHeader(int tableCount, int payloadLength) { buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } + + private int splitDeltaEntriesLength(int splitDeltaStart, int splitDeltaCount) { + if (splitDeltaCount == 0) { + return 0; + } + if (splitDeltaStart != deltaStart || splitDeltaCount != deltaCount) { + throw new IllegalStateException("split delta does not match the staged message" + + " [stagedStart=" + deltaStart + + ", stagedCount=" + deltaCount + + ", splitStart=" + splitDeltaStart + + ", splitCount=" + splitDeltaCount + ']'); + } + return deltaEntriesEnd - deltaEntriesStart; + } + + private int splitMessageSize( + int tableBodyLength, + int splitDeltaStart, + int splitDeltaCount, + int deltaEntriesLength + ) { + long messageSize = (long) HEADER_SIZE + + NativeBufferWriter.varintSize(splitDeltaStart) + + NativeBufferWriter.varintSize(splitDeltaCount) + + deltaEntriesLength + + tableBodyLength; + if (messageSize > Integer.MAX_VALUE) { + throw new OutOfMemoryError("split QWP message size overflow: " + messageSize); + } + return (int) messageSize; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..071020e6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -46,9 +46,11 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderConnectionListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.CharSequenceObjHashMap; @@ -56,6 +58,7 @@ import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; +import io.questdb.client.std.IntList; import io.questdb.client.std.Misc; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -152,7 +155,11 @@ public class QwpWebSocketSender implements Sender { private final QwpWebSocketEncoder encoder; private final List endpoints; // Global symbol dictionary for delta encoding - private final GlobalSymbolDictionary globalSymbolDictionary; + // Not final: seedGlobalDictionaryFromPersisted replaces it with a pre-sized instance + // before anything can observe the original. Nothing retains a reference -- the encoder + // and the persisted dictionary both take it as a per-call parameter -- and the swap + // happens during construction, before the producer or the I/O thread exist. + private GlobalSymbolDictionary globalSymbolDictionary; // Serializes FOREGROUND connect walks only (see buildAndConnect): the // shared-round state in hostTracker (pickNext/beginRound/attempted // bits), roundSeq, roundConnectAttemptSeq, and the foreground lifecycle @@ -168,6 +175,19 @@ public class QwpWebSocketSender implements Sender { // behind a drainer's endpoint walk. private final ReentrantLock connectWalkLock = new ReentrantLock(); private final QwpHostHealthTracker hostTracker; + // Per-table encoded body byte counts captured during flushPendingRows' combined + // encode. flushPendingRowsSplit uses them both for preflight sizing and to walk + // the staged body slices without encoding the batch a second time. Cleared and + // repopulated on every flush; reused to stay zero-GC. + // The non-empty tables of the batch currently being flushed, collected ONCE per + // flush and then iterated by every pass. Each pass used to re-walk tableBuffers.keys() + // and re-probe the hash map per table -- 3 probes per table on a plain flush and 5 on + // a split, on the producer's thread. Reused across flushes to stay zero-GC. Held in + // lockstep so index i names the same table in both, and in the same order as + // splitFrameBodyBytes, which is what lets the split passes index them directly. + private final ObjList flushTableBuffers = new ObjList<>(); + private final ObjList flushTableNames = new ObjList<>(); + private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; @@ -221,6 +241,17 @@ public class QwpWebSocketSender implements Sender { private CursorSendEngine cursorEngine; private CursorWebSocketSendLoop cursorSendLoop; private boolean deferCommit; + // True when the sender emits incremental (delta) symbol dictionaries: each + // message carries only symbol ids not yet sent on the wire, rather than the + // full dictionary from id 0. Enabled in memory-mode (a reconnect replays from + // the in-process ring) and in file-mode store-and-forward when the per-slot + // persisted dictionary opened. In both, the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying, so a non-self-sufficient + // delta frame never dangles an id on a fresh server. Falls back to full + // self-sufficient frames only when the persisted dictionary is unavailable in + // file-mode (recovery/orphan-drain would then have nothing to rebuild the + // deltas from). Set in setCursorEngine. + private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after // startOrphanDrainers) and read at pool-creation time. Null -> drainers @@ -271,11 +302,21 @@ public class QwpWebSocketSender implements Sender { // still terminal. // ASYNC → user thread does not connect at all. The I/O thread runs // the reconnect loop in the background, indefinitely - // (Invariant B); terminal failures (auth/upgrade reject) - // are delivered to the SenderError dispatcher rather than - // thrown from the constructor. + // (Invariant B); endpoint-policy and transport failures stay + // contained in that loop and never reach the producer. private Sender.InitialConnectMode initialConnectMode = Sender.InitialConnectMode.OFF; private boolean ownsCursorEngine; + // Whether close() may let the engine reclaim the parent-anchored LOGICAL slot lock. + // False only while an outer frame holds it: Sender.build() acquires it for the whole + // construct -> connect -> quarantine transition, and connect()'s own rollback closes + // this sender from INSIDE that scope. A fresh slot is "fully drained" by definition + // (publishedFsn < 0), so the default close(true) took the reclaim branch and unlinked + // the very lock file build() was holding -- on POSIX that frees the pathname without + // releasing the flock, so the next acquireLogical creates a SECOND inode and locks it. + // build()'s own careful close(false) calls could not prevent it, because connect() + // closed the engine first. Reset to true once connect() hands ownership back, so a + // later user-initiated close() still retires the lock normally. + private boolean reclaimLogicalSlotLockOnClose = true; private long pendingBytes; // Set true by close() once the SF slot flock has been released (the normal // teardown path). Stays false if close() bailed early with the I/O thread @@ -295,6 +336,12 @@ public class QwpWebSocketSender implements Sender { // maxFrameRejections (connect-string key poison_min_escalation_window_millis). private long poisonMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before an + // orphan drainer may quarantine its slot (connect-string key + // catchup_cap_gap_min_escalation_window_millis). Foreground senders retry forever. See + // CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS. + private long catchUpCapGapMinEscalationWindowMillis = + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; private long reconnectInitialBackoffMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS; private long reconnectMaxBackoffMillis = @@ -313,6 +360,12 @@ public class QwpWebSocketSender implements Sender { // beginRound(true) call. roundSeq=1 is the first round; CONNECTED in the // first round indicates the initial connect. private long roundSeq; + // Highest global symbol id the producer has baked into a frame so far, or -1. + // Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because + // the I/O thread re-registers the full dictionary via a catch-up frame before + // replaying, so the producer's delta baseline stays valid across the wire + // boundary. Used only when deltaDictEnabled; ignored in full-dict mode. + private int sentMaxSymbolId = -1; // When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only // explicit flush() triggers the server-side commit. Enables accumulating // arbitrarily large datasets that exceed the server's recv buffer. @@ -667,7 +720,47 @@ public static QwpWebSocketSender connect( durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, connectionListener, connectionListenerInboxCapacity, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); + } + + /** + * Compatibility overload retained for callers compiled before the + * catch-up cap-gap dwell was added to the master signature. + */ + public static QwpWebSocketSender connect( + List endpoints, + ClientTlsConfiguration tlsConfig, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + String authorizationHeader, + boolean requestDurableAck, + CursorSendEngine cursorEngine, + long closeFlushTimeoutMillis, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + Sender.InitialConnectMode initialConnectMode, + SenderErrorHandler errorHandler, + int errorInboxCapacity, + long durableAckKeepaliveIntervalMillis, + long authTimeoutMs, + int connectTimeoutMs, + SenderConnectionListener connectionListener, + int connectionListenerInboxCapacity, + int maxFrameRejections, + long poisonMinEscalationWindowMillis + ) { + return connect(endpoints, tlsConfig, autoFlushRows, autoFlushBytes, + autoFlushIntervalNanos, authorizationHeader, requestDurableAck, + cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, + initialConnectMode, errorHandler, errorInboxCapacity, + durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, + connectionListener, connectionListenerInboxCapacity, + maxFrameRejections, poisonMinEscalationWindowMillis, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -698,7 +791,8 @@ public static QwpWebSocketSender connect( SenderConnectionListener connectionListener, int connectionListenerInboxCapacity, int maxFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, @@ -716,6 +810,7 @@ public static QwpWebSocketSender connect( sender.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; sender.maxFrameRejections = maxFrameRejections; sender.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + sender.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; sender.initialConnectMode = initialConnectMode == null ? Sender.InitialConnectMode.OFF : initialConnectMode; @@ -732,7 +827,20 @@ public static QwpWebSocketSender connect( } sender.ensureConnected(); } catch (Throwable t) { - sender.close(); + // Preserve t's IDENTITY through the rollback. Sender.build() routes on the + // exception type -- only UnreplayableSlotException reaches its quarantine + // handler -- and close() accumulates cleanup errors and ends in + // rethrowTerminal, so letting a close failure propagate here would REPLACE t + // and silently demote a recoverable slot back to the permanent build() brick. + // This rollback always runs inside Sender.build()'s acquireLogical scope, so + // the logical slot lock is held one frame up. Closing the engine with the + // default reclaim would unlink the lock file build() is still holding. + sender.reclaimLogicalSlotLockOnClose = false; + try { + sender.close(); + } catch (Throwable closeFailure) { + t.addSuppressed(closeFailure); + } throw t; } return sender; @@ -1190,7 +1298,7 @@ public void close() { if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null && !cursorSendLoop.delegateEngineClose()) { try { - cursorEngine.close(); + cursorEngine.close(reclaimLogicalSlotLockOnClose); } catch (Throwable t) { LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); terminalError = captureCloseError(terminalError, t); @@ -1232,7 +1340,7 @@ public void close() { if (ownsCursorEngine && cursorEngine != null) { try { - cursorEngine.close(); + cursorEngine.close(reclaimLogicalSlotLockOnClose); } catch (Throwable t) { LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); terminalError = captureCloseError(terminalError, t); @@ -1762,6 +1870,38 @@ public long getPendingBytes() { return pendingBytes; } + /** + * Snapshot of the producer's symbol prefix whose persisted-dictionary chunks + * have committed. The persisted size advances only after the chunk CRC and + * payload have been written, so this observes the write-ahead boundary without + * reopening the live mmap-backed dictionary (which would attempt recovery-tail + * truncation and is not supported while the producer owns the file on Windows). + * Returns {@code null} in memory mode or when the persisted dictionary is + * unavailable. + */ + @TestOnly + public ObjList getPersistedSymbolsForTest() { + CursorSendEngine engine = cursorEngine; + if (engine == null) { + return null; + } + PersistedSymbolDict persisted = engine.getPersistedSymbolDict(); + if (persisted == null) { + return null; + } + int persistedSize = persisted.size(); + int globalSize = globalSymbolDictionary.size(); + if (persistedSize > globalSize) { + throw new IllegalStateException("persisted symbol dictionary exceeds producer dictionary" + + " [persisted=" + persistedSize + ", producer=" + globalSize + ']'); + } + ObjList snapshot = new ObjList<>(persistedSize); + for (int i = 0; i < persistedSize; i++) { + snapshot.add(globalSymbolDictionary.getSymbol(i)); + } + return snapshot; + } + /** * Server-advertised cap on the per-batch raw byte size. Zero before the * first connect; updated by every successful reconnect via @@ -2215,6 +2355,22 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { } this.cursorEngine = engine; this.ownsCursorEngine = takeOwnership && engine != null; + // Delta encoding is available in memory-mode (in-process catch-up) and in + // file-mode when the persisted dictionary opened (recovery / orphan-drain + // rebuild the dictionary from it). Otherwise fall back to full self- + // sufficient frames. See CursorSendEngine.isDeltaDictEnabled. + this.deltaDictEnabled = engine != null && engine.isDeltaDictEnabled(); + // Recovery: repopulate the producer's global dictionary from the slot's + // persisted dictionary so newly ingested symbols continue from the + // recovered ids (rather than colliding with them at 0), and the delta + // baseline resumes where the crashed session left off. + // NOT gated on deltaDictEnabled. That flag is false exactly when the slot's dictionary + // failed to open -- which is precisely when the frames on disk are the only surviving + // copy of the symbols and the rebuild matters most. Gating the seed on it made the + // rebuild dead code for the very case it was written for. + if (engine != null && engine.wasRecoveredFromDisk()) { + seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict()); + } } /** @@ -2378,7 +2534,8 @@ public synchronized void startOrphanDrainers( requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); ref[0] = drainer; drainerPool.submit(drainer); } @@ -3084,8 +3241,14 @@ private void checkTableSelected() { } } - private int countNonEmptyTables(ObjList keys) { - int tableCount = 0; + /** + * Collects the batch's non-empty tables into {@link #flushTableNames} / + * {@link #flushTableBuffers} and returns how many there are. One walk of the key + * list and one hash probe per table, for the whole flush. + */ + private int collectNonEmptyTables(ObjList keys) { + flushTableNames.clear(); + flushTableBuffers.clear(); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3093,10 +3256,11 @@ private int countNonEmptyTables(ObjList keys) { } QwpTableBuffer tableBuffer = tableBuffers.get(tableName); if (tableBuffer != null && tableBuffer.getRowCount() > 0) { - tableCount++; + flushTableNames.add(tableName); + flushTableBuffers.add(tableBuffer); } } - return tableCount; + return flushTableBuffers.size(); } private Endpoint currentEndpoint() { @@ -3198,14 +3362,24 @@ private void drainOnClose(boolean errorOwnedByCustomHandler) { } if (System.nanoTime() >= deadlineNanos) { long acked = cursorEngine.ackedFsn(); - LOG.warn("close() drain timed out after {}ms [target={} acked={}], pending data may be lost", - closeFlushTimeoutMillis, target, acked); + // Name the outage the I/O thread is riding out, when there is one. A + // foreground sender now retries endpoint-policy rejections indefinitely, + // so a revoked token reaches the operator HERE -- and blaming timeout + // tuning for what is actually an auth failure is how the review found + // this path misdirecting. + CursorWebSocketSendLoop loop = cursorSendLoop; + Throwable outage = loop == null ? null : loop.lastReconnectError(); + LOG.warn("close() drain timed out after {}ms [target={} acked={}], pending data may be lost{}", + closeFlushTimeoutMillis, target, acked, + outage == null ? "" : "; wire is not draining: " + outage.getMessage()); throw new LineSenderException("close() drain timed out after ") .put(closeFlushTimeoutMillis).put(" ms [targetFsn=") .put(target).put(", ackedFsn=").put(acked) .put("] - server did not acknowledge ") .put(target - acked) - .put(" pending batches; data may be lost (use larger closeFlushTimeoutMillis or smaller batches)"); + .put(outage == null + ? " pending batches; data may be lost (use larger closeFlushTimeoutMillis or smaller batches)" + : " pending batches; the wire is not draining: " + outage.getMessage()); } java.util.concurrent.locks.LockSupport.parkNanos(50_000L); } @@ -3277,10 +3451,12 @@ private void ensureConnected() { // Encoder stays at its default (V1 -- the only supported wire // version today). Frames written before the first successful // connect commit to V1 because cursor segments are immutable; - // a future version bump must account for that. Auth/upgrade - // rejects are surfaced via the error inbox by the I/O - // thread, not thrown here; plain connect failures retry - // indefinitely (Invariant B). + // a future version bump must account for that. Transport + // failures retry indefinitely on the I/O thread (Invariant B). + // But a terminal auth, upgrade or capability rejection on this + // initial connect -- before the wire is ever up -- is surfaced + // to the async SenderErrorHandler and latched for a close() + // rethrow, not retried. client = null; break; case OFF: @@ -3300,13 +3476,14 @@ private void ensureConnected() { client, cursorEngine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND); // Plug the async-delivery sink before start() so the I/O thread // never observes a null dispatcher between recordFatal and // notification — the test for null in dispatchError handles @@ -3332,6 +3509,17 @@ private void ensureConnected() { cursorSendLoop.setConnectionDispatcher(connectionDispatcher); cursorSendLoop.start(); } catch (Throwable t) { + // start() (or dispatcher construction) failed after cursorSendLoop was + // assigned. Close it so a caller that retries -- re-entering + // ensureConnected and reassigning cursorSendLoop above -- cannot orphan + // a recovered slot's ctor-seeded native mirror (freed only by close() + // or the I/O loop, neither of which has run). close() is idempotent and + // frees the mirror via its loopNeverRan path; it also closes the shared + // client, so the client.close() below is a safe idempotent no-op. + if (cursorSendLoop != null) { + cursorSendLoop.close(); + cursorSendLoop = null; + } if (client != null) { client.close(); client = null; @@ -3359,18 +3547,18 @@ private void ensureConnected() { host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes); } else { // Async mode: I/O thread will drive the connect. Encoder uses - // its default version (V1). The symbol-dict watermark still gets - // reset for consistency with the sync path; the post-connect replay - // path does not need a producer-side reset signal because every - // cursor frame is self-sufficient. + // its default version (V1). The per-batch symbol-dict watermark still + // gets reset for consistency with the sync path; the post-connect + // replay path needs no producer-side reset signal (see below). Endpoint ep = endpoints.get(0); LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]", ep.host, ep.port, endpoints.size()); } - // Server starts fresh on each connection, so reset the symbol-dict - // watermark. Cursor frames are self-sufficient (every frame carries its - // full inline schema + a symbol-dict delta from id 0), so post-reconnect - // replay needs no producer-side reset signal. + // Server starts fresh on each connection, so reset the per-batch + // symbol-dict watermark. Every frame still carries its full inline schema, + // and the fresh server's dictionary is re-established either by a full-dict + // frame (full-dict mode) or by an I/O-thread catch-up frame before replay + // (delta mode), so post-reconnect replay needs no producer-side reset signal. resetSymbolDictStateForNewConnection(); connectionError.set(null); @@ -3408,7 +3596,7 @@ private void flushPendingRows(boolean deferCommit) { cachedTimestampNanosColumn = null; ObjList keys = tableBuffers.keys(); - int tableCount = countNonEmptyTables(keys); + int tableCount = collectNonEmptyTables(keys); if (tableCount == 0) { pendingBytes = 0; currentTableBufferSnapshotBytes = currentTableBuffer == null @@ -3423,50 +3611,73 @@ private void flushPendingRows(boolean deferCommit) { } ensureActiveBufferReady(); - // Cursor SF requires every on-disk frame to be self-sufficient: - // recorded frames replay to fresh server connections (orphan-slot - // drainers and post-reconnect replay), so always emit the full - // symbol-dict delta from id=0 and the full column schema inline, - // never a back-reference the target server may not have seen. + // In full-dict mode every frame is self-sufficient: it carries the whole + // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh + // server never dangles a symbol id. In delta mode (memory-mode, and + // file-mode store-and-forward once the persisted dictionary opened) each + // frame carries only ids above sentMaxSymbolId; a reconnect re-registers + // the dictionary via an I/O-thread catch-up frame before replay, so the + // producer's monotonic baseline stays valid across the wire boundary. encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) { - continue; - } + symbolDeltaBaseline(), currentBatchMaxSymbolId); + // Record each table's encoded body size (position delta across addTable). + // When the batch needs splitting, these lengths delimit immutable body + // slices in the combined encoder buffer for direct frame assembly. The + // capture is a couple of int ops per table on the common path. + splitFrameBodyBytes.clear(); + int combinedBodyStart = encoder.getBuffer().getPosition(); + int bodyStart = combinedBodyStart; + for (int i = 0; i < tableCount; i++) { + QwpTableBuffer tableBuffer = flushTableBuffers.getQuick(i); if (LOG.isDebugEnabled()) { LOG.debug("Encoding table [name={}, rows={}, batchMaxId={}]", - tableName, tableBuffer.getRowCount(), currentBatchMaxSymbolId); + flushTableNames.getQuick(i), tableBuffer.getRowCount(), currentBatchMaxSymbolId); } encoder.addTable(tableBuffer); + int bodyEnd = encoder.getBuffer().getPosition(); + splitFrameBodyBytes.add(bodyEnd - bodyStart); + bodyStart = bodyEnd; } int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); - if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) { - flushPendingRowsSplit(keys, deferCommit); + // Snapshot the volatile cap ONCE for this whole flush. The I/O thread lowers + // serverMaxBatchSize on a mid-stream failover to a smaller-cap node + // (applyServerBatchSizeLimit); if the split pre-flight and the publish loop + // re-read the field independently, a failover between them would size frames + // against two different caps -- breaking the all-or-nothing guarantee and + // firing the publish-loop assert on a legitimate race. Both use this snapshot; + // the next flush picks up the new cap. + int cap = serverMaxBatchSize; + if (cap > 0 && messageSize > cap) { + // Keep the completed combined frame staged in the encoder while the + // split path copies its delta entries and table-body slices. + flushPendingRowsSplit(deferCommit, combinedBodyStart, cap); return; } + // Write-ahead: durably persist this frame's new symbols BEFORE it is + // published, so a recovered/orphan-drained slot can always rebuild the + // dictionary the (non-self-sufficient) delta frame references. No-op in + // memory mode and when the frame introduces no new symbols. + persistNewSymbolsBeforePublish(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // The frame carrying ids up to currentBatchMaxSymbolId is now on the ring; + // advance the delta baseline so the next frame ships only newer ids. + advanceSentMaxSymbolId(); hasDeferredMessages = deferCommit; if (!deferCommit) { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } - resetTableBuffersAfterFlush(keys); + resetTableBuffersAfterFlush(); } /** @@ -3475,64 +3686,127 @@ private void flushPendingRows(boolean deferCommit) { * own message. All messages except the last carry FLAG_DEFER_COMMIT * so the server appends rows without committing until the final * message arrives. + *

+ * Not atomic across frames. The frames publish one at a time, so a + * publish failure partway through -- {@link #sealAndSwapBuffer()} throwing on + * frame k>1, e.g. a backpressure deadline or the buffer-recycle timeout -- + * leaves frames 1..k-1 already on the ring as deferred (appended, not yet + * committed). The throw propagates past the {@code resetTableBuffersAfterFlush} + * at the end of the loop, so the source rows survive in their table buffers + * and the NEXT flush re-emits the whole batch; the eventual commit then + * commits the already-published prefix alongside the re-sent copies, + * delivering those rows at-least-once (duplicated), not exactly-once. This is + * within store-and-forward's at-least-once contract -- a DEDUP table or a + * durable-ack await absorbs the duplicate, and the symbol-dict state stays + * consistent on the retry (the re-sent frames carry empty deltas and the + * write-ahead persist is a {@code pd.size()} no-op). Making the split atomic + * (rolling back the published prefix, or skipping it on retry) would be a + * larger change. * * @param deferCommit when true, ALL messages (including the last) * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit) { + private void flushPendingRowsSplit( + boolean deferCommit, + int combinedBodyStart, + int cap + ) { if (LOG.isDebugEnabled()) { - LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); - } - - // Collect non-empty table indices so we know which is last. - int nonEmptyCount = 0; - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; + LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", cap, deferCommit); + } + + // Collect non-empty table indices so we know which is last, AND pre-flight + // every split frame's size BEFORE publishing any of them. The split hands + // frames to the ring one at a time (all but the last deferred -- appended but + // uncommitted); if a later table's frame were only found oversized + // mid-publish, the already-published prefix would strand on the ring, a + // subsequent commit would deliver it as a partial batch, and + // resetTableBuffersAfterFlush would discard every source row -- a partial + // commit the caller was told (by the throw) had failed. Checking all sizes up + // front makes the split all-or-nothing: either every frame fits and all + // publish, or none publish and we throw with nothing stranded. + // + // Each split frame's size is derived from the combined encode flushPendingRows + // already performed. simBaseline mirrors the publish loop's baseline advance + // (advanceSentMaxSymbolId), so each size equals the frame the staged-slice + // assembler will build; this pass mutates no delta/persist state. + int nonEmptyCount = flushTableBuffers.size(); + int simBaseline = symbolDeltaBaseline(); + for (int i = 0; i < nonEmptyCount; i++) { + CharSequence tableName = flushTableNames.getQuick(i); + int messageSize = encoder.getSplitMessageSize( + splitFrameBodyBytes.getQuick(i), simBaseline, currentBatchMaxSymbolId); + if (messageSize > cap) { + // The batch stays BUFFERED: this throw precedes every publish, and a + // rejected flush must not silently discard the caller's rows (see + // SelfSufficientFramesTest#testOversizedTableSplitStrandsNothing). So the + // next flush() re-encodes and re-rejects the same batch until either the + // reachable cap grows -- a failover to a larger-cap node, which is the + // case retaining the rows exists to survive -- or the sender is closed, + // which discards them. It cannot drain against a cap this table will + // never fit, so say that here rather than let a caller read the repeat + // rejections as a transient and keep appending to a batch that only + // grows. + throw new LineSenderException("single table batch too large for server batch cap") + .put(" [table=").put(tableName) + .put(", messageSize=").put(messageSize) + .put(", serverMaxBatchSize=").put(cap).put(']') + .put("; the batch is retained for retry and every flush() will " + + "reject it again until a larger-cap node is reached -- " + + "close the sender to discard it, or produce smaller batches"); } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer != null && tableBuffer.getRowCount() > 0) { - nonEmptyCount++; + // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's + // new ids, the remaining frames carry an empty delta above the baseline. + if (deltaDictEnabled && currentBatchMaxSymbolId > simBaseline) { + simBaseline = currentBatchMaxSymbolId; } } - int sent = 0; - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) { - continue; - } + int tableBodyOffset = combinedBodyStart; + for (int i = 0; i < nonEmptyCount; i++) { + CharSequence tableName = flushTableNames.getQuick(i); - sent++; - boolean isLast = (sent == nonEmptyCount); + boolean isLast = (i == nonEmptyCount - 1); boolean deferThis = deferCommit || !isLast; - encoder.setDeferCommit(deferThis); - encoder.beginMessage(1, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); - encoder.addTable(tableBuffer); - int messageSize = encoder.finishMessage(); - QwpBufferWriter buffer = encoder.getBuffer(); - - if (messageSize > serverMaxBatchSize) { - resetTableBuffersAfterFlush(keys); - throw new LineSenderException("single table batch too large for server batch cap") - .put(" [table=").put(tableName) - .put(", messageSize=").put(messageSize) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); - } - + int tableBodyLength = splitFrameBodyBytes.getQuick(i); + // Persist before touching activeBuffer. If the write-ahead fails, the + // caller can retry with both the source rows and active microbatch + // unchanged. The first frame carries the batch's new symbols; later + // frames are no-ops once the baseline has advanced. + persistNewSymbolsBeforePublish(); ensureActiveBufferReady(); - activeBuffer.ensureCapacity(messageSize); - activeBuffer.write(buffer.getBufferPtr(), messageSize); + // The combined encoder buffer remains immutable for the whole split. + // Assemble this frame directly into the active microbatch: patched + // header + staged delta bytes + the staged table-body slice. No row or + // column is encoded a second time. + int messageSize = encoder.copySplitMessage( + activeBuffer, + tableBodyOffset, + tableBodyLength, + deferThis, + symbolDeltaBaseline(), + currentBatchMaxSymbolId + ); + tableBodyOffset += tableBodyLength; + // The pre-flight pass above already verified every split frame fits the + // cap, so none can be found oversized here -- which is what keeps this + // loop from publishing (and stranding) a deferred prefix before an + // oversized table. Both passes size against the SAME snapshot cap, so a + // mid-flush failover cannot make them disagree; the assert therefore only + // catches a genuine divergence between the pre-flight arithmetic and the + // staged assembler (a future bug), not a cap race. It deliberately does NOT + // reset+throw here, because by this point a prefix may already be on the ring. + assert messageSize <= cap + : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName + + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + cap + ']'; + activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // Frame queued: advance so the next split frame's delta starts above + // the ids this one just registered. + advanceSentMaxSymbolId(); } encoder.setDeferCommit(false); @@ -3542,21 +3816,16 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm // committed the whole group. lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } - resetTableBuffersAfterFlush(keys); + resetTableBuffersAfterFlush(); } - private void resetTableBuffersAfterFlush(ObjList keys) { - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) { - continue; - } - tableBuffer.reset(); + private void resetTableBuffersAfterFlush() { + for (int i = 0, n = flushTableBuffers.size(); i < n; i++) { + flushTableBuffers.getQuick(i).reset(); } + // Drop the references; the next flush re-collects. + flushTableNames.clear(); + flushTableBuffers.clear(); currentBatchMaxSymbolId = -1; pendingBytes = 0; currentTableBufferSnapshotBytes = 0; @@ -3573,8 +3842,25 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); + // A commit carries no rows, and it must also carry NO new symbols. Unlike + // the flush paths, sendCommitMessage does NOT write-ahead-persist the + // dictionary, so shipping a symbol here would put an id on the wire that a + // recovered slot cannot rebuild from the persisted .symbol-dict, diverging + // the producer dictionary from the surviving frames and silently + // misattributing reused ids after a crash. currentBatchMaxSymbolId can sit + // ABOVE sentMaxSymbolId (e.g. a cancelled row: cancelRow does not roll back + // currentBatchMaxSymbolId or unregister the symbol), so bound the delta at + // what has already been sent -- and therefore already persisted. In delta + // mode pass sentMaxSymbolId, yielding an empty delta + // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep + // currentBatchMaxSymbolId (reset to -1 by the prior flush, so the commit frame + // carries an empty delta too -- harmless, since the preceding full-dict data + // frames already registered the dictionary on this connection). Any symbol a + // cancelled row leaked is picked up (and persisted) by the next real flush, + // whose persistNewSymbolsBeforePublish resumes from pd.size(). + int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId; encoder.beginMessage(0, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), commitBatchMaxId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); @@ -3586,15 +3872,320 @@ private void sendCommitMessage() { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + /** + * Advances the delta baseline once a frame carrying the current batch's + * symbols has been queued onto the ring. No-op in full-dict mode. Only ever + * moves the baseline forward, so a batch that used no new symbols leaves it + * unchanged. + */ + private void advanceSentMaxSymbolId() { + if (deltaDictEnabled && currentBatchMaxSymbolId > sentMaxSymbolId) { + sentMaxSymbolId = currentBatchMaxSymbolId; + } + } + + /** + * Stops emitting delta dictionaries for the rest of this sender's life, after the + * per-slot {@code .symbol-dict} has proved unwritable. + *

+ * The side-file can stop accepting appends mid-run -- a full disk or an exhausted + * quota, where SF's own segments stay writable because they are pre-allocated mmap + * files while the dictionary is the one thing still growing. Without a way back, + * {@code deltaDictEnabled} is written once at {@code setCursorEngine} and every + * later {@code flush()} re-throws forever: a condition store-and-forward is built + * to survive becomes total, permanent ingestion loss. + *

+ * Full self-sufficient frames need no side file at all -- each carries the whole + * dictionary from id 0, which is exactly what recovery and orphan-drain replay + * against a fresh server. So degrade instead of dying. The producer's monotonic + * baseline stops being consulted ({@link #symbolDeltaBaseline()} returns -1), and + * the write-ahead persist becomes a no-op. + *

+ * Producer-thread only, like every other reader of {@code deltaDictEnabled}. + */ + private void disableDeltaDict(Throwable cause) { + if (!deltaDictEnabled) { + return; + } + deltaDictEnabled = false; + LOG.warn("symbol dictionary persistence failed; this sender has switched to full " + + "self-sufficient frames for the rest of its life (bandwidth cost only -- " + + "no data is at risk, and recovery replays such frames without a side file)", + cause); + } + + /** + * Writes the ids the surviving frames contributed above the persisted prefix back + * into {@code .symbol-dict}, immediately, before any new frame can be published. + *

+ * {@link #seedGlobalDictionaryFromPersisted} can rebuild the producer dictionary from + * TWO sources -- the side-file's intact prefix and the surviving frames' own delta + * sections -- and then resumes {@code sentMaxSymbolId} at the combined tip. When the + * frames contributed anything, that tip is ABOVE {@code pd.size()}, so every frame + * published from here on carries a {@code deltaStart} the side-file cannot describe. + * That breaks the write-ahead invariant the whole design rests on: the persisted + * dictionary must be a superset of every recoverable frame's references. + *

+ * The steady-state write-ahead does NOT close that gap on its own. It persists + * {@code [pd.size() .. currentBatchMaxSymbolId]} and returns early when the batch's + * highest id is below {@code pd.size()}, so it heals only if -- and only as far as -- + * a later batch happens to reference the recovered high ids. Meanwhile the frames + * that carry those ids are the oldest unacked, so they are the FIRST to be acked and + * trimmed. Once they are gone, an ordinary process crash (which store-and-forward + * promises to survive; only the original tear needs a host crash) leaves a slot whose + * frames reference ids nothing holds: recovery marks a gap and {@code build()} + * quarantines it with "resend the affected data". + *

+ * Healing here, eagerly and in full, restores the invariant before the window opens. + */ + private void healPersistedDictionary(PersistedSymbolDict pd) { + if (pd == null || !deltaDictEnabled) { + return; + } + int from = pd.size(); + int to = globalSymbolDictionary.size() - 1; + if (to < from) { + return; // the side-file already covers everything the frames defined + } + try { + pd.appendSymbols(globalSymbolDictionary, from, to); + } catch (Throwable t) { + if (t instanceof Error) { + throw (Error) t; + } + // Do NOT fail recovery: the surviving frames still carry these ids in their + // own deltas, so THIS session replays correctly either way. Only a future + // recovery, after those frames are trimmed, would be affected -- and the + // degrade below removes even that exposure by dropping back to frames that + // need no side file. + disableDeltaDict(t); + } + } + + /** + * Appends the symbols this frame introduces ({@code [sentMaxSymbolId+1 .. + * currentBatchMaxSymbolId]}) to the slot's persisted dictionary BEFORE the + * frame is published to the ring. This write-ahead ordering keeps the + * persisted dictionary a superset of every process-crash-recoverable frame's + * references, so recovery and orphan-drain can re-register it on a fresh + * server. Not fsync'd (see PersistedSymbolDict) -- a host crash that tears it + * is caught by the send loop's replay guard. No-op in memory mode (no + * persisted dictionary) and when the frame introduces no new symbols. + */ + private void persistNewSymbolsBeforePublish() { + if (!deltaDictEnabled || cursorEngine == null) { + return; + } + PersistedSymbolDict pd = cursorEngine.getPersistedSymbolDict(); + if (pd == null) { + return; + } + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write, BEFORE the + // frame is published. + // + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: + // the persist advances pd.size() only after a full write, whereas + // sentMaxSymbolId only advances after the WHOLE frame is published (via + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist + // threw (short write -- disk full/quota) or the publish threw, the frame + // was not published and sentMaxSymbolId stayed put, while the symbols + // before the failure are already on disk. Keying the resume point off + // sentMaxSymbolId+1 would re-append that persisted prefix on the retry, + // duplicating entries and corrupting the dense id->symbol mapping recovery + // relies on (position i must be symbol id i). pd.size() resumes exactly + // past what is already durable, so the write-ahead is idempotent. + int from = pd.size(); + if (currentBatchMaxSymbolId < from) { + return; // nothing new to persist (warm batch, or an idempotent retry) + } + // Fast path: the frame the encoder just built already holds these symbols + // in its delta section as [len][utf8]... -- byte-identical to what + // PersistedSymbolDict stores. In the common case pd.size() equals the + // frame's delta start id (sentMaxSymbolId+1), so persist those bytes + // straight from the frame instead of re-encoding the symbols. After a + // failed publish the durable size has run ahead of the wire baseline, so + // the frame's delta covers MORE than remains to persist; then re-encode + // just the [from .. currentBatchMaxSymbolId] suffix. + try { + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } + } catch (Throwable t) { + // A failed write to the persisted dictionary throws a low-level + // IllegalStateException: in production that is ff.allocate refusing to grow + // the mmap append window (how a full disk / exhausted quota surfaces there), + // and behind an injected facade a short positioned write. Surface it as a + // LineSenderException -- like every other flush-path failure, e.g. the cursor + // append in sealAndSwapBuffer -- so a caller catching LineSenderException + // around flush() also catches a disk-full during the write-ahead persist. The + // persist ran before publish and pd.size() did not advance on the failure, so + // the still-buffered rows re-persist the same range idempotently on retry. + // A JVM Error is never a persist failure; let it propagate. + if (t instanceof Error) { + throw (Error) t; + } + // Degrade before throwing, so this failure is survivable rather than terminal: + // every LATER flush emits full self-sufficient frames, which need no side file + // (see disableDeltaDict). This one flush still has to fail -- beginMessage has + // already baked a delta deltaStart into the staged frame, and publishing it + // would put ids on the ring that the side-file cannot describe. The throw + // precedes every publish, so the caller's rows stay buffered and the next + // flush() re-encodes them from id 0. + disableDeltaDict(t); + throw new LineSenderException("failed to persist symbol dictionary before publish; " + + "this sender has switched to full self-sufficient frames -- retry the flush", t); + } + } + private void resetSymbolDictStateForNewConnection() { - // The new server has an empty symbol dictionary, so the next batch - // must ship a delta starting at id 0. beginMessage() always passes - // confirmedMaxId = -1; resetting the batch watermark here keeps a - // stale value from suppressing re-emission of symbol ids the new - // server has never seen. + // Runs on the foreground (initial) connect only -- NOT on the I/O thread's + // reconnect/failover path. The per-batch watermark is drained state, so + // clearing it here is harmless. sentMaxSymbolId is deliberately left + // untouched: in delta mode the I/O thread re-registers the whole + // dictionary with a catch-up frame on reconnect, so the producer's + // monotonic baseline must survive the wire boundary; resetting it would + // desync the producer from the I/O thread's sent-dictionary count. currentBatchMaxSymbolId = -1; } + /** + * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} so that newly + * ingested symbols continue ABOVE every id the surviving frames already define, and + * resumes the delta baseline at that tip. + *

+ * Seeds from TWO sources, in this order: + *

    + *
  1. the slot's persisted {@code .symbol-dict} -- its intact prefix; then
    2. + *
  2. the surviving frames' OWN delta sections, for every id above that prefix + * ({@link CursorSendEngine#addRecoveredSymbolsTo}).
    3. + *
+ * Those are exactly the two sources, in exactly the order, that the send loop's mirror + * is built from: its constructor seeds {@code sentDictCount} from the same dictionary, + * and {@code accumulateSentDict} then extends it from the same frames as they replay. So + * the producer's {@code sentMaxSymbolId + 1} and the loop's {@code sentDictCount} land on + * the same number BY CONSTRUCTION -- the invariant the torn-dictionary guard rests on -- + * rather than by the two happening to agree. + *

+ * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): the + * persisted dictionary, the on-wire delta and the mirror all key on the entry POSITION + * (id), so the producer's id space must match the recovered entry count exactly. + * {@code getOrAddSymbol} would collapse two source strings that decode to the same + * characters -- only malformed lone UTF-16 surrogates, which UTF-8-encode to {@code '?'} + * -- leaving this dictionary SHORTER than the count and silently misattributing later + * symbols. + *

+ * Why seeding from the frames matters. The dictionary is not fsync'd (see + * {@code PersistedSymbolDict}), so a host/power crash can tear off its newest entries + * while the segment frames that introduced those ids survive -- and those newest frames, + * being the least likely to be acked, are exactly the ones that replay. Seeded from the + * short dictionary alone, this producer would hand its next new symbol an id those frames + * already define, putting two symbols on one id and silently misattributing values. The + * old code detected that and threw, which was safe but far too blunt: it bricked + * {@code build()} for slots the background drainer replays PERFECTLY, because the frames + * carry the torn-off symbols in their own deltas and {@code accumulateSentDict} rebuilds + * the dictionary from them. This method now rebuilds the producer from the same bytes, + * so a torn -- or entirely lost -- dictionary is recoverable whenever the surviving + * frames define the ids themselves. The next flush's write-ahead persist then re-writes + * those ids (it resumes from {@code pd.size()}), healing the side-file on disk. + *

+ * What still fails clean. A genuine GAP: the ids below a surviving frame's delta + * start were introduced by frames that were acked and TRIMMED away, so they lived only in + * the lost dictionary and nothing can rebuild them. + * {@code addRecoveredSymbolsTo} returns -1 for that and we throw. It is the same + * condition the send loop's replay guard ({@code deltaStart > sentDictCount}) trips on, so + * producer and drainer now agree on exactly which slots are recoverable, instead of the + * producer rejecting slots the drainer drains. + */ + private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { + if (cursorEngine == null) { + return; + } + // 1. The dictionary's intact prefix. addRecoveredSymbol appends without de-dup, so + // the producer's size tracks pd.size() exactly -- which is what the send loop's + // mirror also seeds sentDictCount from. + // Pre-size before pouring the recovered symbols in. The default capacity is 64, + // so rebuilding a large dictionary rehashed the map ~log2(n/64) times, each pass + // O(current size) -- roughly doubling the rebuild and touching a growing table + // the whole way. recoveredMaxSymbolId + 1 is the upper bound the seed can reach. + long expected = Math.max(pd == null ? 0L : pd.recoveredSize(), + cursorEngine.recoveredMaxSymbolId() + 1L); + if (expected > globalSymbolDictionary.size() && expected <= Integer.MAX_VALUE) { + globalSymbolDictionary = new GlobalSymbolDictionary((int) expected); + } + int baseline = 0; + if (pd != null && pd.size() > 0) { + pd.addLoadedSymbolsTo(globalSymbolDictionary); + baseline = globalSymbolDictionary.size(); + } + // 2. Everything the surviving frames define above that prefix, straight out of their + // own delta sections -- the same bytes, in the same order, accumulateSentDict will + // feed the mirror as those frames go back on the wire. + long coverage = cursorEngine.addRecoveredSymbolsTo(baseline, globalSymbolDictionary); + if (coverage < 0) { + // A gap: the surviving frames reference ids below their own delta start, + // introduced by frames since acked and trimmed away, and the persisted + // dictionary no longer holds them (a host crash tore its unsynced tail, or it + // could not be opened). That gap only matters for frames that will REPLAY. + // When every recovered committed frame is already acked + // (ackedFsn >= recoveredCommitBoundaryFsn), NOTHING replays: the gap is in + // data the server already has, and the retired orphan-deferred tail above the + // commit boundary is never transmitted. Throwing here would raise a false + // "resend required" for delivered data AND -- because such a slot is fully + // drained -- let build()'s connect-path close unlink the (already-delivered) + // bytes the quarantine claims to preserve. So DON'T throw: seed the intact + // prefix only; addRecoveredSymbolsTo adds nothing on a -1 exactly as the + // send loop's mirror does, so the producer baseline and the mirror's + // sentDictCount still agree by construction. The producer resumes above the + // prefix and the fully-drained slot is cleaned up on close. + long ackedFsn = cursorEngine.ackedFsn(); + long commitBoundaryFsn = cursorEngine.recoveredCommitBoundaryFsn(); + if (ackedFsn >= commitBoundaryFsn) { + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + LOG.info("recovered store-and-forward slot has a torn/incomplete symbol dictionary, " + + "but every committed frame was already acked so nothing needs replaying; " + + "resuming on the intact prefix without quarantine and without data loss " + + "[recoveredPrefixSize={}, ackedFsn={}, commitBoundaryFsn={}]", + baseline, ackedFsn, commitBoundaryFsn); + return; + } + // Genuine loss: unacked committed frames reference ids nothing still holds. + // Typed, because Sender.build() sets such a slot aside instead of failing: this + // is the point at which every source of truth has been tried and none of them + // holds the missing ids. See UnreplayableSlotException. + throw new UnreplayableSlotException( + "recovered store-and-forward symbol dictionary is incomplete and cannot be " + + "rebuilt from the surviving frames (likely a host crash tore its unsynced " + + "tail): the frames reference symbol ids below their own delta start, which " + + "were introduced by frames since acked and trimmed away, so nothing still " + + "holds them; the recovered dictionary holds only " + + (pd == null ? 0 : pd.size()) + " id(s) -- resend the affected data"); + } + // Producer baseline == the coverage the replay will establish == the mirror's + // sentDictCount once those frames have gone out. The first new frame therefore + // starts its delta exactly at the tip, and the replay guard passes. + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + // ...but the baseline now sits ABOVE pd.size() whenever the frames contributed + // ids, so restore the write-ahead invariant right now rather than hoping a later + // batch reaches high enough to do it. See healPersistedDictionary. + healPersistedDictionary(pd); + } + + /** + * The symbol id below which the server already holds every dictionary entry, + * used as {@code confirmedMaxId} when encoding a frame. In delta mode this is + * the producer's monotonic sent watermark; in full-dict mode it is -1 so every + * frame re-ships the dictionary from id 0. + */ + private int symbolDeltaBaseline() { + return deltaDictEnabled ? sentMaxSymbolId : -1; + } + private void rollbackRow() { if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); @@ -3656,7 +4247,7 @@ private void sealAndSwapBuffer() { // back to it; flushPendingRows aborts its post-enqueue state // updates after this throw, so the source rows stay intact and the // next batch re-emits the same rows along with the full inline - // schema and symbol-dict delta from id 0. + // schema and the symbol-dict delta the batch requires. if (toSend.isSending()) { toSend.markRecycled(); } else if (toSend.isSealed()) { @@ -3685,12 +4276,18 @@ private void sendRow() { // batch stay intact. (The check ignores the null-padding bytes // nextRow() will add; that's bounded by numColumns * elemSize and // far below any realistic cap.) - if (serverMaxBatchSize > 0) { + // Snapshot the volatile cap ONCE, as flushPendingRows does. The I/O thread + // lowers serverMaxBatchSize -- or clears it to 0 on a failover to a node that + // advertises no cap -- mid-stream via applyServerBatchSizeLimit. Re-reading the + // field across the guard and the throw could observe it drop to 0 between reads + // and reject the row against a "cap" of 0, which actually means "no cap". + int cap = serverMaxBatchSize; + if (cap > 0) { long rowBytes = currentTableBuffer.getBufferedBytes() - currentTableBufferSnapshotBytes; - if (rowBytes > serverMaxBatchSize) { + if (rowBytes > cap) { throw new LineSenderException("row too large for server batch cap") .put(" [rowBytes=").put(rowBytes) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); + .put(", serverMaxBatchSize=").put(cap).put(']'); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index d79080d4..bae434c8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; /** @@ -44,7 +45,9 @@ *

* Lifecycle: *

    - *
  1. Acquire the slot's {@code .lock}; skip silently on contention.
    2. + *
  2. Acquire the parent-anchored logical-slot lock and revalidate the + * scanner snapshot; skip silently on contention or a stale snapshot.
    3. + *
  3. Acquire the slot's {@code .lock}, then release the logical lock.
    4. *
  4. Open a {@link CursorSendEngine} on the slot — recovery picks up * every {@code .sfa} file already on disk.
    5. *
  5. Open a fresh {@link WebSocketClient} via the supplied factory @@ -75,12 +78,10 @@ public final class BackgroundDrainer implements Runnable { * wall-clock budget {@code reconnectMaxDurationMillis} also caps this * capability-gap loop; whichever is hit first triggers escalation. Both * halves of the budget measure a capability-gap episode: the - * wall clock accumulates only across uninterrupted gap-to-gap intervals - * (never before the first gap is observed, and never across an - * intervening transport window -- an unreachable cluster is not - * "failing to settle"), and an intervening role reject restarts the - * episode -- it proves the topology changed, so the next capability-gap - * error is a fresh episode against a newly promoted node. 16 + * wall clock accumulates only across uninterrupted gap-to-gap intervals. + * Any intervening transport or role state restarts the episode: capability + * gaps separated by an unrelated transient are not consecutive evidence + * of a persistent cluster capability mismatch. 16 * attempts gives the cluster room to settle through a rolling upgrade * (each attempt walks every endpoint internally) without letting a genuine * cluster-wide misconfig hang the drainer forever. @@ -132,6 +133,11 @@ public final class BackgroundDrainer implements Runnable { // Minimum wall-clock dwell before poison escalation, forwarded to every // drain loop; mirrors the owner sender's poison_min_escalation_window_millis. private final long poisonMinEscalationWindowMillis; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before this + // orphan drainer's send loop latches a terminal. Foreground senders retry forever; + // this bounded policy is permitted only so a persistent orphan slot can be + // quarantined for operator intervention. + private final long catchUpCapGapMinEscalationWindowMillis; public BackgroundDrainer( String slotPath, @@ -149,7 +155,33 @@ public BackgroundDrainer( reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); + } + + /** + * Compatibility constructor retained for callers compiled before the + * catch-up cap-gap dwell was added to the master signature. + */ + public BackgroundDrainer( + String slotPath, + long segmentSizeBytes, + long sfMaxTotalBytes, + CursorWebSocketSendLoop.ReconnectFactory clientFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean requestDurableAck, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis + ) { + this(slotPath, segmentSizeBytes, sfMaxTotalBytes, clientFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, requestDurableAck, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -169,7 +201,8 @@ public BackgroundDrainer( boolean requestDurableAck, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { this.slotPath = slotPath; this.segmentSizeBytes = segmentSizeBytes; @@ -182,6 +215,7 @@ public BackgroundDrainer( this.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + this.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; } /** @@ -194,7 +228,7 @@ public BackgroundDrainer( @TestOnly public BackgroundDrainer() { this(null, 0L, 0L, null, 0L, 0L, 0L, false, 0L, - CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L); + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L, 0L); } /** @@ -208,8 +242,9 @@ public BackgroundDrainer() { * durable ack -- i.e. the symptom of a misconfigured cluster or a * rolling-upgrade transient. *

    - * For the foreground sender that condition is loud-fail: the producer - * is actively pushing data. The drainer is asymmetric: source data is + * Blocking foreground startup keeps its fail-fast policy, while asynchronous + * foreground startup and reconnect keep buffering and retrying through a + * rolling capability change. The orphan drainer is asymmetric: source data is * pinned (durable-ack-mode trims only on STATUS_DURABLE_ACK frames, * which the offending endpoints by definition do not send), so we * give the cluster a budget to settle before quarantining the slot. @@ -222,21 +257,17 @@ public BackgroundDrainer() { * {@link QwpDurableAckMismatchException} sweeps only. Transient * conditions -- an all-replica failover window (role reject) or a * transport error -- are retried indefinitely (Invariant B) and never - * consume the budget: the wall-clock half accumulates only across - * uninterrupted gap-to-gap intervals, so a mid-episode transport window - * pauses the clock (without touching the attempt count), and a role - * reject additionally restarts the episode, because it proves the - * topology changed under the rolling upgrade. + * consume the budget. Either transient restarts the attempt count and wall + * clock so only uninterrupted capability-gap sweeps can escalate. * Genuine terminals (auth failure, non-421 upgrade reject) preserve * the original behavior: mark failed, exit. * * @return a fresh durable-ack-capable client, or {@code null} if * {@link #outcome} has been set to FAILED or STOPPED */ - @TestOnly public WebSocketClient connectWithDurableAckRetry() { // run() already set runnerThread; setting it again here is a no-op - // on that path but wires up direct @TestOnly calls so requestStop() + // on that path but wires up direct callers so requestStop() // can unpark them too. runnerThread = Thread.currentThread(); long backoffMillis = reconnectInitialBackoffMillis; @@ -244,25 +275,29 @@ public WebSocketClient connectWithDurableAckRetry() { // QwpDurableAckMismatchException sweeps; the wall-clock half // accumulates ONLY across uninterrupted gap-to-gap intervals, so // transient churn (role reject, transport) can never burn the budget - // -- neither before the first gap is observed nor mid-episode (a - // cluster unreachable for longer than the whole budget that comes - // back still gapped has consumed none of it). An intervening role - // reject resets the episode (topology churn: the offending node is - // gone); a transport error neither increments nor resets the attempt - // count -- a dropped socket does not prove promotion churn, and - // resetting on it would let a flaky-but-misconfigured cluster evade - // the cap forever -- it only pauses the wall clock: the gap-to-gap - // interval spanning the transport window is not charged. + // -- neither before the first gap is observed nor mid-episode. Any + // intervening role or transport state resets the episode: after the + // cluster leaves the capability-gap state, later gaps must establish a + // fresh consecutive run before quarantine is permitted. int capabilityGapAttempts = 0; // Wall-clock time accumulated across uninterrupted gap-to-gap // intervals of the current episode; escalates once it reaches // capabilityGapBudgetNanos (or the attempt cap fires first). long capabilityGapElapsedNanos = 0L; // Timestamp of the previous capability-gap sweep; 0 = the next gap - // charges nothing (episode start, post-role-reject restart, or the - // interval was interrupted by a transport window). + // charges nothing because a fresh episode is starting. long lastCapabilityGapNanos = 0L; - final long capabilityGapBudgetNanos = reconnectMaxDurationMillis * 1_000_000L; + // Saturate rather than multiply: reconnect_max_duration_millis is validated only + // as > 0, so a large value (Long.MAX_VALUE is the natural way to ask for "never + // give up") wraps a raw multiply NEGATIVE. capabilityGapElapsedNanos then clears + // the budget on the FIRST capability-gap sweep -- 0 >= a negative -- and, because + // this gate is an OR with the attempt cap, nothing else holds it back: the slot + // quarantines immediately, skipping the whole 16-sweep settle budget. Asking for + // more tolerance would buy exactly none. TimeUnit clamps at Long.MAX_VALUE, which + // is the intended "effectively unbounded". CursorWebSocketSendLoop's dwell + // conversion guards the same way for the same reason. + final long capabilityGapBudgetNanos = + TimeUnit.MILLISECONDS.toNanos(reconnectMaxDurationMillis); // Observability-only counter for the transient all-replica window; // never consulted for escalation (Invariant B). int roleRejectAttempts = 0; @@ -283,8 +318,7 @@ public WebSocketClient connectWithDurableAckRetry() { } catch (QwpAuthFailedException | WebSocketUpgradeException e) { // Genuinely non-retriable across the cluster (auth 401/403, or a // non-421 upgrade reject): waiting will not fix it, so quarantine - // immediately -- exactly as the live sender's background loop - // (CursorWebSocketSendLoop.connectLoop) halts on these errors. + // immediately under the orphan reconnect policy. String msg = e.getMessage(); LOG.error("drainer terminal upgrade/auth error for slot {}: {}", slotPath, msg); lastErrorMessage = msg; @@ -397,10 +431,11 @@ public WebSocketClient connectWithDurableAckRetry() { // Invariant B -- but it is NOT a transport outage, so log it // truthfully below rather than mislabelling it "cluster unreachable". lastErrorMessage = t.getMessage(); - // Pause the episode wall clock: the gap-to-gap interval this - // window interrupts is never charged. Attempts and elapsed - // already accumulated are preserved (anti-evasion: see the - // budget comment above). + // This unrelated state breaks the consecutive capability-gap + // run. Restart both halves of the settle budget so a later gap + // must establish a fresh episode before quarantine. + capabilityGapAttempts = 0; + capabilityGapElapsedNanos = 0L; lastCapabilityGapNanos = 0L; long nowWarn = System.nanoTime(); if (nowWarn - lastTransportWarnNanos >= 5_000_000_000L) { @@ -521,27 +556,59 @@ private boolean stopRequestedOrInterrupted() { @Override public void run() { runnerThread = Thread.currentThread(); + SlotLock logicalSlotLock = null; CursorSendEngine engine = null; WebSocketClient client = null; CursorWebSocketSendLoop loop = null; try { - // The engine acquires the slot's .lock itself — we don't need - // (and must not) double-lock it. If another sender or drainer - // holds it, the engine constructor throws and we exit silently - // (no .failed sentinel — contention is expected, not an error). + // Scanner results are only snapshots. Serialize adoption against + // a producer's close -> quarantine rename -> fresh-slot recreate + // transition, then revalidate while that stable parent-anchored + // lock is held. The slot's own .lock inode moves with a rename and + // cannot provide this guarantee by itself. + if (slotPath != null) { + try { + logicalSlotLock = SlotLock.acquireLogical(slotPath); + } catch (IllegalStateException t) { + if (isLockContention(t)) { + LOG.info("orphan logical slot already locked, skipping: {} ({})", + slotPath, t.getMessage()); + outcome = DrainOutcome.LOCKED_BY_OTHER; + return; + } + throw t; + } + if (!OrphanScanner.isCandidateOrphan(slotPath)) { + LOG.info("orphan candidate changed before adoption, skipping: {}", slotPath); + outcome = DrainOutcome.SUCCESS; + return; + } + } + + // The engine acquires the directory-local .lock itself. Keep the + // lock order logical -> local, and release the short-lived logical + // lock only after the engine has secured stable ownership. try { engine = new CursorSendEngine(slotPath, segmentSizeBytes, sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS); } catch (IllegalStateException t) { - String msg = t.getMessage(); - if (msg != null && msg.contains("already in use")) { + if (isLockContention(t)) { LOG.info("orphan slot already locked, skipping: {} ({})", - slotPath, msg); + slotPath, t.getMessage()); outcome = DrainOutcome.LOCKED_BY_OTHER; return; } throw t; } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + logicalSlotLock = null; + } + // A recovered deferred-only tail is an aborted transaction and can + // be retired locally once everything below it is already ACKed. + // Do this before opening a socket: auth/upgrade failures must not + // quarantine a slot that has no wire-visible work left. + engine.retireRecoveredOrphanTailIfReady(); long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { LOG.info("orphan slot already drained: {} (acked={} target={})", @@ -569,13 +636,14 @@ public void run() { client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, clientFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN); loop.start(); while (!stopRequestedOrInterrupted()) { @@ -660,10 +728,27 @@ public void run() { LOG.debug("drainer setup failed for slot {}: {}", slotPath, msg, t); } lastErrorMessage = msg; - try { - OrphanScanner.markFailed(slotPath, "setup: " + msg); - } catch (Throwable ignored) { - // best-effort + // Quarantine ONLY a slot this drainer actually adopted. engine is + // assigned after CursorSendEngine has taken the slot's own .lock, so + // a null engine here means the failure happened before adoption -- + // while merely probing a slot another live process still owns. + // + // The failures that reach this point pre-adoption are LOCAL and + // usually transient: acquireLogical rethrows anything that is not + // lock contention, so a permission problem on the shared + // .slot-locks directory or a momentary fd exhaustion lands here. + // Writing the sentinel then would exclude a healthy, in-use slot + // from orphan drain permanently (OrphanScanner.isCandidateOrphan + // treats .failed as disqualifying, and nothing ever removes it), so + // when its real owner later dies its unacked data would be stranded + // until an operator intervened. Leave the slot alone; the next scan + // retries it. + if (engine != null) { + try { + OrphanScanner.markFailed(slotPath, "setup: " + msg); + } catch (Throwable ignored) { + // best-effort + } } outcome = DrainOutcome.FAILED; } finally { @@ -714,12 +799,20 @@ public void run() { + "slot lock releases when it exits", slotPath); } } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + } // Don't let a later requestStop() unpark an unrelated task that // the pool's executor may have scheduled onto this same thread. runnerThread = null; } } + private static boolean isLockContention(IllegalStateException error) { + String message = error.getMessage(); + return message != null && message.contains("already in use"); + } + /** * Plug an observer for durable-ack-related events. {@code null} clears * any previously installed listener. See {@link BackgroundDrainerListener} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..138cb252 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -24,10 +24,13 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import java.util.concurrent.locks.LockSupport; @@ -84,9 +87,9 @@ public final class CursorSendEngine implements QuietCloseable { private final SlotLock slotLock; // True when the constructor recovered an existing on-disk slot rather // than starting fresh. Diagnostic accessor for tests and observability; - // cursor frames are self-sufficient (every frame carries full schema + - // full symbol-dict delta), so producer-side schema reset on recovery - // is not required. + // every frame carries its full inline schema, so producer-side schema reset + // on recovery is not required (the symbol dictionary, which delta frames do + // NOT carry in full, is re-registered by an I/O-thread catch-up instead). private final boolean wasRecoveredFromDisk; // FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame found in a // ring recovered from disk, or -1 for fresh/memory rings and recovered @@ -96,6 +99,27 @@ public final class CursorSendEngine implements QuietCloseable { // covers them. Read by the sender's close-time drain to avoid waiting on // acks that cannot arrive. private long recoveredCommitBoundaryFsn = -1L; + // Highest symbol id any recovered delta frame references, or -1 for + // fresh/memory rings (and recovered rings with no symbol-bearing frame). A + // resuming producer seeds its dictionary baseline from the persisted + // .symbol-dict; if that dictionary was torn below this id by a host crash + // (the side-file is not fsync'd), the producer would re-use ids the surviving + // frames already define. seedGlobalDictionaryFromPersisted compares this + // against the recovered dictionary size to fail clean instead. Computed once + // in the constructor's recovery branch; -1 elsewhere. + private long recoveredMaxSymbolId = -1L; + // How many times the constructor folded the recovered ring. Observable because + // recoveryFramesVisited() cannot be: the full-dict-fallback re-fold REPLACES the + // analysis instance, resetting that counter, so a second fold is invisible through it. + private int recoveryFoldCount; + // Highest deltaStart across the recovered COMMITTED frames; 0 when none carries a symbol + // dictionary. ZERO means every surviving frame is SELF-SUFFICIENT -- it re-registers its + // dictionary from id 0 -- so the slot replays with no dictionary at all and the send loop + // needs no catch-up. ABOVE zero means at least one frame is a true delta whose ids depend + // on registrations it does not itself carry, so the loop must seed its mirror (and ship a + // catch-up) before replaying. Both the full-dict-fallback discard below and the send + // loop's mirror seeding key off this. + private long recoveredMaxSymbolDeltaStart; // FSN of the last frame of a recovered orphaned deferred tail, or -1 when // the recovered ring has no such tail. When >= 0, frames // [recoveredCommitBoundaryFsn + 1 .. recoveredOrphanTipFsn] all carry @@ -110,6 +134,17 @@ public final class CursorSendEngine implements QuietCloseable { // in the constructor, closed by {@link #close()}. The segment manager // writes through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; + // Engine-owned per-slot symbol dictionary file (disk mode only; {@code null} + // in memory mode and if open() failed). Enables delta-encoded SF frames: + // recovery / orphan-drain load it to re-register the dictionary on the fresh + // server before replaying non-self-sufficient frames. Opened in the + // constructor, closed by {@link #close()}. When null in disk mode the engine + // reports delta encoding as unavailable and the sender keeps full-dict frames. + private final PersistedSymbolDict persistedSymbolDict; + // Engine-owned output of the single ordered recovery walk. It is retained + // because both producer seeding and every recycled send loop need the same + // frame-rebuilt symbol suffix. Null for fresh and memory-only engines. + private final RecoveredFrameAnalysis recoveredFrameAnalysis; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -137,9 +172,29 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes) { */ public CursorSendEngine(String sfDir, long segmentSizeBytes, long maxTotalBytes, long appendDeadlineNanos) { + this(sfDir, segmentSizeBytes, maxTotalBytes, appendDeadlineNanos, FilesFacade.INSTANCE); + } + + /** + * As {@link #CursorSendEngine(String, long, long, long)}, but with an explicit + * {@link FilesFacade} for the persisted symbol dictionary. + *

    + * The seam exists so a test can drive a dictionary I/O fault -- a short write from + * a full disk or an exhausted quota -- through the REAL producer path + * ({@code flush()} -> the write-ahead persist), and assert it surfaces as a + * {@code LineSenderException} like every other flush-path failure rather than as a + * raw {@code IllegalStateException} that would sail past every caller's + * {@code catch (LineSenderException)}. Nothing else could reach that translation: + * {@code PersistedSymbolDict} has facade-aware overloads, but the engine used to + * call only the {@code FilesFacade.INSTANCE} ones, so no fault could be injected + * from outside. + */ + @TestOnly + public CursorSendEngine(String sfDir, long segmentSizeBytes, + long maxTotalBytes, long appendDeadlineNanos, FilesFacade dictFf) { this(sfDir, segmentSizeBytes, new SegmentManager(segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes), - true, appendDeadlineNanos); + true, appendDeadlineNanos, dictFf); } /** @@ -148,11 +203,12 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, * ownership of the manager. Uses the default append deadline. */ public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager) { - this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS); + this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS, + FilesFacade.INSTANCE); } private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, - boolean ownsManager, long appendDeadlineNanos) { + boolean ownsManager, long appendDeadlineNanos, FilesFacade dictFf) { // sfDir == null → memory-only mode (non-SF async ingest). Same // cursor architecture, no disk involvement; segments // live in malloc'd native memory. @@ -199,7 +255,17 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // reference instead of orphaning the mmap'd segments + fds. SegmentRing ringInProgress = null; AckWatermark watermarkInProgress = null; + PersistedSymbolDict persistedDictInProgress = null; + RecoveredFrameAnalysis recoveredFrameAnalysisInProgress = null; try { + // v2 segment payloads may depend on a persisted symbol-dictionary + // prefix. Install the rollback barrier before recovery or any new + // append so a v1-only client can never skip the v2 files, treat the + // slot as empty, and silently restart at FSN 0. Current recovery + // recognizes and skips the reserved guard filenames. + if (!memoryMode) { + SegmentRing.installLegacyReaderBarrier(sfDir); + } // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs @@ -257,6 +323,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // mmap doesn't take down the engine -- we just fall // back to the bare lowestBase - 1 seed. watermarkInProgress = AckWatermark.open(sfDir); + // Load the persisted symbol dictionary so delta-encoded frames + // in this recovered slot can be re-registered on the fresh + // server before replay. Null on open failure -> delta disabled. + persistedDictInProgress = PersistedSymbolDict.open(dictFf, sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -276,6 +346,13 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (seed >= 0) { recovered.acknowledge(seed); } + // Fold the whole recovered ring once. The result checkpoints all + // running metadata and raw symbol bytes at each commit-bearing + // frame, so its final snapshot excludes an orphan deferred tail + // without requiring a second bounded scan. + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery( + persistedDictInProgress == null ? 0 : persistedDictInProgress.size()); + recoveryFoldCount++; // Locate the last commit-bearing frame below a potentially // orphaned FLAG_DEFER_COMMIT tail. A producer that crashed (or // closed) mid-transaction leaves deferred frames with no @@ -285,12 +362,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // drainOnClose), and (b) replaying them into a NEW session's // commit would resurrect half a transaction -- see the WARN // below. Computed before the I/O loop or producer append. - this.recoveredCommitBoundaryFsn = recovered.findLastFsnWithoutPayloadFlag( - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DEFER_COMMIT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE - ); + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); if (publishedFsn >= 0 && recoveredCommitBoundaryFsn < publishedFsn) { this.recoveredOrphanTipFsn = publishedFsn; LOG.warn("recovered SF log ends with {} deferred frame(s) whose transaction was never " @@ -300,6 +372,88 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man publishedFsn - Math.max(recoveredCommitBoundaryFsn, -1L), recoveredCommitBoundaryFsn, publishedFsn); } + // Highest symbol id the surviving COMMITTED frames reference. A + // resuming producer compares this against its recovered dictionary + // size (seedGlobalDictionaryFromPersisted) to detect a host-crash + // tear: if a committed frame references an id the (unsynced, torn) + // .symbol-dict no longer holds, resuming would re-use it. The walk is + // bounded to recoveredCommitBoundaryFsn so the aborted orphan-deferred + // tail -- retired without ever being transmitted -- does not inflate + // this and over-reject an otherwise-recoverable slot. maxDeltaEnd() + // returns 0 when no such frame carries a symbol, yielding -1 here. + // Computed before the I/O loop or producer append; single-threaded. + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; + // Full-dict-fallback recovery. When the persisted .symbol-dict is a + // SUBSET of the ids the surviving frames reference + // (recoveredMaxSymbolId >= its size) YET every such frame is + // self-sufficient (maxDeltaStart() == 0 -- a full-dict frame that + // re-registers its dictionary from id 0), the slot was written in + // full-dict fallback: the dictionary never opened when writing, so no + // side-file exists and this recovery opened a FRESH EMPTY one. Those + // frames replay with no dictionary, so discard the empty side-file and + // recover in full-dict mode -- isDeltaDictEnabled() then reports false + // and the producer + send loop both run full-dict, exactly as the slot + // was written. Without this the sender's seed-time guard would treat the + // empty dictionary as a host-crash tear and brick build(), even though + // the orphan drainer drains the same frames fine. A genuine torn DELTA + // dictionary keeps a frame with deltaStart > 0 (maxDeltaStart() > 0) + // and is NOT discarded here: it still fails clean at seed time, since + // the ids its delta frames reference cannot be rebuilt without the lost + // dictionary. The recoveredMaxSymbolId >= size guard means this never + // fires for a slot whose dictionary is intact, nor for an empty slot + // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); + if (persistedDictInProgress != null + && recoveredMaxSymbolId >= persistedDictInProgress.size() + && recoveredMaxSymbolDeltaStart == 0L) { + // Only a NON-EMPTY discarded dictionary can desynchronise the fold. + // The first fold was keyed to this very size, so when it is 0 the + // re-fold below recomputes the identical baseline-0 result -- and + // that is the OVERWHELMINGLY common case, not the rare one the + // comment used to describe: every frame the shipped client ever + // wrote passes confirmedMaxId = -1, hence deltaStart == 0, hence + // maxDeltaStart == 0; and such a slot has no .symbol-dict, so + // recovery creates a fresh EMPTY one. All three conditions therefore + // hold on the FIRST restart after upgrading, for every non-empty + // slot, and the re-fold was a second O(payload bytes) walk -- with a + // byte-at-a-time varint decode over full-dict frames, which carry the + // whole dictionary -- for no result change at all. + int discardedSize = persistedDictInProgress.size(); + persistedDictInProgress.close(); + persistedDictInProgress = null; + // Re-fold at baseline 0. The analysis above was keyed to the + // dictionary's size, and every consumer of it presents the baseline it + // derived the same way -- seedGlobalDictionaryFromPersisted computes + // baseline 0 once pd is gone, and checkedRecoveryAnalysis rejects a + // baseline that disagrees with the fold. Discarding a dictionary that + // held entries (size > 0) therefore desynchronised the two and threw + // "recovery symbol baseline mismatch" out of build(). checkedRecoveryAnalysis + // now raises that as an UnreplayableSlotException, so build() would at least + // set the slot aside rather than rethrow forever -- but quarantining is + // still the wrong outcome HERE, because this slot is fully recoverable + // (its frames carry their whole dictionary inline). Re-folding avoids the + // mismatch entirely instead of trading a permanent brick for a needless + // quarantine plus a "resend the affected data" the operator does not owe. + // + // Reachable on one transient plus one crash: a session whose + // .symbol-dict fails to open (EIO, fd exhaustion, a Windows share + // lock) falls back to full-dict frames and, per the never-recreate + // contract, leaves the previous session's populated side-file intact; + // if that session then crashes, this recovery opens a dictionary with + // size > 0 next to self-sufficient frames that out-reach it. + // + // Re-folding rather than keeping the dictionary preserves the discard's + // whole point -- the slot recovers in full-dict mode, exactly as it was + // written, with producer, mirror and replay guard all anchored at 0. + if (discardedSize > 0) { + recoveredFrameAnalysisInProgress.close(); + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery(0); + recoveryFoldCount++; + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); + } + } } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers @@ -309,6 +463,19 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(sfDir); watermarkInProgress = AckWatermark.open(sfDir); + // A fresh slot MUST start with an EMPTY symbol dictionary. + // Unlike the ack watermark above -- a discardable optimization a + // max() clamp protects -- the dictionary is load-bearing: a + // delta frame referencing an id missing from it is unrecoverable, + // and a STALE dictionary inherited here (the segments are gone, so + // the producer is NOT seeded from it) shifts the dense id->symbol + // mapping and silently misattributes symbols on the next + // reconnect. openClean() truncates any survivor to empty rather + // than trusting a best-effort delete that may have failed (e.g. a + // Windows share lock); if the clean open itself fails, + // persistedSymbolDict stays null and the sender falls back to full + // self-sufficient frames, which is also safe. + persistedDictInProgress = PersistedSymbolDict.openClean(dictFf, sfDir); } MmapSegment initial; String initialPath = null; @@ -333,10 +500,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man manager.start(); } manager.register(ringInProgress, sfDir, watermarkInProgress); - // All construction succeeded — commit the ring and - // watermark references. + // All construction succeeded — commit the ring, watermark and + // symbol-dictionary references. this.ring = ringInProgress; this.watermark = watermarkInProgress; + this.persistedSymbolDict = persistedDictInProgress; + this.recoveredFrameAnalysis = recoveredFrameAnalysisInProgress; } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -362,6 +531,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } catch (Throwable ignored) { } } + if (persistedDictInProgress != null) { + try { + persistedDictInProgress.close(); + } catch (Throwable ignored) { + } + } + if (recoveredFrameAnalysisInProgress != null) { + try { + recoveredFrameAnalysisInProgress.close(); + } catch (Throwable ignored) { + } + } if (acquiredLock != null) { try { acquiredLock.close(); @@ -479,7 +660,24 @@ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos } @Override - public synchronized void close() { + public void close() { + close(true); + } + + /** + * As {@link #close()}, but {@code reclaimLogicalSlotLock == false} skips the + * parent-anchored logical slot-lock unlink at fully-drained retirement. + *

    + * A caller that HOLDS that lock must pass {@code false}. {@code Sender.build()} + * keeps it across engine construction and connect, and closes the engine from + * inside that scope when connect fails -- and a fresh slot is "fully drained" by + * definition ({@code publishedFsn() < 0}), so the default path would unlink the + * lock file while build() still holds the flock on it. On POSIX that frees the + * pathname without releasing the lock, so the next {@code acquireLogical} creates + * a SECOND inode and locks it successfully: two parties owning a lock whose only + * job is serialising the quarantine close->rename->recreate window. + */ + public synchronized void close(boolean reclaimLogicalSlotLock) { if (closed) return; closed = true; // Capture drain state BEFORE closing the ring — once the ring is @@ -541,6 +739,18 @@ public synchronized void close() { } catch (Throwable ignored) { } } + if (persistedSymbolDict != null) { + try { + persistedSymbolDict.close(); + } catch (Throwable ignored) { + } + } + if (recoveredFrameAnalysis != null) { + try { + recoveredFrameAnalysis.close(); + } catch (Throwable ignored) { + } + } if (fullyDrained) { try { unlinkAllSegmentFiles(sfDir); @@ -550,6 +760,26 @@ public synchronized void close() { AckWatermark.removeOrphan(sfDir); } catch (Throwable ignored) { } + try { + // Slot fully drained: the dictionary has no frames behind it. + PersistedSymbolDict.removeOrphan(sfDir); + } catch (Throwable ignored) { + } + if (reclaimLogicalSlotLock) { + try { + // The logical slot lock lives OUTSIDE the slot dir (in the + // shared .slot-locks dir) so it survives a slot rename; the + // fully-drained retirement that removes this slot's other + // side-files must remove it too, or .slot-locks accumulates a + // dead lock+pid pair per distinct slot name for the lifetime of + // sf_dir. Holding the directory-local lock is NOT sufficient to + // make this safe -- it says nothing about the LOGICAL lock, which + // a caller higher in the same stack may hold. Only a caller that + // knows it does not hold it may reclaim, hence the flag. + SlotLock.removeOrphanLogical(sfDir); + } catch (Throwable ignored) { + } + } } } finally { if (slotLock != null) { @@ -562,6 +792,79 @@ public synchronized void close() { } } + /** + * Decodes the cached recovery suffix directly into the producer's global + * dictionary. Recovery always builds the analysis with the persisted + * prefix size as its baseline, so no intermediate cardinality-sized list is + * needed on the production path. + */ + public long addRecoveredSymbolsTo(int baseline, GlobalSymbolDictionary target) { + if (recoveredFrameAnalysis == null) { + return baseline; + } + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + long coverage = analysis.coverage(); + if (coverage >= 0L) { + analysis.addDecodedSymbolsTo(target); + } + return coverage; + } + + long recoveredSymbolCoverage(int baseline) { + return checkedRecoveryAnalysis(baseline).coverage(); + } + + int recoveredSymbolSuffixCount(int baseline) { + return checkedRecoveryAnalysis(baseline).rawCount(); + } + + int recoveredSymbolSuffixLen(int baseline) { + return checkedRecoveryAnalysis(baseline).rawLen(); + } + + void copyRecoveredSymbolSuffix(int baseline, long target) { + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + int len = analysis.rawLen(); + if (len > 0) { + io.questdb.client.std.Unsafe.getUnsafe().copyMemory(analysis.rawAddr(), target, len); + } + } + + @TestOnly + public int recoveryFoldCount() { + return recoveryFoldCount; + } + + @TestOnly + public long recoveryFramesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.framesVisited(); + } + + @TestOnly + public long recoverySymbolEntriesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.symbolEntriesVisited(); + } + + @TestOnly + public int recoverySymbolNativeCapacity() { + return recoveredFrameAnalysis == null ? 0 : recoveredFrameAnalysis.rawCapacity(); + } + + private RecoveredFrameAnalysis checkedRecoveryAnalysis(int baseline) { + if (recoveredFrameAnalysis == null || recoveredFrameAnalysis.baseline() != baseline) { + // UnreplayableSlotException, NOT IllegalStateException: Sender.build() routes + // on the type, and only this type reaches its quarantine handler. A raw + // IllegalStateException escapes build() instead, and because senderId is + // stable and a not-fully-drained slot is retained on close, every restart + // re-recovers the same slot and rethrows -- the application can never + // construct a Sender, so it cannot even BUFFER new rows. + throw new UnreplayableSlotException("recovery symbol baseline mismatch [expected=" + + (recoveredFrameAnalysis == null ? "none" : recoveredFrameAnalysis.baseline()) + + ", actual=" + baseline + ']'); + } + return recoveredFrameAnalysis; + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ @@ -576,6 +879,15 @@ public MmapSegment firstSealed() { return ring.firstSealed(); } + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Number of times {@link #appendBlocking} hit * {@link SegmentRing#BACKPRESSURE_NO_SPARE} on its first attempt and @@ -586,6 +898,18 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } + /** + * Whether the sender may delta-encode symbol dictionaries on this engine. + * Always true in memory mode (the send loop keeps an in-process catch-up + * mirror). In disk mode it requires the persisted dictionary to have opened, + * since delta frames are not self-sufficient and recovery / orphan-drain must + * be able to rebuild the dictionary from disk. When false in disk mode the + * sender falls back to full self-sufficient frames. + */ + public boolean isDeltaDictEnabled() { + return sfDir == null || persistedSymbolDict != null; + } + /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ @@ -650,6 +974,30 @@ public long recoveredCommitBoundaryFsn() { return recoveredCommitBoundaryFsn; } + /** + * Highest symbol id any recovered delta frame references, or {@code -1} for + * fresh/memory rings (and recovered rings with no symbol-bearing frame). A + * resuming producer compares this against its recovered dictionary size to + * detect a host-crash tear of the persisted {@code .symbol-dict}. + */ + public long recoveredMaxSymbolId() { + return recoveredMaxSymbolId; + } + + /** + * Highest {@code deltaStart} across the recovered committed frames; {@code 0} when every + * surviving frame is self-sufficient (or none carries a dictionary at all). + *

    + * The send loop uses this to decide whether it needs a catch-up: at zero, every frame + * re-registers its dictionary from id 0 as it replays, so seeding the mirror -- and + * shipping a catch-up frame off it -- would be pure redundancy. Above zero, at least one + * frame's delta starts above ids it does not itself carry, so the mirror must hold those + * ids before the replay begins or the server null-pads the hole. + */ + public long recoveredMaxSymbolDeltaStart() { + return recoveredMaxSymbolDeltaStart; + } + /** * FSN of the last frame of a recovered orphaned deferred tail, or * {@code -1} when none. See {@link #recoveredCommitBoundaryFsn()}: the @@ -659,6 +1007,31 @@ public long recoveredOrphanTipFsn() { return recoveredOrphanTipFsn; } + /** + * Retires a recovered deferred tail once every frame below it is ACKed. + * The operation is local and idempotent: no wire sequence ever referred + * to these aborted-transaction frames. + * + * @return true if no orphan tail remains, false if lower frames still need + * server ACKs + */ + public boolean retireRecoveredOrphanTailIfReady() { + long orphanTip = recoveredOrphanTipFsn; + if (orphanTip < 0L) { + return true; + } + long orphanStart = recoveredCommitBoundaryFsn + 1L; + if (ackedFsn() < orphanStart - 1L) { + return false; + } + LOG.warn("retiring orphaned deferred tail: {} frame(s) [fsn {}..{}] belong to a transaction " + + "whose commit was never published; aborting them (never transmitted, slots trimmed)", + orphanTip - orphanStart + 1L, orphanStart, orphanTip); + acknowledge(orphanTip); + recoveredOrphanTipFsn = -1L; + return true; + } + /** * Unlinks every {@code .sfa} file under {@code dir}. Called only on * clean shutdown when the ring confirms every published FSN has been diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..f9bf2a47 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -31,14 +31,17 @@ import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.http.client.WebSocketUpgradeException; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.std.CharSequenceLongHashMap; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; @@ -49,6 +52,7 @@ import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; @@ -64,9 +68,13 @@ * can trim fully-acked segments.

    6. *
  6. On wire failure, runs the configured reconnect policy: capped * exponential backoff with jitter, retried indefinitely (Invariant B -- - * a store-and-forward drainer never gives up on a wall-clock budget), - * with only auth-style failures (401/403/non-101 upgrade reject) treated - * as terminal. On reconnect success, repositions the cursor at + * a store-and-forward drainer never gives up on a wall-clock budget). + * Once a foreground sender has connected even once, endpoint-policy + * failures (auth, upgrade, durable-ack capability) become transients it + * keeps retrying, so credential and rolling-capability changes stay + * contained by store-and-forward; they are terminal only for an orphan + * drainer or on a foreground sender's initial connect (before the wire + * has ever come up). On reconnect success, repositions the cursor at * {@code ackedFsn+1} and replays.
    7. *
* No locks on the steady-state path. The producer thread (user) writes @@ -129,11 +137,77 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code LineSenderBuilder.poisonMinEscalationWindowMillis(long)}. */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; + /** + * Default minimum wall-clock dwell (millis) a symbol-dict catch-up CAP GAP must + * persist before an orphan drainer may latch a terminal, even once + * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps have accrued. Same idea as + * {@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}, for the same reason: a + * strike count measures "how many times did we look", not "how long has this been + * true", and only the latter distinguishes a permanent cluster capability gap from + * a node that is briefly away. + *

+ * It matters here because the cap gap's trigger IS a failover. With the reconnect + * backoff capped at {@link #DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS} (5 s), 16 cap + * gaps accrue in roughly two minutes -- comfortably less than an ordinary rolling + * restart of the larger-cap node. A count-only budget could therefore quarantine an + * otherwise drainable orphan slot during a routine cluster operation. Requiring the + * dwell as well means the orphan terminal needs BOTH "we have looked many times" AND + * "it has stayed true for a long time". Foreground senders never apply this terminal + * policy: they retry indefinitely until a larger-cap node returns. + *

+ * 5 minutes -- the same figure as {@link #DEFAULT_RECONNECT_MAX_DURATION_MILLIS}, + * this codebase's existing notion of how long a transient outage may plausibly + * last. Configurable per sender via the + * {@code catchup_cap_gap_min_escalation_window_millis} connect-string key or + * {@code LineSenderBuilder.catchUpCapGapMinEscalationWindowMillis(long)}; {@code 0} + * restores count-only escalation for orphan drainers. + */ + public static final long DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS = 300_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); + // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts + // -- catch-ups that reached a fresh server and found a single dictionary entry + // too large for its advertised batch cap -- may occur consecutively before an + // orphan drainer latches a terminal. This is a + // SANCTIONED orphan-only terminal (a genuine cluster batch-size capability gap), + // the connect-time analog of the orphan drainer's durable-ack capability gap + // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips + // it -- an entry that fit its data frame under a cap always fits its bare catch-up + // frame under that same cap -- so it only affects a heterogeneous / rolling-cap + // cluster, where a failover to a smaller-cap node can hit it for an entry an + // earlier node accepted. A foreground sender retries forever. An orphan drainer + // rides out the transient window until a larger-cap node returns; only a persistent + // gap (this many consecutive cap gaps with no successful catch-up or unrelated + // reconnect state in between) latches. + // + // Budget accounting (satisfies "a transient must never burn the terminal + // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a + // node is reached and an entry is oversized. A successful catch-up ends the + // episode, as does any unrelated reconnect state (connect refusal, catch-up send + // failure, upgrade/role rejection): otherwise its downtime would count toward the + // wall-clock dwell even though no cap gap was observed. The cap-gap exception + // itself does NOT reset the episode, so consecutive small-cap nodes still prove a + // persistent cluster capability gap and can exhaust the orphan policy. + private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; + // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror + // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even + // this needs ~200M+ distinct symbols on a single connection, far past any real + // workload. The guard exists so that pathological growth fails loudly instead + // of overflowing the int capacity math into a heap-corrupting copyMemory. + private static final int MAX_SENT_DICT_BYTES = Integer.MAX_VALUE - 8; /** * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; + // Test seam: when true, recovery mirror seeding throws immediately AFTER + // ensureSentDictCapacity has grown (and therefore taken ownership of) the mirror, + // standing in for the copyRecoveredSymbolSuffix-adjacent failure that leaves a + // freshly malloc'd mirror with no owner. It sits after the grow deliberately: the + // constructor's cleanup only frees an OWNED mirror, so a seam before the grow + // would leave nothing to free and the leak guard would pass with the cleanup + // deleted. Production never sets it; volatile only so a test thread's write is + // visible to the loop under test. + @TestOnly + public static volatile boolean forceMirrorSeedFailureForTest; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -151,8 +225,14 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by the server -- holding stale watermarks across the wire boundary // would falsely advance trim before re-confirmation. private final CharSequenceLongHashMap durableTableWatermarks = new CharSequenceLongHashMap(); + // Pre-converted to nanos. Consulted only by the orphan terminal policy. Zero disables + // the dwell entirely (count-only escalation at MAX_CATCHUP_CAP_GAP_ATTEMPTS); the + // user-facing 5-minute default is applied at the config layer. + private final long catchUpCapGapMinEscalationWindowNanos; + private final CatchUpCapGapPolicy catchUpCapGapPolicy; private final CursorSendEngine engine; private final long parkNanos; + private final ReconnectPolicy reconnectPolicy; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only // when durableAckMode is true. Each entry binds a wireSeq to the per-table // (name, seqTxn) pairs the server reported on the OK frame. The queue is @@ -169,12 +249,6 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final ReconnectFactory reconnectFactory; private final long reconnectInitialBackoffMillis; private final long reconnectMaxBackoffMillis; - // Retained for constructor symmetry and passed in by callers, but NOT - // consulted by the background loop: Invariant B removed the wall-clock - // give-up from connectLoop. The budget still bounds the blocking (non-lazy) - // initial connect via QwpWebSocketSender -> connectWithRetry, which takes it - // as an explicit argument rather than reading this field. - private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); private final CountDownLatch shutdownLatch = new CountDownLatch(1); @@ -200,6 +274,80 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); + // Delta symbol dictionary catch-up state (see swapClient). + // ALWAYS active -- in memory mode, in disk mode, and (critically) even when the + // per-slot persisted dictionary failed to open. sentDictCount is this loop's model + // of how many ids the CURRENT server has been told about, and the torn-dict guard + // in trySendOne reads it, so it must track the wire in every mode; gating it on + // engine.isDeltaDictEnabled() is what once froze it at 0 and terminal'd a slot + // whose frames replay perfectly from id 0 (see the constructor). On a recovered / + // orphan-drained slot the constructor SEEDS sentDict* from the persisted + // dictionary when there is one; otherwise the mirror starts empty and grows from + // the frames themselves. The loop mirrors, in sentDict*, every symbol it has ever + // sent -- the concatenated [len varint][utf8] bytes in global-id order + // (sentDictBytes*) plus the count (sentDictCount) -- so that on reconnect it can + // re-register the whole dictionary on the fresh server (which discards its + // dictionary on every disconnect) before replaying frames whose deltas start above + // id 0. All of this is touched only by the I/O thread. + // Footprint note: this mirror is a SECOND copy of the dictionary -- the same + // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- + // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a + // memory-mode connection's steady-state dictionary footprint is ~2x the symbol + // set. It is bounded by distinct-symbol count (not per-row) and never trimmed for + // the connection's lifetime (a reconnect may need the whole dictionary at any + // moment), so it cannot be dropped; it is an intentional cost of the feature. + private long sentDictBytesAddr; + private int sentDictBytesCapacity; + private int sentDictBytesLen; + // False while the mirror is unallocated, and while a loop of EITHER policy borrows + // the engine's persisted prefix -- both borrow now; neither consumes it. Any growth + // copy-on-writes into loop-owned memory and sets this true; only owned buffers are + // freed. + private boolean sentDictBytesOwned; + private int sentDictCount; + // True when replay frames can start above dictionary id zero and therefore + // depend on a catch-up on a fresh connection. Delta-enabled live engines + // always have this dependency. A recovered delta slot whose dictionary + // failed to open is identified by its non-zero recovered delta start. The + // remaining false case is full-dictionary fallback: every frame re-registers + // from id zero, so sending the mirror first would duplicate the dictionary. + private final boolean hasReplayDictionaryDependency; + // Reusable staging buffer for the small QWP catch-up prefix (fixed header plus + // delta range varints). Symbol bytes stay in sentDictBytesAddr and are passed as a + // second WebSocket payload slice, avoiding a full-dictionary staging copy. + private long catchUpFrameAddr; + private int catchUpFrameCapacity; + private int catchUpFrameGrowthCount; + // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry + // too large for its batch cap -- with no intervening successful catch-up or unrelated + // reconnect state (see MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). + // Foreground retries never increment it. I/O-thread-only. + private int catchUpCapGapAttempts; + // System.nanoTime() of the FIRST cap gap of the current orphan-policy episode. + // Anchors the escalation dwell. Reset together with catchUpCapGapAttempts on a + // successful catch-up or an unrelated reconnect state, so only an uninterrupted run + // of cap-gap observations contributes wall-clock dwell. Anchoring at the FIRST gap + // (not refreshed per attempt) lets consecutive small-cap nodes satisfy the dwell; + // resetting after transport/role states prevents their downtime from doing so. + // I/O-thread-only, like catchUpCapGapAttempts. + // + // -1 marks "no episode open" for a debugger or a heap dump, but it is NOT the test + // for one -- catchUpCapGapAttempts == 0 is (see sendDictCatchUp). A nanoTime instant + // is only meaningful as a difference: its origin is arbitrary and the spec permits + // negative values, so no state may ride on this field's sign. + private long catchUpCapGapFirstNanos = -1L; + // True once a real ring frame (data or commit) has been sent on the CURRENT + // connection, as opposed to only the dictionary catch-up. The catch-up consumes + // wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies "the head + // frame was sent": onClose's poison-strike gate and handleServerRejection's + // pre-send gate key off THIS instead. Without it, a transient outage AFTER the + // catch-up but BEFORE the first data frame (a flapping LB/middlebox that accepts + // the upgrade + catch-up then closes) would be mistaken for a deterministic + // head-frame rejection and escalate to a PROTOCOL_VIOLATION terminal -- breaking + // the store-and-forward "retry a transient outage forever" contract. Reset per + // connection in setWireBaselineWithCatchUp; set in trySendOne after a successful + // send. + private boolean dataFrameSentThisConnection; private WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's @@ -227,11 +375,19 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private long fsnAtZero; // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a - // successful connect attempt in any mode). Once true, stays true. Used to - // distinguish a "never reached the server" terminal failure (looks like a - // config typo or firewall block) from "lost connection after we were - // up" (looks transient). + // successful connect attempt in any mode). Once true, stays true. + // LOAD-BEARING, not observability: it is half of endpointPolicyFailureIsTerminal(), + // which latches an auth / upgrade / durable-ack rejection only for an ORPHAN drainer + // or a sender that has never connected. Relocate or lazily initialise the write and a + // foreground sender's transient auth blip becomes a producer-fatal terminal -- the + // exact failure this policy exists to prevent. private volatile boolean hasEverConnected; + // Cause of the outage the reconnect loop is currently riding out, or null once a + // connect succeeds. Written by the I/O thread, read by the producer thread so a + // backpressure or drain-timeout failure can NAME the reason the wire is not draining + // instead of blaming disk sizing. Was a connectLoop local that never escaped, which + // is why a revoked token surfaced to operators as "sf_max_total_bytes too small". + private volatile Throwable lastReconnectError; private volatile Thread ioThread; // Typed marker for a durable-ack CAPABILITY-GAP terminal: set (before the // terminalError latch, so a checkError() caller that observes the latch is @@ -239,8 +395,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // QwpDurableAckMismatchException. The orphan drainer consults it to route // a mid-drain capability gap into its budgeted settle-retry // (BackgroundDrainer.connectWithDurableAckRetry) instead of quarantining - // the slot on the first sweep; the foreground sender ignores it and keeps - // its spec'd loud-fail (sf-client.md section 8.1). Write-once alongside + // the slot on the first sweep. Foreground reconnects never set this marker; + // they keep retrying after a successful initial connection. Write-once alongside // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; @@ -370,19 +526,18 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code client} may be {@code null} only if {@code reconnectFactory} * is non-null — this is the async-initial-connect path: the I/O thread * runs the same retry loop on its first iteration to obtain a live - * client, and a terminal failure (auth/upgrade reject) is delivered - * through the dispatcher rather than thrown to the constructor's - * caller; plain connect failures are retried indefinitely - * (Invariant B: no wall-clock budget give-up). + * client. Every endpoint-policy and transport failure stays inside that + * background retry loop; it is never delivered to the foreground producer + * (Invariant B: no wall-clock budget give-up). Blocking OFF/SYNC startup + * performs its fail-fast policy before constructing this loop. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis) { this(client, engine, fsnAtZero, parkNanos, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, false); } @@ -398,12 +553,11 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis, boolean durableAckMode) { this(client, engine, fsnAtZero, parkNanos, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, durableAckMode, DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS); } @@ -418,19 +572,18 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis, boolean durableAckMode, long durableAckKeepaliveIntervalMillis) { this(client, engine, fsnAtZero, parkNanos, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, durableAckMode, durableAckKeepaliveIntervalMillis, DEFAULT_MAX_HEAD_FRAME_REJECTIONS); } /** - * Eleven-arg overload — omits the poison-escalation dwell window, which + * Ten-arg overload — omits the poison-escalation dwell window, which * defaults to {@code 0} (legacy: escalate as soon as maxHeadFrameRejections * strikes accrue, with no minimum wall-clock). The user-facing 5s default * ({@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}) is applied at the @@ -441,37 +594,91 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis, boolean durableAckMode, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections) { this(client, engine, fsnAtZero, parkNanos, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, durableAckMode, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, 0L); } /** - * Master constructor — also accepts the poison-frame detector threshold + * Eleven-arg overload — omits the symbol-dict cap-gap escalation dwell, which + * defaults to {@code 0}. This and the twelve-arg overload use the foreground-safe + * retry-forever policy; orphan drainers opt into count-and-dwell escalation through + * the policy-aware master constructor. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, 0L); + } + + /** + * Twelve-arg overload. Catch-up cap gaps are retried indefinitely, which is the + * required policy for a foreground store-and-forward sender. Orphan drainers use the + * policy-aware overload below to opt into bounded terminal escalation. + *

+ * Also accepts the poison-frame detector threshold * ({@code max_frame_rejections}): consecutive server-active rejections of * the same head-of-line frame, with no ack progress in between, before the - * loop escalates to a typed terminal. Must be {@code >= 1}. The final - * argument is the minimum wall-clock dwell (millis) the suspect must stay - * poisoned before escalation ({@code poison_min_escalation_window_millis}; - * {@code >= 0}, where 0 = legacy immediate escalation at the threshold). + * loop escalates to a typed terminal. Must be {@code >= 1}. Then the minimum + * wall-clock dwell (millis) the suspect must stay poisoned before escalation + * ({@code poison_min_escalation_window_millis}; {@code >= 0}, where 0 = legacy + * immediate escalation at the threshold). + *

+ * The final argument is the analogous dwell for the symbol-dict catch-up cap gap. + * It is retained here for source and binary compatibility but is consulted only when + * the policy-aware overload selects + * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET}. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis, boolean durableAckMode, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis) { + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy.RETRY_FOREVER); + } + + /** + * Compatibility policy-aware constructor. New production call sites should + * use the {@link ReconnectPolicy} overload below, which names the foreground + * versus orphan ownership distinction directly. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy catchUpCapGapPolicy) { if (maxHeadFrameRejections < 1) { throw new IllegalArgumentException( "maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections); @@ -480,6 +687,24 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, throw new IllegalArgumentException( "poisonMinEscalationWindowMillis must be >= 0: " + poisonMinEscalationWindowMillis); } + if (catchUpCapGapMinEscalationWindowMillis < 0) { + throw new IllegalArgumentException( + "catchUpCapGapMinEscalationWindowMillis must be >= 0: " + + catchUpCapGapMinEscalationWindowMillis); + } + // TimeUnit conversion saturates at Long.MAX_VALUE. A raw multiply + // wraps large valid millisecond values negative, which makes the + // elapsed-time gate appear satisfied as soon as the strike threshold + // is reached and can quarantine a recoverable orphan slot prematurely. + this.catchUpCapGapMinEscalationWindowNanos = + TimeUnit.MILLISECONDS.toNanos(catchUpCapGapMinEscalationWindowMillis); + if (catchUpCapGapPolicy == null) { + throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null"); + } + this.catchUpCapGapPolicy = catchUpCapGapPolicy; + this.reconnectPolicy = catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER + ? ReconnectPolicy.FOREGROUND + : ReconnectPolicy.ORPHAN; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -489,19 +714,141 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; + this.hasReplayDictionaryDependency = engine.isDeltaDictEnabled() + || engine.recoveredMaxSymbolDeltaStart() > 0L; + // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, + // so seed it from the slot's persisted dictionary. That way the very first + // connection re-registers the whole dictionary (via a catch-up frame) + // before replaying the recovered delta frames. + // + // Deliberately NOT gated on engine.isDeltaDictEnabled(). The mirror, the + // catch-up and the replay guard together are this loop's model of what the + // SERVER's dictionary holds, and that model must track every mode. Gating + // them on that flag is what once bricked a recoverable slot: an unopenable + // .symbol-dict reports isDeltaDictEnabled()=false, yet the frames already on + // disk are still DELTA frames (deltaStart 0,1,2,...). With the mirror gated + // off, sentDictCount froze at 0 while the ungated guard kept comparing + // against it, so the frame at deltaStart=1 tripped a terminal -- even though + // replaying the whole sequence from id 0 rebuilds the dictionary on the + // server contiguously and drains perfectly. isDeltaDictEnabled() decides + // what the PRODUCER emits and persists; it must never decide what this loop + // tracks. When the dictionary could not be opened, pd is null, the mirror + // simply starts empty and grows from the frames themselves. + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + if (pd != null && pd.recoveredSize() > 0) { + int len = pd.loadedEntriesLen(); + if (len > 0) { + // Seed by reference. The foreground loop takes ownership after + // construction succeeds; orphan-drainer loops keep borrowing because + // one engine can create several sessions during capability-gap + // recycling. A borrowed mirror performs copy-on-write if a recovered + // frame suffix must extend it. + sentDictBytesAddr = pd.loadedEntriesAddr(); + sentDictBytesCapacity = len; + sentDictBytesLen = len; + // Set the count only alongside the bytes so sentDictCount can + // never claim symbols the mirror does not hold -- and take it from + // recoveredSize(), NOT the live size(). They differ: the recovery + // heal (QwpWebSocketSender.healPersistedDictionary) appends the + // frame-contributed suffix to the file before this loop is built, so + // size() would already have run past the loaded byte region. + sentDictCount = pd.recoveredSize(); + } + } + // ...and then from the surviving frames' own delta sections, exactly as the producer + // seeds its dictionary. The mirror is the loop's model of what the SERVER holds and it + // is what the catch-up frame ships, so when it stops at the persisted prefix while the + // producer's baseline runs past it, the loop condemns (deltaStart > sentDictCount) a + // slot the producer just seeded successfully. Same two sources, same order, so the two + // land on the same number by construction. + // ...but ONLY when a surviving frame actually depends on ids it does not carry + // (recoveredMaxSymbolDeltaStart > 0). At zero every frame is self-sufficient and + // re-registers its dictionary from id 0 as it replays, so seeding the mirror would buy + // nothing and cost a catch-up frame on every connection -- full-dict slots must stay + // catch-up-free. + // The prefix seed above may still be borrowed from PersistedSymbolDict. + // Extending it inside this cleanup boundary is what makes that safe: + // ensureSentDictCapacity copy-on-writes a borrowed prefix into loop-OWNED + // native memory, so from that point a failure would leak it -- and an + // allocation failure here propagates out of the constructor before any + // caller can own the half-built loop, leaving nothing else able to + // release it. releaseSentDictBytes() in the catch frees exactly that + // mirror, and only when this loop owns it. + try { + if (engine.recoveredMaxSymbolDeltaStart() > 0L) { + int baseline = sentDictCount; + if (engine.recoveredSymbolCoverage(baseline) >= 0L) { + int suffixLen = engine.recoveredSymbolSuffixLen(baseline); + int suffixCount = engine.recoveredSymbolSuffixCount(baseline); + if (suffixLen > 0) { + ensureSentDictCapacity((long) sentDictBytesLen + suffixLen); + if (forceMirrorSeedFailureForTest) { + // Throw AFTER the grow, never before it. ensureSentDictCapacity + // is what copy-on-writes a borrowed prefix into a loop-OWNED + // allocation (sentDictBytesOwned = true), and the catch's + // releaseSentDictBytes() is gated on exactly that flag. A seam + // in front of the grow fires while the mirror is still borrowed + // from PersistedSymbolDict, so the cleanup frees nothing and the + // guard below passes even with the cleanup deleted -- it would + // stop pinning the leak it exists for. + throw new LineSenderException( + "simulated mirror seed allocation failure (test only)"); + } + engine.copyRecoveredSymbolSuffix( + baseline, sentDictBytesAddr + sentDictBytesLen); + sentDictBytesLen += suffixLen; + sentDictCount += suffixCount; + } + } + } + // No per-entry offset index is derived here, and none is kept anywhere: + // the mirror carries its own framing ([len varint][utf8] per entry), so + // sendDictCatchUp -- its only reader -- recovers each entry's span with a + // running pointer as it walks. A recovered slot that drains without ever + // reconnecting, the normal case, therefore never pays that O(n) walk and + // carries no index memory for it. + } catch (Throwable t) { + releaseSentDictBytes(); + throw t; + } + // BOTH policies borrow the engine-owned recovery sources; neither consumes them. + // + // The foreground path used to take ownership here -- pd.takeLoadedEntries() plus + // engine.releaseRecoveredSymbolStorage() -- which made the hand-off ONE-SHOT while + // leaving a second construction reachable: QwpWebSocketSender.ensureConnected's + // catch closes and nulls cursorSendLoop precisely so a caller can retry, and + // `connected` is only set at the very end. takeLoadedEntries zeroes addr/len but + // NOT size, and releaseRecoveredSymbolStorage zeroes the raw buffer but NOT + // baseline, so the retry saw pd.recoveredSize() > 0 with loadedEntriesLen() == 0, + // left sentDictCount at 0, and then either tripped checkedRecoveryAnalysis' + // baseline mismatch or -- when recoveredMaxSymbolDeltaStart == 0 -- latched the + // torn-dictionary terminal ("resend required") on a perfectly healthy slot. + // + // Borrowing removes the state machine instead of trying to make it idempotent. + // Lifetime is safe in both directions: QwpWebSocketSender.close() closes the loop + // before the engine, and BackgroundDrainer's finally does the same, so the buffers + // always outlive their borrower. Any growth copy-on-writes into loop-owned memory + // (ensureSentDictCapacity), and releaseSentDictBytes frees only what the loop owns. this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; this.reconnectFactory = reconnectFactory; - this.reconnectMaxDurationMillis = reconnectMaxDurationMillis; this.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis; this.reconnectMaxBackoffMillis = reconnectMaxBackoffMillis; this.durableAckMode = durableAckMode; + // Saturate, never multiply raw -- the same hazard the cap-gap dwell above + // guards. A raw multiply wraps a large millisecond value NEGATIVE, and both of + // these read as "elapsed >= window", so a negative makes the gate trivially + // true: the keepalive would ping on every loop pass, and the poison detector + // would escalate on the strike count alone -- exactly the transient-to-terminal + // false positive the dwell exists to stop (Invariant B). Both keys are + // user-supplied and validated only as >= 0, so asking for a very long window is + // legal, and asking for a longer one must never buy a shorter one. this.durableAckKeepaliveIntervalNanos = durableAckKeepaliveIntervalMillis > 0 - ? durableAckKeepaliveIntervalMillis * 1_000_000L + ? TimeUnit.MILLISECONDS.toNanos(durableAckKeepaliveIntervalMillis) : 0L; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowNanos = poisonMinEscalationWindowMillis > 0 - ? poisonMinEscalationWindowMillis * 1_000_000L + ? TimeUnit.MILLISECONDS.toNanos(poisonMinEscalationWindowMillis) : 0L; // SYNC/OFF startup hands a live client to the constructor, so we // already know we reached the server at least once. ASYNC startup @@ -519,6 +866,40 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } } + /** + * Policy-aware master constructor. A foreground sender fails fast while + * establishing its first connection, then retries endpoint-policy failures + * indefinitely after it has been live. An orphan drainer returns such failures + * to its owner so the slot can follow its settle/quarantine policy. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + ReconnectPolicy reconnectPolicy) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + catchUpPolicyFor(reconnectPolicy)); + } + + private static CatchUpCapGapPolicy catchUpPolicyFor(ReconnectPolicy reconnectPolicy) { + if (reconnectPolicy == null) { + throw new IllegalArgumentException("reconnectPolicy must be non-null"); + } + return reconnectPolicy == ReconnectPolicy.FOREGROUND + ? CatchUpCapGapPolicy.RETRY_FOREVER + : CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET; + } + /** * Maps a server status byte to a {@link SenderError.Category}. Exposed for unit tests. */ @@ -565,12 +946,21 @@ public static WebSocketClient connectWithRetry( String contextLabel ) { long startNanos = System.nanoTime(); - long deadlineNanos = startNanos + maxDurationMillis * 1_000_000L; + // Saturating budget, compared against ELAPSED rather than an absolute + // startNanos+budget deadline. A large reconnect_max_duration_millis (e.g. + // Long.MAX_VALUE, the natural "retry until the server boots") must not collapse + // the budget to zero: a raw maxDurationMillis*1_000_000L wraps NEGATIVE -- the + // same overflow the drainer's capabilityGapBudgetNanos and the loop dwells were + // hardened against -- so the pre-condition loop below would make ZERO attempts. + // Even TimeUnit.toNanos saturated to Long.MAX_VALUE would overflow an absolute + // startNanos+budget sum; comparing the bounded nanoTime difference against the + // budget stays correct at saturation. + long budgetNanos = TimeUnit.MILLISECONDS.toNanos(maxDurationMillis); long backoffMillis = initialBackoffMillis; int attempts = 0; long lastLogNanos = 0L; Throwable lastError = null; - while (System.nanoTime() < deadlineNanos) { + while (System.nanoTime() - startNanos < budgetNanos) { attempts++; try { WebSocketClient c = factory.reconnect(); @@ -625,7 +1015,7 @@ public static WebSocketClient connectWithRetry( // this > 0. Mirrors BackgroundDrainer's sweep-loop jitter guard. long jitter = ThreadLocalRandom.current().nextLong(Math.max(1L, backoffMillis)); long sleepMillis = backoffMillis + jitter; - long remainingMillis = (deadlineNanos - System.nanoTime()) / 1_000_000L; + long remainingMillis = (budgetNanos - (System.nanoTime() - startNanos)) / 1_000_000L; if (remainingMillis <= 0) { break; } @@ -696,6 +1086,16 @@ public static boolean isTerminalCloseCode(int code) { } } + @TestOnly + public static int maxCatchUpCapGapAttempts() { + return MAX_CATCHUP_CAP_GAP_ATTEMPTS; + } + + @TestOnly + public static int maxSentDictBytes() { + return MAX_SENT_DICT_BYTES; + } + /** * Surfaces any error the I/O thread recorded. Called by the producer * thread (typically from inside its append wrapper) so failures don't @@ -754,6 +1154,12 @@ public synchronized void close() { // after — the latch await is only skipped when the loop never ran. running = false; Thread t = ioThread; + // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets + // freed on ioLoop's exit path. When t == null the loop never ran (start() + // was never called, or t.start() failed before committing ioThread), so + // that free never happens and a seeded (recovery / orphan-drain) mirror + // would leak. Capture that here; free it below, after client teardown. + boolean loopNeverRan = t == null; if (t != null) { LockSupport.unpark(t); // Only await the shutdown latch if the I/O thread actually ran. @@ -812,6 +1218,17 @@ public synchronized void close() { } client = null; } + // Free the I/O-thread-owned symbol-dict mirror ONLY when the loop never + // ran (see loopNeverRan). If it ran, ioLoop's exit already freed it -- and + // on the failed-stop path (interrupted latch await, ioThread left set) the + // thread may still be mid-send, so touching the mirror here would race. + // A duplicate close observes sentDictBytesAddr == 0 and skips. + if (loopNeverRan && sentDictBytesAddr != 0) { + releaseSentDictBytes(); + } + if (loopNeverRan) { + freeCatchUpFrameBuffer(); + } } /** @@ -996,7 +1413,28 @@ public synchronized void start() { // walks back to the lowest unacked frame so sealed-segment data // actually reaches the wire — without it, start() would skip // straight to the active and orphan everything in sealed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // A recovered sender re-registers its dictionary with a catch-up on + // the very first connect. Here that runs on the CALLER thread (sync + // start), so we must NOT let it drive connectLoop -- that would block + // Sender construction forever on a transient outage. Drop the dead + // client instead: the I/O thread then reconnects via + // attemptInitialConnect -> swapClient and re-sends the catch-up off + // this thread. If the failure was already terminal (recordFatal set + // running=false, e.g. an entry too large for the batch cap), the I/O + // thread simply winds down and checkError() surfaces it. + WebSocketClient dead = client; + client = null; + if (dead != null) { + try { + dead.close(); + } catch (Throwable ignored) { + // best-effort + } + } + } Thread t = new Thread(this::ioLoop, "qdb-cursor-ws-io"); t.setDaemon(true); try { @@ -1079,10 +1517,12 @@ private void applyDurableAck() { /** * Drives the very first connect attempt on the I/O thread, used in the * async-initial-connect mode (constructed with {@code client == null}). - * Reuses the same retry+backoff machinery as {@link #fail(Throwable)} — - * connect failures are retried indefinitely (Invariant B), and a - * terminal upgrade reject is delivered through the dispatcher, not - * thrown to the producer. + * Reuses the same retry+backoff machinery as {@link #fail(Throwable)}. + * Transport failures retry indefinitely here (Invariant B). But this path + * runs with {@code !hasEverConnected}, so an auth, upgrade or capability + * rejection is terminal (see {@link #endpointPolicyFailureIsTerminal()}): + * it is dispatched to the async {@code SenderErrorHandler} and latched for + * a {@code close()} rethrow, rather than retried. */ private void attemptInitialConnect() { connectLoop(new LineSenderException( @@ -1104,6 +1544,15 @@ private void clearDurableAckTracking() { lastFrameOrPingNanos = 0L; } + private static boolean isCatchUpCapGap(Throwable t) { + return t instanceof CatchUpSendException && ((CatchUpSendException) t).isCapGap; + } + + private void resetCatchUpCapGapEpisode() { + catchUpCapGapAttempts = 0; + catchUpCapGapFirstNanos = -1L; + } + /** * Shared per-outage retry loop. Used by {@link #fail(Throwable)} for * mid-flight wire failures (phase="reconnect") and by @@ -1118,20 +1567,31 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(initial); return; } + // A wire/role state that interrupted an orphan cap-gap run is not evidence that + // the cluster's batch cap stayed incompatible during the outage. Start a fresh + // episode; the cap-gap exception itself is the one reconnect cause that preserves + // the existing attempt count and dwell anchor. + if (!isCatchUpCapGap(initial)) { + resetCatchUpCapGapEpisode(); + } LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); - // INVARIANT B: a store-and-forward drainer must NEVER terminate on a + // INVARIANT B: a store-and-forward loop must NEVER terminate on a // wall-clock reconnect budget. A replica-only / all-endpoints-replica // window is TRANSIENT -- a replica gets promoted, a primary reappears -- // so this background loop retries for as long as it is running, backing - // off between attempts. The ONLY terminal conditions are a genuinely - // non-retriable upgrade (auth / non-421 upgrade / durable-ack capability - // gap), which return directly below, or the sender being stopped. SF + // off between attempts. Endpoint-policy failures (auth / non-421 + // upgrade / durable-ack capability gap) are terminal only for orphan + // drainers. Foreground senders retry them from asynchronous startup onward + // so a credential or cluster capability rotation cannot stop the producer. SF // exhaustion is surfaced to the PRODUCER as append backpressure, never - // here. reconnect_max_duration_millis is intentionally NOT consulted: it - // bounds only the blocking (non-lazy) initial connect in - // QwpWebSocketSender.buildAndConnect, never this background loop. + // here. reconnect_max_duration_millis is intentionally NOT consulted by + // THIS loop. Its holders pass it explicitly where it does apply: the + // blocking (non-lazy) initial connect hands it to connectWithRetry (see + // QwpWebSocketSender.ensureConnected, the SYNC branch), and + // BackgroundDrainer converts it into the durable-ack capability-gap + // budget. Neither bounds this loop's steady-state reconnect. long backoffMillis = reconnectInitialBackoffMillis; if (paceFirstAttemptMillis > 0 && running) { // NACK-initiated recycle against a reachable server: pace the @@ -1151,7 +1611,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM } int attempts = 0; long lastLogNanos = 0L; - Throwable lastReconnectError = initial; + lastReconnectError = initial; while (running) { attempts++; totalReconnectAttempts.incrementAndGet(); @@ -1187,66 +1647,86 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, fsnAtZero); return; } + // A null factory result is an unsuccessful connect state, not a cap-gap + // observation. Do not let the time spent retrying it satisfy orphan dwell. + resetCatchUpCapGapEpisode(); } catch (QwpAuthFailedException | WebSocketUpgradeException e) { - // Terminal across all configured endpoints per spec sf-client.md - // section 13.3: auth (401/403) bypasses reconnect and surfaces as - // SECURITY_ERROR. WebSocketUpgradeException reaching here is always - // non-421: QwpUpgradeFailures.classify upstream converts a - // 421-with-X-QuestDB-Role to QwpIngressRoleRejectedException, and a - // 421 without that header walks the transport-error path in - // buildAndConnect and lands as a LineSenderException, falling into - // the Throwable branch below. - LOG.error("terminal upgrade error during {} -- won't retry: {}", - phase, e.getMessage()); - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.SECURITY_ERROR, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "ws-upgrade-failed: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; + if (endpointPolicyFailureIsTerminal()) { + // Orphans return control to their quarantine owner. + // WebSocketUpgradeException reaching here is always non-421: + // role rejects are classified into the transient branch below. + LOG.error("terminal upgrade error during {} -- won't retry: {}", + phase, e.getMessage()); + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.SECURITY_ERROR, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "ws-upgrade-failed: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + dispatchRetriedEndpointPolicyFailure( + SenderError.Category.SECURITY_ERROR, "ws-upgrade-failed: " + e.getMessage()); + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground auth/upgrade policy rejected the connection; " + + "retrying while store-and-forward owns the buffered data -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; + } } catch (QwpDurableAckMismatchException e) { - // Per spec sf-client.md section 8.1: the client opted into durable - // ack but the cluster cannot honour it. Loud fail at connect rather - // than silently waiting for ack frames that will never arrive. - // Classified as PROTOCOL_VIOLATION (config/capability mismatch), - // not SECURITY_ERROR -- this is not an auth failure. - LOG.error("durable-ack mismatch during {} -- won't retry: {}", - phase, e.getMessage()); - if (terminalError == null) { - // Mirror recordFatal's first-writer-wins latch: only the - // sweep that owns the terminal may mark the gap, and the - // marker must be visible before the terminalError volatile - // write that checkError() keys on. - capabilityGapTerminal = e; + if (endpointPolicyFailureIsTerminal()) { + // Orphans hand a capability gap back to BackgroundDrainer's + // settle budget. + LOG.error("durable-ack mismatch during {} -- won't retry: {}", + phase, e.getMessage()); + if (terminalError == null) { + // Publish the marker before terminalError, which is the + // volatile first-writer-wins latch observed by the owner. + capabilityGapTerminal = e; + } + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.PROTOCOL_VIOLATION, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "durable-ack-mismatch: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; } - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + dispatchRetriedEndpointPolicyFailure( SenderError.Category.PROTOCOL_VIOLATION, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "durable-ack-mismatch: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; + "durable-ack-mismatch: " + e.getMessage()); + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground durable-ack capability is temporarily " + + "unavailable; retrying while store-and-forward owns the buffered data -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; + } } catch (QwpRoleMismatchException | QwpIngressRoleRejectedException e) { // Role mismatch: every reachable endpoint role-rejected the // upgrade -- right now they are all replicas / primary-catchup. @@ -1263,6 +1743,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM // endpoint, forever. Growing to reconnectMaxBackoffMillis // mirrors the orphan drainer's role-reject path and honours the // documented capped-exponential-backoff contract. + resetCatchUpCapGapEpisode(); lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1286,6 +1767,9 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(e); throw (Error) e; } + if (!isCatchUpCapGap(e)) { + resetCatchUpCapGapEpisode(); + } lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1331,6 +1815,72 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, lastMsg); } + /** + * Whether an endpoint-policy failure (auth, non-421 upgrade, durable-ack + * capability gap) latches a terminal instead of retrying under Invariant B. + *

+ * Two callers own a terminal, for different reasons: + *

+ * Once a FOREGROUND sender has reached the server even once, initialization is + * over and store-and-forward owns the buffered data: every endpoint-policy + * failure is then a transient the drainer must ride out, never a producer-fatal + * terminal (Invariant B). + */ + /** + * Reports an endpoint-policy rejection a FOREGROUND sender is riding out. + *

+ * Retrying is what the store-and-forward contract demands, but until now the retry was + * programmatically invisible: dispatchError ran only in the terminal branches, so a + * revoked token produced nothing but a throttled slf4j WARN -- in a library that ships + * embedded in user applications, frequently with no binding configured. flush() kept + * returning success until SF filled, and the failure then surfaced as "cursor ring + * backpressured ... sf_max_total_bytes too small", pointing the operator at disk + * sizing. That is the same complaint this PR makes to justify keeping STARTUP + * terminal; post-start it was merely delayed by SF capacity, not avoided. + *

+ * RETRIABLE, not TERMINAL: the handler learns the wire is being rejected while the + * producer stays alive and no data is at risk. + */ + private void dispatchRetriedEndpointPolicyFailure(SenderError.Category category, String message) { + long fromFsn = engine.ackedFsn() + 1L; + dispatchError(new SenderError( + category, + SenderError.Policy.RETRIABLE, + SenderError.NO_STATUS_BYTE, + message, + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + Math.max(fromFsn, engine.publishedFsn()), + null, + System.nanoTime() + )); + } + + /** + * Cause of the outage the reconnect loop is riding out, or {@code null} when the wire + * is up. Lets a producer-side failure name the real reason rather than blaming disk + * sizing -- see the field. + */ + public Throwable lastReconnectError() { + return lastReconnectError; + } + + private boolean endpointPolicyFailureIsTerminal() { + return reconnectPolicy == ReconnectPolicy.ORPHAN || !hasEverConnected; + } + /** * Send {@code err} to the async-delivery dispatcher if one is configured. * Producer-side typed throw (TERMINAL) goes through {@code recordFatal} + @@ -1480,8 +2030,8 @@ private void enqueuePendingOk(long wireSeq) { * listener both non-null), enters the per-outage retry loop: capped * exponential backoff with jitter, retried for as long as the loop is * running -- there is NO wall-clock give-up (Invariant B: a store-and- - * forward drainer only terminates on SF exhaustion or a genuinely non- - * retriable auth/upgrade reject). On the first successful reconnect the + * forward drainer does not give up on endpoint or transport state). On the + * first successful reconnect the * I/O loop resumes with reset wire state and replays from * {@code engine.ackedFsn() + 1}. *

@@ -1587,11 +2137,10 @@ private void ioLoop() { // Async-initial-connect path: ctor accepted a null client because // a reconnect factory is wired. Drive the very first connect on // this thread so the producer thread never blocks on it. - // attemptInitialConnect either sets `client` (success) or records - // a terminal failure and clears `running` (auth/upgrade reject; - // plain connect failures retry indefinitely under Invariant B). - // Either way, the main loop below sees the outcome via the - // `running` and `client` fields. + // attemptInitialConnect retries every endpoint/transport failure + // indefinitely under Invariant B and returns only after it installs + // a client or close() clears `running`. The main loop below sees the + // outcome via the `running` and `client` fields. if (client == null && running) { attemptInitialConnect(); } @@ -1669,6 +2218,12 @@ private void ioLoop() { // best-effort } } + // The symbol-dict mirror is I/O-thread-owned; free it here, on the + // owning thread's exit path, after the last send that could touch it. + if (sentDictBytesAddr != 0) { + releaseSentDictBytes(); + } + freeCatchUpFrameBuffer(); shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could // not free the engine safely while this thread was alive, so the @@ -1691,8 +2246,11 @@ private void ioLoop() { /** * Walk the engine's segments to find the one containing {@code targetFsn}, * and set {@code sendOffset} to the byte offset of that frame within it. - * This is called at startup and after every reconnect, after fsnAtZero has - * already been reset to {@code targetFsn} and nextWireSeq to 0. + * This is called at startup and after every reconnect, once + * {@link #setWireBaselineWithCatchUp} has anchored the wire baseline + * ({@code fsnAtZero} / {@code nextWireSeq}) -- which may leave {@code nextWireSeq} + * past the catch-up frames it emitted. This method only positions the byte + * cursor at {@code targetFsn}; it does not touch the wire mapping. *

* If {@code targetFsn} is already published, the method positions the byte * cursor exactly at that frame. If {@code targetFsn} is not published yet, @@ -1832,10 +2390,14 @@ private void sendDurableAckKeepaliveIfDue() { private void swapClient(WebSocketClient newClient) { WebSocketClient old = this.client; this.client = newClient; - // Sticky: once the wire is up, we've reached the server at least - // once for this sender's lifetime. Used downstream to classify a - // subsequent terminal failure as transient vs config-likely. + // Sticky: once the wire is up, we've reached the server at least once + // for this sender's lifetime. Exposed to the owning sender for + // connection-state observability, and it ends initialization: from here + // on endpointPolicyFailureIsTerminal() stops latching endpoint-policy + // failures on a foreground sender and rides them out instead, because + // store-and-forward now owns the buffered data (Invariant B). this.hasEverConnected = true; + this.lastReconnectError = null; if (old != null) { try { old.close(); @@ -1849,8 +2411,6 @@ private void swapClient(WebSocketClient newClient) { // past the tail instead of replaying into it. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; // Snapshot publishedFsn at swap time — frames at FSN ≤ this value // were already on the wire before the drop and will be replayed. // trySendOne resets replayTargetFsn to -1 once we cross the boundary. @@ -1862,9 +2422,666 @@ private void swapClient(WebSocketClient newClient) { // carrying stale state across the wire boundary would either // double-trim or starve the queue. clearDurableAckTracking(); + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } + /** + * Sets the wire-sequence baseline for a fresh connection and, when the symbol + * dictionary mirror is non-empty, emits a full-dictionary catch-up frame first + * so the fresh server (whose dictionary starts empty) can resolve the + * non-self-sufficient delta frames that replay next. + *

+ * The catch-up occupies wire seq 0, which maps to the already-acked FSN just + * below {@code replayStart} (a harmless re-ack); real replay frames then follow + * from wire seq 1. With nothing to catch up (fresh sender, or full-dict mode), + * or before a client exists (async initial connect), keep the plain 1:1 + * {@code fsnAtZero == replayStart} mapping; the catch-up then happens on the + * first real connection via swapClient. + */ + private void setWireBaselineWithCatchUp(long replayStart) { + // Fresh connection: no data frame has been sent on it yet. Reset before the + // catch-up (which sends only dictionary frames) so onClose / + // handleServerRejection can tell "only the catch-up went out" from "the + // head data frame went out". + dataFrameSentThisConnection = false; + // Do not gate solely on isDeltaDictEnabled(): a recovered delta slot can + // lose access to its side-file while its on-disk frames still start above + // zero. hasReplayDictionaryDependency preserves that distinction while + // keeping full-dictionary fallback catch-up-free across every reconnect. + if (client != null && sentDictCount > 0 && hasReplayDictionaryDependency) { + this.nextWireSeq = 0L; + // The catch-up may span several frames when the dictionary exceeds the + // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps + // to an already-acked FSN, so the first real frame still lands on + // replayStart. + int catchUpFrames = sendDictCatchUp(); + this.fsnAtZero = replayStart - catchUpFrames; + } else { + this.fsnAtZero = replayStart; + this.nextWireSeq = 0L; + } + } + + /** + * Returns the symbol-dictionary delta start id of a frame, or -1 when the + * frame carries no delta section. Used by the pre-send torn-dictionary guard. + */ + private int frameDeltaStart(long payloadAddr, int payloadLen) { + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return -1; + } + long encoded = readVarintAt(payloadAddr + QwpConstants.HEADER_SIZE, payloadAddr + payloadLen); + // A malformed start id reads as "no delta section", which the caller's + // `deltaStart >= 0` gate already handles. + return encoded < 0L ? -1 : (int) (encoded >>> 3); + } + + // True only for a well-formed QWP frame this encoder produced that carries a + // delta symbol-dict section. The magic check keeps the dict logic from + // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose + // bytes happen to set the delta flag. + private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) { + return false; + } + byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS); + return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; + } + + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode, for a frame the + * pre-send guard already classified as a delta frame. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + * @param deltaStart the frame's delta start id, already decoded by the + * pre-send guard ({@link #frameDeltaStart}) -- passed in so + * the magic/flags and start-id varint are not re-parsed + */ + private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart) { + long limit = payloadAddr + payloadLen; + // deltaStart is known (the guard decoded it); locate deltaCount just past + // its canonical LEB128 encoding rather than re-reading the header and the + // start-id varint. + long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); + long encodedCount = readVarintAt(p, limit); + if (encodedCount < 0L) { + return; // malformed -- never happens for frames we encoded + } + long deltaCount = encodedCount >>> 3; + p += encodedCount & 7L; + // The mirror holds ids [0, sentDictCount). Accumulate ONLY the part of this + // frame's delta [deltaStart, deltaStart+deltaCount) that extends past the + // tip -- ids [sentDictCount, deltaStart+deltaCount). Cases: + // - deltaStart > sentDictCount: a gap. trySendOne's torn-dictionary guard + // rejects it before send, so it never reaches here; bail defensively + // rather than accumulate past a hole. + // - deltaEnd <= sentDictCount: a pure replay/overlap we already hold -- + // nothing new. + // - deltaStart <= sentDictCount < deltaEnd: extend the mirror by the tail. + // deltaStart == sentDictCount is the steady-state case (skip == 0). + // Handling the partial overlap explicitly -- rather than dropping the whole + // frame whenever deltaStart != sentDictCount -- keeps the mirror complete + // even if a future producer ever emits a delta that overlaps the tip; + // silently dropping the new tail would leave the reconnect catch-up + // incomplete and shift server-side ids. + long deltaEnd = deltaStart + deltaCount; + if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) { + return; + } + // Keeps no per-entry offset index: the mirror stores each entry as + // [len varint][utf8], so the reconnect catch-up (sendDictCatchUp) walks it with + // a running pointer on demand. Maintaining an index here would put a store per + // new symbol on the per-frame I/O path -- and on the workload this feature + // targets (a new symbol per ROW) that is a store per row -- to serve a reconnect + // that may never happen; the catch-up's single walk is dwarfed by shipping the + // same n bytes over the wire. + // + // Walk past the already-held prefix [deltaStart, sentDictCount), then copy + // the new tail [sentDictCount, deltaEnd). + int skip = sentDictCount - deltaStart; + for (int i = 0; i < skip; i++) { + long encoded = readVarintAt(p, limit); + if (encoded < 0L) { + return; // malformed -- bail rather than corrupt the mirror + } + p += (encoded & 7L) + (encoded >>> 3); + if (p > limit) { + return; + } + } + long regionStart = p; + long newCount = deltaEnd - sentDictCount; + // Walk the new tail only to find where it ends; the entry boundaries are not + // recorded here (see above -- the catch-up re-walks the mirror when it needs them). + for (long i = 0; i < newCount; i++) { + long encoded = readVarintAt(p, limit); + if (encoded < 0L) { + // Malformed -- never happens for frames we encoded; bail rather + // than corrupt the mirror. + return; + } + p += (encoded & 7L) + (encoded >>> 3); + if (p > limit) { + return; + } + } + int regionBytes = (int) (p - regionStart); + // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on + // a very-high-cardinality lifetime connection; ensureSentDictCapacity then + // fails loudly rather than overflowing to a negative int (which would make + // the capacity check pass and copyMemory scribble past the buffer). + ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); + Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); + sentDictBytesLen += regionBytes; + sentDictCount += (int) newCount; + } + + private void ensureSentDictCapacity(long required) { + if (sentDictBytesCapacity >= required) { + return; + } + if (required > MAX_SENT_DICT_BYTES) { + // Latch a terminal, do NOT just throw: accumulateSentDict runs AFTER + // the frame's sendBinary, so a bare throw unwinds to ioLoop -> fail() + // -> connectLoop, which (running still true) reconnects and replays the + // same frame, which re-overflows the never-shrinking mirror -- an + // unbounded reconnect livelock rather than the "fails loudly" the + // MAX_SENT_DICT_BYTES ceiling promises. recordFatal flips running=false + // so connectLoop's !running guard winds the loop down and checkError() + // surfaces the terminal, matching sendCatchUpChunk's guard for the same + // ceiling. The throw still unwinds past the pending copyMemory. + LineSenderException err = new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw err; + } + // Grow in long to avoid the capacity*2 int overflow (negative) that would + // otherwise degrade the doubling near 1 GB; clamp to the int ceiling. + long newCap = Math.max((long) sentDictBytesCapacity * 2, Math.max(4096L, required)); + if (newCap > MAX_SENT_DICT_BYTES) { + newCap = MAX_SENT_DICT_BYTES; + } + try { + if (sentDictBytesOwned) { + sentDictBytesAddr = Unsafe.realloc( + sentDictBytesAddr, + sentDictBytesCapacity, + (int) newCap, + MemoryTag.NATIVE_DEFAULT); + } else { + long newAddr = Unsafe.malloc((int) newCap, MemoryTag.NATIVE_DEFAULT); + if (sentDictBytesLen > 0) { + Unsafe.getUnsafe().copyMemory(sentDictBytesAddr, newAddr, sentDictBytesLen); + } + sentDictBytesAddr = newAddr; + sentDictBytesOwned = true; + } + } catch (Throwable t) { + // Latch for exactly the reason the MAX_SENT_DICT_BYTES arm above does, and it + // was asymmetric until now: accumulateSentDict runs AFTER the frame's + // sendBinary, so a bare throw unwinds to ioLoop -> fail() -> connectLoop, + // which (running still true) reconnects and replays the same frame, which + // re-attempts the same allocation. A genuine out-of-memory therefore became an + // unbounded reconnect livelock instead of the loud failure the ceiling arm + // promises. recordFatal flips running=false so connectLoop winds down and + // checkError() surfaces it; the throw still unwinds past the pending copy. + recordFatal(t); + throw t; + } + sentDictBytesCapacity = (int) newCap; + } + + private void releaseSentDictBytes() { + if (sentDictBytesAddr != 0 && sentDictBytesOwned) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + } + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + sentDictBytesOwned = false; + sentDictCount = 0; + } + + /** + * Decodes the varint at {@code [p, limit)} and returns {@code (value << 3) | bytes}, + * or {@code -1} when it is truncated or runs past a canonical length. + *

+ * Packing the consumed length into the return value -- the shape + * {@code RecoveredFrameAnalysis.readVarint} already uses -- drops a store to an + * instance field on every call, and this runs once per SYMBOL: per frame in + * accumulateSentDict, and once per mirror entry in sendDictCatchUp. + *

+ * The {@code -1} also makes truncation DETECTABLE. The field version returned 0 with + * {@code varintEnd == p} when {@code p >= limit}, so a caller computing + * {@code p = varintEnd + len} got {@code p == limit} and its {@code p > limit} + * bail-out could not fire at an exact boundary -- guards that could not guard. + */ + private static long readVarintAt(long p, long limit) { + long value = 0; + int shift = 0; + int bytes = 0; + while (p < limit && bytes < 6) { + byte b = Unsafe.getUnsafe().getByte(p++); + value |= (long) (b & 0x7F) << shift; + bytes++; + if ((b & 0x80) == 0) { + return (value << 3) | bytes; + } + shift += 7; + } + return -1L; + } + + /** + * Re-registers the whole symbol dictionary on a fresh connection, split into + * as many table-less frames as the server's advertised batch cap requires so + * no single frame exceeds it (a large dictionary would otherwise be rejected). + * Each chunk carries a contiguous id range {@code [start .. start+count)}, in + * order, so the server accumulates them exactly as it would the original + * per-frame deltas. Returns the number of frames sent (each consumed a wire + * sequence), so the caller can align {@code fsnAtZero}. Throws {@link + * CatchUpSendException} on a send error (retriable -- the caller reconnects); + * a single entry too large for the cap is non-retriable, so it latches a + * terminal before throwing. + */ + private int sendDictCatchUp() { + int cap = client.getServerMaxBatchSize(); + long fullFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(0) + + NativeBufferWriter.varintSize(sentDictCount) + + (long) sentDictBytesLen; + if (cap <= 0 && fullFrameLen <= MAX_SENT_DICT_BYTES) { + sendCatchUpChunk(0, sentDictCount, sentDictBytesAddr, sentDictBytesLen); + resetCatchUpCapGapEpisode(); + return 1; + } + // The frame ceiling a catch-up chunk must not exceed: the server's + // advertised cap, or -- when the server advertises none (cap <= 0) -- + // MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen (HEADER_SIZE + + // varints + symbolsLen) cannot overflow on a pathological multi-GB + // dictionary (unreachable at real cardinality; defensive). Used by the + // single-entry terminal below, which measures the real solo frame. + int frameLimit = cap > 0 ? cap : MAX_SENT_DICT_BYTES; + // Symbol-bytes budget for PACKING several entries into one chunk, leaving + // room for the 12-byte header and the two delta-section varints. Kept + // deliberately conservative (reserving 16 for the varints): it only makes a + // multi-entry chunk split marginally earlier, never over the cap. It must + // NOT gate the single-entry terminal -- that reserve is larger than the + // minimal data-frame overhead, so an entry the producer already shipped + // under this cap could exceed the reserve yet still fit its own catch-up + // frame; the terminal tests the real solo frame against frameLimit instead. + int budget = cap > 0 + ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) + : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16; + int framesSent = 0; + int chunkStartId = 0; + long chunkStartAddr = sentDictBytesAddr; + int chunkSymbols = 0; + long chunkBytes = 0; + // Walk the mirror once with a running pointer, deriving each entry's byte span + // from its own [len varint][utf8] framing -- no separate offset index is kept + // (see accumulateSentDict). A mirror built from CRC-validated frames is always + // well-formed, so entryEnd can exceed the limit only under memory corruption; + // latch a terminal via recordFatal rather than let a bare throw unwind into + // connectLoop as a transient and reconnect-livelock. + long entryPtr = sentDictBytesAddr; + long mirrorLimit = sentDictBytesAddr + sentDictBytesLen; + for (int entryId = 0; entryId < sentDictCount; entryId++) { + long entryStart = entryPtr; + long encodedLen = readVarintAt(entryPtr, mirrorLimit); + long entryEnd = encodedLen < 0L + ? -1L + : entryPtr + (encodedLen & 7L) + (encodedLen >>> 3); + if (entryEnd < 0L || entryEnd > mirrorLimit) { + LineSenderException err = new LineSenderException( + "invalid symbol dictionary mirror during catch-up [entry=" + + entryId + ", count=" + sentDictCount + ']'); + recordFatal(err); + throw new CatchUpSendException(err); + } + entryPtr = entryEnd; + long entryBytes = entryEnd - entryStart; + // The exact table-less frame sendCatchUpChunk would build for THIS entry + // alone: header + deltaStart varint (the entry's own global id) + + // deltaCount varint (1) + the entry bytes. A cap gap exists only when even + // that solo frame exceeds the cap -- i.e. the entry genuinely cannot be + // re-registered. Testing the real solo frame (not the conservative + // packing budget above) is what keeps a HOMOGENEOUS cluster + // livelock-free: an entry the producer already shipped in a data frame + // under this cap (header + delta varints + entry + schema + >=1 row) is + // strictly larger than its bare catch-up frame, so it always fits here. + long soloFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(chunkStartId + chunkSymbols) + + NativeBufferWriter.varintSize(1) + + entryBytes; + if (soloFrameLen > frameLimit) { + // Cap gap: this entry cannot be re-registered under the fresh + // server's advertised cap. A HOMOGENEOUS cluster never reaches here + // (an entry that fit its data frame under a cap always fits its bare + // catch-up frame under that same cap), so the only way in is a + // heterogeneous / rolling-cap failover to a smaller-cap node. + // + // A foreground sender retries indefinitely: store-and-forward must + // contain a transient heterogeneous-cluster window instead of surfacing + // it to the producer. Only an orphan drainer may apply the settle budget + // below and quarantine its slot after a persistent gap. Under budget the + // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with + // backoff and re-runs the catch-up, which resets the episode on a node + // that accepts it. An unrelated transport/upgrade/role state also resets + // the episode, so its downtime cannot satisfy the orphan dwell. On + // exhaustion latch via recordFatal, NOT fail() -- + // failing from inside the catch-up would re-enter connectLoop (see + // CatchUpSendException); the data must be resent after the cap is raised. + // Escalation needs BOTH the strike count AND a minimum wall-clock dwell + // (see DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS). A count + // alone measures how often we looked, not how long the gap has held -- + // and 16 attempts at the capped backoff take only ~2 minutes, less than + // an ordinary rolling restart of the larger-cap node. Escalating on the + // count alone would therefore hard-fail a live store-and-forward + // producer during a routine cluster operation. + // + // The STRIKE COUNT -- never the anchor's sign -- says whether an episode is + // already open. A System.nanoTime() instant is only meaningful as a + // difference: its origin is arbitrary and the spec permits negative values, + // so a sign test cannot tell an unset anchor from a real one. Wherever it + // misreads a real anchor as unset it re-anchors to now on EVERY strike, + // pinning episodeNanos at ~0, and the dwell can then never be satisfied -- + // the terminal never latches, however long the gap truly persists. + // recordHeadRejectionStrike keys its episode off poisonStrikes for the same + // reason. + if (catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) { + throw new CatchUpSendException(new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ']' + + "; retrying indefinitely -- a larger-cap node may return"), true); + } + + long nowNanos = System.nanoTime(); + if (catchUpCapGapAttempts == 0) { + catchUpCapGapFirstNanos = nowNanos; + } + catchUpCapGapAttempts++; + long episodeNanos = nowNanos - catchUpCapGapFirstNanos; + boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS + && episodeNanos >= catchUpCapGapMinEscalationWindowNanos; + LineSenderException err = new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ", attempt=" + + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + + ", episodeMillis=" + (episodeNanos / 1_000_000L) + + '/' + (catchUpCapGapMinEscalationWindowNanos / 1_000_000L) + ']' + + (exhausted + ? "; the data must be resent after the cap is raised" + : "; retrying -- a larger-cap node may return")); + if (exhausted) { + recordFatal(err); + } + throw new CatchUpSendException(err, true); + } + if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); + framesSent++; + chunkStartId += chunkSymbols; + chunkStartAddr = entryStart; + chunkSymbols = 0; + chunkBytes = 0; + } + chunkSymbols++; + chunkBytes += entryBytes; + } + if (chunkSymbols > 0) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); + framesSent++; + } + // The whole dictionary re-registered without a cap gap: this node accepts + // every entry, so the cap-gap episode (if any) is over -- reset BOTH the strike + // count and the wall-clock anchor. Resetting only the count would leave a stale + // anchor from an old episode, so the very first strike of a LATER episode would + // already satisfy the dwell. + resetCatchUpCapGapEpisode(); + return framesSent; + } + + /** + * Sends one table-less catch-up frame carrying dictionary ids + * {@code [deltaStart .. deltaStart+deltaCount)}. Throws {@link + * CatchUpSendException} on a send error instead of calling {@link #fail} + * (see that type for why the catch-up must not re-enter the reconnect loop); + * the caller turns it into a single, non-re-entrant reconnect. + */ + private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + // Compute the frame size in long and fail loud if it would overflow the int + // size math into a negative Unsafe.malloc. sendDictCatchUp already caps each + // chunk's symbol bytes under the budget, so this is unreachable at real + // cardinality -- but the mirror-side ensureSentDictCapacity guards the same + // math, and a future caller must not be able to overflow this one silently. + long payloadLenL = (long) NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount) + + symbolsLen; + long frameLenL = QwpConstants.HEADER_SIZE + payloadLenL; + if (frameLenL > MAX_SENT_DICT_BYTES) { + LineSenderException err = new LineSenderException( + "symbol dictionary catch-up frame exceeds the maximum size [" + + "frameLen=" + frameLenL + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw new CatchUpSendException(err); + } + int payloadLen = (int) payloadLenL; + int prefixLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount); + ensureCatchUpFrameCapacity(prefixLen); + long frame = catchUpFrameAddr; + try { + Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); + Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W'); + Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); + Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); + Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + // FLAG_DEFER_COMMIT: the catch-up carries dictionary entries but NO + // rows, so it must never trigger a server-side commit. Today it is + // always the first frame on a fresh (empty-transaction) connection, so + // committing nothing is a no-op -- but that invariant is load-bearing + // and unasserted. Deferring the (empty) commit removes the dependency: + // a future mid-stream catch-up cannot prematurely commit an in-flight + // deferred transaction. The dictionary delta still registers (as any + // deferred data frame's does); only the row commit is deferred, and the + // next real frame commits it. + Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT + | QwpConstants.FLAG_DEFER_COMMIT)); + Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount + Unsafe.getUnsafe().putInt(frame + 8, payloadLen); + long q = NativeBufferWriter.writeVarint(frame + QwpConstants.HEADER_SIZE, deltaStart); + NativeBufferWriter.writeVarint(q, deltaCount); + client.sendBinary(frame, prefixLen, symbolsAddr, symbolsLen); + } catch (Throwable t) { + // Do NOT fail() here -- see CatchUpSendException. Signal the failure + // up so exactly one non-re-entrant reconnect follows. A JVM Error is + // never a transient reconnect case; let it propagate as-is so the + // I/O loop latches it terminal rather than looping on it. + if (t instanceof Error) { + throw (Error) t; + } + throw new CatchUpSendException(t); + } + nextWireSeq++; // this catch-up chunk consumed a wire sequence + lastFrameOrPingNanos = System.nanoTime(); + totalFramesSent.incrementAndGet(); + } + + @TestOnly + public int catchUpFrameGrowthCount() { + return catchUpFrameGrowthCount; + } + + @TestOnly + public int catchUpCapGapAttempts() { + return catchUpCapGapAttempts; + } + + @TestOnly + public long catchUpCapGapFirstNanos() { + return catchUpCapGapFirstNanos; + } + + @TestOnly + public long catchUpCapGapMinEscalationWindowNanos() { + return catchUpCapGapMinEscalationWindowNanos; + } + + @TestOnly + public long fsnAtZero() { + return fsnAtZero; + } + + @TestOnly + public long poisonFsn() { + return poisonFsn; + } + + @TestOnly + public int poisonStrikes() { + return poisonStrikes; + } + + @TestOnly + public long sentDictBytesAddr() { + return sentDictBytesAddr; + } + + @TestOnly + public int sentDictBytesLen() { + return sentDictBytesLen; + } + + @TestOnly + public boolean sentDictBytesOwned() { + return sentDictBytesOwned; + } + + @TestOnly + public int sentDictCount() { + return sentDictCount; + } + + @TestOnly + public int zeroProgressRecycles() { + return zeroProgressRecycles; + } + + @TestOnly + public void accumulateSentDictForTest(long payloadAddr, int payloadLen, int deltaStart) { + accumulateSentDict(payloadAddr, payloadLen, deltaStart); + } + + @TestOnly + public void connectLoopForTest(Throwable initial, String phase, long paceFirstAttemptMillis) { + connectLoop(initial, phase, paceFirstAttemptMillis); + } + + @TestOnly + public void deliverCloseForTest(int code, String reason) { + responseHandler.onClose(code, reason); + } + + @TestOnly + public void deliverResponseForTest(long payloadPtr, int payloadLen) { + responseHandler.onBinaryMessage(payloadPtr, payloadLen); + } + + @TestOnly + public void ensureSentDictCapacityForTest(long required) { + ensureSentDictCapacity(required); + } + + @TestOnly + public void positionCursorForStartForTest() { + positionCursorForStart(); + } + + @TestOnly + public void seedSentDictMirrorForTest(long addr, int bytes, int symbolCount) { + sentDictBytesAddr = addr; + sentDictBytesCapacity = bytes; + sentDictBytesLen = bytes; + sentDictCount = symbolCount; + sentDictBytesOwned = true; + } + + @TestOnly + public void sendCatchUpChunkForTest(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + sendCatchUpChunk(deltaStart, deltaCount, symbolsAddr, symbolsLen); + } + + @TestOnly + public void setCatchUpCapGapFirstNanosForTest(long nanos) { + catchUpCapGapFirstNanos = nanos; + } + + @TestOnly + public void setDataFrameSentThisConnectionForTest(boolean value) { + dataFrameSentThisConnection = value; + } + + @TestOnly + public void setFsnAtZeroForTest(long value) { + fsnAtZero = value; + } + + @TestOnly + public void setNextWireSeqForTest(long value) { + nextWireSeq = value; + } + + @TestOnly + public void setRunningForTest(boolean value) { + running = value; + } + + @TestOnly + public void setWireBaselineWithCatchUpForTest(long replayStart) { + setWireBaselineWithCatchUp(replayStart); + } + + @TestOnly + public boolean trySendOneForTest() { + return trySendOne(); + } + + private void ensureCatchUpFrameCapacity(int required) { + if (catchUpFrameCapacity >= required) { + return; + } + long newCapacity = Math.max(required, Math.max(4096L, (long) catchUpFrameCapacity * 2L)); + if (newCapacity > MAX_SENT_DICT_BYTES) { + newCapacity = MAX_SENT_DICT_BYTES; + } + catchUpFrameAddr = Unsafe.realloc( + catchUpFrameAddr, + catchUpFrameCapacity, + (int) newCapacity, + MemoryTag.NATIVE_DEFAULT); + catchUpFrameCapacity = (int) newCapacity; + catchUpFrameGrowthCount++; + } + + private void freeCatchUpFrameBuffer() { + if (catchUpFrameAddr != 0L) { + Unsafe.free(catchUpFrameAddr, catchUpFrameCapacity, MemoryTag.NATIVE_DEFAULT); + catchUpFrameAddr = 0L; + catchUpFrameCapacity = 0; + } + } + private boolean tryReceiveAcks() { boolean any = false; try { @@ -1895,10 +3112,32 @@ private boolean trySendOne() { return false; } if (nextWireSeq == 0) { - // Nothing sent on this connection yet: re-anchor in place past - // the retired tail. The wireSeq<->FSN mapping is untouched - // because no wire sequence has been consumed. - positionCursorForStart(); + // Reached only when the tail was NOT retirable at connection setup -- + // both setup sites (swapClient, positionCursorForStart) call + // tryRetireOrphanTail first -- which means frames below it still needed + // acks, which means they were SENT on this connection. So nextWireSeq is + // never 0 here in practice and this arm is effectively dead; it is kept + // as a cheap correctness guard rather than removed. + // + // NB the mapping is untouched only while NO wire sequence has been + // consumed. A dictionary catch-up consumes sequences 0..n-1 before any + // data frame, so after one this test is false and the recycle below -- + // which re-anchors the mapping from scratch -- is the correct path, not + // an avoidable cost. + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // Re-anchor's catch-up send failed. fail() here is a fresh, + // non-re-entrant connectLoop entry from the I/O loop body -- + // the same recovery a normal trySendOne send failure takes. + // Preserve the wrapper on a cap gap so connectLoop's + // isCatchUpCapGap keeps the orphan settle episode (attempt count + // + dwell anchor) alive across the re-anchor recycle; an ordinary + // failure unwraps to the raw cause and restarts the episode, like + // any normal send failure. + fail(isCatchUpCapGap(e) ? e : e.getCause()); + return false; + } return true; } // Frames were already sent on this connection: the linear @@ -1949,12 +3188,55 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + // Torn-dictionary guard. sentDictCount is this loop's model of how many ids + // the CURRENT server has been told about: seeded from the persisted + // dictionary, re-registered by the catch-up, and extended by + // accumulateSentDict as each frame goes out. A frame whose delta starts ABOVE + // that coverage references ids the server was never given; replaying it would + // make the server null-pad the hole (QwpMessageCursor grows the dict with + // nulls to deltaStart+deltaCount) and land rows with SILENTLY NULL symbols. + // Fail terminally instead; the unreplayable data must be resent. + // + // The guard, the mirror and the catch-up are ONE mechanism and are all + // ungated (see the constructor). A frame at deltaStart == sentDictCount is + // contiguous and extends the coverage, so a slot whose frames start at id 0 + // replays cleanly with no persisted dictionary at all -- which is exactly why + // the mirror below must keep advancing even when the dictionary is missing. + // A gap here therefore means genuine loss: a host/power crash tore the + // unsynced .symbol-dict below the ids the surviving frames still reference + // (SF is process-crash but not host-crash durable). + int deltaStart = frameDeltaStart(frameAddr, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; + } try { - client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + client.sendBinary(frameAddr, payloadLen); } catch (Throwable t) { fail(t); return false; } + // A real ring frame (data or commit) has now gone out on this connection, + // as opposed to only the dictionary catch-up. onClose / handleServerRejection + // key their poison-strike vs pre-send decision off this, not off nextWireSeq + // (which the catch-up advances). + dataFrameSentThisConnection = true; + if (deltaStart >= 0) { + // Mirror the symbols this frame just registered on the server, so a later + // reconnect can rebuild the whole dictionary and the guard above keeps an + // accurate view of the server's coverage. Idempotent on replay: a frame + // whose delta we already hold advances nothing. + // + // Ungated (see the constructor): this is the ONLY thing that advances + // sentDictCount from the frames themselves, so gating it while leaving the + // guard ungated froze the coverage at 0 and terminal'd a slot that replays + // perfectly from id 0. + accumulateSentDict(frameAddr, payloadLen, deltaStart); + } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; long fsnSent = fsnAtZero + nextWireSeq; @@ -1984,8 +3266,11 @@ void positionCursorForStart() { // starts past it. Zero wire cost, no recycle. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; + // Recovery / orphan-drain seed the dictionary mirror, so the initial + // connection may also need a catch-up (client is non-null in the + // sync-start and drainer paths; null in async-initial, where swapClient + // handles it on the first connect). + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } @@ -2011,31 +3296,79 @@ private boolean tryRetireOrphanTail() { if (orphanSkipTipFsn < 0) { return true; } - if (engine.ackedFsn() < orphanSkipStartFsn - 1L) { + if (!engine.retireRecoveredOrphanTailIfReady()) { return false; } - LOG.warn("retiring orphaned deferred tail: {} frame(s) [fsn {}..{}] belong to a transaction " - + "whose commit was never published; aborting them (never transmitted, slots trimmed)", - orphanSkipTipFsn - orphanSkipStartFsn + 1, orphanSkipStartFsn, orphanSkipTipFsn); - engine.acknowledge(orphanSkipTipFsn); orphanSkipStartFsn = -1L; orphanSkipTipFsn = -1L; return true; } + /** + * Determines whether an oversized symbol-dictionary catch-up entry is always + * retriable or may become terminal after the orphan settle budget. + */ + public enum CatchUpCapGapPolicy { + /** Keep reconnecting until a node with a compatible batch cap returns. */ + RETRY_FOREVER, + /** Quarantine an orphan slot after both the attempt and dwell limits expire. */ + TERMINAL_AFTER_SETTLE_BUDGET + } + + /** Identifies who owns data while the reconnect loop is unavailable. */ + public enum ReconnectPolicy { + /** A live producer keeps buffering and retries endpoint-policy failures. */ + FOREGROUND, + /** An orphan drainer returns terminal states to its quarantine owner. */ + ORPHAN + } + /** * Factory used by the I/O loop to build a fresh, connected, upgraded * {@link WebSocketClient} after a wire failure. Implementations close * the old client (if needed), build a new one with the same auth/TLS * config, connect, perform the WebSocket upgrade, and return it ready - * to send. Throw on a terminal failure (auth rejection, etc.) — the - * I/O loop will treat the throw as fatal. + * to send. The loop's {@link ReconnectPolicy} decides whether endpoint-policy + * failures are retried or returned as terminal. */ @FunctionalInterface public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; } + /** + * Signals that a symbol-dictionary catch-up frame could not be sent on the + * current connection. Thrown by {@link #sendDictCatchUp}/{@link + * #sendCatchUpChunk} instead of calling {@link #fail}: the catch-up runs + * inside {@link #connectLoop} (via {@link #swapClient}) and, on the initial + * connect, inside {@link #start()} / {@link #trySendOne} on the caller + * thread. Calling {@code fail()} from there would re-enter {@code + * connectLoop} -- corrupting the {@code fsnAtZero}/{@code nextWireSeq} wire + * mapping (a subsequent ACK then trims un-acked frames) and growing the + * stack until it overflows into a terminal, breaking the "retry a transient + * outage forever" invariant -- or run {@code connectLoop} on the caller + * thread and block {@code Sender} construction indefinitely. Each catch + * site instead turns it into ONE non-re-entrant reconnect: {@code + * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()} + * from the I/O loop body (trySendOne path), or dropping the dead client so + * the I/O thread reconnects (start path). A JVM {@code Error} is never + * wrapped -- it must stay terminal. The {@code capGap} marker lets the reconnect + * loop preserve only consecutive incompatible-cap observations; ordinary catch-up + * send failures restart the orphan settle episode. + */ + private static final class CatchUpSendException extends RuntimeException { + private final boolean isCapGap; + + CatchUpSendException(Throwable cause) { + this(cause, false); + } + + CatchUpSendException(Throwable cause, boolean isCapGap) { + super(cause); + this.isCapGap = isCapGap; + } + } + /** * One slot in the pendingDurable FIFO. Holds a wireSeq plus the per-table * (name, seqTxn) pairs from its OK frame. Empty entries (tableCount = 0) @@ -2173,7 +3506,7 @@ public void onClose(int code, String reason) { || code == WebSocketCloseCode.GOING_AWAY; LineSenderException cause = new LineSenderException( "WebSocket closed by server: code=" + code + " reason=" + reason); - if (!orderly && nextWireSeq > 0) { + if (!orderly && dataFrameSentThisConnection) { if (recordHeadRejectionStrike(Math.max(engine.ackedFsn(), highestOkFsn) + 1L)) { haltOnPoisonedFrame("ws-close[" + code + ' ' + WebSocketCloseCode.describe(code) + "]: " + reason, @@ -2280,17 +3613,21 @@ private void handleServerRejection(long wireSeq) { // value is only used to attribute an FSN to the error report -- // a rejection never advances the watermark. long highestSent = nextWireSeq - 1L; - if (highestSent < 0L) { - // Pre-send rejection: server emitted an error frame before - // we sent anything on this connection (typical after a - // fresh swapClient — auth failure, server-initiated halt, - // etc.). The server-named wireSeq does not correspond to - // any frame we sent, so clamping it to 0 and acknowledging - // fsnAtZero would silently advance ackedFsn past a real - // unsent batch (fsnAtZero == ackedFsn + 1 right after a - // swap). Skip the watermark advance entirely; still surface - // the error so the user's handler sees it and HALT errors - // remain producer-observable. + if (!dataFrameSentThisConnection) { + // Pre-send rejection: the server emitted an error frame before we + // sent any DATA frame on this connection (typical after a fresh + // swapClient -- auth failure, server-initiated halt, or a rejection + // of the dictionary catch-up itself). nextWireSeq may be > 0 here + // because the catch-up consumed wire sequences, so this keys off + // dataFrameSentThisConnection, not highestSent >= 0 -- otherwise a + // transient NACK of a catch-up frame would take the post-send + // poison-strike path and could escalate a transient outage to a + // terminal. The server-named wireSeq does not correspond to any + // data frame we sent, so clamping it to 0 and acknowledging + // fsnAtZero would silently advance ackedFsn past a real unsent + // batch. Skip the watermark advance entirely; still surface the + // error so the user's handler sees it and HALT errors remain + // producer-observable. handlePreSendRejection(wireSeq, status, category, policy); return; } @@ -2300,6 +3637,32 @@ private void handleServerRejection(long wireSeq) { wireSeq, highestSent); } long fsn = fsnAtZero + cappedSeq; + if (fsn <= engine.ackedFsn()) { + // The clamped wire seq maps at or below the replay head, so this + // NACK is for a dictionary catch-up frame -- which occupies the + // already-acked wire sequences below replayStart -- not a data frame + // this connection sent. (dataFrameSentThisConnection can be true here + // because trySendOne ships the head data frame before tryReceiveAcks + // reads the catch-up's NACK in the same loop iteration.) Attributing + // it a data FSN would key recordHeadRejectionStrike() off a + // below-baseline FSN -- negative when replayStart < catchUpFrames -- + // colliding with the poisonFsn == -1 "no suspect" sentinel and + // laundering a genuine poison run, and would report a bogus FSN. + // Treat it exactly like a pre-send rejection: surface + recycle, no + // poison strike, no watermark advance. Symmetric with the success + // path, where engine.acknowledge() no-ops at or below ackedFsn. A + // real replayed data frame is at fsn > ackedFsn, so it is never + // caught here. + // + // Catch-up frames therefore sit OUTSIDE the poison detector: a + // deterministically-NACKed catch-up recycles forever (paced, so no + // busy-loop). That is acceptable -- a catch-up only re-registers + // symbols the cluster already accepted, so a persistent NACK of one + // is a server bug, not a poison-frame the client can quarantine, and + // Invariant B's "retry a transient outage forever" applies. + handlePreSendRejection(wireSeq, status, category, policy); + return; + } // Best-effort table attribution: the parser populates // response.tableNames on error frames the same way it does on // STATUS_OK. If exactly one table was named, surface it; if diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 78e5db9f..27569029 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -44,7 +44,7 @@ *

* On-disk layout — header and frame format: * 

- *   [u32 magic 'SF01'] [u8 ver=1] [u8 flags=0] [u16 reserved=0]
+ *   [u32 magic 'SF01'] [u8 ver]   [u8 flags=0] [u16 reserved=0]
  *   [u64 baseSeq]       [u64 createdMicros]                       24-byte header
  *   frame, frame, ...                                              each frame:
  *                                                                  [u32 crc32c]
@@ -63,8 +63,8 @@ public final class MmapSegment implements QuietCloseable {
     public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian
     public static final int FRAME_HEADER_SIZE = 8;   // u32 crc + u32 payloadLen
     public static final int HEADER_SIZE = 24;
-    public static final byte VERSION = 1;
-    private static final int[] CRC32C_TABLE = buildCrc32cTable();
+    public static final byte LEGACY_VERSION = 1;
+    public static final byte VERSION = 2;
     private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class);
 
     private final String path;
@@ -74,6 +74,7 @@ public final class MmapSegment implements QuietCloseable {
     // memory-backed segments — same cursor architecture, no disk involvement.
     // close() and msync() branch on this flag.
     private final boolean memoryBacked;
+    private final byte version;
     // appendCursor: written only by the producer thread, never read by anyone else
     // — it's the reservation cursor. Plain field is fine.
     private long appendCursor;
@@ -104,7 +105,7 @@ public final class MmapSegment implements QuietCloseable {
 
     private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
                         long baseSeq, long initialCursor, long frameCount,
-                        boolean memoryBacked, long tornTailBytes) {
+                        boolean memoryBacked, long tornTailBytes, byte version) {
         this.path = path;
         this.fd = fd;
         this.mmapAddress = mmapAddress;
@@ -115,6 +116,7 @@ private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
         this.frameCount = frameCount;
         this.memoryBacked = memoryBacked;
         this.tornTailBytes = tornTailBytes;
+        this.version = version;
     }
 
     /**
@@ -162,6 +164,59 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long
      * per-call {@code byte[]} + native-malloc the way the String overload does.
      */
     public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) {
+        return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, VERSION);
+    }
+
+    /**
+     * Creates a tiny v1 segment containing one deliberately non-QWP payload.
+     * Two such files with non-contiguous base sequences form the rollback
+     * barrier installed by {@link SegmentRing}: a v1-only reader opens both
+     * and fails its mandatory FSN-contiguity check before replay. If a process
+     * dies after creating only one guard, that reader still adopts a non-empty
+     * log and sends the invalid one-byte head frame first, so it cannot silently
+     * discard the v2 files and start a fresh slot.
+     */
+    static void createLegacyReaderGuard(String path, long baseSeq) {
+        long pathPtr = FilesFacade.INSTANCE.allocNativePath(path);
+        long payload = 0L;
+        try (MmapSegment segment = create(
+                FilesFacade.INSTANCE,
+                pathPtr,
+                path,
+                baseSeq,
+                HEADER_SIZE + FRAME_HEADER_SIZE + 1L,
+                LEGACY_VERSION
+        )) {
+            payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT);
+            Unsafe.getUnsafe().putByte(payload, (byte) 0);
+            if (segment.tryAppend(payload, 1) != HEADER_SIZE) {
+                throw new MmapSegmentException("could not append legacy-reader guard frame: " + path);
+            }
+            // Unlike the segment log this file guards, the guard IS msync'd. It is a
+            // rollback barrier, so it has to be durable before the data it protects
+            // becomes durable -- a guard still sitting in the page cache when the host
+            // dies leaves zeroes on disk beside written-back v2 segments, and a
+            // rolled-back v1 reader skips the guard (bad magic) and the segments (bad
+            // version) alike, sees an empty slot, and truncates the log. It is 33 bytes
+            // written once per slot lifetime, not per flush, so the sync costs nothing
+            // measurable and buys the barrier the durability its whole purpose assumes.
+            segment.msync();
+        } finally {
+            if (payload != 0L) {
+                Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT);
+            }
+            FilesFacade.INSTANCE.freeNativePath(pathPtr);
+        }
+    }
+
+    private static MmapSegment create(
+            FilesFacade ff,
+            long pathPtr,
+            String displayPath,
+            long baseSeq,
+            long sizeBytes,
+            byte version
+    ) {
         if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) {
             throw new IllegalArgumentException(
                     "sizeBytes too small for header + one minimal frame: " + sizeBytes);
@@ -192,12 +247,13 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat
             }
             // Header goes straight into the mapping — no separate write syscall.
             Unsafe.getUnsafe().putInt(addr, FILE_MAGIC);
-            Unsafe.getUnsafe().putByte(addr + 4, VERSION);
+            Unsafe.getUnsafe().putByte(addr + 4, version);
             Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L);
+            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, false, 0L, version);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT);
@@ -234,7 +290,8 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) {
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0);
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L);
+            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, true, 0L, VERSION);
         } catch (Throwable t) {
             Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT);
             throw t;
@@ -278,6 +335,23 @@ public static MmapSegment openExisting(String path) {
      * {@link FilesFacade#length(String)}.
      */
     public static MmapSegment openExisting(FilesFacade ff, String path) {
+        return openExisting(ff, path, null);
+    }
+
+    /**
+     * As {@link #openExisting(FilesFacade, String)}, but also publishes the constructed
+     * segment into {@code inFlight[0]} immediately before returning it.
+     * 

+     * Pre-JDK-21 HotSpot delivers an unsafe-access fault ASYNCHRONOUSLY -- at the next
+     * return or safepoint check rather than at the faulting instruction (JDK-8283699).
+     * A fault taken while scanning this segment can therefore surface at this method's
+     * RETURN, in the caller's frame, so the caller's {@code seg = openExisting(...)}
+     * assignment never happens even though the fd and the whole-file mapping were
+     * already acquired -- and nothing is left holding them. The holder gives the
+     * caller's {@code finally} something to close. Callers MUST clear {@code inFlight[0]}
+     * wherever they transfer ownership, or they will close a segment they just handed on.
+     */
+    static MmapSegment openExisting(FilesFacade ff, String path, MmapSegment[] inFlight) {
         long fileSize = ff.length(path);
         if (fileSize < HEADER_SIZE) {
             throw new MmapSegmentException("file shorter than header: " + path + " size=" + fileSize);
@@ -298,7 +372,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                         "bad magic in " + path + ": 0x" + Integer.toHexString(magic));
             }
             byte version = Unsafe.getUnsafe().getByte(addr + 4);
-            if (version != VERSION) {
+            if (version != LEGACY_VERSION && version != VERSION) {
                 throw new MmapSegmentException("unsupported version in " + path + ": " + version);
             }
             long baseSeq = Unsafe.getUnsafe().getLong(addr + 8);
@@ -313,8 +387,9 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                 throw new MmapSegmentException(
                         "bad baseSeq in " + path + ": " + baseSeq);
             }
-            long lastGood = scanFrames(addr, fileSize);
-            long count = countFrames(addr, lastGood);
+            FrameScan scan = scanFrames(addr, fileSize);
+            long lastGood = scan.lastGood;
+            long count = scan.frameCount;
             long tornTail = detectTornTail(addr, lastGood, fileSize);
             if (tornTail > 0) {
                 LOG.warn("SF segment {}: torn tail of {} bytes at offset {} "
@@ -324,7 +399,12 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                                 + "Investigate disk health or unexpected writer crash.",
                         path, tornTail, lastGood, fileSize, count);
             }
-            return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail);
+            MmapSegment segment = new MmapSegment(path, fd, addr, fileSize, baseSeq,
+                    lastGood, count, false, tornTail, version);
+            if (inFlight != null) {
+                inFlight[0] = segment;
+            }
+            return segment;
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT);
@@ -438,6 +518,11 @@ public long sizeBytes() {
         return sizeBytes;
     }
 
+    /** On-disk format version read from or written to this segment. */
+    public byte version() {
+        return version;
+    }
+
     /**
      * Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]}
      * starting at the current append cursor, then advances both cursors
@@ -482,44 +567,17 @@ public long tryAppend(long payloadAddr, int payloadLen) {
     }
 
     /**
-     * Walks every published frame in this segment and returns the FSN of the
-     * LAST frame whose payload does NOT carry the given flag bit, or {@code -1}
-     * when every frame carries it (or the segment is empty).
-     * 

-     * A frame counts as carrying the flag ONLY when it positively parses as a
-     * message of the expected protocol: payload at least {@code minPayloadLen}
-     * bytes AND the little-endian u32 at payload offset 0 equals
-     * {@code headerMagic} AND the byte at {@code flagsOffset} has
-     * {@code flagMask} set. Anything else -- short frames, foreign payloads,
-     * magic mismatches -- counts as NOT carrying the flag. This direction is
-     * deliberate: the caller retires (trims) frames ABOVE the returned FSN,
-     * so a frame we cannot positively identify must act as a retirement
-     * barrier, never as trimmable. Misclassifying an unknown frame as
-     * deferred would silently discard data that should replay.
-     * 

-     * Producer-thread only, and only meaningful before new appends race the
-     * walk (recovery time). Used to locate the last commit-bearing QWP frame
-     * below a potentially orphaned FLAG_DEFER_COMMIT tail: frames above the
-     * returned FSN all carry the flag, i.e. they belong to a transaction whose
-     * commit frame was never published.
+     * Feeds every recovered frame in this segment to the engine's single-pass
+     * recovery fold. Segments are visited oldest-first by {@link SegmentRing}.
      */
-    public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) {
-        long best = -1L;
+    void scanRecovery(RecoveredFrameAnalysis analysis) {
         long off = HEADER_SIZE;
         long frames = frameCount;
         for (long i = 0; i < frames; i++) {
             int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4);
-            long payload = mmapAddress + off + FRAME_HEADER_SIZE;
-            boolean flagSet = payloadLen >= minPayloadLen
-                    && payloadLen > flagsOffset
-                    && Unsafe.getUnsafe().getInt(payload) == headerMagic
-                    && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagMask) != 0;
-            if (!flagSet) {
-                best = baseSeq + i;
-            }
+            analysis.accept(baseSeq + i, mmapAddress + off + FRAME_HEADER_SIZE, payloadLen);
             off += FRAME_HEADER_SIZE + payloadLen;
         }
-        return best;
     }
 
     /**
@@ -571,8 +629,23 @@ public long tornTailBytes() {
      * fragment keeps the guard effective on JDK 8/11/17 as well as 21+, while
      * still being specific enough that a genuine VirtualMachineError (real OOM,
      * StackOverflow) is never swallowed.
+     * 

+     * Delivery is NOT precise before JDK 21, so callers must guard too.
+     * Pre-21 HotSpot records the fault and raises the {@code InternalError} at
+     * the next return or safepoint check rather than at the faulting
+     * instruction, so it can surface in a CALLER's frame -- past every handler
+     * in this class. Observed on JDK 8/17: a fault taken inside
+     * {@link #scanFrames} arrives only after {@link #openExisting} has already
+     * returned its segment, so neither the scan's own catch nor
+     * {@code openExisting}'s outer catch ever sees it. JDK-8283699 made
+     * delivery precise in 21+, which is why the same case is fully contained
+     * there. The in-class catches therefore handle the common (precise) case
+     * only; {@link SegmentRing#openExisting} applies this same predicate at the
+     * per-file boundary so a late-delivered fault still skips one {@code .sfa}
+     * instead of aborting recovery of the whole slot. Package-private for that
+     * caller.
      */
-    private static boolean isMmapAccessFault(Throwable t) {
+    static boolean isMmapAccessFault(Throwable t) {
         if (!(t instanceof InternalError)) {
             return false;
         }
@@ -581,14 +654,15 @@ private static boolean isMmapAccessFault(Throwable t) {
     }
 
     /**
-     * Forward scan that returns the offset just past the last frame whose
-     * CRC verifies. A torn-tail frame (declared length runs past EOF, or
-     * CRC mismatch) leaves both cursors at the start of that frame; the
-     * next {@link #tryAppend} will overwrite it. The scan only reads from
-     * the mapping — no syscalls.
+     * Forward scan that returns the offset just past the last frame whose CRC
+     * verifies and the number of verified frames. A torn-tail frame (declared
+     * length runs past EOF, or CRC mismatch) leaves both cursors at the start of
+     * that frame; the next {@link #tryAppend} will overwrite it. The scan only
+     * reads from the mapping — no syscalls.
      */
-    private static long scanFrames(long addr, long fileSize) {
+    private static FrameScan scanFrames(long addr, long fileSize) {
         long pos = HEADER_SIZE;
+        long frameCount = 0L;
         try {
             while (pos + FRAME_HEADER_SIZE <= fileSize) {
                 int crcRead = Unsafe.getUnsafe().getInt(addr + pos);
@@ -596,7 +670,7 @@ private static long scanFrames(long addr, long fileSize) {
                 // Defensive: a corrupt length field could be enormous or negative,
                 // both of which would otherwise overrun the mapping.
                 if (payloadLen < 0 || pos + FRAME_HEADER_SIZE + payloadLen > fileSize) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 // CRC over the contiguous (payloadLen, payload) pair, folded
                 // via Unsafe reads rather than the native Crc32c.update.
@@ -615,13 +689,18 @@ private static long scanFrames(long addr, long fileSize) {
                 // the same scan). Folding over Unsafe keeps every fault
                 // catchable -- handled below as the boundary of recoverable
                 // data; a page that instead reads back as zeroes just fails the
-                // CRC check and ends the scan. Recovery is cold, so the slower
-                // table CRC here is immaterial.
-                int crcCalc = crc32cRecovery(addr + pos + 4, 4L + payloadLen);
+                // CRC check and ends the scan. The shared Java/Unsafe
+                // slice-by-8 path keeps those faults catchable without imposing
+                // a scalar byte-at-a-time scan on large recovery backlogs.
+                int crcCalc = Crc32c.updateUnsafe(
+                        Crc32c.INIT,
+                        addr + pos + 4,
+                        4L + payloadLen);
                 if (crcCalc != crcRead) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 pos += FRAME_HEADER_SIZE + payloadLen;
+                frameCount++;
             }
         } catch (InternalError e) {
             // The read at `pos` hit a mapped page that is not backed by real
@@ -647,43 +726,7 @@ private static long scanFrames(long addr, long fileSize) {
                             + "if this recurs.",
                     pos, fileSize);
         }
-        return pos;
-    }
-
-    /**
-     * CRC-32C (Castagnoli) of {@code [addr, addr + len)} read through
-     * {@link Unsafe}, seeded like {@code Crc32c.update(Crc32c.INIT, addr, len)}
-     * and bit-identical to it (verified) -- but every byte load is an Unsafe
-     * intrinsic, so a fault on an unbacked mapped page is a catchable
-     * {@link InternalError} instead of the uncatchable JNI SIGBUS the native
-     * {@link Crc32c} would raise. Byte-at-a-time via a precomputed table
-     * ({@code ~0.5 GiB/s}); used only on the cold recovery scan, never on the
-     * append hot path (which stays on the native, hardware-friendly path).
-     */
-    private static int crc32cRecovery(long addr, long len) {
-        int crc = ~Crc32c.INIT;
-        for (long i = 0; i < len; i++) {
-            crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ Unsafe.getUnsafe().getByte(addr + i)) & 0xFF];
-        }
-        return ~crc;
-    }
-
-    /**
-     * Standard reflected CRC-32C byte table (polynomial {@code 0x82F63B78}),
-     * matching {@code crc32c_table[0]} in the native {@code crc32c.c}. Computed
-     * at class init to avoid 256 hand-transcribed literals; drives
-     * {@link #crc32cRecovery}.
-     */
-    private static int[] buildCrc32cTable() {
-        int[] table = new int[256];
-        for (int n = 0; n < 256; n++) {
-            int c = n;
-            for (int k = 0; k < 8; k++) {
-                c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1);
-            }
-            table[n] = c;
-        }
-        return table;
+        return new FrameScan(pos, frameCount);
     }
 
     /**
@@ -723,19 +766,13 @@ private static long detectTornTail(long addr, long lastGood, long fileSize) {
         return 0L;
     }
 
-    /**
-     * Counts frames in {@code [HEADER_SIZE, lastGood)}. Walks the framing in
-     * lockstep with {@link #scanFrames} (which already validated CRCs); so
-     * this is just length-driven traversal, no CRC re-check.
-     */
-    private static long countFrames(long addr, long lastGood) {
-        long pos = HEADER_SIZE;
-        long count = 0;
-        while (pos < lastGood) {
-            int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4);
-            pos += FRAME_HEADER_SIZE + payloadLen;
-            count++;
+    private static final class FrameScan {
+        private final long frameCount;
+        private final long lastGood;
+
+        private FrameScan(long lastGood, long frameCount) {
+            this.lastGood = lastGood;
+            this.frameCount = frameCount;
         }
-        return count;
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
index 833960ec..e98541d9 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
@@ -61,6 +61,20 @@ public final class OrphanScanner {
 
     /** Name of the sentinel that disqualifies a slot from auto-drain. */
     public static final String FAILED_SENTINEL_NAME = ".failed";
+    /**
+     * Reserved infix for a slot {@code Sender.build()} set aside as unreplayable. Such a
+     * copy is kept for forensics and a manual resend; it is NEVER drainable, so the
+     * scanner excludes it BY NAME.
+     * 

+     * Excluding it by name rather than by the {@code .failed} sentinel alone is what makes
+     * the exclusion reliable. {@code markFailed} is best-effort and returns silently when
+     * it cannot open the sentinel -- and the condition that makes a slot unreplayable (a
+     * full or read-only disk) is exactly the condition that makes writing the sentinel
+     * fail. Without the name check, the SAME build() that quarantined the slot dispatches
+     * its orphan drainers twelve lines later and re-adopts it: unbounded churn against
+     * data explicitly declared human-in-the-loop, repeating on every restart.
+     */
+    public static final String QUARANTINE_SLOT_INFIX = ".unreplayable-";
 
     private OrphanScanner() {
     }
@@ -271,12 +285,29 @@ public static boolean isCandidateOrphan(String slotPath) {
         if (!Files.exists(slotPath)) {
             return false;
         }
+        if (isQuarantinedSlotName(slotPath)) {
+            return false;
+        }
         if (Files.exists(slotPath + "/" + FAILED_SENTINEL_NAME)) {
             return false;
         }
         return hasAnySegmentFile(slotPath);
     }
 
+    /**
+     * Whether {@code slotPath}'s last path element names a quarantined slot. Independent
+     * of the {@code .failed} sentinel, which is best-effort -- see
+     * {@link #QUARANTINE_SLOT_INFIX}.
+     */
+    public static boolean isQuarantinedSlotName(String slotPath) {
+        if (slotPath == null) {
+            return false;
+        }
+        int slash = Math.max(slotPath.lastIndexOf('/'), slotPath.lastIndexOf('\\'));
+        String name = slash < 0 ? slotPath : slotPath.substring(slash + 1);
+        return name.contains(QUARANTINE_SLOT_INFIX);
+    }
+
     /**
      * Drops a {@link #FAILED_SENTINEL_NAME} file in {@code slotPath}.
      * Idempotent — touching an existing sentinel is a no-op (its presence
@@ -327,7 +358,17 @@ private static boolean hasAnySegmentFile(String slotPath) {
             while (rc > 0) {
                 String name = Files.utf8ToString(Files.findName(find));
                 rc = Files.findNext(find);
-                if (name != null && name.endsWith(".sfa")) {
+                // The legacy-reader barriers are named .sfa on purpose (a rolled-back v1
+                // reader must not skip them), but they are a barrier, not data. Counting
+                // them here would call a slot that never held a byte an orphan with
+                // unacked data: CursorSendEngine plants them FIRST, before recovery or
+                // any segment is created, and its cleanup does not unlink them, so a
+                // construction that fails afterwards (ENOSPC on the initial segment)
+                // leaves a directory holding nothing else. A drainer would then adopt it,
+                // fail its own build under the same disk pressure, and quarantine an
+                // empty slot with a permanent .failed sentinel plus an ERROR for an
+                // operator to chase.
+                if (name != null && name.endsWith(".sfa") && !SegmentRing.isLegacyReaderGuard(name)) {
                     return true;
                 }
             }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
new file mode 100644
index 00000000..b4508662
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
@@ -0,0 +1,1229 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.client.NativeBufferWriter;
+import io.questdb.client.std.Crc32c;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.str.Utf8s;
+import org.jetbrains.annotations.TestOnly;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Append-only, per-slot persistence of the global symbol dictionary that a
+ * store-and-forward sender ships to the server with delta encoding. Lives at
+ * {@code /.symbol-dict} alongside the segment files, the slot lock and
+ * {@code .ack-watermark}.
+ * 

+ * Delta-encoded SF frames are NOT self-sufficient: a frame carries only the
+ * symbols it introduces, so recovering (process restart) or draining (orphan
+ * adoption) a slot requires re-registering the whole dictionary on the fresh
+ * server before those frames replay. This file is that dictionary. Unlike
+ * {@link AckWatermark} -- a discardable optimization protected by a
+ * {@code max()} clamp -- this file is load-bearing: a surviving frame
+ * that references an id missing from it is unrecoverable. It is therefore held
+ * to a stronger durability contract, and {@link #open} never destroys it (see
+ * "Never recreate over an existing file" below).
+ * 

+ * Layout (little-endian):
+ * 
+ *   offset 0: u32 magic = 'SYD1'
+ *   offset 4: u8  version = 3
+ *   offset 5: 3 bytes reserved (zero)
+ *   offset 8: chunks, each
+ *             [entryCount: varint][entryBytes: varint][entries][crc32c: u32]
+ *             where entries = [len: varint][utf8] repeated entryCount times,
+ *             occupying exactly entryBytes bytes, and the CRC-32C covers the
+ *             two header varints AND the entry region.
+ * 

+ * A chunk is one append -- i.e. exactly the set of symbols one frame
+ * introduces, since the producer persists a frame's new symbols in a single call
+ * before publishing it. Symbol id {@code i} is the {@code i}-th entry across all
+ * chunks (ids are dense and assigned sequentially from 0), so no id is stored.
+ * 

+ * Why the checksum is per chunk, not per entry. The only consumer of the
+ * recovered prefix is the send loop's replay guard, which compares a surviving
+ * frame's {@code deltaStart} against the recovered dictionary size -- and every
+ * {@code deltaStart} is a chunk boundary, because chunks and frame deltas are
+ * written one-for-one. A tear inside a chunk therefore invalidates exactly the
+ * frames a per-entry checksum would have invalidated anyway: per-entry
+ * granularity buys no extra recoverable prefix. It costs a great deal, though.
+ * {@link Crc32c#update} is a native call, so checksumming per entry put one JNI
+ * transition -- plus one sub-cache-line copy and one redundant varint decode --
+ * on the producer thread for every new symbol. On the high-cardinality batch this
+ * feature exists to serve (one new symbol per row), that is a thousand native
+ * calls per flush where the chunk needs one.
+ * 

+ * Durability / write-ahead ordering: the producer appends the symbols a
+ * frame introduces BEFORE that frame is published to the ring, but does NOT
+ * fsync -- matching the rest of store-and-forward, which is page-cache (not
+ * disk) durable. This ordering is sufficient for a process/JVM crash: the
+ * page cache survives, so both the dictionary and the frames survive and the
+ * dictionary is a superset of every recoverable frame's references. It is NOT
+ * sufficient for a host/power crash, where unflushed pages can be lost out
+ * of order and the dictionary may end up torn relative to the frames it serves --
+ * exactly as the segment frames themselves may be lost on a host crash. Two
+ * layers keep a host-crash tear from silently corrupting data:
+ * 

+ * Together these turn every detectable host-crash tear into a fail-clean
+ * "resend required" instead of a silent symbol misattribution -- the same
+ * CRC-32C protection the segment frames carry. A tear that happened to leave a
+ * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32
+ * collision per corrupted chunk, no weaker than the frames' own checksum.
+ * 

+ * A torn trailing chunk from a crash mid-append is self-healing: {@link #open}
+ * stops parsing at the first incomplete chunk and the next append overwrites it.
+ * 

+ * Never recreate over an existing file. {@link #open} -- the RECOVERY
+ * entry point -- returns {@code null} when an existing file cannot be read or
+ * parsed, and NEVER falls back to recreating it empty. Recreating would mean
+ * {@code O_TRUNC} over the only copy of load-bearing state, so a single transient
+ * read error (an EIO on a flaky disk, a short read) would permanently destroy the
+ * dictionary the surviving delta frames reference -- turning a recoverable outage
+ * into unrecoverable data. A {@code null} instead degrades the sender to full
+ * self-sufficient frames and leaves every byte on disk, so a later attempt, once
+ * the transient clears, can still recover the slot in full. Only
+ * {@link #openClean} -- the FRESH-slot path, where discarding is the whole point
+ * -- truncates.
+ * 

+ * Lifecycle: single-writer (the producer / user thread) for appends. Read
+ * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The
+ * owner (the engine) closes it, and {@code close()} is callable from any thread
+ * (a shutdown hook, test cleanup). {@code close()} and the append methods are
+ * therefore {@code synchronized}: without that, a close racing an in-flight append
+ * could unmap the production append region (or free the fault-test scratch buffer),
+ * close the fd, and let an in-flight write corrupt memory or land on a descriptor
+ * the OS has reused for another file. Not thread-safe for concurrent writers.
+ */
+public final class PersistedSymbolDict implements QuietCloseable {
+
+    /**
+     * Filename within the slot directory. Dot-prefixed so directory
+     * enumerators that filter by the {@code .sfa} suffix (segment recovery,
+     * OrphanScanner, trim) skip it automatically.
+     */
+    public static final String FILE_NAME = ".symbol-dict";
+    static final int CRC_SIZE = 4; // u32 CRC-32C trailing every chunk
+    static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian
+    static final int HEADER_SIZE = 8;
+    // One bounded, segment-sized append window avoids the allocate/unmap/mmap
+    // cycle every 64 KiB without geometrically reserving up to 2x a large
+    // dictionary. close() truncates the unused tail back to appendOffset.
+    static final int APPEND_MAP_CAPACITY = 4 * 1024 * 1024;
+    /**
+     * Upper bound on a chunk's two header varints ({@code entryCount} and
+     * {@code entryBytes}): each is at most 5 bytes for a 32-bit value. The
+     * encoders reserve this much in front of the entry region so the header can
+     * be back-filled once the region's exact size is known, keeping header,
+     * entries and CRC one contiguous run.
+     */
+    static final int MAX_CHUNK_HEADER_SIZE = 10;
+    /**
+     * Ceiling for the append scratch buffer, mirroring
+     * {@code CursorWebSocketSendLoop.MAX_SENT_DICT_BYTES}: the capacity math is
+     * int-typed, so a larger buffer cannot be addressed. Exceeding it throws --
+     * {@link #ensureScratch} never silently under-allocates.
+     */
+    static final int MAX_SCRATCH_BYTES = Integer.MAX_VALUE - 8;
+    static final byte VERSION = 3; // v3 moved the CRC-32C from per-entry to per-chunk
+    private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class);
+    private final int fd;
+    // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files);
+    // tests inject a fault facade to exercise recovery I/O failures (a truncate
+    // that cannot drop a torn tail, a short write) without a real broken disk.
+    private final FilesFacade ff;
+    // Slot-qualified path, retained purely so diagnostics can name WHICH dictionary they
+    // are about: one JVM routinely holds many (a foreground sender plus N orphan
+    // drainers), so a warning carrying only the bare filename is unattributable exactly
+    // when it fires.
+    private final String filePath;
+    // Production writes directly into segmented append mappings. Wrapping facades retain the
+    // positioned-write path by default so fault tests can inject short writes through ff.write;
+    // mmap-specific fault facades opt in through FilesFacade.isMmapAllowed().
+    private final boolean mappedAppend;
+    // True only when recovery parsed the file through a temporary read-only
+    // mmap instead of allocating a second native buffer as large as the file.
+    // Test-visible so the peak-memory regression has an observable contract.
+    private final boolean mappedRecoveryInput;
+    // Entry count that corresponds EXACTLY to loadedEntriesAddr/loadedEntriesLen,
+    // fixed at open. Distinct from the live `size`, which appends advance -- including
+    // the recovery-time heal in QwpWebSocketSender.healPersistedDictionary, which runs
+    // BEFORE the send loop is constructed. The loop seeds its mirror from the loaded
+    // BYTES, so it must take its count from here; pairing those bytes with the live
+    // size would let sentDictCount claim symbols the mirror does not hold.
+    private final int recoveredSize;
+    private long appendMapAddr;
+    private long appendMapCapacity;
+    private long appendMapOffset;
+    private int appendMapGrowthCount;
+    private long appendOffset;
+    private long appendWriteCount;
+    private boolean closed;
+    // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and
+    // CRCs stripped -- populated only when open() recovered existing chunks
+    // (recovery / orphan-drain). Zero/empty for a freshly created file. READ (not
+    // consumed) to seed the producer's id map and to seed the send loop's catch-up
+    // mirror. Foreground construction transfers ownership to its sole loop after
+    // producer seeding; orphan-drainer loops borrow it because one engine may create
+    // several wire sessions. If ownership was not transferred, close() frees it.
+    private long loadedEntriesAddr;
+    private int loadedEntriesLen;
+    private long scratchAddr;
+    private int scratchCap;
+    private int size;
+
+    private PersistedSymbolDict(
+            FilesFacade ff,
+            String filePath,
+            int fd,
+            long appendOffset,
+            int size,
+            long loadedEntriesAddr,
+            int loadedEntriesLen,
+            boolean mappedRecoveryInput
+    ) {
+        this.ff = ff;
+        this.filePath = filePath;
+        this.fd = fd;
+        this.mappedAppend = ff.isMmapAllowed();
+        this.mappedRecoveryInput = mappedRecoveryInput;
+        this.appendOffset = appendOffset;
+        this.size = size;
+        this.recoveredSize = size;
+        this.loadedEntriesAddr = loadedEntriesAddr;
+        this.loadedEntriesLen = loadedEntriesLen;
+    }
+
+    /**
+     * Opens the dictionary file in {@code slotDir} for RECOVERY, creating it only
+     * when it does not already exist. An existing file is parsed and its complete,
+     * CRC-valid chunks are loaded into memory (see {@link #loadedEntriesAddr()}).
+     * 

+     * Returns {@code null} on any I/O or parse failure -- including an existing file
+     * that cannot be read, carries an unknown version, or fails its checksums. The
+     * caller then falls back to full-dictionary (self-sufficient) frames for this
+     * slot, so a broken side-file degrades gracefully rather than aborting the
+     * sender. Crucially, a {@code null} return NEVER destroys the file: see the
+     * class-level "Never recreate over an existing file" note.
+     */
+    public static PersistedSymbolDict open(String slotDir) {
+        return open(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #open(String)}. Production passes
+     * {@link FilesFacade#INSTANCE}; tests inject a fault facade to drive recovery
+     * I/O failures (e.g. a truncate that cannot drop a torn tail).
+     */
+    public static PersistedSymbolDict open(FilesFacade ff, String slotDir) {
+        String filePath = slotDir + "/" + FILE_NAME;
+        boolean exists = ff.exists(filePath);
+        long existing = exists ? ff.length(filePath) : -1L;
+        if (exists && existing < 0) {
+            // The file is present but its length could not be stat'd (a transient EIO
+            // on a flaky disk). Do NOT fall through to openFresh below, which O_TRUNCs:
+            // truncating the only copy of load-bearing state on a TRANSIENT fault is the
+            // exact destruction the class-level "Never recreate over an existing file"
+            // note forbids -- and unlike the openExisting read path, this routing check
+            // otherwise has no guard. A genuine sub-header stub reports a length in
+            // [0, HEADER_SIZE); only a stat error reports < 0, so the two are
+            // distinguishable. Degrade to full self-sufficient frames and leave every
+            // byte on disk for a later attempt, once the transient clears.
+            LOG.warn("symbol dict {} exists but its length could not be read; "
+                    + "falling back to full-dictionary frames (file left intact)", filePath);
+            return null;
+        }
+        if (existing >= HEADER_SIZE) {
+            // Chunk lengths and the retained contiguous entry region are int-sized,
+            // so a dictionary at or past Integer.MAX_VALUE cannot be represented
+            // safely even though production recovery maps rather than reads it.
+            if (existing >= Integer.MAX_VALUE) {
+                LOG.warn("symbol dict {} too large ({} bytes) to reopen; "
+                        + "falling back to full-dictionary frames (file left intact)", filePath, existing);
+                return null;
+            }
+            // NEVER recreate over an existing file on the recovery path: openFresh
+            // truncates, and these bytes are the only copy of state the surviving
+            // delta frames reference. A null degrades this slot to full
+            // self-sufficient frames and preserves the file for a later attempt.
+            //
+            // The holder catches what openExisting's own catch structurally cannot.
+            // Recovery reads the file through a mapping (Unsafe loads in
+            // scanAndCopyRecoveredChunks), and pre-JDK-21 HotSpot delivers an
+            // unsafe-access fault ASYNCHRONOUSLY -- at the next return or safepoint,
+            // which can be openExisting's own return, in THIS frame. The instance is
+            // fully built by then and owns an fd plus a buffer as large as the file,
+            // but the assignment never happens, so nothing else can release them.
+            // MmapSegment.openExisting takes an inFlight[] for exactly this shape; the
+            // dictionary is at least as exposed, because ensureAppendMap grows the file
+            // with ff.allocate and the reserve truncate is skipped after a crash, so a
+            // sparse tail is routine. Without this, the fault also escapes open()
+            // entirely -- past CursorSendEngine's constructor and out of Sender.build()
+            // -- turning the documented "degrade to full-dictionary frames" into a
+            // sender that cannot be constructed at all.
+            PersistedSymbolDict[] inFlight = new PersistedSymbolDict[1];
+            try {
+                return openExisting(ff, filePath, existing, inFlight);
+            } catch (Throwable t) {
+                if (inFlight[0] != null) {
+                    inFlight[0].close();
+                }
+                LOG.warn("symbol dict {} recovery faulted late; falling back to "
+                        + "full-dictionary frames (file left intact)", filePath, t);
+                return null;
+            }
+        }
+        // Absent, or a sub-header stub left by a crash inside openFresh: no
+        // load-bearing content to lose, so create it.
+        return openFresh(ff, filePath);
+    }
+
+    /**
+     * Opens the dictionary in {@code slotDir} as a FRESH, EMPTY file, discarding
+     * any surviving content. This is the fresh-start counterpart to {@link #open}:
+     * a slot with no recovered segments must start with an empty dictionary, so a
+     * dictionary left by a prior lifecycle -- a fully-drained slot whose
+     * best-effort delete failed, or a crash in the close window -- must NOT be
+     * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for
+     * recovery/orphan-drain replay and never destroys it, this truncates: the
+     * fresh-start producer is not seeded from the dictionary, so trusting a survivor
+     * would leave the producer's ids diverged from the dictionary the send loop
+     * replays and silently misattribute symbols on the next reconnect. Truncating
+     * (rather than relying on an unlink succeeding first) closes the gap even when
+     * the delete is refused -- e.g. a Windows share lock. Returns {@code null} on
+     * I/O failure, so the caller falls back to full self-sufficient frames exactly
+     * as {@link #open} does.
+     */
+    public static PersistedSymbolDict openClean(String slotDir) {
+        return openClean(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #openClean(String)}.
+     */
+    public static PersistedSymbolDict openClean(FilesFacade ff, String slotDir) {
+        return openFresh(ff, slotDir + "/" + FILE_NAME);
+    }
+
+    /**
+     * Best-effort removal of a stale dictionary file. Used at fully-drained close
+     * (the slot is empty, nothing references the dictionary any more), mirroring
+     * {@link AckWatermark#removeOrphan}. The fresh-start path deliberately does NOT
+     * use this -- it opens a clean dictionary via {@link #openClean} instead, so a
+     * failed delete cannot leave a stale dictionary a new session would trust.
+     */
+    public static void removeOrphan(String slotDir) {
+        removeOrphan(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #removeOrphan(String)}.
+     */
+    public static void removeOrphan(FilesFacade ff, String slotDir) {
+        ff.remove(slotDir + "/" + FILE_NAME);
+    }
+
+    /**
+     * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the
+     * symbol-dict delta section the frame encoder just wrote -- as ONE chunk.
+     * 

+     * The consistency walk below decodes each entry's length varint, but the bytes
+     * themselves are copied in a SINGLE {@code copyMemory} and checksummed by a
+     * SINGLE {@link Crc32c#update} covering the whole chunk. A per-entry checksum
+     * would put one JNI transition, one sub-cache-line copy and one redundant varint
+     * decode on the producer thread per new symbol; the chunk needs one of each.
+     * 

+     * Advances {@code size} by {@code count}. Same durability/idempotency contract
+     * as {@link #appendSymbols}: no fsync, and a short write throws WITHOUT
+     * advancing {@code size}/{@code appendOffset}, so a retry keyed off
+     * {@link #size()} re-persists the same range at the same offset. No-op when the
+     * range is empty or the dictionary is closed.
+     */
+    public synchronized void appendRawEntries(long addr, int len, int count) {
+        if (closed || count <= 0 || len <= 0) {
+            return;
+        }
+        // Validate the (addr, len, count) triple BEFORE writing anything: an
+        // inconsistent triple would record a chunk whose stored entryCount disagreed
+        // with the entries it holds, shifting the dense id->symbol map on recovery.
+        // The sole caller derives count and len from one beginMessage, so this cannot
+        // fire today -- but the file this guards is the one the "resend required"
+        // contract rests on, so keep the structural guard. Gated behind assert: it
+        // re-walks every entry's length prefix on the common flush path, and the client
+        // library runs without -ea in production (embedded in user apps), so this holds
+        // the guard in the client's own -ea test suite without the per-flush cost in
+        // production.
+        assert validateRawEntries(addr, len, count);
+        int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len);
+        if (mappedAppend) {
+            long recLen = (long) hdrLen + len + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, count);
+            NativeBufferWriter.writeVarint(p, len);
+            Unsafe.getUnsafe().copyMemory(addr, recStart + hdrLen, len);
+            commitMappedChunk(recStart, hdrLen, len, count);
+            return;
+        }
+        ensureScratch((long) hdrLen + len + CRC_SIZE);
+        long p = NativeBufferWriter.writeVarint(scratchAddr, count);
+        NativeBufferWriter.writeVarint(p, len);
+        Unsafe.getUnsafe().copyMemory(addr, scratchAddr + hdrLen, len);
+        flushChunk(scratchAddr, hdrLen, len, count);
+    }
+
+    /**
+     * Appends one symbol as a single-entry chunk, extending the on-disk dictionary.
+     * The caller appends a frame's new symbols BEFORE publishing that frame, so the
+     * write ordering (dictionary entry before referencing frame) holds; no fsync is
+     * performed (see the class-level durability note). Assigns the next dense id
+     * implicitly (the entry's position).
+     * 

+     * Test-only: production persists a frame's whole new-symbol range in one chunk
+     * via {@link #appendSymbols} / {@link #appendRawEntries}. Tests use this to
+     * build a dictionary one entry at a time.
+     */
+    @TestOnly
+    public synchronized void appendSymbol(CharSequence symbol) {
+        if (closed) {
+            return;
+        }
+        int utf8Len = Utf8s.utf8Bytes(symbol);
+        int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+        if (mappedAppend) {
+            int hdrLen = NativeBufferWriter.varintSize(1) + NativeBufferWriter.varintSize(wireLen);
+            long recLen = (long) hdrLen + wireLen + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, 1);
+            p = NativeBufferWriter.writeVarint(p, wireLen);
+            p = NativeBufferWriter.writeVarint(p, utf8Len);
+            if (utf8Len > 0) {
+                Utf8s.strCpyUtf8(symbol, p, utf8Len);
+            }
+            commitMappedChunk(recStart, hdrLen, wireLen, 1);
+            return;
+        }
+        ensureScratch((long) MAX_CHUNK_HEADER_SIZE + wireLen + CRC_SIZE);
+        long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE;
+        long p = NativeBufferWriter.writeVarint(entryStart, utf8Len);
+        if (utf8Len > 0) {
+            Utf8s.strCpyUtf8(symbol, p, utf8Len);
+        }
+        writeChunkFromScratch(wireLen, 1);
+    }
+
+    /**
+     * Appends the dense id range {@code [from .. to]} as one chunk with one
+     * checksum. This is the RE-ENCODE path: the steady-state persist ships a frame's
+     * pre-encoded delta bytes through {@link #appendRawEntries}, and only a retry
+     * after a failed publish rebuilds a range straight from the dictionary. The
+     * mapped path stages the entry region in the scratch buffer with a single UTF-8
+     * walk per symbol, then bulk-copies it into the append window after the header,
+     * and still commits WITHOUT a positioned-write syscall. A direct encode into the
+     * window would walk each symbol's UTF-8 length twice -- the exact entriesLen sizes
+     * both the header varint and the mmap reserve -- and the back-to-back chunk format
+     * leaves no room to reserve-and-back-fill the header in place. The
+     * positioned-write fallback runs only behind an injected filesystem facade so
+     * short-write recovery tests retain their fault seam. Callers pass the dictionary
+     * and the range so the ids resolve to their symbol strings.
+     * 

+     * Same durability and idempotency contract as {@link #appendSymbol}: no fsync,
+     * and a short write throws WITHOUT advancing {@code size}/{@code appendOffset},
+     * so a retry keyed off {@link #size()} re-encodes the same range and overwrites
+     * at the same offset. No-op when the range is empty or the dictionary is closed.
+     */
+    public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) {
+        if (closed || to < from) {
+            return;
+        }
+        int count = to - from + 1;
+        if (mappedAppend) {
+            // Stage the entry region in scratch with ONE UTF-8 walk per symbol, then
+            // bulk-copy it into the append window after the header (see the method
+            // javadoc for why a direct in-window encode would have to walk each symbol
+            // twice). ensureScratch enforces the same MAX_SCRATCH_BYTES ceiling the old
+            // sizing pass did, throwing before size/appendOffset advance.
+            int entriesLen = 0;
+            for (int id = from; id <= to; id++) {
+                CharSequence symbol = dict.getSymbol(id);
+                int utf8Len = Utf8s.utf8Bytes(symbol);
+                int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+                ensureScratch((long) entriesLen + wireLen);
+                long q = NativeBufferWriter.writeVarint(scratchAddr + entriesLen, utf8Len);
+                if (utf8Len > 0) {
+                    Utf8s.strCpyUtf8(symbol, q, utf8Len);
+                }
+                entriesLen += wireLen;
+            }
+            int hdrLen = NativeBufferWriter.varintSize(count)
+                    + NativeBufferWriter.varintSize(entriesLen);
+            long recLen = (long) hdrLen + entriesLen + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, count);
+            NativeBufferWriter.writeVarint(p, entriesLen);
+            if (entriesLen > 0) {
+                Unsafe.getUnsafe().copyMemory(scratchAddr, recStart + hdrLen, entriesLen);
+            }
+            commitMappedChunk(recStart, hdrLen, entriesLen, count);
+            return;
+        }
+        int entriesLen = 0;
+        for (int id = from; id <= to; id++) {
+            CharSequence symbol = dict.getSymbol(id);
+            int utf8Len = Utf8s.utf8Bytes(symbol);
+            int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+            ensureScratch((long) MAX_CHUNK_HEADER_SIZE + entriesLen + wireLen + CRC_SIZE);
+            long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE + entriesLen;
+            long p = NativeBufferWriter.writeVarint(entryStart, utf8Len);
+            if (utf8Len > 0) {
+                Utf8s.strCpyUtf8(symbol, p, utf8Len);
+            }
+            entriesLen += wireLen;
+        }
+        writeChunkFromScratch(entriesLen, count);
+    }
+
+    @Override
+    public synchronized void close() {
+        if (closed) {
+            return;
+        }
+        closed = true;
+        // Each step in its own try/catch, as CursorSendEngine.close() already does.
+        // `closed` is set first, so a throw from any step short-circuits every retry --
+        // stranding the scratch buffer and the fd for the process's lifetime. Unreachable
+        // with FilesFacade.INSTANCE (munmap and truncate are native calls returning
+        // int/boolean), but the ff seam exists precisely so a test CAN inject a throw,
+        // and a close path must not depend on its own steps never failing.
+        if (loadedEntriesAddr != 0L) {
+            try {
+                Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT);
+            } catch (Throwable ignored) {
+            }
+            // Null after freeing (like scratchAddr below) so a CONSTRUCTION-PHASE reader
+            // that runs after close observes 0 rather than a dangling pointer. This is
+            // NOT a cross-thread guard, and the getters' own javadoc is the accurate
+            // statement: they are unsynchronised reads of non-volatile fields, so a
+            // caller on another thread has no guarantee of seeing this write and can
+            // still dereference freed memory. Ordering, not nulling, is what makes the
+            // borrow safe -- every owner closes its loop before the engine.
+            loadedEntriesAddr = 0L;
+            loadedEntriesLen = 0;
+        }
+        if (appendMapAddr != 0L) {
+            long mapAddr = appendMapAddr;
+            long mapCapacity = appendMapCapacity;
+            appendMapAddr = 0L;
+            appendMapCapacity = 0L;
+            appendMapOffset = 0L;
+            try {
+                ff.munmap(mapAddr, mapCapacity, MemoryTag.MMAP_DEFAULT);
+            } catch (Throwable ignored) {
+            }
+            try {
+                // The active window reserves space past the logical end. Return that tail on
+                // orderly close; after a crash open() treats the zero-filled reserve as
+                // a torn trailing chunk and truncates it to the same appendOffset.
+                if (!ff.truncate(fd, appendOffset)) {
+                    LOG.warn("symbol dict {} could not trim mmap reserve to {}; recovery will "
+                                    + "discard the zero-filled tail on the next open",
+                            filePath, appendOffset);
+                }
+            } catch (Throwable ignored) {
+            }
+        }
+        if (scratchAddr != 0L) {
+            try {
+                Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT);
+            } catch (Throwable ignored) {
+            }
+            scratchAddr = 0L;
+            scratchCap = 0;
+        }
+        if (fd >= 0) {
+            try {
+                ff.close(fd);
+            } catch (Throwable ignored) {
+            }
+        }
+    }
+
+    @TestOnly
+    public synchronized int appendMapGrowthCount() {
+        return appendMapGrowthCount;
+    }
+
+    @TestOnly
+    public synchronized long appendWriteCount() {
+        return appendWriteCount;
+    }
+
+    /**
+     * Base address of the loaded entry region -- the concatenated
+     * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly as a
+     * delta section carries them (chunk headers and CRCs stripped). Zero when
+     * nothing was recovered.
+     * 

+     * Construction-phase only. This hands out a raw pointer into native
+     * memory that {@link #close()} frees and nulls, with no closed-guard and no
+     * synchronization. It is safe to read only BEFORE the slot's I/O thread and
+     * any producer append start -- i.e. while the send loop is being constructed
+     * or an orphan-drain is seeding its mirror, both of which happen-before those
+     * threads. A caller that reads it from a running thread races {@code close()}
+     * and can dereference freed memory (use-after-free).
+     */
+    public long loadedEntriesAddr() {
+        return loadedEntriesAddr;
+    }
+
+    /**
+     * Byte length of {@link #loadedEntriesAddr()}. Construction-phase only, for
+     * the same reason -- see {@link #loadedEntriesAddr()}.
+     */
+    public int loadedEntriesLen() {
+        return loadedEntriesLen;
+    }
+
+    /**
+     * Number of symbols {@link #open} recovered from disk -- the exact entry count of
+     * {@link #loadedEntriesAddr()} / {@link #loadedEntriesLen()}. Unlike {@link #size()}
+     * this never advances, so a caller seeding from the loaded bytes stays in lockstep
+     * with them even after the producer has appended (the recovery heal does exactly
+     * that, before the send loop is built).
+     */
+    public int recoveredSize() {
+        return recoveredSize;
+    }
+
+    @TestOnly
+    public boolean usedMappedRecoveryInput() {
+        return mappedRecoveryInput;
+    }
+
+    /**
+     * Decodes the recovered entries directly into {@code target} in ascending-id
+     * order. This avoids materialising a cardinality-sized temporary list that
+     * the producer would immediately copy into the global dictionary.
+     * Construction-phase only; see {@link #loadedEntriesAddr()}.
+     */
+    public void addLoadedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeLoadedSymbols(target, null);
+    }
+
+    /**
+     * Materialises the loaded entries as symbol strings in ascending-id order.
+     * Retained for recovery-format tests; production decodes directly through
+     * {@link #addLoadedSymbolsTo(GlobalSymbolDictionary)}.
+     */
+    @TestOnly
+    public ObjList readLoadedSymbols() {
+        ObjList out = new ObjList<>(Math.max(recoveredSize, 1));
+        decodeLoadedSymbols(null, out);
+        return out;
+    }
+
+    private void decodeLoadedSymbols(GlobalSymbolDictionary target, ObjList out) {
+        long p = loadedEntriesAddr;
+        long limit = p + loadedEntriesLen;
+        // open() CRC-validated these bytes and copyRecoveredEntries laid down exactly
+        // `size` entries, so this decode runs on trusted, self-consistent input. Any
+        // mismatch means an internal invariant broke: fail loud like the rest of this
+        // file (copyRecoveredEntries throws too) instead of silently under-populating
+        // the dictionary, which would shift every id above the short point.
+        // recoveredSize, not the live size: this decodes the LOADED region, whose entry
+        // count is fixed at open. Appends (including the recovery heal) advance size
+        // without extending that region, so keying off size would over-read.
+        for (int i = 0; i < recoveredSize; i++) {
+            long len = 0;
+            int shift = 0;
+            boolean terminated = false;
+            while (p < limit) {
+                byte b = Unsafe.getUnsafe().getByte(p++);
+                len |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    terminated = true;
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    break; // over-long run -- a canonical length varint is <= 5 bytes
+                }
+            }
+            if (!terminated || p + len > limit) {
+                // UnreplayableSlotException so Sender.build() can set the slot aside
+                // instead of rethrowing forever -- see CursorSendEngine's twin throw.
+                throw new UnreplayableSlotException("truncated loaded symbol dictionary entry to "
+                        + FILE_NAME + " [entry=" + i + ", size=" + recoveredSize + ']');
+            }
+            String symbol = Utf8s.stringFromUtf8Bytes(p, p + len);
+            if (target != null) {
+                target.addRecoveredSymbol(symbol);
+            } else {
+                out.add(symbol);
+            }
+            p += len;
+        }
+        if (p != limit) {
+            throw new UnreplayableSlotException("loaded symbol dictionary has trailing bytes to "
+                    + FILE_NAME + " [consumed=" + (p - loadedEntriesAddr) + ", length=" + loadedEntriesLen + ']');
+        }
+    }
+
+    /**
+     * Number of symbols the dictionary holds (highest id + 1).
+     */
+    public int size() {
+        return size;
+    }
+
+    /**
+     * @param inFlight single-slot holder the caller must close when this method throws.
+     *                 Set immediately before the return, so a late-delivered mmap access
+     *                 fault -- which pre-JDK-21 HotSpot may raise at this method's RETURN,
+     *                 in the caller's frame, past the catch below -- still leaves the fully
+     *                 built instance (fd plus loaded-entry buffer) reachable by someone.
+     */
+    private static PersistedSymbolDict openExisting(
+            FilesFacade ff, String filePath, long fileLen, PersistedSymbolDict[] inFlight) {
+        int fd = ff.openRW(filePath);
+        if (fd < 0) {
+            LOG.warn("symbol dict {} could not be opened (rc={}); "
+                    + "falling back to full-dictionary frames (file left intact)", filePath, fd);
+            return null;
+        }
+        int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE)
+        boolean mappedInput = ff.isMmapAllowed();
+        long inputAddr = 0L;
+        long entriesAddr = 0L;
+        int entriesCapacity = 0;
+        int entriesLen = 0;
+        try {
+            if (mappedInput) {
+                inputAddr = ff.mmap(fd, fileLen, 0L, Files.MAP_RO, MemoryTag.MMAP_DEFAULT);
+                if (inputAddr == Files.FAILED_MMAP_ADDRESS) {
+                    inputAddr = 0L;
+                    throw new IllegalStateException("could not mmap symbol dictionary for recovery");
+                }
+            } else {
+                // Fault-injection facades retain the positioned-read path so tests
+                // can still force short reads without mapping around the seam.
+                inputAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+                long read = ff.read(fd, inputAddr, len, 0);
+                if (read != len) {
+                    throw new IllegalStateException("short read while recovering symbol dictionary "
+                            + "[expected=" + len + ", actual=" + read + ']');
+                }
+            }
+            if (Unsafe.getUnsafe().getInt(inputAddr) != FILE_MAGIC
+                    || Unsafe.getUnsafe().getByte(inputAddr + 4) != VERSION) {
+                throw new IllegalStateException("bad magic or unknown symbol dictionary version");
+            }
+            // ONE pass: validate each chunk's CRC and, once proven good, copy its
+            // entries straight out. The entry region is by construction a subset of
+            // the file, so `len` is a safe upper bound to allocate against -- it
+            // overshoots only by the chunk headers and CRCs -- and the exact size is
+            // reclaimed by the shrink below. The previous shape walked the file
+            // TWICE, re-decoding both header varints of every chunk on the second
+            // pass purely to size one allocation it could have bounded for free.
+            entriesCapacity = len;
+            entriesAddr = Unsafe.malloc(entriesCapacity, MemoryTag.NATIVE_DEFAULT);
+            RecoveryScan scan = scanAndCopyRecoveredChunks(inputAddr, len, entriesAddr);
+            entriesLen = scan.entriesLen;
+            if (entriesLen == 0) {
+                Unsafe.free(entriesAddr, entriesCapacity, MemoryTag.NATIVE_DEFAULT);
+                entriesAddr = 0L;
+                entriesCapacity = 0;
+            } else if (entriesLen < entriesCapacity) {
+                entriesAddr = Unsafe.realloc(
+                        entriesAddr, entriesCapacity, entriesLen, MemoryTag.NATIVE_DEFAULT);
+                entriesCapacity = entriesLen;
+            }
+            if (mappedInput) {
+                ff.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+            } else {
+                Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+            }
+            inputAddr = 0L;
+            // Drop any torn/stale trailing bytes so a LATER, shorter append cannot
+            // leave residue past its own end. The truncate result IS checked: a file
+            // we cannot trim could still expose stale post-end bytes whose
+            // (self-consistent) chunk CRC the parse would accept at a shifted
+            // position, so a failed truncate makes the file untrusted -- return null
+            // (the sender falls back to full self-sufficient frames) and, per the
+            // never-destroy contract, leave every byte on disk.
+            if (scan.validLen < len && !ff.truncate(fd, scan.validLen)) {
+                throw new IllegalStateException("could not drop torn/stale symbol dictionary tail");
+            }
+            PersistedSymbolDict dict = new PersistedSymbolDict(
+                    ff, filePath, fd, scan.validLen, scan.count, entriesAddr, entriesLen, mappedInput);
+            // Publish to the holder BEFORE returning: from here on the caller owns the
+            // cleanup, including when the return itself faults. See the parameter doc.
+            inFlight[0] = dict;
+            return dict;
+        } catch (Throwable t) {
+            if (inputAddr != 0L) {
+                if (mappedInput) {
+                    ff.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+                } else {
+                    Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+                }
+            }
+            if (entriesAddr != 0L) {
+                // capacity, not entriesLen: the shrink may not have happened yet.
+                Unsafe.free(entriesAddr, entriesCapacity, MemoryTag.NATIVE_DEFAULT);
+            }
+            if (fd >= 0) {
+                ff.close(fd);
+            }
+            // Pass the throwable as a trailing argument with no matching placeholder so
+            // slf4j prints the stack trace: this WARN is the only forensic record of why
+            // a load-bearing dictionary was abandoned.
+            LOG.warn("symbol dict {} recovery failed; falling back to full-dictionary frames "
+                    + "(file left intact)", filePath, t);
+            return null;
+        }
+    }
+
+    /**
+     * Validates every chunk and copies the entries of each proven-good one into
+     * {@code dstAddr}, in a single pass. {@code dstAddr} must have room for {@code len}
+     * bytes -- the entry region is a subset of the file, so that always suffices.
+     * Stops at the first chunk that is torn, fails its CRC, or is internally
+     * inconsistent, exactly as the two-pass version did, so the trusted prefix is
+     * unchanged.
+     */
+    private static RecoveryScan scanAndCopyRecoveredChunks(long inputAddr, int len, long dstAddr) {
+        Varint v = new Varint();
+        int count = 0;
+        int diskPos = HEADER_SIZE;
+        long entriesLen = 0L;
+        while (diskPos < len) {
+            if (!v.decode(inputAddr, diskPos, len)) {
+                break;
+            }
+            long entryCount = v.value;
+            if (!v.decode(inputAddr, v.end, len)) {
+                break;
+            }
+            long entryBytes = v.value;
+            int entriesStart = v.end;
+            long chunkEnd = (long) entriesStart + entryBytes;
+            if (chunkEnd + CRC_SIZE > len) {
+                break;
+            }
+            int chunkEndI = (int) chunkEnd;
+            int crcStored = Unsafe.getUnsafe().getInt(inputAddr + chunkEndI);
+            int crcCalc = Crc32c.updateUnsafe(
+                    Crc32c.INIT,
+                    inputAddr + diskPos,
+                    chunkEndI - diskPos);
+            if (crcCalc != crcStored
+                    || entryCount <= 0
+                    // Every entry costs at least its own length varint, so a positive
+                    // entryCount inside a zero-byte region is self-contradictory. The CRC
+                    // proves the bytes are what was WRITTEN, never that the triple is
+                    // consistent, and decodeLoadedSymbols would only discover it later --
+                    // as a throw two layers up rather than a trimmed trusted prefix.
+                    || entryBytes <= 0
+                    || (long) count + entryCount > Integer.MAX_VALUE
+                    || entriesLen + entryBytes > Integer.MAX_VALUE) {
+                break;
+            }
+            if (!isConsistentEntryRegion(inputAddr, entriesStart, entryBytes, entryCount)) {
+                break;
+            }
+            Unsafe.getUnsafe().copyMemory(inputAddr + entriesStart, dstAddr + entriesLen, entryBytes);
+            entriesLen += entryBytes;
+            diskPos = chunkEndI + CRC_SIZE;
+            count += (int) entryCount;
+        }
+        return new RecoveryScan(count, (int) entriesLen, diskPos);
+    }
+
+    /**
+     * Whether {@code [start, start + bytes)} holds exactly {@code count} well-formed
+     * {@code [len varint][utf8]} entries, consuming the region exactly.
+     * 

+     * The chunk CRC proves the bytes are what was WRITTEN; it says nothing about whether
+     * the header triple is self-consistent. The only write-side guard,
+     * {@code validateRawEntries}, sits behind an {@code assert} -- and this library ships
+     * embedded in user applications, which run without {@code -ea}. So a producer bug or
+     * a torn write that happens to re-checksum could record a chunk whose stored
+     * entryCount disagrees with its entries, shifting the dense id->symbol map for
+     * everything above it.
+     * 

+     * Checking here ends the trusted prefix at that chunk -- the same treatment a CRC
+     * failure gets -- instead of letting decodeLoadedSymbols discover it later and throw
+     * two layers up, which quarantines the whole slot rather than salvaging its intact
+     * prefix. It costs one varint decode per entry on a cold path that already walks
+     * every entry immediately afterwards.
+     */
+    private static boolean isConsistentEntryRegion(long addr, int start, long bytes, long count) {
+        long p = start;
+        long limit = start + bytes;
+        for (long i = 0; i < count; i++) {
+            long len = 0;
+            int shift = 0;
+            boolean terminated = false;
+            while (p < limit) {
+                byte b = Unsafe.getUnsafe().getByte(addr + p++);
+                len |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    terminated = true;
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    return false; // over-long run: a canonical length varint is <= 5 bytes
+                }
+            }
+            if (!terminated || len < 0 || p + len > limit) {
+                return false;
+            }
+            p += len;
+        }
+        return p == limit;
+    }
+
+    private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) {
+        int fd = ff.openCleanRW(filePath);
+        if (fd < 0) {
+            LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd);
+            return null;
+        }
+        long hdr = 0L;
+        try {
+            hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+            Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC);
+            Unsafe.getUnsafe().putByte(hdr + 4, VERSION);
+            Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0);
+            Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0);
+            Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0);
+            long written = ff.write(fd, hdr, HEADER_SIZE, 0);
+            if (written != HEADER_SIZE) {
+                int fdToClose = fd;
+                fd = -1; // relinquish before close so the catch cannot double-close if close throws
+                ff.close(fdToClose);
+                ff.remove(filePath); // drop the headerless stub rather than litter
+                LOG.warn("symbol dict {} header write failed; proceeding without it", filePath);
+                return null;
+            }
+        } catch (Throwable t) {
+            // Unreachable with FilesFacade.INSTANCE (Files.write is native and returns
+            // -1 rather than throwing; the Unsafe puts target a valid 8-byte buffer and
+            // an 8-byte malloc cannot realistically OOM), but the ff seam exists so
+            // tests CAN inject a throwing facade -- close the fd and drop the stub so
+            // neither leaks.
+            if (fd >= 0) { // the header-write branch relinquished fd to -1 before closing
+                int fdToClose = fd;
+                fd = -1;
+                ff.close(fdToClose);
+                ff.remove(filePath);
+            }
+            LOG.warn("symbol dict {} creation failed; proceeding without it", filePath, t);
+            return null;
+        } finally {
+            if (hdr != 0L) {
+                Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+            }
+        }
+        return new PersistedSymbolDict(ff, filePath, fd, HEADER_SIZE, 0, 0L, 0, false);
+    }
+
+    /**
+     * Verifies that {@code count} wire entries ({@code [len varint][utf8]}) occupy
+     * exactly {@code len} bytes from {@code addr}. Returns {@code true} when the triple
+     * is consistent; throws {@link IllegalStateException} (naming the offending entry /
+     * count / consumed bytes) otherwise. Called only from an {@code assert} in
+     * {@link #appendRawEntries}: it guards an internal invariant the sole caller cannot
+     * violate today, so it runs under the test suite's {@code -ea} but is elided from
+     * the per-flush production path (client apps run without {@code -ea}).
+     */
+    private static boolean validateRawEntries(long addr, int len, int count) {
+        long src = addr;
+        long srcLimit = addr + len;
+        for (int i = 0; i < count; i++) {
+            long symLen = 0;
+            int shift = 0;
+            while (src < srcLimit) {
+                byte b = Unsafe.getUnsafe().getByte(src++);
+                symLen |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    // A canonical entry-length varint is <= 5 bytes; a longer
+                    // continuation run is corrupt. The bound check below rejects it.
+                    break;
+                }
+            }
+            src += symLen; // src was just past the len varint
+            if (src > srcLimit) {
+                throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME
+                        + " [entry=" + i + ", count=" + count + ']');
+            }
+        }
+        if (src != srcLimit) {
+            throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to "
+                    + FILE_NAME + " [count=" + count + ", len=" + len
+                    + ", consumed=" + (int) (src - addr) + ']');
+        }
+        return true;
+    }
+
+    private long checkedRequiredOffset(long recordLen) {
+        long required = appendOffset + recordLen;
+        if (recordLen < 0L || required < appendOffset) {
+            throw new IllegalStateException("symbol dict append offset overflow to "
+                    + FILE_NAME + " [offset=" + appendOffset + ", recordLen=" + recordLen + ']');
+        }
+        return required;
+    }
+
+    /**
+     * Commits a chunk already assembled directly in the append mmap. The logical
+     * offset and symbol count advance last, after the CRC and a store fence, so an
+     * interrupted process leaves either a complete chunk or a tail that open()
+     * rejects and truncates.
+     */
+    private void commitMappedChunk(long recStart, int hdrLen, int entriesLen, int count) {
+        long bodyLen = (long) hdrLen + entriesLen;
+        long recLen = bodyLen + CRC_SIZE;
+        // updateUnsafe, NOT the native update: this checksums bytes inside the append
+        // MAPPING. ensureAppendMap grows the file with ff.allocate, whose native fallback
+        // is ftruncate, so the window can cover blocks the filesystem has not committed
+        // (ENOSPC, a quota, a sparse tail left by a crash) -- and touching one raises
+        // SIGBUS. Inside JNI that ABORTS THE JVM with no recovery; at an Unsafe intrinsic
+        // site HotSpot converts it to a catchable InternalError. MmapSegment.scanFrames
+        // already uses updateUnsafe over the same class of mapping for exactly this
+        // reason. Costs nothing measurable and drops a JNI transition per flush.
+        Unsafe.getUnsafe().putInt(
+                recStart + bodyLen,
+                Crc32c.updateUnsafe(Crc32c.INIT, recStart, bodyLen));
+        Unsafe.getUnsafe().storeFence();
+        appendOffset += recLen;
+        size += count;
+    }
+
+    /**
+     * Ensures the production append mmap covers the absolute file offset
+     * {@code required}. The log is mapped in 4 MiB-aligned windows: small flushes
+     * share a segment-sized window, while a large existing dictionary does not force
+     * the process to reserve and remap its whole prefix or geometrically over-allocate it.
+     */
+    private void ensureAppendMap(long required) {
+        if (appendMapAddr != 0L
+                && required >= appendMapOffset
+                && required <= appendMapOffset + appendMapCapacity) {
+            return;
+        }
+        // Page-align, not APPEND_MAP_CAPACITY-align: mmap only requires a page-aligned
+        // file offset. Rounding the START down to a 4 MiB boundary while sizing `needed`
+        // to the record's END meant a chunk straddling that boundary produced a window
+        // spanning BOTH -- 8 MiB mapped and re-allocated, whose lower half covers bytes
+        // already written and never touched again. Steady state then advanced 4 MiB per
+        // remap while always mapping 8.
+        long pageMask = Files.PAGE_SIZE - 1;
+        long newOffset = appendOffset & ~pageMask;
+        long needed = required - newOffset;
+        long newCapacity = Math.max(APPEND_MAP_CAPACITY, needed);
+        long remainder = newCapacity % APPEND_MAP_CAPACITY;
+        if (remainder != 0L) {
+            long padding = APPEND_MAP_CAPACITY - remainder;
+            if (newCapacity > Long.MAX_VALUE - padding) {
+                throw new IllegalStateException("symbol dict mmap capacity overflow to "
+                        + FILE_NAME + " [required=" + required + ']');
+            }
+            newCapacity += padding;
+        }
+        if (newOffset > Long.MAX_VALUE - newCapacity) {
+            throw new IllegalStateException("symbol dict mmap file offset overflow to "
+                    + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']');
+        }
+        long newFileSize = newOffset + newCapacity;
+        if (!ff.allocate(fd, newFileSize)) {
+            throw new IllegalStateException("could not grow mmap append region for "
+                    + FILE_NAME + " [required=" + required + ", fileSize=" + newFileSize + ']');
+        }
+        if (appendMapAddr != 0L) {
+            long oldAddr = appendMapAddr;
+            long oldCapacity = appendMapCapacity;
+            appendMapAddr = 0L;
+            appendMapCapacity = 0L;
+            appendMapOffset = 0L;
+            ff.munmap(oldAddr, oldCapacity, MemoryTag.MMAP_DEFAULT);
+        }
+        long newAddr = ff.mmap(
+                fd, newCapacity, newOffset, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
+        if (newAddr == Files.FAILED_MMAP_ADDRESS) {
+            throw new IllegalStateException("could not mmap append region for "
+                    + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']');
+        }
+        appendMapAddr = newAddr;
+        appendMapCapacity = newCapacity;
+        appendMapOffset = newOffset;
+        appendMapGrowthCount++;
+    }
+
+    /**
+     * Grows the append scratch buffer to hold at least {@code required} bytes.
+     * 

+     * Throws when {@code required} exceeds {@link #MAX_SCRATCH_BYTES} rather than
+     * clamping to it: a clamp would hand back a buffer SMALLER than the caller asked
+     * for and return normally, and every caller then writes {@code required} bytes
+     * into it -- turning a clean out-of-memory into a silent native-heap overflow, on
+     * the very write path the dictionary's integrity rests on. This is the same
+     * loud-failure shape {@code CursorWebSocketSendLoop.ensureSentDictCapacity} uses
+     * on the same bound. Unreachable at any realistic cardinality (it needs a ~2 GiB
+     * dictionary section in a single frame, itself bounded by the server's batch
+     * cap), but a size guard on a data-integrity write path must never
+     * under-allocate.
+     */
+    private void ensureScratch(long required) {
+        if (scratchCap >= required) {
+            return;
+        }
+        if (required > MAX_SCRATCH_BYTES) {
+            throw new IllegalStateException("symbol dict scratch buffer exceeds the maximum size to "
+                    + FILE_NAME + " [required=" + required + ", max=" + MAX_SCRATCH_BYTES + ']');
+        }
+        // Double in long: scratchCap * 2 as an int overflows negative past ~1 GB and
+        // would make the realloc size negative.
+        long newCap = Math.max(required, Math.max(256L, (long) scratchCap * 2));
+        if (newCap > MAX_SCRATCH_BYTES) {
+            newCap = MAX_SCRATCH_BYTES;
+        }
+        scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, (int) newCap, MemoryTag.NATIVE_DEFAULT);
+        scratchCap = (int) newCap;
+    }
+
+    /**
+     * Checksums {@code [recStart, recStart + hdrLen + entriesLen)} in ONE native
+     * call, appends the CRC, and issues ONE positioned write. Advances
+     * {@code size}/{@code appendOffset} only on a complete write, so a short write
+     * throws and a retry keyed off {@link #size()} re-persists the same range at the
+     * same offset.
+     */
+    private void flushChunk(long recStart, int hdrLen, int entriesLen, int count) {
+        // long math, matching commitMappedChunk: hdrLen + entriesLen can reach
+        // Integer.MAX_VALUE (entriesLen is bounded only by MAX_SCRATCH_BYTES), and an
+        // int sum would wrap negative and hand a bogus length to the CRC and the write.
+        long bodyLen = (long) hdrLen + entriesLen;
+        long recLen = bodyLen + CRC_SIZE;
+        Unsafe.getUnsafe().putInt(recStart + bodyLen, Crc32c.update(Crc32c.INIT, recStart, bodyLen));
+        appendWriteCount++;
+        long written = ff.write(fd, recStart, recLen, appendOffset);
+        if (written != recLen) {
+            throw new IllegalStateException("short write to " + FILE_NAME
+                    + " [expected=" + recLen + ", actual=" + written + ']');
+        }
+        appendOffset += recLen;
+        size += count;
+    }
+
+    /**
+     * Writes one chunk whose entry region is ALREADY encoded in scratch at offset
+     * {@link #MAX_CHUNK_HEADER_SIZE}. Back-fills the header immediately in front of
+     * the entries -- the header is at most {@code MAX_CHUNK_HEADER_SIZE} bytes, so it
+     * always fits the reserve -- leaving header, entries and CRC one contiguous run
+     * for a single checksum and a single write.
+     */
+    private void writeChunkFromScratch(int entriesLen, int count) {
+        int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(entriesLen);
+        long recStart = scratchAddr + MAX_CHUNK_HEADER_SIZE - hdrLen;
+        long p = NativeBufferWriter.writeVarint(recStart, count);
+        NativeBufferWriter.writeVarint(p, entriesLen);
+        flushChunk(recStart, hdrLen, entriesLen, count);
+    }
+
+    private static final class RecoveryScan {
+        private final int count;
+        private final int entriesLen;
+        private final int validLen;
+
+        private RecoveryScan(int count, int entriesLen, int validLen) {
+            this.count = count;
+            this.entriesLen = entriesLen;
+            this.validLen = validLen;
+        }
+    }
+
+    /**
+     * Zero-allocation LEB128 decoder, one instance per recovery pass -- not one
+     * per chunk. The previous {@code long[]}-returning decoder allocated once per
+     * entry, so a million-symbol recovery churned a million throwaway arrays.
+     */
+    private static final class Varint {
+        int end;
+        long value;
+
+        /**
+         * Decodes the varint at {@code buf[pos..limit)}. Returns false -- leaving
+         * {@code value}/{@code end} undefined -- when the varint is truncated (a torn
+         * tail) or runs longer than a canonical 5-byte length.
+         */
+        boolean decode(long buf, int pos, int limit) {
+            long v = 0;
+            int shift = 0;
+            int cur = pos;
+            while (cur < limit) {
+                byte b = Unsafe.getUnsafe().getByte(buf + cur);
+                cur++;
+                v |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    value = v;
+                    end = cur;
+                    return true;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    return false; // implausible for a chunk header; treat as torn
+                }
+            }
+            return false;
+        }
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
new file mode 100644
index 00000000..790db93b
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
@@ -0,0 +1,388 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.str.Utf8s;
+
+/**
+ * Engine-owned result of the one ordered frame walk performed during recovery.
+ * It folds commit-boundary detection, delta extrema, gap detection and the raw
+ * symbol suffix into the same pass. Running state is checkpointed only at a
+ * commit-bearing frame, so a deferred orphan tail can be scanned without ever
+ * leaking its metadata or symbols into the committed result.
+ */
+final class RecoveredFrameAnalysis implements QuietCloseable {
+
+    private static final int MAX_RAW_BYTES = Integer.MAX_VALUE - 8;
+    private final long ackedFsn;
+    private final int baseline;
+    private long committedBoundaryFsn = -1L;
+    private long committedCoverage;
+    private boolean committedGap;
+    private long committedMaxDeltaEnd;
+    private long committedMaxDeltaStart;
+    private int committedRawCount;
+    private int committedRawLen;
+    private long framesVisited;
+    // Set when a gap-reset rewinds the raw write cursor, cleared at every commit
+    // checkpoint. Still set at finish() => the committed snapshot's bytes were
+    // overwritten by an uncommitted tail and must not be published. See finish().
+    private boolean hasRewoundSinceCommit;
+    private boolean runningGap;
+    private boolean runningUnackedGap;
+    private long runningMaxDeltaEnd;
+    private long runningMaxDeltaStart;
+    private long symbolEntriesVisited;
+    private long rawAddr;
+    private int rawCapacity;
+    private int runningRawCount;
+    private int runningRawLen;
+    private long runningCoverage;
+
+    RecoveredFrameAnalysis(int baseline, long ackedFsn) {
+        this.baseline = baseline;
+        this.ackedFsn = ackedFsn;
+        this.runningCoverage = baseline;
+        this.committedCoverage = baseline;
+    }
+
+    void accept(long fsn, long payload, int payloadLen) {
+        framesVisited++;
+        boolean isQwp = payloadLen >= QwpConstants.HEADER_SIZE
+                && payloadLen > QwpConstants.HEADER_OFFSET_FLAGS
+                && Unsafe.getUnsafe().getInt(payload) == QwpConstants.MAGIC_MESSAGE;
+        byte flags = isQwp
+                ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS)
+                : 0;
+        if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) {
+            foldDelta(fsn, payload + QwpConstants.HEADER_SIZE, payload + payloadLen);
+        }
+
+        // Only a positively identified deferred QWP frame can belong to an
+        // uncommitted tail. Short, foreign or otherwise non-QWP payloads remain
+        // retirement barriers.
+        if (!isQwp || (flags & QwpConstants.FLAG_DEFER_COMMIT) == 0) {
+            committedBoundaryFsn = fsn;
+            committedCoverage = runningCoverage;
+            committedGap = runningGap;
+            committedMaxDeltaEnd = runningMaxDeltaEnd;
+            committedMaxDeltaStart = runningMaxDeltaStart;
+            committedRawLen = runningRawLen;
+            committedRawCount = runningRawCount;
+            hasRewoundSinceCommit = false;
+        }
+    }
+
+    void addDecodedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeSymbols(target);
+    }
+
+    private void decodeSymbols(GlobalSymbolDictionary target) {
+        long p = rawAddr;
+        long limit = rawAddr + committedRawLen;
+        for (int i = 0; i < committedRawCount; i++) {
+            long encoded = readVarint(p, limit);
+            if (encoded < 0L) {
+                throw new IllegalStateException("malformed cached symbol dictionary suffix");
+            }
+            int varintLen = (int) (encoded & 7L);
+            long symbolLen = encoded >>> 3;
+            p += varintLen;
+            if (symbolLen > limit - p) {
+                throw new IllegalStateException("truncated cached symbol dictionary suffix");
+            }
+            target.addRecoveredSymbol(Utf8s.stringFromUtf8Bytes(p, p + symbolLen));
+            p += symbolLen;
+        }
+        if (p != limit) {
+            throw new IllegalStateException("overfilled cached symbol dictionary suffix");
+        }
+    }
+
+    int baseline() {
+        return baseline;
+    }
+
+    long commitBoundaryFsn() {
+        return committedBoundaryFsn;
+    }
+
+    long coverage() {
+        return committedGap ? -1L : committedCoverage;
+    }
+
+    long framesVisited() {
+        return framesVisited;
+    }
+
+    /**
+     * Discards native bytes accumulated only while scanning an uncommitted
+     * deferred tail. Call exactly once after the ordered recovery walk.
+     */
+    void finish() {
+        if (hasRewoundSinceCommit) {
+            // A gap-reset rewound the write cursor after the last commit checkpoint, so
+            // the deferred tail has overwritten the bytes committedRawLen/Count still
+            // describe. Publishing them would decode the TAIL's entries under the
+            // committed ids -- exactly the silent misattribution this analysis exists to
+            // prevent. Fail clean instead: no suffix, and a gap so coverage() reports -1
+            // and the producer falls back to the persisted prefix (or, when unacked
+            // frames depend on the missing ids, quarantines).
+            committedRawLen = 0;
+            committedRawCount = 0;
+            committedGap = true;
+        }
+        if (rawCapacity == committedRawLen) {
+            return;
+        }
+        if (committedRawLen == 0) {
+            Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawAddr = 0L;
+            rawCapacity = 0;
+            return;
+        }
+        rawAddr = Unsafe.realloc(
+                rawAddr,
+                rawCapacity,
+                committedRawLen,
+                MemoryTag.NATIVE_DEFAULT);
+        rawCapacity = committedRawLen;
+    }
+
+    long maxDeltaEnd() {
+        return committedMaxDeltaEnd;
+    }
+
+    long maxDeltaStart() {
+        return committedMaxDeltaStart;
+    }
+
+    long rawAddr() {
+        return rawAddr;
+    }
+
+    int rawCount() {
+        return committedRawCount;
+    }
+
+    int rawLen() {
+        return committedRawLen;
+    }
+
+    int rawCapacity() {
+        return rawCapacity;
+    }
+
+    /**
+     * Releases the cached suffix after a foreground producer and its one send
+     * loop have both consumed it. Recovery metadata remains available, but the
+     * raw entries must not be requested again.
+     */
+    void releaseRawStorage() {
+        if (rawAddr != 0L) {
+            Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawAddr = 0L;
+            rawCapacity = 0;
+        }
+        runningRawLen = 0;
+        runningRawCount = 0;
+        committedRawLen = 0;
+        committedRawCount = 0;
+    }
+
+    long symbolEntriesVisited() {
+        return symbolEntriesVisited;
+    }
+
+    @Override
+    public void close() {
+        releaseRawStorage();
+    }
+
+    /**
+     * Appends one contiguous run of {@code count} wire entries -- {@code [len][utf8]}
+     * repeated, exactly as a delta section carries them -- in a single copy. Callers
+     * pass a whole frame's new-symbol suffix at once; see {@link #foldDelta} for why
+     * that suffix is always contiguous.
+     */
+    private void appendRaw(long addr, int len, int count) {
+        long required = (long) runningRawLen + len;
+        if (required > MAX_RAW_BYTES) {
+            throw new IllegalStateException("recovered symbol dictionary suffix exceeds maximum size "
+                    + "[required=" + required + ", max=" + MAX_RAW_BYTES + ']');
+        }
+        if (required > rawCapacity) {
+            long newCapacity = Math.max(required, Math.max(4_096L, (long) rawCapacity * 2L));
+            if (newCapacity > MAX_RAW_BYTES) {
+                newCapacity = MAX_RAW_BYTES;
+            }
+            rawAddr = Unsafe.realloc(rawAddr, rawCapacity, (int) newCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawCapacity = (int) newCapacity;
+        }
+        Unsafe.getUnsafe().copyMemory(addr, rawAddr + runningRawLen, len);
+        runningRawLen += len;
+        runningRawCount += count;
+    }
+
+    private void foldDelta(long fsn, long p, long limit) {
+        long encodedStart = readVarint(p, limit);
+        if (encodedStart < 0L) {
+            markGap(fsn);
+            return;
+        }
+        int startLen = (int) (encodedStart & 7L);
+        long deltaStart = encodedStart >>> 3;
+        p += startLen;
+        if (deltaStart > runningMaxDeltaStart) {
+            runningMaxDeltaStart = deltaStart;
+        }
+
+        long encodedCount = readVarint(p, limit);
+        if (encodedCount < 0L) {
+            markGap(fsn);
+            return;
+        }
+        int countLen = (int) (encodedCount & 7L);
+        long deltaCount = encodedCount >>> 3;
+        p += countLen;
+        long deltaEnd = deltaCount > Long.MAX_VALUE - deltaStart
+                ? Long.MAX_VALUE
+                : deltaStart + deltaCount;
+        if (deltaEnd > runningMaxDeltaEnd) {
+            runningMaxDeltaEnd = deltaEnd;
+        }
+        if (runningGap) {
+            // A full dictionary is a new self-sufficient epoch, but it may only
+            // repair a gap that is entirely behind the durable ACK watermark.
+            // If any gapped frame will replay first, accepting this reset would
+            // hide the unsafe wire-order gap and let the server observe missing
+            // ids before it reaches the full frame.
+            if (deltaStart != 0L || runningUnackedGap) {
+                return;
+            }
+            runningGap = false;
+            runningCoverage = baseline;
+            // Rewinding the write cursor to 0 means the NEXT appendRaw overwrites
+            // [0, committedRawLen) in place -- bytes the committed snapshot still
+            // counts. That is harmless when a commit-bearing frame follows (accept()
+            // re-checkpoints and clears this flag), but a reset inside an uncommitted
+            // deferred TAIL never gets that refresh, and finish() would then hand out
+            // the old counts over the tail's bytes: silent symbol misattribution.
+            hasRewoundSinceCommit = true;
+            runningRawLen = 0;
+            runningRawCount = 0;
+        }
+        if (deltaStart > runningCoverage) {
+            markGap(fsn);
+            return;
+        }
+        // The segment scan already CRC-validated this frame. When its entire
+        // range is covered, entry parsing cannot extend recovery state, so skip
+        // the cardinality-proportional payload walk.
+        if (deltaEnd <= runningCoverage) {
+            return;
+        }
+
+        // runningCoverage is loop-invariant here -- it only advances after the walk --
+        // and id ascends from deltaStart, so `id >= runningCoverage` is a step
+        // predicate: the entries this frame contributes are always ONE contiguous run
+        // at the tail of its delta section. Note where that run starts and copy it in a
+        // single memcpy once the walk succeeds, rather than paying a bound check, a
+        // capacity check and a ~12-byte copyMemory per symbol. Recovery walks the whole
+        // backlog, and the workload this feature exists for introduces a new symbol per
+        // ROW, so that is millions of stub-dispatched small copies where one bulk copy
+        // per frame does the job.
+        //
+        // Deferring the copy past the markGap bail-outs also drops the partial prefix
+        // the per-entry version used to leave behind. That residue was already
+        // unreachable -- a gap pins coverage() at -1, and a later self-sufficient frame
+        // resets runningRawLen/runningRawCount -- so not writing it is equivalent, and
+        // leaves less state to reason about.
+        long id = deltaStart;
+        long suffixStart = 0L;
+        int suffixCount = 0;
+        for (long i = 0; i < deltaCount; i++, id++) {
+            symbolEntriesVisited++;
+            long entryStart = p;
+            long encodedLen = readVarint(p, limit);
+            if (encodedLen < 0L) {
+                markGap(fsn);
+                return;
+            }
+            int varintLen = (int) (encodedLen & 7L);
+            long symbolLen = encodedLen >>> 3;
+            p += varintLen;
+            if (symbolLen > limit - p) {
+                markGap(fsn);
+                return;
+            }
+            p += symbolLen;
+            if (id >= runningCoverage) {
+                if (suffixCount == 0) {
+                    suffixStart = entryStart;
+                }
+                suffixCount++;
+            }
+        }
+        if (suffixCount > 0) {
+            appendRaw(suffixStart, (int) (p - suffixStart), suffixCount);
+        }
+        if (deltaEnd > runningCoverage) {
+            runningCoverage = deltaEnd;
+        }
+    }
+
+    private void markGap(long fsn) {
+        runningGap = true;
+        if (fsn > ackedFsn) {
+            runningUnackedGap = true;
+        }
+    }
+
+    /**
+     * Returns {@code (value << 3) | encodedByteCount}, or {@code -1} for an
+     * unterminated/oversized 32-bit protocol varint.
+     */
+    private static long readVarint(long p, long limit) {
+        long value = 0L;
+        int shift = 0;
+        int bytes = 0;
+        while (p < limit && bytes < 5) {
+            byte b = Unsafe.getUnsafe().getByte(p++);
+            value |= (long) (b & 0x7F) << shift;
+            bytes++;
+            if ((b & 0x80) == 0) {
+                return (value << 3) | bytes;
+            }
+            shift += 7;
+        }
+        return -1L;
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
index c8516c4c..73b3467d 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
@@ -25,6 +25,7 @@
 package io.questdb.client.cutlass.qwp.client.sf.cursor;
 
 import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
 import io.questdb.client.std.ObjList;
 import io.questdb.client.std.QuietCloseable;
 import org.jetbrains.annotations.TestOnly;
@@ -62,6 +63,8 @@ public final class SegmentRing implements QuietCloseable {
     public static final long BACKPRESSURE_NO_SPARE = -1L;
     /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */
     public static final long PAYLOAD_TOO_LARGE = -2L;
+    static final String LEGACY_READER_GUARD_A = ".qwp-v2-guard-a.sfa";
+    static final String LEGACY_READER_GUARD_B = ".qwp-v2-guard-b.sfa";
     private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class);
     // Tally of baseSeq comparisons performed by sortByBaseSeq across every
     // openExisting() recovery on this JVM. Used by SegmentRingTest to
@@ -69,6 +72,7 @@ public final class SegmentRing implements QuietCloseable {
     // (CI runner variance makes elapsed-millisecond bounds flaky). Cheap
     // in production: one volatile-free add per partition pass, dwarfed by
     // the mmap I/O the recovery does on every segment.
+    private static final int SEALED_PREFIX_COMPACTION_THRESHOLD = 64;
     private static long sortComparisons;
     private final long maxBytesPerSegment;
     // Sealed segments in baseSeq order, oldest first. Active is held separately.
@@ -78,6 +82,17 @@ public final class SegmentRing implements QuietCloseable {
     // looks at sealedSegments after observing a higher ackedFsn, by which
     // point the producer thread's add to sealedSegments has retired.
     private final ObjList sealedSegments = new ObjList<>();
+    // Logical deque head. Trimming nulls and advances this cursor instead of
+    // shifting the remaining ObjList on every ACK. When the dead prefix is at
+    // least half the backing list (and non-trivially large), one bulk compaction
+    // reclaims it. Each live reference is therefore moved at most once per
+    // halving, making sequential trim O(N) overall instead of O(N^2).
+    private int sealedHead;
+    // Deterministic performance probes: unlike elapsed-time assertions these
+    // let tests pin binary successor search and amortized prefix compaction.
+    private int lastFindSegmentComparisons;
+    private int lastNextSealedSearchComparisons;
+    private long sealedCompactionMoves;
     // High-water byte offset within the active segment at which we proactively
     // ask the segment manager to provision a spare (if one isn't already
     // installed). Computed once as 3/4 of segment capacity -- leaves the manager
@@ -155,6 +170,21 @@ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) {
      * depends on every segment being readable.
      */
     public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
+        return openExisting(FilesFacade.INSTANCE, sfDir, maxBytesPerSegment);
+    }
+
+    /**
+     * Facade-aware variant of {@link #openExisting(String, long)}. Only the
+     * per-segment {@link MmapSegment#openExisting(FilesFacade, String)} call
+     * takes the facade -- directory enumeration and the empty-segment unlink
+     * stay on {@link Files}, because the seam exists for one purpose: letting a
+     * facade report an inflated stat length so a segment maps past
+     * end-of-file and its recovery reads fault deterministically on ANY
+     * filesystem. That is what regression-tests the per-file mmap-fault skip
+     * below, which on a real deployment only triggers on a sparse
+     * pre-allocation tail (e.g. ZFS) or a file truncated under the mapping.
+     */
+    public static SegmentRing openExisting(FilesFacade ff, String sfDir, long maxBytesPerSegment) {
         if (!Files.exists(sfDir)) {
             return null;
         }
@@ -176,13 +206,20 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
         try {
             try {
                 int rc = 1;
+                // Reused across files: MmapSegment.openExisting publishes the segment
+                // here before returning, so a pre-21 late-delivered unsafe-access fault
+                // that steals the return value still leaves us something to close.
+                final MmapSegment[] inFlight = new MmapSegment[1];
                 while (rc > 0) {
                     String name = Files.utf8ToString(Files.findName(find));
-                    if (name != null && name.endsWith(".sfa")) {
+                    if (name != null
+                            && name.endsWith(".sfa")
+                            && !isLegacyReaderGuard(name)) {
                         String path = sfDir + "/" + name;
                         MmapSegment seg = null;
+                        inFlight[0] = null;
                         try {
-                            seg = MmapSegment.openExisting(path);
+                            seg = MmapSegment.openExisting(ff, path, inFlight);
                             // Filter out empty leftovers -- typically hot-spare
                             // segments the manager pre-allocated for a prior
                             // session that never got rotated into active. They
@@ -207,6 +244,7 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
                                 long torn = seg.tornTailBytes();
                                 seg.close();
                                 seg = null;
+                                inFlight[0] = null;
                                 if (torn > 0) {
                                     Files.rename(path, path + ".corrupt");
                                 } else {
@@ -215,8 +253,9 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
                             } else {
                                 opened.add(seg);
                                 seg = null;
+                                inFlight[0] = null;
                             }
-                        } catch (MmapSegmentException t) {
+                        } catch (Throwable t) {
                             // Per-file data error (bad magic, bad header,
                             // unsupported version, mmap rejection on this one
                             // file). Don't take down recovery for one corrupt
@@ -228,6 +267,24 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
                             // an OOM would just fail again on the next file
                             // while silently leaking the segments we managed
                             // to recover before it.
+                            //
+                            // The mmap-access-fault arm catches Throwable
+                            // rather than just MmapSegmentException because
+                            // pre-21 HotSpot delivers an unsafe-access fault
+                            // ASYNCHRONOUSLY -- the InternalError can surface
+                            // here, in MmapSegment.openExisting's caller,
+                            // instead of inside the handlers that class
+                            // installs around its own reads (see
+                            // MmapSegment.isMmapAccessFault). An unbacked or
+                            // sparse page in ONE segment must still skip only
+                            // that segment; letting the raw InternalError reach
+                            // the outer catch would abort recovery of the whole
+                            // slot and strand every frame the other segments
+                            // still hold.
+                            if (!(t instanceof MmapSegmentException)
+                                    && !MmapSegment.isMmapAccessFault(t)) {
+                                throw t;
+                            }
                             LOG.warn("openExisting: skipping {} -- {}", path, t.toString());
                         } finally {
                             // Close any seg whose ownership wasn't transferred
@@ -236,9 +293,16 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
                             // open and transfer -- most importantly an OOM
                             // from ObjList.add growing its backing array
                             // after the mmap+fd were already acquired.
-                            if (seg != null) {
+                            // seg may still be null while the segment was fully
+                            // constructed: a late-delivered mmap fault can surface at
+                            // openExisting's RETURN, before the assignment. inFlight is
+                            // cleared at every ownership transfer above, so this can
+                            // never close a segment `opened` now holds.
+                            MmapSegment orphan = seg != null ? seg : inFlight[0];
+                            inFlight[0] = null;
+                            if (orphan != null) {
                                 try {
-                                    seg.close();
+                                    orphan.close();
                                 } catch (Throwable closeErr) {
                                     LOG.warn("openExisting: error closing in-flight segment {}",
                                             path, closeErr);
@@ -290,21 +354,29 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
             // the next appendOrFsn returns BACKPRESSURE_NO_SPARE, the manager
             // installs a hot spare, the producer rotates. Same fast path as a
             // mid-life ring.
+            //
+            // Ownership transfers to the ring in ONE step, at the very end. Detaching
+            // the active segment from `opened` any earlier -- as an `opened.remove(last)`
+            // before the ring is fully built did -- puts it beyond the reach of the catch
+            // below while that catch can still fire: `new SegmentRing` and the
+            // `sealedSegments.add` loop both allocate, so an ObjList growth IOOBE or a
+            // native OOM lands there with the active segment referenced by nothing. Its
+            // whole-segment mapping (up to sf_max_bytes) and its fd would leak.
             int last = opened.size() - 1;
-            MmapSegment active = opened.get(last);
-            opened.remove(last);
-            SegmentRing ring = new SegmentRing(active, maxBytesPerSegment);
+            SegmentRing ring = new SegmentRing(opened.get(last), maxBytesPerSegment);
             // Older segments become sealed in baseSeq order.
-            for (int i = 0, n = opened.size(); i < n; i++) {
+            for (int i = 0; i < last; i++) {
                 ring.sealedSegments.add(opened.get(i));
             }
+            // Every segment now belongs to `ring`. Release our references LAST, so a
+            // throw anywhere above still finds all of them in `opened`.
+            opened.clear();
             return ring;
         } catch (Throwable t) {
-            // Close every recovered MmapSegment that's still in `opened`.
-            // After the success path, `opened` no longer contains the active
-            // segment (removed above), but the sealed segments transferred to
-            // ring.sealedSegments are still owned by the ring once it's
-            // returned -- so this catch only fires before the return statement.
+            // Close every recovered MmapSegment that's still in `opened`. On the success
+            // path `opened` was cleared only after the ring took ownership of every
+            // segment, so reaching here always means the ring was never returned and
+            // `opened` is still the sole owner -- including of the active segment.
             for (int i = 0, n = opened.size(); i < n; i++) {
                 try {
                     opened.get(i).close();
@@ -325,6 +397,58 @@ public long ackedFsn() {
         return ackedFsn;
     }
 
+    /**
+     * Installs an on-disk rollback barrier before any v2 segment can be
+     * created or appended. The guards are valid v1 segments so a legacy
+     * reader cannot skip them as an unknown format. Their deliberately
+     * non-contiguous FSN ranges make its recovery fail closed before replay;
+     * current readers recognize the reserved names and omit them from the
+     * logical ring.
+     */
+    static void installLegacyReaderBarrier(String sfDir) {
+        installLegacyReaderGuardIfDamaged(sfDir + '/' + LEGACY_READER_GUARD_A, Long.MAX_VALUE - 4L);
+        installLegacyReaderGuardIfDamaged(sfDir + '/' + LEGACY_READER_GUARD_B, Long.MAX_VALUE - 1L);
+    }
+
+    /**
+     * Verifies the guard at {@code path} and rewrites it only when it is missing or
+     * damaged.
+     * 

+     * A guard's content never changes, so re-creating one that is already intact buys
+     * nothing -- and costs the only window in which the barrier can be lost. Creation
+     * goes through {@code openCleanRW} (which truncates) and does not fsync, so an
+     * unconditional rewrite on every engine open re-opens that window every time. A host
+     * crash inside it leaves a zero-filled guard next to v2 segment data that is already
+     * durable; a rolled-back v1 reader then skips the guard (bad magic) AND the v2
+     * segments (bad version), concludes the slot is empty, and truncates the unacked log
+     * the barrier exists to protect. Keeping an intact guard means the steady state never
+     * enters that window at all.
+     */
+    private static void installLegacyReaderGuardIfDamaged(String path, long baseSeq) {
+        if (isIntactLegacyReaderGuard(path, baseSeq)) {
+            return;
+        }
+        MmapSegment.createLegacyReaderGuard(path, baseSeq);
+    }
+
+    private static boolean isIntactLegacyReaderGuard(String path, long baseSeq) {
+        if (!Files.exists(path)) {
+            return false;
+        }
+        try (MmapSegment guard = MmapSegment.openExisting(path)) {
+            // Anything short of the exact shape createLegacyReaderGuard writes -- a
+            // zero-filled survivor of a torn creation included -- is rewritten.
+            return guard != null
+                    && guard.version() == MmapSegment.LEGACY_VERSION
+                    && guard.baseSeq() == baseSeq
+                    && guard.frameCount() == 1
+                    && guard.tornTailBytes() == 0L;
+        } catch (Throwable t) {
+            LOG.warn("legacy-reader guard {} could not be verified ({}); rewriting it", path, t.toString());
+            return false;
+        }
+    }
+
     /**
      * I/O thread (or anyone tracking ACK) advances the ACK cursor. {@code seq}
      * is cumulative -- the server has confirmed every FSN up to and including
@@ -439,13 +563,14 @@ public synchronized void close() {
             hotSpare.close();
             hotSpare = null;
         }
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+        for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
             MmapSegment s = sealedSegments.get(i);
             if (s != null) {
                 s.close();
             }
         }
         sealedSegments.clear();
+        sealedHead = 0;
     }
 
     /**
@@ -462,9 +587,10 @@ public synchronized ObjList drainTrimmable() {
         // Sealed segments are in baseSeq order, oldest first; once we hit one
         // that isn't fully acked, none of the later ones can be either.
         // Synchronized so the I/O thread's snapshotSealedSegments() can't
-        // race against the remove(0) shuffling slots underneath it.
-        while (sealedSegments.size() > 0) {
-            MmapSegment s = sealedSegments.get(0);
+        // race against logical-head advancement or a bulk prefix compaction.
+        int n = sealedSegments.size();
+        while (sealedHead < n) {
+            MmapSegment s = sealedSegments.get(sealedHead);
             long lastSeq = s.baseSeq() + s.frameCount() - 1;
             if (lastSeq > acked) {
                 break;
@@ -473,8 +599,9 @@ public synchronized ObjList drainTrimmable() {
                 out = new ObjList<>();
             }
             out.add(s);
-            sealedSegments.remove(0);
+            sealedSegments.setQuick(sealedHead++, null);
         }
+        compactSealedPrefixIfNeeded();
         return out;
     }
 
@@ -490,8 +617,28 @@ public synchronized ObjList drainTrimmable() {
      * scan cost doesn't matter.
      */
     public synchronized MmapSegment findSegmentContaining(long fsn) {
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
-            MmapSegment s = sealedSegments.get(i);
+        // Binary search, for the same reason nextSealedAfter got one: the sealed list
+        // is sorted by baseSeq and its ranges are disjoint, and its length is bounded
+        // only by the documented ~16K-segment ceiling -- so the linear scan this
+        // replaces cost up to 16K comparisons under the ring monitor, on the reconnect
+        // path, right beside a sibling doing the identical predicate in log N.
+        int lo = sealedHead;
+        int hi = sealedSegments.size();
+        int comparisons = 0;
+        while (lo < hi) {
+            int mid = (lo + hi) >>> 1;
+            comparisons++;
+            if (sealedSegments.get(mid).baseSeq() <= fsn) {
+                lo = mid + 1;
+            } else {
+                hi = mid;
+            }
+        }
+        lastFindSegmentComparisons = comparisons;
+        // lo is the first segment whose baseSeq exceeds fsn, so the candidate is the
+        // one before it -- the only sealed segment whose range can contain fsn.
+        if (lo > sealedHead) {
+            MmapSegment s = sealedSegments.get(lo - 1);
             long base = s.baseSeq();
             if (fsn >= base && fsn < base + s.frameCount()) {
                 return s;
@@ -513,37 +660,27 @@ public synchronized MmapSegment findSegmentContaining(long fsn) {
      * fallback -- see {@link #nextSealedAfter(MmapSegment)}.
      */
     public synchronized MmapSegment firstSealed() {
-        return sealedSegments.size() > 0 ? sealedSegments.get(0) : null;
+        return sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null;
     }
 
     /** Active segment -- exposed for the I/O thread's "send next batch" path. */
     /**
-     * Walks every published frame in the ring (sealed segments plus the active
-     * segment) and returns the FSN of the LAST frame whose payload does NOT
-     * carry the given flag bit, or {@code -1} when every published frame
-     * carries it (or the ring is empty). All frames above the returned FSN
-     * carry the flag.
-     * 

-     * Recovery-time helper: locates the last commit-bearing QWP frame below a
-     * potentially orphaned FLAG_DEFER_COMMIT tail left behind by a producer
-     * that crashed (or closed) mid-transaction. Call before the I/O loop and
-     * producer start appending; the walk is not synchronized against appends
-     * into the active segment. See
-     * {@link MmapSegment#findLastFrameFsnWithoutPayloadFlag} for the
-     * positive-identification contract: frames that do not parse as protocol
-     * messages count as commit-bearing (retirement barriers), never as
-     * trimmable.
+     * Performs the one ordered recovery fold across sealed segments and the
+     * active segment. The returned native suffix remains owned by the caller.
      */
-    public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) {
-        long best = -1L;
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
-            long fsn = sealedSegments.get(i).findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen);
-            if (fsn > best) {
-                best = fsn;
+    synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) {
+        RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline, ackedFsn);
+        try {
+            for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
+                sealedSegments.get(i).scanRecovery(analysis);
             }
+            active.scanRecovery(analysis);
+            analysis.finish();
+            return analysis;
+        } catch (Throwable t) {
+            analysis.close();
+            throw t;
         }
-        long fsn = active.findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen);
-        return Math.max(best, fsn);
     }
 
     public MmapSegment getActive() {
@@ -556,7 +693,8 @@ public MmapSegment getActive() {
      * concurrent rotation. Cross-thread readers (typically the I/O loop)
      * should use {@link #snapshotSealedSegments(MmapSegment[])} instead.
      */
-    public ObjList getSealedSegments() {
+    public synchronized ObjList getSealedSegments() {
+        compactSealedPrefix();
         return sealedSegments;
     }
 
@@ -608,13 +746,20 @@ public boolean needsHotSpare() {
      */
     public synchronized MmapSegment nextSealedAfter(MmapSegment current) {
         long currentBase = current.baseSeq();
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
-            MmapSegment s = sealedSegments.get(i);
-            if (s.baseSeq() > currentBase) {
-                return s;
+        int lo = sealedHead;
+        int hi = sealedSegments.size();
+        int comparisons = 0;
+        while (lo < hi) {
+            int mid = (lo + hi) >>> 1;
+            comparisons++;
+            if (sealedSegments.get(mid).baseSeq() <= currentBase) {
+                lo = mid + 1;
+            } else {
+                hi = mid;
             }
         }
-        return null;
+        lastNextSealedSearchComparisons = comparisons;
+        return lo < sealedSegments.size() ? sealedSegments.get(lo) : null;
     }
 
     /**
@@ -663,15 +808,15 @@ public void setManagerWakeup(Runnable wakeup) {
      * the I/O loop is about to do.
      */
     public synchronized int snapshotSealedSegments(MmapSegment[] target) {
-        int n = sealedSegments.size();
+        int n = sealedSegments.size() - sealedHead;
         if (n > target.length) {
             for (int i = 0; i < target.length; i++) {
-                target[i] = sealedSegments.get(i);
+                target[i] = sealedSegments.get(sealedHead + i);
             }
             return -1;
         }
         for (int i = 0; i < n; i++) {
-            target[i] = sealedSegments.get(i);
+            target[i] = sealedSegments.get(sealedHead + i);
         }
         return n;
     }
@@ -689,13 +834,28 @@ public synchronized long totalSegmentBytes() {
         if (a != null) total += a.sizeBytes();
         MmapSegment hs = hotSpare;
         if (hs != null) total += hs.sizeBytes();
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+        for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
             MmapSegment s = sealedSegments.get(i);
             if (s != null) total += s.sizeBytes();
         }
         return total;
     }
 
+    @TestOnly
+    public synchronized int getLastFindSegmentComparisons() {
+        return lastFindSegmentComparisons;
+    }
+
+    @TestOnly
+    public synchronized int getLastNextSealedSearchComparisons() {
+        return lastNextSealedSearchComparisons;
+    }
+
+    @TestOnly
+    public synchronized long getSealedCompactionMoves() {
+        return sealedCompactionMoves;
+    }
+
     /**
      * Returns the cumulative count of baseSeq comparisons performed by
      * {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()}
@@ -717,6 +877,38 @@ public static void resetSortComparisons() {
         sortComparisons = 0;
     }
 
+    private void compactSealedPrefix() {
+        if (sealedHead == 0) {
+            return;
+        }
+        int size = sealedSegments.size();
+        int remaining = size - sealedHead;
+        if (remaining == 0) {
+            // Every retired slot was nulled while draining, so resetting pos
+            // neither retains references nor has to clear the backing array.
+            sealedSegments.setPos(0);
+        } else {
+            sealedCompactionMoves += remaining;
+            for (int i = 0; i < remaining; i++) {
+                sealedSegments.setQuick(i, sealedSegments.get(sealedHead + i));
+            }
+            for (int i = remaining; i < size; i++) {
+                sealedSegments.setQuick(i, null);
+            }
+            sealedSegments.setPos(remaining);
+        }
+        sealedHead = 0;
+    }
+
+    private void compactSealedPrefixIfNeeded() {
+        int size = sealedSegments.size();
+        if (sealedHead == size
+                || (sealedHead >= SEALED_PREFIX_COMPACTION_THRESHOLD
+                && sealedHead >= size - sealedHead)) {
+            compactSealedPrefix();
+        }
+    }
+
     /**
      * In-place quicksort over {@code list[lo, hi)} keyed by ascending
      * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²)
@@ -767,6 +959,18 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) {
         }
     }
 
+    /**
+     * Whether {@code name} is one of the reserved rollback-barrier files this class
+     * plants in every disk-mode slot. They carry the {@code .sfa} suffix so a rolled-back
+     * v1 reader cannot skip them, which means every directory walk that reasons about
+     * "does this slot hold data" has to exclude them -- they are a barrier, not a log.
+     * Package-private so {@link OrphanScanner} shares this one definition rather than
+     * matching the names itself.
+     */
+    static boolean isLegacyReaderGuard(String name) {
+        return LEGACY_READER_GUARD_A.equals(name) || LEGACY_READER_GUARD_B.equals(name);
+    }
+
     private static void swap(ObjList list, int i, int j) {
         if (i == j) return;
         MmapSegment tmp = list.get(i);
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
index 0b8379de..e03a7996 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
@@ -26,17 +26,25 @@
 
 import io.questdb.client.std.Compat;
 import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.QuietCloseable;
 import io.questdb.client.std.Unsafe;
+import org.jetbrains.annotations.TestOnly;
 
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
- * Advisory exclusive lock for a single SF slot directory.
+ * Advisory exclusive locks for a single SF slot.
  * 

- * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx}
- * for the entire lifetime of the engine that owns the slot. The lock is
+ * {@link #acquire(String)} locks a {@code .lock} file inside the slot directory
+ * for the entire lifetime of the engine that owns it. {@link #acquireLogical(String)}
+ * locks a sibling file under the parent SF directory for short-lived pathname
+ * transitions and orphan adoption; because it is outside the slot directory,
+ * it remains stable if that directory is renamed. Both use
+ * {@code flock}/{@code LockFileEx}. A lock is
  * automatically released when the fd is closed — including on hard process
  * exit, since the kernel cleans up file locks for terminated processes.
  * 

@@ -58,10 +66,13 @@ public final class SlotLock implements QuietCloseable {
 
     private static final String LOCK_FILE_NAME = ".lock";
     private static final String LOCK_PID_FILE_NAME = ".lock.pid";
+    private static final String LOGICAL_LOCK_DIR_NAME = ".slot-locks";
+    private final FilesFacade ff;
     private final String slotDir;
     private int fd;
 
-    private SlotLock(String slotDir, int fd) {
+    private SlotLock(FilesFacade ff, String slotDir, int fd) {
+        this.ff = ff;
         this.slotDir = slotDir;
         this.fd = fd;
     }
@@ -75,42 +86,185 @@ private SlotLock(String slotDir, int fd) {
      *                               or lock contention.
      */
     public static SlotLock acquire(String slotDir) {
+        validateSlotDir(slotDir);
+        // DIR_MODE_DEFAULT is right here: one process creates its own slot
+        // directory and only that process writes inside it.
+        ensureDirectory(FilesFacade.INSTANCE, slotDir, "slot dir", Files.DIR_MODE_DEFAULT);
+        String lockPath = slotDir + "/" + LOCK_FILE_NAME;
+        String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME;
+        return acquireAt(FilesFacade.INSTANCE, slotDir, lockPath, pidPath);
+    }
+
+    /**
+     * Acquires the stable logical lock for {@code slotDir}. Unlike the
+     * directory-local {@code .lock}, this lock is anchored in the parent SF
+     * directory, so renaming the slot cannot move the lock inode away from the
+     * logical slot name it guards.
+     * 

+     * Callers use this as a short-lived transition/adoption lock, always before
+     * acquiring the directory-local lock. In particular it must cover an
+     * unreplayable slot's close -> rename -> recreate transition, preventing a
+     * queued orphan drainer from adopting the renamed inode and later touching
+     * the fresh directory through the old pathname.
+     */
+    public static SlotLock acquireLogical(String slotDir) {
+        return acquireLogical(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /** Facade-aware variant used to exercise logical-lock I/O failures. */
+    @TestOnly
+    public static SlotLock acquireLogical(FilesFacade ff, String slotDir) {
+        validateSlotDir(slotDir);
+        String[] paths = resolveLogicalLock(slotDir);
+        if (paths == null) {
+            throw new IllegalArgumentException(
+                    "slotDir must contain a parent and slot name: " + slotDir);
+        }
+        // DIR_MODE_SHARED, not DIR_MODE_DEFAULT: unlike a slot directory -- which
+        // one process creates and only that process writes -- this directory is
+        // shared by every sender under the same sf_dir, and each of them must
+        // CREATE its own lock file inside it. At 0755 the first process to start
+        // owns it and a sender running as a different uid cannot create its lock
+        // file at all, so build() throws before it can open the slot. Passing
+        // 0777 hands the sharing policy to the deployment's umask, which is what
+        // already governs sf_dir itself: an unset umask leaves this identical to
+        // the old 0755, while a shared-group deployment (umask 002) gets the
+        // group-writable directory it needs.
+        ensureDirectory(ff, paths[0], "logical slot lock dir", Files.DIR_MODE_SHARED);
+        return acquireAt(ff, slotDir, paths[1], paths[2]);
+    }
+
+    /**
+     * Best-effort removal of the parent-anchored logical lock files
+     * ({@code /.slot-locks/.lock} and its {@code .lock.pid}
+     * sidecar) for {@code slotDir}, mirroring {@link AckWatermark#removeOrphan}
+     * and {@link PersistedSymbolDict#removeOrphan} for the slot's in-directory
+     * side-files. The fully-drained close that permanently retires a slot calls
+     * this: the logical lock lives OUTSIDE the slot dir (so it survives a slot
+     * rename), so nothing else reclaims it, and without this
+     * {@code /.slot-locks} accumulates one dead lock+pid pair per
+     * distinct slot name for the lifetime of {@code sf_dir} -- unbounded under
+     * rotating {@code senderId}s.
+     * 

+     * The unlink is best-effort and safe: the retiring engine still holds the
+     * slot's directory-local {@link #acquire} lock (the real multi-writer guard)
+     * across this cleanup, and fully-drained retirement performs no rename, so
+     * the logical lock -- which only guards the close/rename/recreate transition
+     * -- is not in use. An orphan drainer momentarily mid-{@link #acquireLogical}
+     * fails its immediately-following candidacy / directory-lock check (the slot
+     * is gone) and backs off. Unlike {@link #acquireLogical}, an unusable
+     * {@code slotDir} is a silent no-op here rather than a throw.
+     */
+    public static void removeOrphanLogical(String slotDir) {
+        removeOrphanLogical(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /** Facade-aware variant of {@link #removeOrphanLogical(String)}. */
+    public static void removeOrphanLogical(FilesFacade ff, String slotDir) {
         if (slotDir == null || slotDir.isEmpty()) {
-            throw new IllegalArgumentException("slotDir must not be empty");
+            return;
         }
-        if (!Files.exists(slotDir)) {
-            int rc = Files.mkdir(slotDir, Files.DIR_MODE_DEFAULT);
-            if (rc != 0) {
-                throw new IllegalStateException(
-                        "could not create slot dir: " + slotDir + " rc=" + rc);
-            }
+        String[] paths = resolveLogicalLock(slotDir);
+        if (paths == null) {
+            return;
         }
-        String lockPath = slotDir + "/" + LOCK_FILE_NAME;
-        String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME;
-        int fd = Files.openRW(lockPath);
+        // Unlink ONLY while holding the lock. Removing a lock file another party holds
+        // is precisely the attack Files.DIR_MODE_SHARED's javadoc describes: the unlink
+        // does not release that party's flock, but it does free the pathname, so the
+        // next acquirer creates a SECOND inode and locks it successfully -- two owners
+        // of a lock whose only job is mutual exclusion.
+        //
+        // The retiring engine's directory-local lock does not stand in for this. It says
+        // nothing about the LOGICAL lock, which Sender.build() holds across its whole
+        // construct -> connect -> quarantine transition, in a frame ABOVE this one: an
+        // ordinary failed connect closes the engine from inside that scope, reaches here,
+        // and used to unlink the very file build() was holding. Acquiring first turns
+        // that into a no-op instead of silent double-ownership.
+        SlotLock guard;
+        try {
+            guard = acquireAt(ff, slotDir, paths[1], paths[2]);
+        } catch (Throwable t) {
+            // Contended (someone holds it, possibly a caller above us) or unopenable.
+            // Leaving the pair on disk is the safe outcome -- a live holder's lock must
+            // outlive our cleanup. The next fully-drained retirement reclaims it.
+            return;
+        }
+        try {
+            ff.remove(paths[1]);
+            ff.remove(paths[2]);
+        } finally {
+            guard.close();
+        }
+    }
+
+    private static SlotLock acquireAt(FilesFacade ff, String slotDir, String lockPath, String pidPath) {
+        int fd = ff.openRW(lockPath);
         if (fd < 0) {
             throw new IllegalStateException(
                     "could not open slot lock file: " + lockPath);
         }
         boolean ok = false;
         try {
-            int rc = Files.lock(fd);
+            int rc = ff.lock(fd);
             if (rc != 0) {
                 String holder = readHolder(pidPath);
                 throw new IllegalStateException(
                         "sf slot already in use by another process [slot="
                                 + slotDir + ", holder=" + holder + "]");
             }
-            writePid(pidPath);
+            writePid(ff, pidPath);
             ok = true;
-            return new SlotLock(slotDir, fd);
+            return new SlotLock(ff, slotDir, fd);
         } finally {
             if (!ok) {
-                Files.close(fd);
+                ff.close(fd);
             }
         }
     }
 
+    private static void ensureDirectory(FilesFacade ff, String path, String description, int mode) {
+        if (!ff.exists(path)) {
+            int rc = ff.mkdir(path, mode);
+            // Multiple senders may create the shared parent lock directory
+            // concurrently. Treat EEXIST as success, just as the builder does
+            // for the SF root itself.
+            if (rc != 0 && !ff.exists(path)) {
+                throw new IllegalStateException(
+                        "could not create " + description + ": " + path + " rc=" + rc);
+            }
+        }
+    }
+
+    /**
+     * Resolves the parent-anchored logical lock layout for {@code slotDir}:
+     * {@code [0]} the {@code .slot-locks} directory, {@code [1]} the
+     * {@code .lock} path, {@code [2]} the {@code .lock.pid} path.
+     * Returns {@code null} when {@code slotDir} has no usable parent or name.
+     * Shared by {@link #acquireLogical} and {@link #removeOrphanLogical} so the
+     * two can never target different files.
+     */
+    private static String[] resolveLogicalLock(String slotDir) {
+        Path slotPath = Paths.get(slotDir);
+        Path parentPath = slotPath.getParent();
+        Path slotNamePath = slotPath.getFileName();
+        if (parentPath == null || slotNamePath == null || slotNamePath.toString().isEmpty()) {
+            return null;
+        }
+        String logicalLockDir = parentPath + "/" + LOGICAL_LOCK_DIR_NAME;
+        String slotName = slotNamePath.toString();
+        return new String[]{
+                logicalLockDir,
+                logicalLockDir + "/" + slotName + LOCK_FILE_NAME,
+                logicalLockDir + "/" + slotName + LOCK_PID_FILE_NAME
+        };
+    }
+
+    private static void validateSlotDir(String slotDir) {
+        if (slotDir == null || slotDir.isEmpty()) {
+            throw new IllegalArgumentException("slotDir must not be empty");
+        }
+    }
+
     /** Slot dir this lock guards. */
     public String slotDir() {
         return slotDir;
@@ -122,7 +276,7 @@ public void close() {
         // file or the .lock.pid sidecar — a stale PID is harmless (next
         // acquirer overwrites .lock.pid on success).
         if (fd >= 0) {
-            Files.close(fd);
+            ff.close(fd);
             fd = -1;
         }
     }
@@ -152,7 +306,7 @@ private static String readHolder(String pidPath) {
         }
     }
 
-    private static void writePid(String pidPath) {
+    private static void writePid(FilesFacade ff, String pidPath) {
         long pid;
         try {
             pid = Compat.currentPid();
@@ -160,25 +314,25 @@ private static void writePid(String pidPath) {
             // Diagnostic-only — never block lock acquisition on it.
             pid = -1L;
         }
-        int wfd = Files.openRW(pidPath);
+        int wfd = ff.openRW(pidPath);
         if (wfd < 0) {
             // Diagnostic-only — never block lock acquisition on it.
             return;
         }
         try {
-            Files.truncate(wfd, 0L);
+            ff.truncate(wfd, 0L);
             byte[] payload = (pid + "\n").getBytes(StandardCharsets.UTF_8);
             long addr = Unsafe.malloc(payload.length, MemoryTag.NATIVE_DEFAULT);
             try {
                 for (int i = 0; i < payload.length; i++) {
                     Unsafe.getUnsafe().putByte(addr + i, payload[i]);
                 }
-                Files.write(wfd, addr, payload.length, 0L);
+                ff.write(wfd, addr, payload.length, 0L);
             } finally {
                 Unsafe.free(addr, payload.length, MemoryTag.NATIVE_DEFAULT);
             }
         } finally {
-            Files.close(wfd);
+            ff.close(wfd);
         }
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java
new file mode 100644
index 00000000..00c0d351
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java
@@ -0,0 +1,51 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+
+/**
+ * A recovered store-and-forward slot whose symbol dictionary cannot be rebuilt at all --
+ * neither from the persisted side-file's intact prefix nor from the surviving frames' own
+ * delta sections. Its frames reference ids that nothing still holds, so replaying them
+ * would null-pad the gap on the server and silently misattribute symbol values.
+ * 

+ * This is the ONE verdict {@code seedGlobalDictionaryFromPersisted} reaches after it has
+ * tried every source of truth it has, so it is the only signal that may quarantine a slot.
+ * A distinct type, rather than a message match, because the difference between "this slot
+ * is unreplayable" and any other {@link LineSenderException} out of the connect path is the
+ * difference between setting a slot aside and silently discarding one that was fine.
+ * 

+ * It stays a {@code LineSenderException} so that a caller which does NOT handle it -- the
+ * background drainer, a test constructing the sender directly -- keeps the old fail-clean
+ * behaviour rather than seeing a new checked type. Only {@code Sender.build()} treats it as
+ * recoverable, by setting the slot aside and starting the producer on a fresh one.
+ */
+public class UnreplayableSlotException extends LineSenderException {
+
+    public UnreplayableSlotException(CharSequence message) {
+        super(message);
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
index a255e638..7097f3cd 100644
--- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
+++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
@@ -77,6 +77,7 @@ public final class ConfigSchema {
         // (one vocabulary, side-owned application).
         str("max_frame_rejections", Side.INGRESS);
         str("poison_min_escalation_window_millis", Side.INGRESS);
+        str("catchup_cap_gap_min_escalation_window_millis", Side.INGRESS);
         str("max_name_len", Side.INGRESS);
         str("reconnect_initial_backoff_millis", Side.INGRESS);
         str("reconnect_max_backoff_millis", Side.INGRESS);
diff --git a/core/src/main/java/io/questdb/client/std/Crc32c.java b/core/src/main/java/io/questdb/client/std/Crc32c.java
index d0a2e6a8..738efb1b 100644
--- a/core/src/main/java/io/questdb/client/std/Crc32c.java
+++ b/core/src/main/java/io/questdb/client/std/Crc32c.java
@@ -46,6 +46,7 @@
 public final class Crc32c {
     /** Seed value to start a fresh CRC-32C accumulation. */
     public static final int INIT = 0;
+    private static final int[] CRC32C_TABLE = buildCrc32cTable();
 
     private Crc32c() {
     }
@@ -65,6 +66,73 @@ private Crc32c() {
      */
     public static native int update(int seed, long addr, long len);
 
+    /**
+     * Java/Unsafe slice-by-8 CRC-32C for memory that can fault while it is read,
+     * such as a recovery mmap over a sparse or concurrently truncated file.
+     * Keeping every load at an {@link Unsafe} intrinsic site lets HotSpot turn an
+     * mmap access fault into a catchable {@link InternalError}; the native
+     * {@link #update} path cannot provide that guarantee because a SIGBUS raised
+     * inside JNI aborts the JVM.
+     * 

+     * The hot loop consumes each 8-byte block with two {@code getInt}s rather than
+     * eight {@code getByte}s -- 9 loads per block instead of 16, for the same table
+     * work. {@code getInt} is an {@link Unsafe} intrinsic exactly as {@code getByte}
+     * is, so the fault-catchability above is unaffected. It does make the loop
+     * little-endian, which costs nothing here: the segment and dictionary formats this
+     * checksums are already little-endian throughout (their headers are read back with
+     * {@code getInt}), and the native twin this must agree with asserts the same.
+     *
+     * @param seed previous CRC value, or {@link #INIT} to start fresh
+     * @param addr off-heap address of at least {@code len} readable bytes
+     * @param len  number of bytes to consume
+     * @return the new CRC value
+     */
+    public static int updateUnsafe(int seed, long addr, long len) {
+        assert len >= 0L : "CRC length must be non-negative";
+        int crc = ~seed;
+        int[] table = CRC32C_TABLE;
+        while (len >= Long.BYTES) {
+            // Little-endian: lo holds bytes 0..3 and hi bytes 4..7, ascending from the
+            // low byte, so the per-byte table lookups below stay in wire order.
+            int lo = Unsafe.getUnsafe().getInt(addr) ^ crc;
+            int hi = Unsafe.getUnsafe().getInt(addr + Integer.BYTES);
+            crc = table[7 * 256 + (lo & 0xFF)]
+                    ^ table[6 * 256 + ((lo >>> 8) & 0xFF)]
+                    ^ table[5 * 256 + ((lo >>> 16) & 0xFF)]
+                    ^ table[4 * 256 + (lo >>> 24)]
+                    ^ table[3 * 256 + (hi & 0xFF)]
+                    ^ table[2 * 256 + ((hi >>> 8) & 0xFF)]
+                    ^ table[256 + ((hi >>> 16) & 0xFF)]
+                    ^ table[hi >>> 24];
+            addr += Long.BYTES;
+            len -= Long.BYTES;
+        }
+        while (len-- > 0L) {
+            crc = (crc >>> 8) ^ table[(crc ^ Unsafe.getUnsafe().getByte(addr++)) & 0xFF];
+        }
+        return ~crc;
+    }
+
+    private static int[] buildCrc32cTable() {
+        int[] table = new int[8 * 256];
+        for (int n = 0; n < 256; n++) {
+            int c = n;
+            for (int k = 0; k < 8; k++) {
+                c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1);
+            }
+            table[n] = c;
+        }
+        for (int slice = 1; slice < 8; slice++) {
+            int previousOffset = (slice - 1) * 256;
+            int offset = slice * 256;
+            for (int n = 0; n < 256; n++) {
+                int previous = table[previousOffset + n];
+                table[offset + n] = (previous >>> 8) ^ table[previous & 0xFF];
+            }
+        }
+        return table;
+    }
+
     static {
         Os.init();
     }
diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java
index 02756c37..d3f91fb4 100644
--- a/core/src/main/java/io/questdb/client/std/Files.java
+++ b/core/src/main/java/io/questdb/client/std/Files.java
@@ -75,6 +75,39 @@ public final class Files {
      * others may traverse and read but not modify. 0755 octal
      */
     public static final int DIR_MODE_DEFAULT = 493;
+    /**
+     * Creation mode for a directory that processes running as DIFFERENT users must all
+     * create entries in: the store-and-forward root and its logical slot-lock directory.
+     * 01777 octal -- 0777 plus the sticky bit.
+     * 

+     * The rwx widening is the load-bearing part and it works: the deployment's umask
+     * decides the sharing policy, so the usual 022 yields 0755 (identical to
+     * {@link #DIR_MODE_DEFAULT}, the common case unchanged) while a shared-group deployment
+     * (umask 002) gets the 0775 a second uid needs to create its own entries here.
+     * 

+     * The sticky bit is best-effort, NOT a guarantee this client relies on. On
+     * Linux {@code mkdir(2)} sets the new directory to {@code (mode & ~umask & 0777)} -- the
+     * {@code & 0777} strips {@code S_ISVTX}, so the bit is silently dropped on the primary
+     * deployment platform (POSIX leaves sticky-on-mkdir unspecified; some platforms honor
+     * it, Linux does not). It is therefore wrong to depend on restricted-deletion semantics
+     * for correctness. Mutual exclusion of the slot lock does NOT rest on it: the safety
+     * comes from never unlinking a lock file this process does not hold
+     * ({@code SlotLock.removeOrphanLogical} acquires the lock before removing it), which
+     * holds regardless of directory permissions. Defending a multi-tenant host against a
+     * hostile local uid unlinking another process's lock would need the sticky bit applied
+     * by an explicit {@code chmod} after {@code mkdir} (and on an already-existing directory
+     * on upgrade, since {@code mkdir} does not touch it) -- there is no {@code chmod} binding
+     * in this class today, so that protection is not provided and must not be assumed.
+     * 

+     * This must be applied to EVERY directory on the path a foreign uid has to create in.
+     * Applying it only to {@code .slot-locks} -- while {@code sf_dir} itself stays 0755 --
+     * is a half-measure: umask can only clear bits, so under umask 002 the lock directory
+     * becomes group-writable while sf_dir does not, and the second uid still cannot create
+     * its slot directory. build() then fails one level before the problem this mode
+     * exists to solve. Do NOT use it for a directory only its creator writes to; the slot
+     * directory itself stays {@link #DIR_MODE_DEFAULT}.
+     */
+    public static final int DIR_MODE_SHARED = 1023;
 
     /** {@code dirent.d_type} sentinel: type unknown (filesystem doesn't fill it). */
     public static final int DT_UNKNOWN = 0;
diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java
index 1b408cf4..c5e6afc7 100644
--- a/core/src/main/java/io/questdb/client/std/FilesFacade.java
+++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java
@@ -85,6 +85,16 @@ public interface FilesFacade {
 
     int fsync(int fd);
 
+    /**
+     * Whether callers should use this facade's mmap path. The production facade
+     * returns {@code true}; wrapping fault facades retain positioned I/O unless
+     * they explicitly opt in, preserving their ability to inject short reads and
+     * writes.
+     */
+    default boolean isMmapAllowed() {
+        return this == INSTANCE;
+    }
+
     long length(int fd);
 
     /**
@@ -107,6 +117,19 @@ public interface FilesFacade {
 
     int mkdir(String path, int mode);
 
+    /**
+     * Maps a file region. Kept on the facade so mmap failures can be injected
+     * without relying on platform-specific filesystem behavior.
+     */
+    default long mmap(int fd, long len, long offset, int flags, int memoryTag) {
+        return Files.mmap(fd, len, offset, flags, memoryTag);
+    }
+
+    /** Releases a region returned by {@link #mmap(int, long, long, int, int)}. */
+    default void munmap(long address, long len, int memoryTag) {
+        Files.munmap(address, len, memoryTag);
+    }
+
     int openCleanRW(String path);
 
     /**
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java
index a44c57f2..93a380f8 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java
@@ -32,6 +32,7 @@
 import io.questdb.client.network.PlainSocketFactory;
 import io.questdb.client.network.Socket;
 import io.questdb.client.network.SocketReadinessWaiter;
+import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.Unsafe;
 import org.junit.Assert;
 import org.junit.Test;
@@ -312,6 +313,41 @@ public void onClose(int code, String reason) {
         });
     }
 
+    @Test
+    public void testSendBinaryAssemblesTwoPayloadSlicesIntoOneMaskedFrame() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingSocket socket = new CapturingSocket();
+            long first = Unsafe.malloc(3, MemoryTag.NATIVE_DEFAULT);
+            long second = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT);
+            try (CapturingWebSocketClient client = new CapturingWebSocketClient(socket)) {
+                setUpgradedTrue(client);
+                for (int i = 0; i < 3; i++) {
+                    Unsafe.getUnsafe().putByte(first + i, (byte) (i + 1));
+                }
+                for (int i = 0; i < 4; i++) {
+                    Unsafe.getUnsafe().putByte(second + i, (byte) (i + 4));
+                }
+
+                client.sendBinary(first, 3, second, 4);
+
+                byte[] frame = socket.sent;
+                Assert.assertNotNull(frame);
+                Assert.assertEquals((byte) 0x82, frame[0]);
+                Assert.assertEquals((byte) (0x80 | 7), frame[1]);
+                byte[] payload = new byte[7];
+                for (int i = 0; i < payload.length; i++) {
+                    payload[i] = (byte) (frame[6 + i] ^ frame[2 + (i & 3)]);
+                }
+                Assert.assertArrayEquals(
+                        new byte[]{1, 2, 3, 4, 5, 6, 7},
+                        payload);
+            } finally {
+                Unsafe.free(first, 3, MemoryTag.NATIVE_DEFAULT);
+                Unsafe.free(second, 4, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
     @Test
     public void testSendCloseFrameDoesNotClobberSendBuffer() throws Exception {
         assertMemoryLeak(() -> {
@@ -463,6 +499,34 @@ public boolean wantsTlsWrite() {
         }
     }
 
+    private static class CapturingSocket extends FakeSocket {
+        private byte[] sent;
+
+        @Override
+        public int send(long bufferPtr, int bufferLen) {
+            sent = new byte[bufferLen];
+            for (int i = 0; i < bufferLen; i++) {
+                sent[i] = Unsafe.getUnsafe().getByte(bufferPtr + i);
+            }
+            return bufferLen;
+        }
+    }
+
+    private static class CapturingWebSocketClient extends WebSocketClient {
+
+        CapturingWebSocketClient(CapturingSocket socket) {
+            super(DefaultHttpClientConfiguration.INSTANCE, (nf, log) -> socket);
+        }
+
+        @Override
+        protected void ioWait(int timeout, int op) {
+        }
+
+        @Override
+        protected void setupIoWait() {
+        }
+    }
+
     /**
      * Socket that serves a fixed byte sequence from recv() and reports
      * every send() as fully written (so close-frame sends succeed).
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
index 2a266212..639b7c8d 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
@@ -30,6 +30,7 @@
 import io.questdb.client.SenderErrorHandler;
 import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
 import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
 import org.jetbrains.annotations.NotNull;
 import org.junit.Assert;
@@ -37,14 +38,18 @@
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * Pins both branches of {@code QwpWebSocketSender.close()}'s safety-net
- * rethrow, strictly — unlike the close assertions in
- * {@link InitialConnectAsyncTest}, which tolerate either outcome:
+ * rethrow using a deterministic server-side parse rejection:
  *