diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 85c4d1a59d1d..3c30c039d02a 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -32,8 +32,10 @@ import { ServerConfig } from "../config.ts"; import { gitCommandDuration } from "../observability/Metrics.ts"; import { makeGitVcsDriverCore, + makeStatusRemoteRefreshLedger, parseGitCheckoutProgressLine, splitNullSeparatedGitStdoutPaths, + STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY, } from "./GitVcsDriverCore.ts"; import * as GitVcsDriver from "./GitVcsDriver.ts"; @@ -818,10 +820,481 @@ it.effect("backs off and logs failed fetch attempts across linked worktrees", () yield* readRemoteStatus(cwd); assert.equal(yield* Ref.get(fetchAttempts), 3); assert.lengthOf(warnings, 3); + + // Fast failures keep backing off. They do not stop the remote. + yield* TestClock.adjust("119 seconds"); + yield* readRemoteStatus(worktreePath); + assert.equal(yield* Ref.get(fetchAttempts), 3); + + yield* TestClock.adjust("1 second"); + yield* readRemoteStatus(cwd); + assert.equal(yield* Ref.get(fetchAttempts), 4); }), ).pipe(Effect.provide(ServerConfigLayer.pipe(Layer.provideMerge(NodeServices.layer)))), ); +/** + * Builds a repository whose background status fetch can be hung or released. + * Used to check that repeated timeouts stop polling and that pull, push, or + * fetch lets that poll run again. + */ +const makeUpstreamRefreshFixture = () => + Effect.gen( + /** + * Creates a repository and a spawner that can hang the background status fetch. + */ + function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const fetchAttempts = yield* Ref.make(0); + const hangStatusFetches = yield* Ref.make(true); + const fetchStartedRef = yield* Ref.make(yield* Deferred.make()); + const statusFetches: Array<{ + readonly args: ReadonlyArray; + readonly env: Record | undefined; + }> = []; + const pullCommands: Array<{ + readonly args: ReadonlyArray; + readonly env: Record | undefined; + }> = []; + const warnings: string[] = []; + const logger = Logger.make(({ message }) => { + warnings.push(String(message)); + }); + const spawner = ChildProcessSpawner.make( + /** + * Hangs status fetches until the test clock hits the timeout, and runs other Git commands for real. + */ + (command) => + Effect.gen( + /** + * Records status-fetch and pull commands, then either hangs or returns success. + */ + function* () { + if (!ChildProcess.isStandardCommand(command)) { + return yield* Effect.die("expected a standard Git command"); + } + const isStatusFetch = + command.args.includes("fetch") && command.args.includes("--no-auto-gc"); + if (isStatusFetch) { + statusFetches.push({ args: command.args, env: command.options.env }); + yield* Ref.update(fetchAttempts, (count) => count + 1); + if (yield* Ref.get(hangStatusFetches)) { + yield* Deferred.succeed(yield* Ref.get(fetchStartedRef), undefined).pipe( + Effect.ignore, + ); + return ChildProcessSpawner.makeHandle({ + ...makeNonRepositoryHandle(), + exitCode: Effect.never, + }); + } + return makeSuccessfulHandle(""); + } + if (command.args[0] === "pull") { + pullCommands.push({ args: command.args, env: command.options.env }); + } + return yield* delegate.spawn(command); + }, + ), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + /** + * Runs a setup Git command through the driver. + */ + const runGit = (workingDirectory: string, args: ReadonlyArray) => + driver.execute({ + operation: "GitVcsDriver.test.upstreamRefreshStop", + cwd: workingDirectory, + args, + timeoutMs: 10_000, + }); + yield* driver.initRepo({ cwd }); + yield* runGit(cwd, ["config", "user.email", "test@test.com"]); + yield* runGit(cwd, ["config", "user.name", "Test"]); + yield* writeTextFile(cwd, "README.md", "# test\n"); + yield* runGit(cwd, ["add", "."]); + yield* runGit(cwd, ["commit", "-m", "initial commit"]); + const initialBranch = (yield* runGit(cwd, ["branch", "--show-current"])).stdout.trim(); + yield* runGit(remote, ["init", "--bare"]); + yield* runGit(cwd, ["remote", "add", "origin", remote]); + yield* runGit(cwd, ["push", "-u", "origin", initialBranch]); + yield* Ref.set(fetchAttempts, 0); + statusFetches.length = 0; + + /** + * Captures background-fetch warnings from a status read. + */ + const withLogger = (effect: Effect.Effect) => + effect.pipe(Effect.provideService(Logger.CurrentLoggers, new Set([logger]))); + /** + * Reads status while the status fetch is hung, then advances the test clock + * until that fetch hits the status timeout. + */ + const readUntilFetchTimeout = () => + Effect.gen( + /** + * Waits until the hung status fetch starts, then fires its timeout. + */ + function* () { + const started = yield* Deferred.make(); + yield* Ref.set(fetchStartedRef, started); + const fiber = yield* withLogger(driver.statusDetailsRemote(cwd)).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(started); + // The timeout is armed when the hung fetch is spawned. A second tick covers + // the case where that registration lands after the first adjustment. + yield* TestClock.adjust("5 seconds"); + yield* TestClock.adjust("5 seconds"); + return yield* Fiber.join(fiber); + }, + ); + + const previousEnv = { + GCM_INTERACTIVE: process.env.GCM_INTERACTIVE, + GIT_TERMINAL_PROMPT: process.env.GIT_TERMINAL_PROMPT, + }; + process.env.GCM_INTERACTIVE = "always"; + process.env.GIT_TERMINAL_PROMPT = "1"; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + for (const key of ["GCM_INTERACTIVE", "GIT_TERMINAL_PROMPT"] as const) { + const previous = previousEnv[key]; + if (previous === undefined) delete process.env[key]; + else process.env[key] = previous; + } + }), + ); + + return { + driver, + cwd, + fetchAttempts, + hangStatusFetches, + fetchStartedRef, + warnings, + statusFetches, + pullCommands, + withLogger, + readUntilFetchTimeout, + }; + }, + ); + +/** + * Drives three consecutive background status timeouts so later steps can check + * that polling has stopped. + */ +const stopUpstreamRefresh = ( + fixture: Effect.Success>, +) => + Effect.gen( + /** + * Times out the background status fetch three times, the point where polling stops. + */ + function* () { + yield* fixture.readUntilFetchTimeout(); + yield* TestClock.adjust("31 seconds"); + yield* fixture.readUntilFetchTimeout(); + yield* TestClock.adjust("61 seconds"); + yield* fixture.readUntilFetchTimeout(); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 3); + }, + ); + +it.effect( + "evicts idle upstream refresh generations and keeps in-flight ones", + /** + * Checks idle generation entries stay within capacity while an in-flight fetch keeps its generation. + */ + () => + Effect.sync( + /** + * Fills the ledger, pins an in-flight generation, and rejects a stale failure after eviction. + */ + () => { + const ledger = makeStatusRemoteRefreshLedger(STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY); + for (let index = 0; index < STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY + 1; index += 1) { + ledger.resumeStatusRemoteRefresh(`repo-${index}`); + } + assert.equal( + ledger.statusRemoteRefreshLedgerSize(), + STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY, + ); + assert.equal(ledger.statusRemoteRefreshGeneration("repo-0"), null); + assert.equal( + ledger.statusRemoteRefreshGeneration(`repo-${STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY}`), + 1, + ); + + const pinned = makeStatusRemoteRefreshLedger(1); + const captured = pinned.beginStatusRemoteRefresh("pinned"); + pinned.beginStatusRemoteRefresh("other"); + assert.equal(pinned.statusRemoteRefreshLedgerSize(), 2); + assert.equal(pinned.statusRemoteRefreshGeneration("pinned"), captured); + pinned.resumeStatusRemoteRefresh("pinned"); + assert.equal(pinned.recordStatusRemoteRefreshFailure("pinned", true, captured), null); + assert.equal(pinned.statusRemoteRefreshNeedsAttention("pinned"), false); + assert.equal(pinned.statusRemoteRefreshGeneration("pinned"), captured + 1); + + const streak = makeStatusRemoteRefreshLedger(2); + const generation = streak.beginStatusRemoteRefresh("repo"); + assert.equal(streak.recordStatusRemoteRefreshFailure("repo", true, generation), 1); + assert.equal(streak.recordStatusRemoteRefreshFailure("repo", true, generation), 2); + assert.equal(streak.recordStatusRemoteRefreshFailure("repo", false, generation), 0); + assert.equal(streak.statusRemoteRefreshNeedsAttention("repo"), false); + assert.equal(streak.recordStatusRemoteRefreshFailure("repo", true, generation), 1); + assert.equal(streak.recordStatusRemoteRefreshFailure("repo", true, generation), 2); + assert.equal(streak.recordStatusRemoteRefreshFailure("repo", true, generation), 3); + assert.equal(streak.statusRemoteRefreshNeedsAttention("repo"), true); + streak.finishStatusRemoteRefresh("repo"); + streak.resumeStatusRemoteRefresh("repo"); + assert.equal(streak.statusRemoteRefreshNeedsAttention("repo"), false); + + const evicted = makeStatusRemoteRefreshLedger(1); + const evictedGeneration = evicted.beginStatusRemoteRefresh("repo"); + evicted.finishStatusRemoteRefresh("repo"); + evicted.resumeStatusRemoteRefresh("other"); + assert.equal(evicted.statusRemoteRefreshGeneration("repo"), null); + assert.equal( + evicted.recordStatusRemoteRefreshFailure("repo", true, evictedGeneration), + null, + ); + }, + ), +); + +it.effect( + "stops background upstream refresh after repeated timeouts without clearing credential helpers", + /** + * Checks three timeouts stop polling and the status fetch still leaves credential helpers alone. + */ + () => + Effect.scoped( + Effect.gen( + /** + * Stops the remote, then checks the fetch arguments and that another wait does not fetch. + */ + function* () { + const fixture = yield* makeUpstreamRefreshFixture(); + yield* stopUpstreamRefresh(fixture); + + assert.equal( + fixture.warnings.filter((warning) => + warning.includes("upstream status needs attention"), + ).length, + 1, + ); + const statusFetch = fixture.statusFetches[0]; + assert.ok(statusFetch); + assert.equal(statusFetch.args[0], "--git-dir"); + assert.deepEqual(statusFetch.args.slice(-5), [ + "fetch", + "--quiet", + "--no-tags", + "--no-auto-gc", + "origin", + ]); + assert.isFalse(statusFetch.args.includes("-c")); + assert.isFalse(statusFetch.args.some((arg) => arg.includes("credential.helper"))); + assert.equal(statusFetch.env?.GCM_INTERACTIVE, "never"); + assert.equal(statusFetch.env?.GIT_ASKPASS, ""); + assert.equal(statusFetch.env?.GIT_TERMINAL_PROMPT, "0"); + assert.equal(statusFetch.env?.SSH_ASKPASS, ""); + assert.equal(statusFetch.env?.SSH_ASKPASS_REQUIRE, "never"); + assert.deepEqual( + Object.keys(statusFetch.env ?? {}).filter( + (key) => key.startsWith("GIT_CONFIG_KEY_") && process.env[key] === undefined, + ), + [], + ); + + yield* TestClock.adjust("15 minutes"); + yield* fixture.withLogger(fixture.driver.statusDetailsRemote(fixture.cwd)); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 3); + }, + ), + ).pipe(Effect.provide(ServerConfigLayer.pipe(Layer.provideMerge(NodeServices.layer)))), +); + +it.effect( + "resets the background fetch timeout streak after a successful fetch", + /** + * Checks a successful background fetch clears the timeout streak so later timeouts start over. + */ + () => + Effect.scoped( + Effect.gen( + /** + * Times out twice, succeeds once, then times out again without reaching the stop threshold. + */ + function* () { + const fixture = yield* makeUpstreamRefreshFixture(); + yield* fixture.readUntilFetchTimeout(); + yield* TestClock.adjust("31 seconds"); + yield* fixture.readUntilFetchTimeout(); + yield* TestClock.adjust("61 seconds"); + yield* Ref.set(fixture.hangStatusFetches, false); + + yield* fixture.withLogger(fixture.driver.statusDetailsRemote(fixture.cwd)); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 3); + assert.equal(fixture.warnings.length, 2); + + // A successful fetch is fresh for 15s and clears the timeout streak, so + // later timeouts start over instead of stopping the remote. + yield* TestClock.adjust("31 seconds"); + yield* Ref.set(fixture.hangStatusFetches, true); + yield* fixture.readUntilFetchTimeout(); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 4); + + yield* TestClock.adjust("31 seconds"); + yield* fixture.readUntilFetchTimeout(); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 5); + + yield* TestClock.adjust("15 minutes"); + yield* Ref.set(fixture.hangStatusFetches, false); + yield* fixture.withLogger(fixture.driver.statusDetailsRemote(fixture.cwd)); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 6); + }, + ), + ).pipe(Effect.provide(ServerConfigLayer.pipe(Layer.provideMerge(NodeServices.layer)))), +); + +it.effect( + "resumes background upstream refresh after an explicit pull", + /** + * Checks a successful explicit pull lets background status fetch run again. + */ + () => + Effect.scoped( + Effect.gen( + /** + * Stops polling, pulls without the status-fetch environment, then reads status again. + */ + function* () { + const fixture = yield* makeUpstreamRefreshFixture(); + yield* stopUpstreamRefresh(fixture); + yield* Ref.set(fixture.hangStatusFetches, false); + + const pulled = yield* fixture.driver.pullCurrentBranch(fixture.cwd); + assert.equal(pulled.status, "skipped_up_to_date"); + assert.equal(fixture.pullCommands.length, 1); + assert.deepEqual(fixture.pullCommands[0]?.args, ["pull", "--ff-only"]); + assert.equal(fixture.pullCommands[0]?.env?.GIT_TERMINAL_PROMPT, "1"); + assert.equal(fixture.pullCommands[0]?.env?.GCM_INTERACTIVE, "always"); + assert.isFalse( + fixture.pullCommands[0]?.args.some((arg) => arg.includes("credential.helper")), + ); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 3); + + yield* fixture.withLogger(fixture.driver.statusDetailsRemote(fixture.cwd)); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 4); + }, + ), + ).pipe(Effect.provide(ServerConfigLayer.pipe(Layer.provideMerge(NodeServices.layer)))), +); + +it.effect( + "resumes background upstream refresh after an explicit fetch", + /** + * Checks an explicit remote fetch lets background status fetch run again. + */ + () => + Effect.scoped( + Effect.gen( + /** + * Stops polling, fetches origin, then reads status and expects one new background fetch. + */ + function* () { + const fixture = yield* makeUpstreamRefreshFixture(); + yield* stopUpstreamRefresh(fixture); + yield* Ref.set(fixture.hangStatusFetches, false); + + yield* fixture.driver.fetchRemote({ cwd: fixture.cwd, remoteName: "origin" }); + yield* fixture.withLogger(fixture.driver.statusDetailsRemote(fixture.cwd)); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 4); + }, + ), + ).pipe(Effect.provide(ServerConfigLayer.pipe(Layer.provideMerge(NodeServices.layer)))), +); + +it.effect( + "resumes background upstream refresh after a completed push and not a skipped push", + /** + * Checks a skipped push leaves polling stopped and a completed push resumes it. + */ + () => + Effect.scoped( + Effect.gen( + /** + * Confirms an up-to-date push does not fetch, then pushes a new commit and reads status. + */ + function* () { + const fixture = yield* makeUpstreamRefreshFixture(); + yield* stopUpstreamRefresh(fixture); + + const skipped = yield* fixture.driver.pushCurrentBranch(fixture.cwd, null); + assert.equal(skipped.status, "skipped_up_to_date"); + yield* TestClock.adjust("15 minutes"); + yield* fixture.withLogger(fixture.driver.statusDetailsRemote(fixture.cwd)); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 3); + + yield* Ref.set(fixture.hangStatusFetches, false); + yield* fixture.driver.execute({ + operation: "GitVcsDriver.test.upstreamRefreshPush", + cwd: fixture.cwd, + args: ["commit", "--allow-empty", "-m", "ahead"], + timeoutMs: 10_000, + }); + const pushed = yield* fixture.driver.pushCurrentBranch(fixture.cwd, null); + assert.equal(pushed.status, "pushed"); + yield* fixture.withLogger(fixture.driver.statusDetailsRemote(fixture.cwd)); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 4); + }, + ), + ).pipe(Effect.provide(ServerConfigLayer.pipe(Layer.provideMerge(NodeServices.layer)))), +); + +it.effect( + "ignores a background timeout that finishes after an explicit fetch", + /** + * Checks a timeout that finishes after an explicit fetch cannot restore the stopped state. + */ + () => + Effect.scoped( + Effect.gen( + /** + * Resumes from an explicit fetch while a background timeout is still in flight. + */ + function* () { + const fixture = yield* makeUpstreamRefreshFixture(); + yield* fixture.readUntilFetchTimeout(); + yield* TestClock.adjust("31 seconds"); + yield* fixture.readUntilFetchTimeout(); + yield* TestClock.adjust("61 seconds"); + + const started = yield* Deferred.make(); + yield* Ref.set(fixture.fetchStartedRef, started); + const inFlight = yield* fixture + .withLogger(fixture.driver.statusDetailsRemote(fixture.cwd)) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(started); + yield* Ref.set(fixture.hangStatusFetches, false); + yield* fixture.driver.fetchRemote({ cwd: fixture.cwd, remoteName: "origin" }); + yield* TestClock.adjust("5 seconds"); + yield* TestClock.adjust("5 seconds"); + yield* Fiber.join(inFlight); + + yield* fixture.withLogger(fixture.driver.statusDetailsRemote(fixture.cwd)); + assert.equal(yield* Ref.get(fixture.fetchAttempts), 4); + }, + ), + ).pipe(Effect.provide(ServerConfigLayer.pipe(Layer.provideMerge(NodeServices.layer)))), +); + for (const scenario of [ { name: "HTTPS credentials", diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 0dd73af687f9..9e55695e7fc7 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -67,7 +67,12 @@ const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5); const STATUS_UPSTREAM_REFRESH_FAILURE_BASE_COOLDOWN = Duration.seconds(30); const STATUS_UPSTREAM_REFRESH_FAILURE_MAX_COOLDOWN = Duration.minutes(15); -const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048; +export const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048; +const GIT_COMMAND_TIMED_OUT_DETAIL = "Git command timed out."; +// One slow fetch is not enough to stop. Three consecutive kills at the status +// timeout match a credential helper blocked on a GUI prompt: each attempt can +// raise a macOS keychain dialog and ends before Always Allow can be stored. +const STATUS_UPSTREAM_REFRESH_TIMEOUT_STOP_AFTER = 3; const REPOSITORY_PATHS_CACHE_CAPACITY = 2_048; const REPOSITORY_PATHS_CACHE_TTL = Duration.minutes(10); const REPOSITORY_PATHS_REFRESH_COALESCE_TTL = Duration.seconds(5); @@ -78,6 +83,10 @@ const LIST_REFS_REFRESH_COALESCE_TTL = Duration.seconds(5); const LIST_REFS_REFRESH_FAILURE_COOLDOWN = Duration.seconds(30); const STATUS_DEFAULT_BRANCH_CACHE_TTL = Duration.minutes(5); const STATUS_ORIGIN_EXISTS_CACHE_TTL = Duration.minutes(5); +// Suppress Git Credential Manager, terminal prompts, and OpenSSH askpass. +// credential.helper stays intact so a silent keychain or `gh auth git-credential` +// lookup can still succeed. A helper that blocks on a GUI is stopped after +// repeated timeouts instead of being disabled on every background fetch. const STATUS_UPSTREAM_REFRESH_ENV = Object.freeze({ GCM_INTERACTIVE: "never", GIT_ASKPASS: "", @@ -122,6 +131,10 @@ class StatusRemoteRefreshCacheKey extends Data.Class<{ remoteName: string; }> {} +/** + * Backs off a failed background upstream fetch. The first failure waits 30s and + * each later failure doubles that, capped at 15 minutes. + */ function statusUpstreamRefreshFailureCooldown(consecutiveFailures: number): Duration.Duration { const exponent = Math.max(0, consecutiveFailures - 1); const cooldownMs = @@ -129,6 +142,174 @@ function statusUpstreamRefreshFailureCooldown(consecutiveFailures: number): Dura return Duration.min(Duration.millis(cooldownMs), STATUS_UPSTREAM_REFRESH_FAILURE_MAX_COOLDOWN); } +interface StatusRemoteRefreshRecord { + consecutiveFailures: number; + consecutiveTimeouts: number; + generation: number; + inFlight: number; +} + +/** + * Tracks background upstream-refresh failures for one process. + * + * Explicit pull, push, and fetch advance `generation` so a background timeout + * that finishes later cannot restore the stopped state. Idle entries, including + * generations remembered for a successful explicit fetch, are evicted at + * `capacity`. An entry with an in-flight fetch stays until that fetch finishes, + * so the fetch still observes the generation it captured. + */ +export function makeStatusRemoteRefreshLedger(capacity: number) { + const records = new Map(); + + /** + * Drops the oldest idle entries once the ledger is past `capacity`. + * In-flight fetches stay. `preserveKey` also stays when its fetch just + * finished, so cache TTL can still read that failure streak. + */ + function evictIdleStatusRemoteRefreshRecords(preserveKey?: string) { + while (records.size > capacity) { + let evicted = false; + for (const [key, record] of records) { + if (key === preserveKey || record.inFlight > 0) continue; + records.delete(key); + evicted = true; + break; + } + if (!evicted) return; + } + } + + /** + * Pins a remote's generation for one background fetch and returns the + * generation that fetch must present when it finishes. + */ + function beginStatusRemoteRefresh(key: string) { + const record = records.get(key) ?? { + consecutiveFailures: 0, + consecutiveTimeouts: 0, + generation: 0, + inFlight: 0, + }; + record.inFlight += 1; + records.delete(key); + records.set(key, record); + evictIdleStatusRemoteRefreshRecords(key); + return record.generation; + } + + /** + * Drops the in-flight pin for a finished background fetch without evicting + * that remote's record yet. + */ + function finishStatusRemoteRefresh(key: string) { + const record = records.get(key); + if (record === undefined || record.inFlight === 0) return; + record.inFlight -= 1; + evictIdleStatusRemoteRefreshRecords(key); + } + + /** + * Counts a finished background failure when `generation` is still current. + * A timeout continues the streak that stops polling. Any other failure + * clears that streak and keeps the backoff. Returns null when recovery + * already superseded this attempt. + */ + function recordStatusRemoteRefreshFailure(key: string, timedOut: boolean, generation: number) { + const record = records.get(key); + if (record === undefined || record.generation !== generation) return null; + record.consecutiveFailures += 1; + record.consecutiveTimeouts = timedOut ? record.consecutiveTimeouts + 1 : 0; + records.delete(key); + records.set(key, record); + evictIdleStatusRemoteRefreshRecords(key); + return record.consecutiveTimeouts; + } + + /** + * Forgets backoff after a background fetch that is still the current generation. + */ + function recordStatusRemoteRefreshSuccess(key: string, generation: number) { + const record = records.get(key); + if (record === undefined || record.generation !== generation) return; + record.consecutiveFailures = 0; + record.consecutiveTimeouts = 0; + records.delete(key); + records.set(key, record); + evictIdleStatusRemoteRefreshRecords(key); + } + + /** + * Clears backoff and advances the generation after an explicit pull, push, + * or fetch. An in-flight background fetch keeps its captured generation and + * cannot write a new failure streak. + */ + function resumeStatusRemoteRefresh(key: string) { + const record = records.get(key) ?? { + consecutiveFailures: 0, + consecutiveTimeouts: 0, + generation: 0, + inFlight: 0, + }; + record.generation += 1; + record.consecutiveFailures = 0; + record.consecutiveTimeouts = 0; + records.delete(key); + records.set(key, record); + evictIdleStatusRemoteRefreshRecords(record.inFlight > 0 ? key : undefined); + } + + /** + * Reports whether background polling has stopped for this remote. + */ + function statusRemoteRefreshNeedsAttention(key: string) { + return ( + (records.get(key)?.consecutiveTimeouts ?? 0) >= STATUS_UPSTREAM_REFRESH_TIMEOUT_STOP_AFTER + ); + } + + /** + * Returns the cached failure count used to back off the next background fetch. + */ + function statusRemoteRefreshFailureCount(key: string) { + return records.get(key)?.consecutiveFailures ?? 0; + } + + /** + * Returns the recovery generation for a remote, or null when that entry was evicted. + */ + function statusRemoteRefreshGeneration(key: string) { + const generation = records.get(key)?.generation; + return generation === undefined ? null : generation; + } + + /** + * Returns how many remotes currently hold refresh state. + */ + function statusRemoteRefreshLedgerSize() { + return records.size; + } + + return { + beginStatusRemoteRefresh, + finishStatusRemoteRefresh, + recordStatusRemoteRefreshFailure, + recordStatusRemoteRefreshSuccess, + resumeStatusRemoteRefresh, + statusRemoteRefreshNeedsAttention, + statusRemoteRefreshFailureCount, + statusRemoteRefreshGeneration, + statusRemoteRefreshLedgerSize, + }; +} + +/** + * Distinguishes a killed background fetch from a fast Git error. + * Timeouts are what leave git-credential-osxkeychain blocked on a password dialog. + */ +function isStatusUpstreamRefreshTimeout(error: GitCommandError) { + return error.detail === GIT_COMMAND_TIMED_OUT_DETAIL; +} + class GitRefsSnapshotCacheKey extends Data.Class<{ gitCommonDir: string; epoch: number; @@ -824,6 +1005,10 @@ const collectOutput = Effect.fnUntraced(function* ( }; }); +/** + * Builds the Git driver. Background upstream refresh stops after repeated + * timeouts and resumes after a successful explicit pull, push, or fetch. + */ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* () { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -831,6 +1016,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const { worktreesDir } = yield* ServerConfig; const crypto = yield* Crypto.Crypto; + /** + * Spawns Git for driver operations. A timeout fails with + * `GIT_COMMAND_TIMED_OUT_DETAIL`, which background status refresh uses to + * tell a hung credential prompt from other Git failures. + */ const executeRaw: GitVcsDriver.GitVcsDriver["Service"]["execute"] = Effect.fnUntraced( function* (input) { const commandInput = { @@ -841,6 +1031,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; const appendTruncationMarker = input.appendTruncationMarker ?? false; + /** + * Runs one Git process and fails with `GIT_COMMAND_TIMED_OUT_DETAIL` when + * the command outlives its timeout. + */ const runGitCommand = Effect.fn("runGitCommand")(function* () { const trace2Monitor = yield* createTrace2Monitor(commandInput, input.progress).pipe( Effect.provideService(Path.Path, path), @@ -953,7 +1147,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* () => new GitCommandError({ ...gitCommandContext(commandInput), - detail: "Git command timed out.", + detail: GIT_COMMAND_TIMED_OUT_DETAIL, }), ), ), @@ -1330,63 +1524,141 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); }); - const statusRemoteRefreshFailureCounts = new Map(); - const statusRemoteRefreshFailureKey = (cacheKey: StatusRemoteRefreshCacheKey) => + const statusRemoteRefreshLedger = makeStatusRemoteRefreshLedger( + STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY, + ); + /** + * Keys refresh state by repository and remote. Linked worktrees share a Git + * common directory, so they share one backoff. + */ + const statusRemoteRefreshKey = (cacheKey: StatusRemoteRefreshCacheKey) => `${cacheKey.gitCommonDir}\0${cacheKey.remoteName}`; - const recordStatusRemoteRefreshFailure = (cacheKey: StatusRemoteRefreshCacheKey) => { - const key = statusRemoteRefreshFailureKey(cacheKey); - const nextCount = (statusRemoteRefreshFailureCounts.get(key) ?? 0) + 1; - statusRemoteRefreshFailureCounts.delete(key); - statusRemoteRefreshFailureCounts.set(key, nextCount); - if (statusRemoteRefreshFailureCounts.size > STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY) { - const oldestKey = statusRemoteRefreshFailureCounts.keys().next().value; - if (oldestKey !== undefined) { - statusRemoteRefreshFailureCounts.delete(oldestKey); - } + /** + * Logs when a remote's timeout streak reaches the stop threshold. + * A superseded attempt passes null and logs nothing. + */ + const logStatusUpstreamRefreshStop = (consecutiveTimeouts: number | null) => { + if ( + consecutiveTimeouts === null || + consecutiveTimeouts < STATUS_UPSTREAM_REFRESH_TIMEOUT_STOP_AFTER + ) { + return Effect.void; } + return Effect.logWarning( + "Stopped background Git fetch after repeated timeouts; upstream status needs attention", + ).pipe(Effect.annotateLogs({ consecutiveTimeouts })); }; - const clearStatusRemoteRefreshFailures = (cacheKey: StatusRemoteRefreshCacheKey) => { - statusRemoteRefreshFailureCounts.delete(statusRemoteRefreshFailureKey(cacheKey)); - }; + /** + * Fetches upstream refs for one remote and counts timeouts toward the stop. + * A late failure whose generation was already resumed does not count. + */ const refreshStatusRemoteCacheEntry = Effect.fn("refreshStatusRemoteCacheEntry")(function* ( cacheKey: StatusRemoteRefreshCacheKey, ) { + const key = statusRemoteRefreshKey(cacheKey); + const generation = statusRemoteRefreshLedger.beginStatusRemoteRefresh(key); return yield* fetchRemoteForStatus(cacheKey.gitCommonDir, cacheKey.remoteName).pipe( - Effect.tap(() => Effect.sync(() => clearStatusRemoteRefreshFailures(cacheKey))), - Effect.tapError(() => Effect.sync(() => recordStatusRemoteRefreshFailure(cacheKey))), + Effect.tap(() => + Effect.sync(() => { + statusRemoteRefreshLedger.recordStatusRemoteRefreshSuccess(key, generation); + }), + ), + Effect.tapError((error) => + logStatusUpstreamRefreshStop( + statusRemoteRefreshLedger.recordStatusRemoteRefreshFailure( + key, + isStatusUpstreamRefreshTimeout(error), + generation, + ), + ), + ), Effect.tapCause((cause) => Effect.logWarning("Background Git fetch failed", cause)), + Effect.ensuring( + Effect.sync(() => { + statusRemoteRefreshLedger.finishStatusRemoteRefresh(key); + }), + ), Effect.as(true as const), ); }); + /** + * How long a cached background fetch stays fresh. Repeated timeouts do not + * expire, so a credential helper blocked on a GUI prompt is not spawned again. + */ + const statusRemoteRefreshCacheTtl = ( + exit: Exit.Exit, + cacheKey: StatusRemoteRefreshCacheKey, + ) => { + if (Exit.isSuccess(exit)) return STATUS_UPSTREAM_REFRESH_INTERVAL; + const key = statusRemoteRefreshKey(cacheKey); + if (statusRemoteRefreshLedger.statusRemoteRefreshNeedsAttention(key)) { + return Duration.infinity; + } + const consecutiveFailures = statusRemoteRefreshLedger.statusRemoteRefreshFailureCount(key); + return statusUpstreamRefreshFailureCooldown( + consecutiveFailures === 0 ? 1 : consecutiveFailures, + ); + }; const statusRemoteRefreshCache = yield* Cache.makeWith(refreshStatusRemoteCacheEntry, { capacity: STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY, - // A failed background fetch is intentionally cached and exponentially - // backed off. Status reads swallow this failure and use the last fetched - // refs, so repeated thread mounts cannot turn a slow or unavailable remote - // into a repository-wide Git subprocess storm. - timeToLive: (exit, cacheKey) => - Exit.isSuccess(exit) - ? STATUS_UPSTREAM_REFRESH_INTERVAL - : statusUpstreamRefreshFailureCooldown( - statusRemoteRefreshFailureCounts.get(statusRemoteRefreshFailureKey(cacheKey)) ?? 1, - ), + // A failed background fetch is cached and exponentially backed off. After + // repeated timeouts the entry does not expire, so a credential helper that + // blocks on a GUI prompt is not spawned again. Status reads keep the last + // fetched refs. A successful explicit pull, push, or fetch clears that stop. + timeToLive: statusRemoteRefreshCacheTtl, + }); + + /** + * Lets background refresh poll a remote again after an explicit fetch of it. + */ + const releaseStatusUpstreamRefresh = Effect.fn("releaseStatusUpstreamRefresh")(function* ( + cwd: string, + remoteName: string, + ) { + const gitCommonDir = yield* resolveGitCommonDir(cwd).pipe( + Effect.catchTags({ GitCommandError: () => Effect.succeed(null) }), + ); + if (gitCommonDir === null) return; + const cacheKey = new StatusRemoteRefreshCacheKey({ gitCommonDir, remoteName }); + statusRemoteRefreshLedger.resumeStatusRemoteRefresh(statusRemoteRefreshKey(cacheKey)); + yield* Cache.invalidate(statusRemoteRefreshCache, cacheKey); }); + /** + * Lets background refresh poll the current branch's upstream after pull or push. + */ + const releaseTrackedUpstreamStatusRefresh = Effect.fn("releaseTrackedUpstreamStatusRefresh")( + function* (cwd: string) { + const upstream = yield* resolveCurrentUpstream(cwd).pipe( + Effect.catchTags({ GitCommandError: () => Effect.succeed(null) }), + ); + if (!upstream) return; + yield* releaseStatusUpstreamRefresh(cwd, upstream.remoteName); + }, + ); + + /** + * Refreshes upstream refs when the cached fetch is stale. A remote that has + * already stopped on repeated timeouts is left alone. + */ const refreshStatusUpstreamIfStale = Effect.fn("refreshStatusUpstreamIfStale")(function* ( cwd: string, ) { const upstream = yield* resolveCurrentUpstream(cwd); if (!upstream) return; const gitCommonDir = yield* resolveGitCommonDir(cwd); + const cacheKey = new StatusRemoteRefreshCacheKey({ + gitCommonDir, + remoteName: upstream.remoteName, + }); + if ( + statusRemoteRefreshLedger.statusRemoteRefreshNeedsAttention(statusRemoteRefreshKey(cacheKey)) + ) { + return; + } // The cache loader logs failed attempts; cache hits keep using the last fetched refs. - yield* Cache.get( - statusRemoteRefreshCache, - new StatusRemoteRefreshCacheKey({ - gitCommonDir, - remoteName: upstream.remoteName, - }), - ).pipe(Effect.ignore); + yield* Cache.get(statusRemoteRefreshCache, cacheKey).pipe(Effect.ignore); }); const resolveDefaultBranchName = ( @@ -3665,6 +3937,69 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), ); + /** + * Resumes background upstream refresh when a push updated the remote. + * A skipped push did not replace remote refs, so polling stays where it was. + */ + const resumeUpstreamRefreshIfPushed = (cwd: string, result: GitVcsDriver.GitPushResult) => + result.status === "pushed" ? releaseTrackedUpstreamStatusRefresh(cwd) : Effect.void; + /** + * Pushes the current branch. A completed push lets background status fetch + * that upstream again. + */ + const pushCurrentBranchAndResumeRefresh: GitVcsDriver.GitVcsDriver["Service"]["pushCurrentBranch"] = + (cwd, fallbackBranch, options) => + withListRefsInvalidation( + cwd, + pushCurrentBranch(cwd, fallbackBranch, options).pipe( + Effect.tap((result) => resumeUpstreamRefreshIfPushed(cwd, result)), + ), + ); + /** + * Pulls the current branch, then lets background status fetch run again. + * Pull keeps the interactive Git timeout so a keychain prompt can be approved. + */ + const pullCurrentBranchAndResumeRefresh: GitVcsDriver.GitVcsDriver["Service"]["pullCurrentBranch"] = + (cwd) => + withListRefsInvalidation( + cwd, + pullCurrentBranch(cwd).pipe(Effect.tap(() => releaseTrackedUpstreamStatusRefresh(cwd))), + ); + /** + * Fetches a remote outside the status poll, then lets that poll run again. + */ + const fetchRemoteAndResumeRefresh: GitVcsDriver.GitVcsDriver["Service"]["fetchRemote"] = ( + input, + ) => + withListRefsInvalidation( + input.cwd, + fetchRemote(input).pipe( + Effect.tap(() => releaseStatusUpstreamRefresh(input.cwd, input.remoteName)), + ), + ); + /** + * Fetches one remote branch, then lets background status refresh poll that remote. + */ + const fetchRemoteBranchAndResumeRefresh: GitVcsDriver.GitVcsDriver["Service"]["fetchRemoteBranch"] = + (input) => + withListRefsInvalidation( + input.cwd, + fetchRemoteBranch(input).pipe( + Effect.tap(() => releaseStatusUpstreamRefresh(input.cwd, input.remoteName)), + ), + ); + /** + * Fetches a remote-tracking branch, then lets background status refresh poll that remote. + */ + const fetchRemoteTrackingBranchAndResumeRefresh: GitVcsDriver.GitVcsDriver["Service"]["fetchRemoteTrackingBranch"] = + (input) => + withListRefsInvalidation( + input.cwd, + fetchRemoteTrackingBranch(input).pipe( + Effect.tap(() => releaseStatusUpstreamRefresh(input.cwd, input.remoteName)), + ), + ); + return GitVcsDriver.GitVcsDriver.of({ execute, status, @@ -3674,9 +4009,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* prepareCommitContext, commit: (cwd, subject, body, options) => withListRefsInvalidation(cwd, commit(cwd, subject, body, options)), - pushCurrentBranch: (cwd, fallbackBranch, options) => - withListRefsInvalidation(cwd, pushCurrentBranch(cwd, fallbackBranch, options)), - pullCurrentBranch: (cwd) => withListRefsInvalidation(cwd, pullCurrentBranch(cwd)), + pushCurrentBranch: pushCurrentBranchAndResumeRefresh, + pullCurrentBranch: pullCurrentBranchAndResumeRefresh, readRangeContext, getReviewDiffPreview, getReviewDiffFileContents, @@ -3693,13 +4027,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ensureRemote: (input) => withListRefsInvalidation(input.cwd, ensureRemote(input)), resolvePrimaryRemoteName, resolveDefaultBranchName, - fetchRemote: (input) => withListRefsInvalidation(input.cwd, fetchRemote(input)), + fetchRemote: fetchRemoteAndResumeRefresh, remoteExists, remoteBranchExists, resolveRemoteTrackingCommit, - fetchRemoteBranch: (input) => withListRefsInvalidation(input.cwd, fetchRemoteBranch(input)), - fetchRemoteTrackingBranch: (input) => - withListRefsInvalidation(input.cwd, fetchRemoteTrackingBranch(input)), + fetchRemoteBranch: fetchRemoteBranchAndResumeRefresh, + fetchRemoteTrackingBranch: fetchRemoteTrackingBranchAndResumeRefresh, setBranchUpstream: (input) => withListRefsInvalidation(input.cwd, setBranchUpstream(input)), removeWorktree: (input) => withListRefsInvalidation(input.cwd, removeWorktree(input)), pruneWorktrees: (input) => withListRefsInvalidation(input.cwd, pruneWorktrees(input)),