diff --git a/GraphcodeKit/Sources/Domain/RemoteProjectLocation.swift b/GraphcodeKit/Sources/Domain/RemoteProjectLocation.swift index 6b5c08d..e319b1b 100644 --- a/GraphcodeKit/Sources/Domain/RemoteProjectLocation.swift +++ b/GraphcodeKit/Sources/Domain/RemoteProjectLocation.swift @@ -143,8 +143,10 @@ public struct RemoteProjectLocation: Equatable, Sendable { /// is where the sporadic "ssh failed, retrying" noise on healthy networks came from. /// A mux channel over a live master cannot fail in transport the way a fresh dial can. /// `ControlPersist` keeps the master up between ticks; a dead master is redialed by - /// whichever command comes next. If the socket directory is missing ssh just warns and - /// dials directly, so this degrades to the old behaviour, never to a failure. + /// whichever command comes next. A missing socket directory is a hard failure, not a + /// warning: OpenSSH 10.3 exits 255 with `unix_listener: cannot bind to path` and sends + /// nothing (measured on the loopback rig), so every spawn site calls + /// `prepareControlSocketDirectory()` before dialing. /// /// A Codespace dials through `gh codespace ssh -c -- ` /// instead — everything after `--` reaches gh's underlying ssh untouched, so the diff --git a/GraphcodeKit/Sources/ProjectPersistence+Export.swift b/GraphcodeKit/Sources/ProjectPersistence+Export.swift index 5c360e4..8b9ba78 100644 --- a/GraphcodeKit/Sources/ProjectPersistence+Export.swift +++ b/GraphcodeKit/Sources/ProjectPersistence+Export.swift @@ -14,6 +14,10 @@ extension ProjectPersistence { /// Import is *not* the mirror of this call: the daemon owns the live graph, so a /// bundle goes back in through `GraphCommand.importNodes`, never by writing the /// graph file from a client. + /// + /// Sessions are read off this Mac's disk — right for a local project. A remote + /// project's sessions are on its host, and this call would leave them behind; the + /// `async` twin below fetches them over the ssh dial. public func createExportBundle( for nodeIDs: [UUID], from graph: LoopGraph, @@ -22,95 +26,173 @@ extension ProjectPersistence { includeMemory: Bool = true, createdBy: String? = nil ) -> GraphExportBundle? { - var nodeIDsToExport = Set(nodeIDs) + guard let slice = slice(of: graph, for: nodeIDs, includeChildren: includeChildren) + else { return nil } + return bundle( + slice, projectPath: projectPath, includeMemory: includeMemory, createdBy: createdBy, + sessions: Self.sessionArtifacts(for: slice.nodes, projectPath: projectPath)) + } + + /// The same bundle, with sessions collected wherever the project actually lives: a + /// remote project's are fetched from its host (`SessionTransplant.exportRemoteArtifact`), + /// a local project's are read off this disk exactly as the synchronous call does. The + /// app and the CLI export through this one so an ssh:// or codespace:// export carries + /// its conversations (issue #333). + public func createExportBundle( + for nodeIDs: [UUID], + from graph: LoopGraph, + projectPath: String, + includeChildren: Bool = true, + includeMemory: Bool = true, + createdBy: String? = nil + ) async -> GraphExportBundle? { + guard let slice = slice(of: graph, for: nodeIDs, includeChildren: includeChildren) + else { return nil } + let sessions = await Self.remoteAwareSessionArtifacts( + for: slice.nodes, projectPath: projectPath) + return bundle( + slice, projectPath: projectPath, includeMemory: includeMemory, createdBy: createdBy, + sessions: sessions) + } + + /// Exports an entire graph as a shareable bundle. Sessions come off this disk, as in + /// `createExportBundle`; the `async` twin reaches a remote project's host. + public func createFullGraphExportBundle( + for graph: LoopGraph, + projectPath: String, + createdBy: String? = nil + ) -> GraphExportBundle { + bundle( + Slice(graph: graph, nodes: Array(graph.nodes), isFullGraph: true, includesChildren: true), + projectPath: projectPath, includeMemory: true, createdBy: createdBy, + sessions: Self.sessionArtifacts(for: graph.nodes, projectPath: projectPath)) + } + + /// The whole graph with sessions collected from wherever the project lives — see the + /// `async` `createExportBundle`. + public func createFullGraphExportBundle( + for graph: LoopGraph, + projectPath: String, + createdBy: String? = nil + ) async -> GraphExportBundle { + let sessions = await Self.remoteAwareSessionArtifacts( + for: graph.nodes, projectPath: projectPath) + return bundle( + Slice(graph: graph, nodes: Array(graph.nodes), isFullGraph: true, includesChildren: true), + projectPath: projectPath, includeMemory: true, createdBy: createdBy, sessions: sessions) + } + + /// Each exported loop's backend conversation, where one exists and the backend can + /// carry it — see `SessionTransplant`. Local disk only. + static func sessionArtifacts( + for nodes: some Sequence, projectPath: String + ) -> [String: SessionTransplant.Artifact] { + var artifacts: [String: SessionTransplant.Artifact] = [:] + for node in nodes { + if let artifact = SessionTransplant.exportArtifact(forNode: node, projectPath: projectPath) { + artifacts[node.id.uuidString] = artifact + } + } + return artifacts + } + + /// How many remote fetches run at once. Each is its own dial — multiplexed over one + /// connection for a plain host, a fresh `gh` tunnel per loop for a Codespace — plus + /// a `tar` and a watchdog; a thirty-loop Codespace graph fetched all at once would be + /// thirty tunnels racing to start a stopped codespace. Four keeps the export quick on + /// a live host without turning it into that. + static let remoteSessionFetchConcurrency = 4 + + /// `sessionArtifacts` for a project on any host. A remote project's loops are fetched + /// `remoteSessionFetchConcurrency` at a time — the next dial starts as one finishes — + /// and a loop whose fetch comes back empty is simply exported without a session: the + /// export never fails on one. + static func remoteAwareSessionArtifacts( + for nodes: some Sequence, projectPath: String + ) async -> [String: SessionTransplant.Artifact] { + guard let remote = RemoteProjectLocation.parse(projectPath: projectPath) else { + return sessionArtifacts(for: nodes, projectPath: projectPath) + } + return await withTaskGroup(of: (String, SessionTransplant.Artifact?).self) { group in + var artifacts: [String: SessionTransplant.Artifact] = [:] + var inFlight = 0 + for node in nodes { + if inFlight == remoteSessionFetchConcurrency, let (nodeID, artifact) = await group.next() { + inFlight -= 1 + if let artifact { artifacts[nodeID] = artifact } + } + inFlight += 1 + group.addTask { + ( + node.id.uuidString, + await SessionTransplant.exportRemoteArtifact(forNode: node, at: remote) + ) + } + } + for await (nodeID, artifact) in group { + if let artifact { artifacts[nodeID] = artifact } + } + return artifacts + } + } + + /// The part of a graph an export names: the snapshot to write and the loops it holds. + private struct Slice { + var graph: LoopGraph + var nodes: [LoopNode] + var isFullGraph: Bool + var includesChildren: Bool + } + private func slice(of graph: LoopGraph, for nodeIDs: [UUID], includeChildren: Bool) -> Slice? { + var nodeIDsToExport = Set(nodeIDs) if includeChildren { for nodeID in nodeIDs { nodeIDsToExport.formUnion(descendants(of: nodeID, in: graph)) } } - let exportedNodes = graph.nodes.filter { nodeIDsToExport.contains($0.id) } guard !exportedNodes.isEmpty else { return nil } - let exportedEdges = graph.edges.filter { nodeIDsToExport.contains($0.from) && nodeIDsToExport.contains($0.to) } - let exportGraph = LoopGraph( id: graph.id, scope: graph.scope, nodes: IdentifiedArray(uniqueElements: exportedNodes), edges: IdentifiedArray(uniqueElements: exportedEdges) ) + return Slice( + graph: exportGraph, nodes: Array(exportedNodes), + isFullGraph: Set(graph.nodes.map(\.id)) == nodeIDsToExport, + includesChildren: includeChildren) + } + private func bundle( + _ slice: Slice, projectPath: String, includeMemory: Bool, createdBy: String?, + sessions: [String: SessionTransplant.Artifact] + ) -> GraphExportBundle { var memoryByNodeID: [String: [String]] = [:] if includeMemory { - for nodeID in nodeIDsToExport { - let entries = NodeMemory.entries(forProjectPath: projectPath, nodeID: nodeID) + for node in slice.nodes { + let entries = NodeMemory.entries(forProjectPath: projectPath, nodeID: node.id) if !entries.isEmpty { - memoryByNodeID[nodeID.uuidString] = entries + memoryByNodeID[node.id.uuidString] = entries } } } - let contents = ExportContents( - nodeIDs: nodeIDsToExport.map(\.uuidString), - includesChildren: includeChildren, - isFullGraph: Set(graph.nodes.map(\.id)) == nodeIDsToExport, + nodeIDs: slice.nodes.map(\.id.uuidString), + includesChildren: slice.includesChildren, + isFullGraph: slice.isFullGraph, sourceProject: projectPath, includesMemory: includeMemory ) - - return GraphExportBundle( - manifest: ExportManifest(createdBy: createdBy, contents: contents), - graphSnapshot: exportGraph, - memoryByNodeID: memoryByNodeID, - sessionsByNodeID: Self.sessionArtifacts(for: exportedNodes, projectPath: projectPath) - ) - } - - /// Each exported loop's backend conversation, where one exists and the backend can - /// carry it — see `SessionTransplant`. - static func sessionArtifacts( - for nodes: some Sequence, projectPath: String - ) -> [String: SessionTransplant.Artifact] { - var artifacts: [String: SessionTransplant.Artifact] = [:] - for node in nodes { - if let artifact = SessionTransplant.exportArtifact(forNode: node, projectPath: projectPath) { - artifacts[node.id.uuidString] = artifact - } - } - return artifacts - } - - /// Exports an entire graph as a shareable bundle. - public func createFullGraphExportBundle( - for graph: LoopGraph, - projectPath: String, - createdBy: String? = nil - ) -> GraphExportBundle { - var memoryByNodeID: [String: [String]] = [:] - for node in graph.nodes { - let entries = NodeMemory.entries(forProjectPath: projectPath, nodeID: node.id) - if !entries.isEmpty { - memoryByNodeID[node.id.uuidString] = entries - } - } - - let contents = ExportContents( - nodeIDs: graph.nodes.map { $0.id.uuidString }, - includesChildren: true, - isFullGraph: true, - sourceProject: projectPath, - includesMemory: true - ) - return GraphExportBundle( manifest: ExportManifest(createdBy: createdBy, contents: contents), - graphSnapshot: graph, + graphSnapshot: slice.graph, memoryByNodeID: memoryByNodeID, - sessionsByNodeID: Self.sessionArtifacts(for: graph.nodes, projectPath: projectPath) + sessionsByNodeID: sessions ) } diff --git a/GraphcodeKit/Sources/Sessions/SessionTransplant.swift b/GraphcodeKit/Sources/Sessions/SessionTransplant.swift index 132ec76..63cc2d9 100644 --- a/GraphcodeKit/Sources/Sessions/SessionTransplant.swift +++ b/GraphcodeKit/Sources/Sessions/SessionTransplant.swift @@ -92,6 +92,219 @@ public enum SessionTransplant { } } + // MARK: - Remote export + + /// The remote twin of `exportArtifact`. A remote loop's session lives on the host it + /// runs on — its id banked at the file `PresenceHooks.remoteSessionIDExpression` + /// names, its transcript beside it — so reading this Mac's home directory found + /// nothing and every remote export shipped without sessions (issue #333). The host + /// locates the session and streams it back as `tar` on the ssh dial's stdout, the + /// mirror of `restoreRemote`'s tar-in; the stream is unpacked here and re-keyed to + /// exactly the artifact the local export produces, so `restore` and `restoreRemote` + /// need no remote-aware branch of their own. + /// + /// Nil for anything short of a whole session: nothing banked, the files gone, the + /// link dying mid-stream, the deadline (`remoteExportDeadlineSeconds`) passing. A loop + /// whose fetch fails is exported without a session — the shape a local loop with + /// nothing banked already has — never as a failed export. + public static func exportRemoteArtifact( + forNode node: LoopNode, at location: RemoteProjectLocation + ) async -> Artifact? { + guard let script = remoteExportScript(forNode: node, at: location) else { return nil } + let staging = FileManager.default.temporaryDirectory + .appendingPathComponent("graphcode-export-\(UUID().uuidString)", isDirectory: true) + guard + (try? FileManager.default.createDirectory(at: staging, withIntermediateDirectories: true)) + != nil + else { return nil } + let status = remoteExportStatusFile(besideStaging: staging) + defer { + try? FileManager.default.removeItem(at: staging) + try? FileManager.default.removeItem(at: status) + } + RemoteProjectLocation.prepareControlSocketDirectory() + guard + await runShell(remoteExportPipeline(remoteScript: script, staging: staging, at: location)), + (try? String(contentsOf: status, encoding: .utf8)) == "0" + else { return nil } + return artifact( + fromFetched: filesUnder(staging), backend: node.backend, + workingDirectory: remoteWorkingDirectory(forNode: node, at: location)) + } + + /// The local half of the transfer: the dial's stdout straight into `tar -x`, so the + /// bytes never pass through a PTY or a `String` — a transcript is arbitrary bytes at + /// megabytes, the reason `deliver` is a pipeline too. Two verdicts, both required: + /// the untar's, which is the pipeline's own status and rejects a stream the link + /// truncated; and the dial's, recorded to a file beside the staging directory because + /// a `tar -c` that lost a member mid-archive still emits a complete archive of the + /// rest and exits non-zero — bytes plus a failure is a *partial* session, not one to + /// carry (measured on the loopback rig, GNU and bsd tar alike). `gh codespace ssh` + /// flattens every remote exit to 1, which is still non-zero, so the rule holds there. + /// Recorded by a POSIX group rather than `set -o pipefail`, which `/bin/sh` is not + /// guaranteed to know (dash rejects it and exits). A host that found nothing sends an + /// empty stream and exits 0, which `tar` accepts and extracts nothing from — an empty + /// staging directory is "nothing to carry", the local export's answer for a loop with + /// nothing banked. + static func remoteExportPipeline( + remoteScript: String, staging: URL, at location: RemoteProjectLocation + ) -> String { + let status = RemoteProjectLocation.shellQuoted( + remoteExportStatusFile(besideStaging: staging).path) + let pipeline = + "{ " + location.sshCommandLine(remoteCommand: remoteScript) + + "; printf %s \"$?\" > \(status); }" + + " | tar -xf - -C \(RemoteProjectLocation.shellQuoted(staging.path))" + return bounded(pipeline, seconds: remoteExportDeadlineSeconds) + } + + /// How long one loop's fetch may take before it is abandoned: generous enough for a + /// stopped Codespace, which `gh` starts on the way in and which can take five minutes + /// to deliver its first byte, and finite so that a live-but-silent remote or a wedged + /// ssh master can never hang an export. On the deadline the loop is exported without + /// a session, never as a failed export. + static let remoteExportDeadlineSeconds = 600 + + /// `pipeline` under a watchdog. Job control (`set -m`) puts the pipeline in a process + /// group of its own, and the deadline kills that *group*: terminating only the shell + /// would orphan `ssh` and `tar`, still joined by their pipe, holding the connection + /// open for as long as the remote stayed silent. The watchdog's own group is killed + /// on the way out so a fetch that finished in a second leaves no ten-minute `sleep` + /// behind. Measured on a silent pipeline: dead at the deadline with exit 143 and no + /// process left. `dash` runs this with job control off and a warning, so on a Linux + /// host the deadline ends the wait but not the children — acceptable for the one + /// caller, whose runner only ever runs on the Mac. + /// + /// The job's stdin is `/dev/null`, and that is load-bearing: a background process + /// group that reads the controlling terminal is stopped with `SIGTTIN`, and `ssh` + /// reads its inherited stdin. From the CLI in a Terminal that stdin *is* the tty, so + /// without the redirect the dial stopped silently, the watchdog fired at the deadline, + /// and the loop exported with no session (found in review; every measurement here had + /// run without a tty). Nothing is ever sent to the host on this path. + /// + /// The trap is the other half of owning those groups: a signal to the shell alone — + /// the app terminating the export, Ctrl-C in a Terminal, which reaches only the CLI's + /// own group — would otherwise leave `ssh`, `tar`, the subshells and the ten-minute + /// `sleep` running to the deadline (measured on the rig: all alive after the shell + /// was TERMed). Both groups are killed and the shell exits 143, the status a TERM + /// would have given it. + static func bounded(_ pipeline: String, seconds: Int) -> String { + "set -m; { \(pipeline); } /dev/null & gc_w=$!; " + + "trap 'kill -TERM -- -$gc_p -$gc_w 2>/dev/null; exit 143' INT TERM HUP; " + + "wait $gc_p; gc_s=$?; kill -TERM -- -$gc_w 2>/dev/null; exit $gc_s" + } + + /// Where the pipeline leaves the dial's exit status: beside the staging directory, + /// never inside it, or `filesUnder` would carry it as part of the session. + static func remoteExportStatusFile(besideStaging staging: URL) -> URL { + URL(fileURLWithPath: staging.path + ".status") + } + + /// What the host runs to find the loop's session and stream it out — the mirror of + /// `remoteInstallScript`, and like it a pure function so its shape is testable. Each + /// backend's lookup is the one graphcode already trusts elsewhere: + /// + /// - Claude Code: the banked id — the file the ensure's resume branch consumes — then + /// the transcript by id across every project directory, because a worktree-bound + /// loop's transcript lives under the worktree's slug (`findClaudeTranscript`). + /// - Copilot: the banked id, else the directory whose `workspace.yaml` names the zmx + /// session graphcode launched it as — the walk `remoteIDBankFragment` does. + /// - Codex: the newest rollout whose header opened in the loop's working directory, + /// the match `CodexSessionLog.remoteSummaryInvocation` makes. + /// - OpenCode: nothing, for the reason the local export carries nothing. + /// + /// The archive's first path component is the session's identity — `.jsonl`, + /// `/…`, `rollout-….jsonl` — which is how the id reaches this side without a + /// second channel; `artifact(fromFetched:)` reads it back. A session that is not there + /// exits 0 having written nothing. Not wrapped in the login shell the probes use: an + /// interactive `zsh -i` may print from its rc files, and stdout here *is* the archive. + static func remoteExportScript( + forNode node: LoopNode, at location: RemoteProjectLocation + ) -> String? { + let idFile = PresenceHooks.remoteSessionIDExpression(forNodeID: node.id) + switch node.backend { + case .claudeCode: + return "S=$(cat \(idFile) 2>/dev/null); [ -n \"$S\" ] || exit 0; " + + "F=$(ls -t \"$HOME\"/.claude/projects/*/\"$S\".jsonl 2>/dev/null | head -1); " + + "[ -n \"$F\" ] || exit 0; exec tar -cf - -C \"$(dirname \"$F\")\" \"$S.jsonl\"" + case .copilotCLI: + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + return "S=$(cat \(idFile) 2>/dev/null); if [ -z \"$S\" ]; then " + + "for d in $(ls -t \"$HOME/.copilot/session-state/\" 2>/dev/null); do " + + "if grep -qx 'name: \(name)' " + + "\"$HOME/.copilot/session-state/$d/workspace.yaml\" 2>/dev/null; " + + "then S=\"$d\"; break; fi; done; fi; " + + "[ -n \"$S\" ] && [ -d \"$HOME/.copilot/session-state/$S\" ] || exit 0; " + + "exec tar -cf - -C \"$HOME/.copilot/session-state\" \"$S\"" + case .codex: + let directory = RemoteProjectLocation.shellQuoted( + remoteWorkingDirectory(forNode: node, at: location)) + return "W=\(directory); F=''; " + + "for f in $(ls -t \"$HOME\"/.codex/sessions/*/*/*/rollout-*.jsonl 2>/dev/null" + + " | head -40); do " + + "if head -c 65536 \"$f\" 2>/dev/null | grep -q \"\\\"cwd\\\":\\\"$W\\\"\"; " + + "then F=\"$f\"; break; fi; done; " + + "[ -n \"$F\" ] || exit 0; exec tar -cf - -C \"$(dirname \"$F\")\" \"$(basename \"$F\")\"" + case .openCode: + return nil + } + } + + /// A remote loop's working directory: its worktree if bound, else the project folder + /// on that host — `CodexSessionLog.summary`'s answer, because the local one checks the + /// path exists on this Mac and a remote one never does. + static func remoteWorkingDirectory( + forNode node: LoopNode, at location: RemoteProjectLocation + ) -> String { + node.worktreeBinding?.worktreePath ?? location.remotePath + } + + /// The fetched archive as the artifact the *local* export would have produced for the + /// same session — same keys, same id, so nothing downstream can tell where a session + /// came from. The first path component carries the identity: for Claude and Codex a + /// single file named by it, for Copilot the session directory, stripped from every key. + /// Anything else — two transcripts, a stray file beside the directory — is not a + /// session this side knows how to restore, and is refused rather than guessed at. + static func artifact( + fromFetched files: [String: Data], backend: CLISessionBackendKind, workingDirectory: String + ) -> Artifact? { + switch backend { + case .claudeCode: + guard let only = singleFile(in: files) else { return nil } + return Artifact( + backend: .claudeCode, sessionID: String(only.name.dropLast(".jsonl".count)), + sourceWorkingDirectory: workingDirectory, files: ["transcript.jsonl": only.data]) + case .copilotCLI: + guard let first = files.keys.sorted().first, let slash = first.firstIndex(of: "/") + else { return nil } + let sessionID = String(first[.. (name: String, data: Data)? { + guard files.count == 1, let entry = files.first, !entry.key.contains("/"), + entry.key.hasSuffix(".jsonl") + else { return nil } + return (entry.key, entry.value) + } + // MARK: - Restore /// Installs an exported session for a freshly imported node, under a fresh identity, diff --git a/graphcode-cli/Sources/main.swift b/graphcode-cli/Sources/main.swift index c9df34c..5315e26 100644 --- a/graphcode-cli/Sources/main.swift +++ b/graphcode-cli/Sources/main.swift @@ -131,6 +131,22 @@ func openProject(_ projectPath: String) throws -> LoopGraph? { /// it prints: one loop's unread slice, one post, or the whole room. A refusal (the /// project not open, a path the daemon does not know) arrives as an error and stops /// here, the way `openProject`'s does. +/// Runs an export's async bundle build to completion from this synchronous top level — +/// the `reap`/`importNodes` semaphore pattern — because a remote project's sessions +/// arrive over ssh. +func awaitBundle(_ build: @escaping @Sendable () async -> GraphExportBundle?) -> GraphExportBundle? +{ + final class Box: @unchecked Sendable { var bundle: GraphExportBundle? } + let box = Box() + let built = DispatchSemaphore(value: 0) + Task { + box.bundle = await build() + built.signal() + } + built.wait() + return box.bundle +} + func fetchMailbox(_ projectPath: String, _ query: MailboxQuery) throws -> Mailbox { try sendCommand(.mailbox(projectPath: projectPath, query: query)) phase = "waiting for the mailbox answer" @@ -551,15 +567,20 @@ do { case .exportNode(let projectPath, let nodeID, let output, let includeChildren): guard let graph = try openProject(projectPath) else { fail("Could not load graph") } + // Async behind a semaphore, the `importNodes` pattern in reverse: a remote + // project's sessions are fetched from its host over ssh, and a loop whose fetch + // fails is exported without one — the count printed last is what actually rode. let persistence = ProjectPersistence(baseDirectory: SupportDirectory.url) guard - let bundle = persistence.createExportBundle( - for: [nodeID], - from: graph, - projectPath: projectPath, - includeChildren: includeChildren, - createdBy: ProcessInfo.processInfo.environment["USER"] - ) + let bundle = awaitBundle({ + await persistence.createExportBundle( + for: [nodeID], + from: graph, + projectPath: projectPath, + includeChildren: includeChildren, + createdBy: ProcessInfo.processInfo.environment["USER"] + ) + }) else { fail("Could not create export bundle") } guard let zipPath = bundle.writeToZip(at: output) else { @@ -569,16 +590,21 @@ do { print("Exported to \(zipPath)") print("Nodes: \(bundle.manifest.contents.nodeIDs.count)") print("Memory logs: \(bundle.memoryByNodeID.count)") + print("Sessions: \(bundle.sessionsByNodeID.count)") case .exportGraph(let projectPath, let output): guard let graph = try openProject(projectPath) else { fail("Could not load graph") } let persistence = ProjectPersistence(baseDirectory: SupportDirectory.url) - let bundle = persistence.createFullGraphExportBundle( - for: graph, - projectPath: projectPath, - createdBy: ProcessInfo.processInfo.environment["USER"] - ) + guard + let bundle = awaitBundle({ + await persistence.createFullGraphExportBundle( + for: graph, + projectPath: projectPath, + createdBy: ProcessInfo.processInfo.environment["USER"] + ) + }) + else { fail("Could not create export bundle") } guard let zipPath = bundle.writeToZip(at: output) else { fail("Could not write ZIP file to \(output)") @@ -588,6 +614,7 @@ do { print("Nodes: \(bundle.manifest.contents.nodeIDs.count)") print("Edges: \(graph.edges.count)") print("Memory logs: \(bundle.memoryByNodeID.count)") + print("Sessions: \(bundle.sessionsByNodeID.count)") case .importNodes(let projectPath, let fromZip, let asChildOf): guard let bundle = GraphExportBundle.readFromZip(at: fromZip) else { diff --git a/graphcode/Sources/Features/Project/ProjectFeature.swift b/graphcode/Sources/Features/Project/ProjectFeature.swift index 0dae5ed..fe39274 100644 --- a/graphcode/Sources/Features/Project/ProjectFeature.swift +++ b/graphcode/Sources/Features/Project/ProjectFeature.swift @@ -964,32 +964,36 @@ extension ProjectFeature { /// /// Export is read-only, so unlike import it never goes near the daemon: the graph in /// hand is the daemon's own latest broadcast, and memory logs are read straight off - /// disk. The finished zip is revealed in Finder — that reveal *is* the success - /// feedback, pointing at the file the user is about to go share. + /// disk. A remote project's sessions are fetched from its host over ssh — off the + /// main actor, since that is round-trips and the panel is long dismissed. The finished + /// zip is revealed in Finder — that reveal *is* the success feedback. private func exportBundle( from graph: LoopGraph, projectPath: String, nodeIDs: [UUID]?, suggestedName: String ) -> Effect { .run { _ in - await MainActor.run { + let destination = await MainActor.run { () -> URL? in let panel = NSSavePanel() panel.allowedContentTypes = [.zip] panel.nameFieldStringValue = suggestedName.replacingOccurrences(of: "/", with: "-") + ".zip" panel.message = "Export loops as a shareable bundle" - guard panel.runModal() == .OK, let url = panel.url else { return } - - let persistence = ProjectPersistence(baseDirectory: SupportDirectory.url) - let bundle: GraphExportBundle? = - if let nodeIDs { - persistence.createExportBundle( - for: nodeIDs, from: graph, projectPath: projectPath, createdBy: NSUserName()) - } else { - persistence.createFullGraphExportBundle( - for: graph, projectPath: projectPath, createdBy: NSUserName()) - } - guard let bundle, bundle.writeToZip(at: url.path) != nil else { return } - NSWorkspace.shared.activateFileViewerSelecting([url]) + guard panel.runModal() == .OK else { return nil } + return panel.url } + guard let url = destination else { return } + + let persistence = ProjectPersistence(baseDirectory: SupportDirectory.url) + let createdBy = NSUserName() + let bundle: GraphExportBundle? = + if let nodeIDs { + await persistence.createExportBundle( + for: nodeIDs, from: graph, projectPath: projectPath, createdBy: createdBy) + } else { + await persistence.createFullGraphExportBundle( + for: graph, projectPath: projectPath, createdBy: createdBy) + } + guard let bundle, bundle.writeToZip(at: url.path) != nil else { return } + await MainActor.run { NSWorkspace.shared.activateFileViewerSelecting([url]) } } } diff --git a/graphcode/Tests/RemoteSessionExportTests.swift b/graphcode/Tests/RemoteSessionExportTests.swift new file mode 100644 index 0000000..6bde17f --- /dev/null +++ b/graphcode/Tests/RemoteSessionExportTests.swift @@ -0,0 +1,281 @@ +import Foundation +import Testing + +@testable import GraphcodeKit + +/// What `SessionTransplant.exportRemoteArtifact` asks the host for, and how what comes +/// back becomes an artifact. The local export reads this Mac's home directory, which a +/// remote loop never wrote to — so an ssh:// or codespace:// export carried the graph and +/// memory logs and no sessions (issue #333). The remote fetch has to read the id the +/// host banked itself, find the backend's files by that id where the backend keeps them, +/// and hand back exactly the artifact the local export would have built — the same +/// keys — so `restore` and `restoreRemote` stay untouched. +@Suite +struct RemoteSessionExportTests { + private let location = RemoteProjectLocation( + user: "dev", host: "buildbox", remotePath: "/srv/widget") + + private func node( + _ backend: CLISessionBackendKind, worktree: String? = nil + ) -> LoopNode { + var node = LoopNode(title: "Worker", loopType: .goalBased, goal: GoalSpec(summary: "ship")) + node.backend = backend + if let worktree { + node.worktreeBinding = WorktreeRef( + id: "wt", repositoryPath: "/srv/widget", worktreePath: worktree, branch: "topic") + } + return node + } + + private func script(_ node: LoopNode) throws -> String { + try #require(SessionTransplant.remoteExportScript(forNode: node, at: location)) + } + + // MARK: - Script shape per backend + + @Test + func claudeFetchReadsTheBankedIDThenTheTranscriptByID() throws { + let node = node(.claudeCode) + let script = try script(node) + + // The id decides which transcript; the directory is found, not reconstructed — a + // worktree-bound loop's transcript lives under the worktree's slug. + #expect( + script.contains("S=$(cat \(PresenceHooks.remoteSessionIDExpression(forNodeID: node.id))")) + #expect(script.contains("\"$HOME\"/.claude/projects/*/\"$S\".jsonl")) + #expect(script.hasSuffix("exec tar -cf - -C \"$(dirname \"$F\")\" \"$S.jsonl\"")) + } + + @Test + func fetchScriptsStreamNothingAndExitCleanlyWhenNothingIsBanked() throws { + for backend in [CLISessionBackendKind.claudeCode, .copilotCLI, .codex] { + let script = try script(node(backend)) + #expect(script.contains("|| exit 0"), "\(backend)") + // The archive is the whole of stdout: nothing may print before `tar` does. + #expect(!script.contains("echo"), "\(backend)") + #expect(!script.contains("printf"), "\(backend)") + #expect(!script.hasPrefix("exec zsh"), "\(backend)") + } + } + + @Test + func copilotFetchFallsBackToTheDirectoryNamedAfterTheZmxSession() throws { + let node = node(.copilotCLI) + let script = try script(node) + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + + // Banked id first, then the same `workspace.yaml` walk the ensure's bank fragment + // does — Copilot has no hook to bank its own id, so a session may be unbanked. + let bank = try #require(script.range(of: "cat \"$HOME/.graphcode/sessions\"")) + let walk = try #require(script.range(of: "grep -qx 'name: \(name)'")) + #expect(bank.lowerBound < walk.lowerBound) + #expect(script.contains("/.copilot/session-state/$d/workspace.yaml")) + #expect(script.hasSuffix("exec tar -cf - -C \"$HOME/.copilot/session-state\" \"$S\"")) + } + + @Test + func codexFetchMatchesTheNewestRolloutForTheLoopsWorkingDirectory() throws { + let project = try script(node(.codex)) + let bound = try script(node(.codex, worktree: "/srv/widget-wt/topic")) + + // The working directory is the only handle on a Codex rollout, and it is the + // loop's own worktree when it has one, the project folder on the host otherwise. + #expect(project.contains("W='/srv/widget';")) + #expect(bound.contains("W='/srv/widget-wt/topic';")) + #expect(project.contains("\"$HOME\"/.codex/sessions/*/*/*/rollout-*.jsonl")) + #expect(project.contains("ls -t")) + #expect(project.contains("\\\"cwd\\\":\\\"$W\\\"")) + #expect( + project.hasSuffix("exec tar -cf - -C \"$(dirname \"$F\")\" \"$(basename \"$F\")\"")) + } + + @Test + func openCodeHasNoRemoteFetch() { + #expect(SessionTransplant.remoteExportScript(forNode: node(.openCode), at: location) == nil) + } + + @Test + func bankPathMatchesWhatTheEnsureConsumes() throws { + // The reader here and the reader in `remoteCreateScript` must name the same file + // the remote `SessionStart` hook wrote; drifting apart would not fail loudly — + // every remote export would just silently carry no sessions, the exact bug this + // path exists to end. Same shape as the install-side parity test. + let node = node(.claudeCode) + let script = try script(node) + let consumed = PresenceHooks.remoteSessionIDExpression(forNodeID: node.id) + let read = try #require(script.range(of: "S=$(cat ")) + let end = try #require( + script.range(of: " 2>/dev/null)", range: read.upperBound.. '/tmp/graphcode-export-x.status'; }")) + #expect(pipeline.contains("} | tar -xf - -C '/tmp/graphcode-export-x'; }")) + #expect(!pipeline.contains("'-t'")) + #expect(!pipeline.contains("zsh")) + } + + @Test + func pipelineIsBoundedAndKillsItsOwnProcessGroupOnTheDeadline() throws { + let pipeline = SessionTransplant.remoteExportPipeline( + remoteScript: "x", staging: URL(fileURLWithPath: "/tmp/s"), at: location) + + // Ten minutes: a stopped Codespace can take five to its first byte; a silent remote + // or a wedged ssh master must not hold an export forever. The whole group dies, or + // ssh and tar would outlive the shell still joined by their pipe. + #expect(SessionTransplant.remoteExportDeadlineSeconds == 600) + #expect(pipeline.hasPrefix("set -m; ")) + // A background group reading the controlling tty is stopped with SIGTTIN, and ssh + // reads its inherited stdin: from a Terminal the dial would sit stopped until the + // deadline. The job never sends anything to the host, so its stdin is /dev/null. + #expect(pipeline.contains("; } /dev/null; exit 143' INT TERM HUP; ")) + let wait = try #require(pipeline.range(of: "wait $gc_p")) + #expect(trap.upperBound <= wait.lowerBound) + } + + @Test + func remoteFetchesAreCappedAtFourInFlight() { + // A thirty-loop Codespace graph must not become thirty gh tunnels racing to start + // a stopped codespace, each with its own tar and watchdog. + #expect(ProjectPersistence.remoteSessionFetchConcurrency == 4) + } + + @Test + func codespacePipelineDialsThroughGh() { + let codespace = RemoteProjectLocation( + host: "fluffy-space", remotePath: "/workspaces/widget", isCodespace: true) + let pipeline = SessionTransplant.remoteExportPipeline( + remoteScript: "exec tar -cf - x", staging: URL(fileURLWithPath: "/tmp/s"), at: codespace) + + #expect(pipeline.contains("'codespace' 'ssh' '-c' 'fluffy-space' '--'")) + #expect(pipeline.contains("} | tar -xf - -C '/tmp/s'; }")) + } + + // MARK: - Fetched files become the local export's artifact + + @Test + func claudeArchiveBecomesTheTranscriptArtifact() throws { + let transcript = Data("{\"type\":\"user\"}\n".utf8) + let artifact = try #require( + SessionTransplant.artifact( + fromFetched: ["aaaa-bbbb.jsonl": transcript], backend: .claudeCode, + workingDirectory: "/srv/widget")) + + #expect(artifact.backend == .claudeCode) + #expect(artifact.sessionID == "aaaa-bbbb") + #expect(artifact.sourceWorkingDirectory == "/srv/widget") + #expect(artifact.files == ["transcript.jsonl": transcript]) + } + + @Test + func copilotArchiveIsReKeyedRelativeToItsSessionDirectory() throws { + let artifact = try #require( + SessionTransplant.artifact( + fromFetched: [ + "c0ffee/events.jsonl": Data("e".utf8), + "c0ffee/workspace.yaml": Data("w".utf8), + "c0ffee/checkpoints/1.md": Data("c".utf8), + ], backend: .copilotCLI, workingDirectory: "/srv/widget")) + + #expect(artifact.sessionID == "c0ffee") + #expect(Set(artifact.files.keys) == ["events.jsonl", "workspace.yaml", "checkpoints/1.md"]) + } + + @Test + func codexArchiveBecomesTheRolloutArtifact() throws { + let name = "rollout-2026-09-09T10-00-00-0f1e2d3c-4b5a-6978-8a9b-0c1d2e3f4a5b.jsonl" + let artifact = try #require( + SessionTransplant.artifact( + fromFetched: [name: Data("r".utf8)], backend: .codex, workingDirectory: "/srv/widget")) + + #expect(artifact.sessionID == name) + #expect(artifact.files == ["rollout.jsonl": Data("r".utf8)]) + } + + @Test + func anythingButOneWholeSessionIsRefused() { + let empty: [String: Data] = [:] + #expect( + SessionTransplant.artifact(fromFetched: empty, backend: .claudeCode, workingDirectory: "/") + == nil) + #expect( + SessionTransplant.artifact( + fromFetched: ["a.jsonl": Data(), "b.jsonl": Data()], backend: .claudeCode, + workingDirectory: "/") == nil) + #expect( + SessionTransplant.artifact( + fromFetched: ["one/events.jsonl": Data(), "two/events.jsonl": Data()], + backend: .copilotCLI, workingDirectory: "/") == nil) + #expect( + SessionTransplant.artifact( + fromFetched: ["x.jsonl": Data()], backend: .openCode, workingDirectory: "/") == nil) + } + + @Test + func anUnreachableHostYieldsNoSessionRatherThanAFailure() async { + // Port 1 on loopback refuses at once, so the dial dies before a byte arrives — the + // runner must hand back nil, and the bundle builder must carry on without it. + let unreachable = RemoteProjectLocation( + user: "nobody", host: "127.0.0.1", port: 1, remotePath: "/srv/none") + let node = node(.claudeCode) + + let artifact = await SessionTransplant.exportRemoteArtifact(forNode: node, at: unreachable) + let sessions = await ProjectPersistence.remoteAwareSessionArtifacts( + for: [node], projectPath: unreachable.projectPath) + + #expect(artifact == nil) + #expect(sessions.isEmpty) + } + + // MARK: - The bundle builders + + @Test + func localProjectsCollectSessionsTheSameWayThroughEitherBuilder() async throws { + // The async builders exist for remote projects; a local path must take the same + // disk read the synchronous ones do, or the app and CLI would export local loops + // differently from before. + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("remote-export-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + let persistence = ProjectPersistence(baseDirectory: directory) + let project = ProjectRef(path: directory.path, name: "local") + let graph = LoopGraph(project: project, nodes: [node(.claudeCode)]) + + // From a synchronous closure: in an async context the async overload wins, which is + // the resolution the app and CLI rely on. + let sync = { + persistence.createExportBundle( + for: [graph.nodes[0].id], from: graph, projectPath: project.path) + }() + let async = await persistence.createExportBundle( + for: [graph.nodes[0].id], from: graph, projectPath: project.path) + let full = await persistence.createFullGraphExportBundle( + for: graph, projectPath: project.path) + + #expect(sync?.sessionsByNodeID.keys == async?.sessionsByNodeID.keys) + #expect(sync?.manifest.contents.nodeIDs == async?.manifest.contents.nodeIDs) + #expect(full.manifest.contents.isFullGraph) + #expect(full.manifest.contents.nodeIDs == [graph.nodes[0].id.uuidString]) + } +}