From 97514ea34d8b2c601b802bfda260965769f8d571 Mon Sep 17 00:00:00 2001 From: Vasek - Tom C Date: Tue, 11 Aug 2026 15:35:46 +0200 Subject: [PATCH] feat(module): generate CLI 1.0 modules in the SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route Mod.generate to this SDK's generator for dagger-module.toml modules, and keep delegating to the engine for dagger.json ones. A workspace module builds from its committed files, so what generate writes *is* what the runtime executes; a legacy module regenerates at call time anyway, and generating it here would only leave a second, differently-versioned copy on disk. Both routes run through generateStaged so `mod generate` and generate-all cannot pick different generators. Both keep the generateLocalDependencies staging: a dependent's schema is only loadable once its local deps' generated files exist, and dep generation may cross SDKs. That staging is scaffolding, not output, and each route has to take it back out a different way. The SDK route builds its changeset out of the unstaged workspace, so the deps never enter it. The engine route hands back a whole workspace instead, so this takes the module's own directory out of that result and lays it over the unstaged base — diffing the whole thing against the base would write the dependencies' generated files into the user's tree alongside the module they asked for, and diffing against the staged workspace instead cancels the module's own output whenever the engine has already generated it for a dependent. generate-legacy-deps covers the case; generate-deps already covered the SDK side. Staging the generated tree over the module's existing one, rather than in place of it, is what keeps the module's source and config out of the changeset — withNewDirectory swaps the whole directory, and staging only what we generate reports everything else as deleted. Applying to disk hides that; staging into a workspace does not, and the engine generates a dependency by doing exactly that. The cost of layering is that a file we stop generating is left behind, so regeneration prunes the `*.gen.ts` it owns first. Without it a dependency dropped from a module's config keeps its .gen.ts, exporting an API against a module that is no longer served, and a default tsc rejects the directory. Pruned in both the baseline and the workspace, for the same reason client generation does both. The typescript pin gets a check that can fail. Asserting it on a fixture that arrives pinned only proves generation left it alone, so the dependency fixture drops to a bare package.json — the shape a module has before its first generate — and the check asserts the pin appears. Signed-off-by: Tom Chauveau --- .../fixtures/generate-deps/dep/package.json | 5 +- .../generate-legacy-deps/app/.gitattributes | 2 + .../generate-legacy-deps/app/.gitignore | 5 + .../generate-legacy-deps/app/dagger.json | 13 ++ .../generate-legacy-deps/app/package.json | 6 + .../generate-legacy-deps/app/src/index.ts | 9 ++ .../generate-legacy-deps/app/tsconfig.json | 13 ++ .../generate-legacy-deps/dep/.gitattributes | 2 + .../generate-legacy-deps/dep/.gitignore | 5 + .../generate-legacy-deps/dep/dagger.json | 7 + .../generate-legacy-deps/dep/package.json | 6 + .../generate-legacy-deps/dep/src/index.ts | 9 ++ .../generate-legacy-deps/dep/tsconfig.json | 13 ++ .dagger/modules/e2e/generate.dang | 143 ++++++++++++++++++ .dagger/modules/e2e/util.dang | 8 + dagger.toml | 9 ++ design/module-gen.md | 52 +++++-- mod.dang | 52 ++++++- typescript-sdk.dang | 118 +++++++++++++-- 19 files changed, 444 insertions(+), 33 deletions(-) create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/app/.gitattributes create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/app/.gitignore create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/app/dagger.json create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/app/package.json create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/app/src/index.ts create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/app/tsconfig.json create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/dep/.gitattributes create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/dep/.gitignore create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/dep/dagger.json create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/dep/package.json create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/dep/src/index.ts create mode 100644 .dagger/modules/e2e/fixtures/generate-legacy-deps/dep/tsconfig.json diff --git a/.dagger/modules/e2e/fixtures/generate-deps/dep/package.json b/.dagger/modules/e2e/fixtures/generate-deps/dep/package.json index b5e4d24..3dbc1ca 100644 --- a/.dagger/modules/e2e/fixtures/generate-deps/dep/package.json +++ b/.dagger/modules/e2e/fixtures/generate-deps/dep/package.json @@ -1,6 +1,3 @@ { - "type": "module", - "dependencies": { - "typescript": "5.9.3" - } + "type": "module" } diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/.gitattributes b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/.gitattributes new file mode 100644 index 0000000..8eb4cdc --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/.gitattributes @@ -0,0 +1,2 @@ +/sdk/** linguist-generated +/__dagger.entrypoint.ts linguist-generated diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/.gitignore b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/.gitignore new file mode 100644 index 0000000..b1d836b --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/.gitignore @@ -0,0 +1,5 @@ +/sdk +/**/node_modules/** +/**/.pnpm-store/** +/.env +/__dagger.entrypoint.ts diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/dagger.json b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/dagger.json new file mode 100644 index 0000000..4256493 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/dagger.json @@ -0,0 +1,13 @@ +{ + "name": "legacy-deps-app", + "engineVersion": "latest", + "sdk": { + "source": "typescript" + }, + "dependencies": [ + { + "name": "legacydep", + "source": "../dep" + } + ] +} diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/package.json b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/package.json new file mode 100644 index 0000000..b5e4d24 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/package.json @@ -0,0 +1,6 @@ +{ + "type": "module", + "dependencies": { + "typescript": "5.9.3" + } +} diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/src/index.ts b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/src/index.ts new file mode 100644 index 0000000..be16f63 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/src/index.ts @@ -0,0 +1,9 @@ +import { object, func } from "@dagger.io/dagger" + +@object() +export class LegacyDepsApp { + @func() + hello(): string { + return "hello" + } +} diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/tsconfig.json b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/tsconfig.json new file mode 100644 index 0000000..4ec0c2d --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/app/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "moduleResolution": "Node", + "experimentalDecorators": true, + "strict": true, + "skipLibCheck": true, + "paths": { + "@dagger.io/dagger": ["./sdk/index.ts"], + "@dagger.io/dagger/telemetry": ["./sdk/telemetry.ts"] + } + } +} diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/.gitattributes b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/.gitattributes new file mode 100644 index 0000000..8eb4cdc --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/.gitattributes @@ -0,0 +1,2 @@ +/sdk/** linguist-generated +/__dagger.entrypoint.ts linguist-generated diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/.gitignore b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/.gitignore new file mode 100644 index 0000000..b1d836b --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/.gitignore @@ -0,0 +1,5 @@ +/sdk +/**/node_modules/** +/**/.pnpm-store/** +/.env +/__dagger.entrypoint.ts diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/dagger.json b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/dagger.json new file mode 100644 index 0000000..9970214 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/dagger.json @@ -0,0 +1,7 @@ +{ + "name": "legacydep", + "engineVersion": "latest", + "sdk": { + "source": "typescript" + } +} diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/package.json b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/package.json new file mode 100644 index 0000000..b5e4d24 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/package.json @@ -0,0 +1,6 @@ +{ + "type": "module", + "dependencies": { + "typescript": "5.9.3" + } +} diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/src/index.ts b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/src/index.ts new file mode 100644 index 0000000..0f226e9 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/src/index.ts @@ -0,0 +1,9 @@ +import { object, func } from "@dagger.io/dagger" + +@object() +export class Legacydep { + @func() + value(): string { + return "dep" + } +} diff --git a/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/tsconfig.json b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/tsconfig.json new file mode 100644 index 0000000..4ec0c2d --- /dev/null +++ b/.dagger/modules/e2e/fixtures/generate-legacy-deps/dep/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "moduleResolution": "Node", + "experimentalDecorators": true, + "strict": true, + "skipLibCheck": true, + "paths": { + "@dagger.io/dagger": ["./sdk/index.ts"], + "@dagger.io/dagger/telemetry": ["./sdk/telemetry.ts"] + } + } +} diff --git a/.dagger/modules/e2e/generate.dang b/.dagger/modules/e2e/generate.dang index 85936ac..9f4196c 100644 --- a/.dagger/modules/e2e/generate.dang +++ b/.dagger/modules/e2e/generate.dang @@ -69,6 +69,149 @@ type GenerateChecks { null } + """ + A CLI 1.0 module is generated by this SDK, and what it writes has to satisfy + the engine runtime by itself: that runtime does no codegen for a + dagger-module.toml module, it mounts sdk/ as @dagger.io/dagger and runs the + committed entrypoint. + + So this asserts the whole contract rather than a single marker file — the + bundled library the bindings import, the bindings themselves, the dispatch + entrypoint, and the config the runtime reads. A missing piece here is a module + that fails to load at `dagger call` time, long after generate. + """ + generateWorkspaceModuleCheck(ws: Workspace!): Void @check { + let path = fixtures.depAppModule + let changes = typescriptSdk.mod(ws, path: path).generate(ws) + let tree = changes.after.directory(path) + + Asserts.generated(changes, path + "/sdk/client.gen.ts") + Asserts.generated(changes, path + "/sdk/core.js") + Asserts.generated(changes, path + "/sdk/core.d.ts") + Asserts.generated(changes, path + "/sdk/index.ts") + Asserts.generated(changes, path + "/sdk/telemetry.ts") + Asserts.generated(changes, path + "/__dagger.entrypoint.ts") + + # package.json and tsconfig.json are asserted on the resulting tree rather + # than the changeset: this fixture already carries correct ones, so a + # correctly-behaving generate leaves them untouched and out of the diff. + + # The bindings must reach the runtime through the bundle sitting beside + # them, not through the npm package: sdk/ *is* @dagger.io/dagger here. + Asserts.stringContains( + tree.file("sdk/client.gen.ts").contents, + "from \"./core.js\"", + "module bindings should import the bundled runtime", + ) + + # The entrypoint dispatches by importing the user's classes from where they + # were declared, which only the source scan knows. + Asserts.stringContains( + tree.file("__dagger.entrypoint.ts").contents, + "from \"./src/index\"", + "entrypoint should import the module's own source", + ) + + # This fixture arrives already pinned, so the pin only proves generation + # preserved it. That generation *writes* one is generateTypescriptPinCheck. + Asserts.stringContains( + tree.file("package.json").contents, + "\"typescript\"", + "package.json should keep the module's typescript pin", + ) + Asserts.stringContains( + tree.file("tsconfig.json").contents, + "./sdk/index.ts", + "tsconfig should alias @dagger.io/dagger to the generated sdk", + ) + + null + } + + """ + Generating a module that has not pinned typescript should add the pin. + + Run against the dependency fixture, whose package.json carries nothing but + `type: module` — the shape a module has before its first generate. That is the + only way to tell writing from preserving: a fixture that arrives pinned goes + on passing after generation stops writing the pin. The cost of losing it is + silent — the runtime mounts its prebuilt compiler + only when the version matches its default, so an unpinned module installs + typescript on every single call. + """ + generateTypescriptPinCheck(ws: Workspace!): Void @check { + let path = fixtures.depLibModule + let changes = typescriptSdk.mod(ws, path: path).generate(ws) + + Asserts.stringNotContains( + ws.directory("/" + path).file("package.json").contents, + "typescript", + "fixture must start unpinned for this check to mean anything", + ) + Asserts.stringContains( + changes.after.directory(path).file("package.json").contents, + "\"typescript\"", + "generation should pin typescript for a module that has not", + ) + + null + } + + """ + Regenerating should remove bindings for dependencies the module no longer has. + + Layering the generated tree over the existing one is what keeps a module's own + source and config out of the diff, but it also leaves behind any file we no + longer generate: drop a dependency and its `.gen.ts` stays, exporting an + API against a module that is no longer served. A default `tsc` compiles the + whole sdk/ directory and rejects it. + + Staged rather than fixtured because sdk/ is gitignored in every module, so + there is no committed stale file to start from — the check plants one and + asserts regeneration takes it away. + """ + generatePrunesStaleBindingsCheck(ws: Workspace!): Void @check { + let path = fixtures.depAppModule + let stale = path + "/sdk/departed.gen.ts" + + let dirty = ws.withNewFile("/" + stale, "export const gone = true\n") + let changes = typescriptSdk.mod(dirty, path: path).generate(dirty) + + Asserts.generated(changes, path + "/sdk/gendep.gen.ts") + Asserts.assert( + Asserts.contains(changes.removedPaths, stale), + "regeneration should remove bindings for a dependency that is gone, got removed [" + + changes.removedPaths.join(", ") + "]", + ) + + null + } + + """ + Generating a legacy module with a local dependency should write that module's + files and nothing else. + + The dependency closure has to be staged before generation — a dependency's + schema is only loadable once its own generated files exist — but that staging + is scaffolding, not output. A legacy module routes to the engine, which hands + back a whole workspace rather than a changeset, so what is taken out of that + workspace is the only thing keeping the dependency's freshly generated files + from being written into the user's tree alongside the module they asked for. + """ + generateLegacyDependencyCheck(ws: Workspace!): Void @check { + let changes = typescriptSdk.mod(ws, path: fixtures.legacyDepAppModule).generate(ws) + + Asserts.generated(changes, fixtures.legacyDepAppModule + "/sdk/index.ts") + + Asserts.notAdded( + changes, + fixtures.legacyDepLibModule + "/sdk/index.ts", + "generating a legacy dependent should not also write its dependency's files", + ) + + null + } + """ Generating a module with a local dependency must resolve that dependency and emit its typed bindings alongside the module's own — otherwise the module's diff --git a/.dagger/modules/e2e/util.dang b/.dagger/modules/e2e/util.dang index 523fbbe..4a8a441 100644 --- a/.dagger/modules/e2e/util.dang +++ b/.dagger/modules/e2e/util.dang @@ -130,6 +130,12 @@ type Fixtures { let depAppModule: String! = root + "/generate-deps/app" let depLibModule: String! = root + "/generate-deps/dep" + # The same edge in the legacy format: a dagger.json module with a local + # dagger.json dependency. Generation routes to the engine here, and its + # dependency staging is what a changeset must not carry back. + let legacyDepAppModule: String! = root + "/generate-legacy-deps/app" + let legacyDepLibModule: String! = root + "/generate-legacy-deps/dep" + # Module config in /.dagger/modules/app, implementation source in # /ci — the split layout `dagger setup` migration produces. let parentSourceRoot: String! = root + "/parent-source" @@ -185,6 +191,8 @@ type Fixtures { managedTomlModule, depAppModule, depLibModule, + legacyDepAppModule, + legacyDepLibModule, parentSourceModule, ] } diff --git a/dagger.toml b/dagger.toml index e641702..fe5e438 100644 --- a/dagger.toml +++ b/dagger.toml @@ -43,6 +43,15 @@ path = ".dagger/modules/e2e/fixtures/generate-deps/app" [[modules.typescript-sdk.as-sdk.modules]] path = ".dagger/modules/e2e/fixtures/generate-deps/dep" +# The legacy-format counterpart of generate-deps: a dagger.json module with a +# local dagger.json dependency. Generation routes to the engine for these, the +# one path where the SDK hands back a workspace it did not assemble itself. +[[modules.typescript-sdk.as-sdk.modules]] +path = ".dagger/modules/e2e/fixtures/generate-legacy-deps/app" + +[[modules.typescript-sdk.as-sdk.modules]] +path = ".dagger/modules/e2e/fixtures/generate-legacy-deps/dep" + [[modules.typescript-sdk.as-sdk.modules]] path = ".dagger/modules/e2e/fixtures/parent-source/.dagger/modules/app" diff --git a/design/module-gen.md b/design/module-gen.md index 24ec49f..b021b4e 100644 --- a/design/module-gen.md +++ b/design/module-gen.md @@ -559,15 +559,40 @@ engine builds its own from whatever it resolves. This is the cutover's safety net and it is cheap while both paths exist. -**Phase 3 — cutover, for `dagger-module.toml` modules only.** Flip -`Mod.generate` to the local path when the module's config is -`dagger-module.toml`; keep delegating to `polyfill.moduleSource(...).generate` -for `dagger.json` ones (§8). Keep `generateLocalDependencies` staging (still -required: a dependent's schema can only be built if its local deps' generated -files exist, and dep generation may cross SDKs). Update the e2e assertions from -`sdk/index.ts` to the full expected tree, and add `__dagger.entrypoint.ts` + -`sdk/core.js` assertions. Verify `dagger call` on a generated fixture actually -runs — the runtime contract in §2.2 is only really proven by executing a module. +**Phase 3 — cutover, for `dagger-module.toml` modules only.** ✅ Done. +`Mod.generate` takes the local path when the module's config is +`dagger-module.toml` and keeps delegating to +`polyfill.moduleSource(...).generate` for `dagger.json` ones (§8). +`generateLocalDependencies` staging is kept for both: a dependent's schema can +only be loaded once its local deps' generated files exist, and dep generation +may cross SDKs. + +Validated three ways rather than one: + +- **Byte-for-byte against the engine.** Generating the same fixture both ways + produces identical `client.gen.ts`, `__dagger.entrypoint.ts`, `index.ts`, + `telemetry.ts`, `core.d.ts`, `package.json` and `tsconfig.json`. The one + exception is `core.js`, which is *ours* by construction: built from the + vendored lockfile at 4.3 MB against the engine's unpinned 5.4 MB (§4.2). + Getting there required three generator fixes the diff surfaced (§ the + generator was ported from an older upstream commit than the engine we pin). +- **By running a module.** A `dagger call` on a generated fixture returns its + value, exercising our bindings, bundle and entrypoint through the engine + runtime, dependency bindings included. The runtime contract in §2.2 is only + really proven by executing a module. +- **By an e2e check** asserting the whole tree contract, not one marker file. + +**VCS files are not written** (decided): no `.gitignore`, no `.gitattributes`. +The engine appends to both around codegen, but for a workspace module the +ignore list is reduced to `node_modules`/`.pnpm-store` anyway, which is the +user's business rather than codegen's. + +Worth knowing, since it looks like our bug when it happens: a module whose +`.gitignore` still ignores `sdk/` — a legacy `dagger.json` module migrated to +`.toml` — fails to load with *"committed generated file sdk/client.gen.ts is +missing"* even though the file is on disk, because the ignore keeps it out of +the module context. The engine has the same problem: it only avoids *adding* +generated paths for toml modules, it never removes one already there. **Phase 4 — upstream cleanup** (separate `dagger/dagger` PR, §9). @@ -650,15 +675,16 @@ late): Bundled without `--compile`, with the trimmed compiler at `node_modules/typescript`, it scans a fixture module and emits a `typedef.json` with the `location` data the entrypoint needs. -2. **Does our `module` mode reproduce the engine's `sdk/client.gen.ts` - byte-for-byte** for a fixture, given `ModuleSource.introspectionSchemaJSON`? - This is the differential check of §7 Phase 2, run by hand once, first. +2. ~~**Does our `module` mode reproduce the engine's `sdk/client.gen.ts` + byte-for-byte**~~ — **yes**, once the generator was synced with the engine + it targets (§7 Phase 3). It did not before: enum members were miscased, + `arguments` went unescaped, and the entrypoint carried no source maps. 3. ~~**Is `bun build` output reproducible enough**~~ — **yes**, with the image pinned by digest and the vendored lockfile in place: a second packager run over an unchanged tree reports no changes. Dropping the lockfile is what breaks it (§4.2). -Only the differential check is left, and it belongs to Phase 2 anyway. +All three are now answered; the work they were guarding is done. (For the record on the fetch alternative in §4.1: dang does support `@cache(policy:, ttl:)` → `withCachePolicy`, but a plain container exec is already content-addressed by the engine, so the decorator would mostly buy a TTL diff --git a/mod.dang b/mod.dang index aea1428..259dd35 100644 --- a/mod.dang +++ b/mod.dang @@ -93,7 +93,57 @@ type Mod { if (skipGenerate(ws)) { ws.changes(ws) } else { - ws.moduleSource("/" + rootPath).generate(ws).changes(ws) + # Stage the local dependency closure so this module's codegen sees + # up-to-date dependency bindings before generating it. A dependency's + # schema can only be loaded once its own generated files exist, so this + # has to happen before the schema is read either way. + let stagedWs = ws.withChanges(ws.moduleSource("/" + rootPath).generateLocalDependencies(ws)) + + generateStaged(stagedWs, ws) + } + } + + """ + Generate this module against a workspace that already has its dependency + closure staged, choosing the generator by config format. + + Split from `generate` so generate-all can reuse the routing without repeating + the staging: the engine dispatches the SDK's @generate rollup for each local + dependency, so generate-all already runs inside another module's staging. + """ + let generateStaged(ws: Workspace!, base: Workspace!): Changeset! { + if (isWorkspaceManaged(ws)) { + TypescriptSdk().moduleFiles(ws, rootPath, sourcePath, base) + } else { + # A dagger.json module stays with the engine's built-in TypeScript + # runtime end to end: it regenerates everything at call time anyway, so + # generating it here would only put a second, differently-versioned copy + # of the same files on disk. + # + # Take this module's own directory out of the engine's result and lay it + # over the unstaged base, the same shape moduleFiles builds. The engine + # hands back a whole workspace, and that workspace still carries the + # dependency closure staged above — diffing it against `base` writes the + # dependencies' generated files into the user's tree alongside the module + # they asked for. Diffing against the staged workspace instead is not the + # answer either: staging a module the engine has already generated for a + # dependent cancels its own output and emits nothing at all. + let generated = ws.moduleSource("/" + rootPath).generate(ws) + base.withNewDirectory( + "/" + sourcePath, + generated.directory("/" + sourcePath), + ).changes(base) } } + + """ + Whether this module uses the CLI 1.0 config format, whose modules build from + their committed generated files rather than regenerating at call time. That + makes what `generate` writes the artifact the runtime executes — and makes + this SDK, not the engine, the thing that has to write it. + """ + let isWorkspaceManaged(ws: Workspace!): Boolean! { + let path = if (rootPath == ".") { "dagger-module.toml" } else { rootPath + "/dagger-module.toml" } + ws.directory("/", include: [path]).exists(path) + } } diff --git a/typescript-sdk.dang b/typescript-sdk.dang index 8c08972..910afc4 100644 --- a/typescript-sdk.dang +++ b/typescript-sdk.dang @@ -591,26 +591,102 @@ type TypescriptSdk { } """ - Generate a module's SDK files in this SDK rather than through the engine's - built-in TypeScript runtime. + Generate a module's files and stage them at its source directory. - Staged behind `Mod.generate` while the two are compared: see - e2e:generate:local-matches-engine-check, which generates a fixture both ways - and diffs everything but the bundle. + Takes the module's paths rather than a Mod so it can be called with a + workspace that already has the module's dependency closure staged — resolving + the module afresh from that workspace is what makes its schema loadable. """ - generateModuleLocal(ws: Workspace!, path: String!): Changeset! { - let mod = mod(ws, path: path, findUp: false) - let modSrc = ws.moduleSource("/" + mod.rootPath) - let sourcePath = mod.sourcePath + let moduleFiles(ws: Workspace!, rootPath: String!, sourcePath: String!, base: Workspace!): Changeset! { + let modSrc = ws.moduleSource("/" + rootPath) let sourcePrefix = if (sourcePath == ".") { "" } else { sourcePath + "/" } - ws.withNewDirectory("/" + sourcePath, moduleDirectory( + # Everything the module is generated *from* reads the staged workspace, so + # dependency bindings are current. Everything the changeset is measured + # *against* reads the unstaged one: the engine may have just generated this + # module while staging it for a dependent, and diffing against that would + # report its own output as already present and emit nothing. + let existingSource = if (sourcePath == ".") { base.directory("/") } else { base.directory("/" + sourcePath) } + let existing = existingSource.withoutFiles(moduleBindings(base, sourcePath).map { entry => "sdk/" + entry }) + + let generated = moduleDirectory( modSrc.introspectionSchemaJSON.contents, modSrc.moduleOriginalName, - ws.directory("/" + sourcePrefix + "src"), - existingModuleConfig(ws, sourcePath), - mod.config.runtime, - )).changes(ws) + base.directory("/" + sourcePrefix + "src"), + existingModuleConfig(base, sourcePath), + moduleRuntime(base, sourcePath), + ) + + # Stage the generated files over the module's existing tree, not in place of + # it. withNewDirectory swaps the whole directory, so staging only what we + # generate says everything else — the module's config and its source — was + # deleted. Applying to disk hides that, but staging into a workspace does + # not: the engine generates a dependency by running this against a workspace + # it then reuses, and the next module's dependency resolution finds the + # dependency's config gone. + withoutModuleBindings(base, sourcePath) + .withNewDirectory("/" + sourcePath, existing.withDirectory(".", generated)) + .changes(base) + } + + """ + Workspace-relative path of a module's generated sdk/ directory. + """ + let moduleSdkPath(sourcePath: String!): String! { + if (sourcePath == ".") { "sdk" } else { sourcePath + "/sdk" } + } + + """ + The `*.gen.ts` bindings sitting in a module's sdk/ directory, by name. + + Regeneration owns that set and nothing else: a dependency that has left the + module's closure must lose its file, while the rest of sdk/ — the bundle and + the wrappers — is rewritten in place every time. + """ + let moduleBindings(ws: Workspace!, sourcePath: String!): [String!]! { + existingDir(ws, moduleSdkPath(sourcePath)).entries + .filter { entry => entry.trimSuffix(".gen.ts") != entry } + } + + """ + The workspace with a module's generated bindings removed. + + Layering the generated tree over the existing one is what keeps the module's + own source and config in the changeset, but it also means a file we no longer + generate is simply left behind — so a dependency dropped from the config keeps + its .gen.ts, exporting an API against a module that is no longer served, + and a default tsc compiles the whole directory and rejects it. + + Pruned in both places for the same reason client generation is (see + withoutClientBindings): withNewDirectory replaces the directory it writes on a + local-directory workspace and merges into it on a git-loaded one, so taking + the bindings out of the baseline covers the first and out of the workspace the + second. + """ + let withoutModuleBindings(ws: Workspace!, sourcePath: String!): Workspace! { + let sdkPath = moduleSdkPath(sourcePath) + moduleBindings(ws, sourcePath).reduce(ws) { pruned, entry => + pruned.withoutFile("/" + sdkPath + "/" + entry) + } + } + + """ + Which TypeScript runtime a module's source directory is configured for, + detected from the files it carries — the same rule the engine applies when it + loads the module. + """ + let moduleRuntime(ws: Workspace!, sourcePath: String!): Runtime! { + ModConfig(sourcePath: sourcePath, ws: ws).runtime + } + + """ + Generate a module's files through this SDK, bypassing `Mod.generate`'s choice + of path. Exists so a check can generate the same module both ways and compare + the results; `Mod.generate` is what callers should use. + """ + generateModuleLocal(ws: Workspace!, path: String!): Changeset! { + let mod = mod(ws, path: path, findUp: false) + moduleFiles(ws, mod.rootPath, mod.sourcePath, ws) } """ @@ -797,7 +873,19 @@ type TypescriptSdk { generateAllModule(ws: Workspace!): Changeset! @generate { let changes = modules(ws) .filter { mod => mod.skipGenerate(ws) == false } - .map { mod => ws.moduleSource("/" + mod.rootPath).generate(ws).changes(ws) } + .map { mod => + # Stage this module's local dependency closure first (leaf-first, possibly + # across SDKs) so its codegen sees up-to-date dependency bindings. The dep + # codegen is ephemeral: taking the changeset against the staged workspace + # cancels it out, leaving only each module's own changes. + let stagedWs = ws.withChanges(ws.moduleSource("/" + mod.rootPath).generateLocalDependencies(ws)) + + # Generate through Mod so this and `mod generate` cannot pick different + # generators — but only its staged half: Mod.generate would stage a + # second time, and the engine already re-enters this function under each + # dependency's own staging. + mod.generateStaged(stagedWs, ws) + } # Force the per-module codegen to evaluate concurrently: selecting a field on # the whole list resolves every element in one pass, where folding them into