diff --git a/.dagger/modules/e2e/fixtures/clients/app-self/App.java b/.dagger/modules/e2e/fixtures/clients/app-self/App.java new file mode 100644 index 0000000..d0ed0ee --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/app-self/App.java @@ -0,0 +1,42 @@ +package io.dagger.modules.app; + +import static io.dagger.client.app.App.app; +import static io.dagger.client.dep.Dep.dep; +import static io.dagger.client.greeter.Greeter.greeter; +import static io.dagger.sdk.Dagger.dag; + +import io.dagger.module.annotation.Function; +import io.dagger.module.annotation.Object; +import io.dagger.sdk.exception.DaggerQueryException; +import java.util.concurrent.ExecutionException; + +@Object +public class App { + /** A call on a dependency, through its generated client. */ + @Function + public String greetViaDep(String name) + throws InterruptedException, ExecutionException, DaggerQueryException { + return dep(dag()).greet(name); + } + + /** A core type returned by the dependency's client, used through core: the same Java type. */ + @Function + public String depFileViaCore() + throws InterruptedException, ExecutionException, DaggerQueryException { + return dep(dag()).scratch().file("dep.txt").contents(); + } + + /** The same dependency under an alias: a second client, on the same session. */ + @Function + public String greetViaAlias(String name) + throws InterruptedException, ExecutionException, DaggerQueryException { + return greeter(dag()).greet(name); + } + + /** A self call, through this module's own generated client. */ + @Function + public String greetSelf(String name) + throws InterruptedException, ExecutionException, DaggerQueryException { + return app(dag()).greetViaDep(name); + } +} diff --git a/.dagger/modules/e2e/fixtures/clients/app/dagger-module.toml b/.dagger/modules/e2e/fixtures/clients/app/dagger-module.toml new file mode 100644 index 0000000..f930219 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/app/dagger-module.toml @@ -0,0 +1,15 @@ +name = "app" +engineVersion = "v1.0.0-beta.10" + +[runtime] + source = "java" + +[[dependencies]] + name = "dep" + source = "../dep" + +# The same module under another name: the engine applies the alias with +# withName, so its client has to chain and serve "greeter", not "dep". +[[dependencies]] + name = "greeter" + source = "../dep" diff --git a/.dagger/modules/e2e/fixtures/clients/app/src/main/java/io/dagger/modules/app/App.java b/.dagger/modules/e2e/fixtures/clients/app/src/main/java/io/dagger/modules/app/App.java new file mode 100644 index 0000000..73416be --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/app/src/main/java/io/dagger/modules/app/App.java @@ -0,0 +1,26 @@ +package io.dagger.modules.app; + +import static io.dagger.client.dep.Dep.dep; +import static io.dagger.sdk.Dagger.dag; + +import io.dagger.module.annotation.Function; +import io.dagger.module.annotation.Object; +import io.dagger.sdk.exception.DaggerQueryException; +import java.util.concurrent.ExecutionException; + +@Object +public class App { + /** A call on a dependency, through its generated client. */ + @Function + public String greetViaDep(String name) + throws InterruptedException, ExecutionException, DaggerQueryException { + return dep(dag()).greet(name); + } + + /** A core type returned by the dependency's client, used through core: the same Java type. */ + @Function + public String depFileViaCore() + throws InterruptedException, ExecutionException, DaggerQueryException { + return dep(dag()).scratch().file("dep.txt").contents(); + } +} diff --git a/.dagger/modules/e2e/fixtures/clients/dep-client/Main.java b/.dagger/modules/e2e/fixtures/clients/dep-client/Main.java new file mode 100644 index 0000000..1d25bf8 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/dep-client/Main.java @@ -0,0 +1,15 @@ +package io.dagger.clients.depclient; + +import static io.dagger.client.dep.Dep.dep; + +import io.dagger.sdk.AutoCloseableClient; +import io.dagger.sdk.Dagger; + +/** A standalone client: opens its own session and reaches the module through the preamble. */ +public class Main { + public static void main(String[] args) throws Exception { + try (AutoCloseableClient dag = Dagger.connect()) { + System.out.println(dep(dag).greet("client")); + } + } +} diff --git a/.dagger/modules/e2e/fixtures/clients/dep/dagger-module.toml b/.dagger/modules/e2e/fixtures/clients/dep/dagger-module.toml new file mode 100644 index 0000000..4e12fbe --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/dep/dagger-module.toml @@ -0,0 +1,5 @@ +name = "dep" +engineVersion = "v1.0.0-beta.10" + +[runtime] + source = "java" diff --git a/.dagger/modules/e2e/fixtures/clients/dep/src/main/java/io/dagger/modules/dep/Dep.java b/.dagger/modules/e2e/fixtures/clients/dep/src/main/java/io/dagger/modules/dep/Dep.java new file mode 100644 index 0000000..a492741 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/dep/src/main/java/io/dagger/modules/dep/Dep.java @@ -0,0 +1,21 @@ +package io.dagger.modules.dep; + +import static io.dagger.sdk.Dagger.dag; + +import io.dagger.core.Directory; +import io.dagger.module.annotation.Function; +import io.dagger.module.annotation.Object; + +@Object +public class Dep { + @Function + public String greet(String name) { + return "hello " + name; + } + + /** A core type handed across the client boundary. */ + @Function + public Directory scratch() { + return dag().directory().withNewFile("dep.txt", "from dep"); + } +} diff --git a/.dagger/modules/e2e/fixtures/clients/workspace.toml b/.dagger/modules/e2e/fixtures/clients/workspace.toml new file mode 100644 index 0000000..27e262a --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/workspace.toml @@ -0,0 +1,15 @@ +# Placed at the workspace root by the e2e checks, so that both the SDK's module +# list and the engine's generator rollup for local dependencies see the same +# config. Not named dagger.toml so that find-up from another fixture never +# reads it. +[modules.java-sdk] +source = "." + +[modules.java-sdk.as-sdk] +name = "java" + +[[modules.java-sdk.as-sdk.modules]] +path = ".dagger/modules/e2e/fixtures/clients/dep" + +[[modules.java-sdk.as-sdk.modules]] +path = ".dagger/modules/e2e/fixtures/clients/app" diff --git a/.dagger/modules/e2e/main.dang b/.dagger/modules/e2e/main.dang index cf989b8..e8761cb 100644 --- a/.dagger/modules/e2e/main.dang +++ b/.dagger/modules/e2e/main.dang @@ -14,6 +14,12 @@ type E2e { let managedTomlModulePath: String! = fixtureRoot + "/managed-toml/app" # A module using a different SDK; this SDK must never manage it. let nonJavaModulePath: String! = fixtureRoot + "/lookup/not-java" + # Two real Java modules, app depending on dep, registered by a workspace + # config of their own (fixtures/clients/workspace.toml, placed at the root by + # clientsWS) so the module inventory the discovery checks assert stays as it is. + let clientsRoot: String! = fixtureRoot + "/clients" + let depModulePath: String! = clientsRoot + "/dep" + let appModulePath: String! = clientsRoot + "/app" """ Fail the current check when a condition is false. @@ -235,7 +241,7 @@ type E2e { .withDirectory(".", initialized.layer) # Module-relative paths of the two artifacts generation stages. - let vendoredClient = "sdk/src/main/java/io/dagger/client/Dagger.java" + let vendoredClient = "sdk/src/main/java/io/dagger/sdk/Dagger.java" let entrypoint = "src/generated/java/io/dagger/gen/entrypoint/Entrypoint.java" # From the module's parent the module is one level down, so its changes are @@ -301,9 +307,9 @@ type E2e { let nullableReturnSource: String! { "package io.dagger.modules.generateapp;\n" + "\n" - + "import static io.dagger.client.Dagger.dag;\n" + + "import static io.dagger.sdk.Dagger.dag;\n" + "\n" - + "import io.dagger.client.Directory;\n" + + "import io.dagger.core.Directory;\n" + "import io.dagger.module.annotation.Function;\n" + "import io.dagger.module.annotation.Object;\n" + "import java.util.Optional;\n" @@ -319,4 +325,223 @@ type E2e { + " }\n" + "}\n" } + + """ + The clients fixtures as buildable modules: both initialized in memory from the + default template, their committed sources and config overlaid, the + fixture-wide skip marker removed so generation runs, and the clients workspace + config at the root. The root is where the engine reads the generator rollup + it scopes a local dependency's generation to, so it has to be the same config + the SDK's own module list comes from; the other fixtures' config is left out + so find-up from the app cannot reach it first. + """ + let clientsWS(ws: Workspace!): Directory! { + let depInit = javaSdk.initModule(ws, name: "dep", path: depModulePath) + let appInit = javaSdk.initModule(ws, name: "app", path: appModulePath) + testWS(ws) + .directory("/", exclude: [fixtureRoot + "/.dagger-java-sdk-skip-generate", fixtureRoot + "/dagger.toml"]) + .withFile("dagger.toml", currentModule.source.file("fixtures/clients/workspace.toml")) + # initModule's changeset is rooted at the workspace, not at the module + .withDirectory(".", depInit.layer) + .withDirectory(".", appInit.layer) + .withDirectory(depModulePath, currentModule.source.directory("fixtures/clients/dep")) + .withDirectory(appModulePath, currentModule.source.directory("fixtures/clients/app")) + } + + let generatedRoot: String! = "sdk/src/generated/java/io/dagger" + + let mavenImage: String! = "maven:3.9.9-eclipse-temurin-21-alpine@sha256:4cbb8bf76c46b97e028998f2486ed014759a8e932480431039bdb93dffe6813e" + + """ + A tree with uniform permissions, so two of them can be compared on their bytes. + + Directory.digest covers permissions, and a changeset layer does not carry the + ones generation produced: the engine writes a module's generated tree into the + workspace at 0666/0777 while a standalone client's lands at 0644/0755, from + identical 0644 input. Verified by exporting both trees. Normalizing here keeps + the comparison a digest — every byte and the whole shape — without asserting + on a mode the workspace, not this SDK, decides. + """ + let sameModes(tree: Directory!): Directory! { + container + .from(mavenImage) + .withoutEntrypoint + .withDirectory("/tree", tree) + .withExec(["sh", "-c", "find /tree -type d -exec chmod 0755 {} + ; find /tree -type f -exec chmod 0644 {} +"]) + .directory("/tree") + } + + """ + Dependencies become clients, and a module gets one for itself. Generating the + app fixture — which depends on dep — from nothing must produce core, the dep + client and app's own client; with that client in place app can call itself, + and generating again picks the new function up; and a run with no edits must + change nothing. + """ + clientsGenerateCheck(ws: Workspace!): Void @check { + let root1 = clientsWS(ws) + let first = javaSdk.generateAll(root1.asWorkspace(cwd: appModulePath)) + assertAdded(first, generatedRoot + "/core/Container.java") + assertAdded(first, generatedRoot + "/client/dep/Dep.java") + assertAdded(first, generatedRoot + "/client/greeter/Greeter.java") + assertAdded(first, generatedRoot + "/client/app/App.java") + assertAdded(first, "src/generated/java/io/dagger/gen/entrypoint/Entrypoint.java") + assert( + contains(first.addedPaths, generatedRoot + "/core/Host.java") == false, + "Host must stay hidden from module code", + ) + + let dep = first.layer.file(generatedRoot + "/client/dep/Dep.java").contents + assertContains(dep, "public static Dep from(Client dag)", "the dependency client should expose from(Client)") + assertContains(dep, "public static Dep dep(Client dag)", "the dependency client should expose the static-import alias") + assertContains( + dep, + "ModuleBinding.ensureServed(root, \"dep\", \"LOCAL_SOURCE\", \"/" + depModulePath + "\", \"\")", + "the dependency client should serve dep by its workspace path", + ) + assertContains(dep, "import io.dagger.core.Directory;", "a core type returned by the dependency should resolve to io.dagger.core") + + # app declares dep twice, the second time as "greeter": the alias is the + # module's final name, so the aliased client chains and serves that name and + # both clients live on one session. + let greeter = first.layer.file(generatedRoot + "/client/greeter/Greeter.java").contents + assertContains(greeter, "public static Greeter greeter(Client dag)", "the aliased dependency should be named after the alias") + assertContains( + greeter, + "ModuleBinding.ensureServed(root, \"greeter\", \"LOCAL_SOURCE\", \"/" + depModulePath + "\", \"\")", + "the aliased client should serve the alias, from the aliased module's own path", + ) + + let selfClient = first.layer.file(generatedRoot + "/client/app/App.java").contents + assertContains(selfClient, "public static App app(Client dag)", "the module should get a client for itself") + assertContains(selfClient, "greetViaDep(", "the self client should expose the module's functions") + + # With the self client vendored, the module can call itself; the previous + # self client is carried through the first pass so this compiles. The stale + # package stands in for a dependency dropped from dagger-module.toml: it is + # committed, this generation does not produce it, and it has to go. + # A changeset reports a directory that went away as one removed path, not as + # one per file it held. + let stalePath = generatedRoot + "/client/gone/" + let root2 = root1 + .withDirectory(appModulePath, first.layer) + .withNewFile( + appModulePath + "/" + stalePath + "Gone.java", + "package io.dagger.client.gone;\n\npublic class Gone {}\n", + ) + .withFile( + appModulePath + "/src/main/java/io/dagger/modules/app/App.java", + currentModule.source.file("fixtures/clients/app-self/App.java"), + ) + let second = javaSdk.generateAll(root2.asWorkspace(cwd: appModulePath)) + assertContains( + second.layer.file(generatedRoot + "/client/app/App.java").contents, + "greetSelf(", + "the self client should pick up a function added since the last generation", + ) + assert( + contains(second.removedPaths, stalePath), + "a committed client package this generation does not produce should be removed", + ) + + # A changeset removes as well as adds, and withDirectory only merges, so the + # removals are replayed before the layer is applied. + let root3 = second + .removedPaths + .reduce(root2) { dir, removed => dir.withoutDirectory(appModulePath + "/" + removed) } + .withDirectory(appModulePath, second.layer) + let third = javaSdk.generateAll(root3.asWorkspace(cwd: appModulePath)) + assert( + third.addedPaths.length == 0 and third.modifiedPaths.length == 0 and third.removedPaths.length == 0, + "generating an unchanged module again should change nothing", + ) + null + } + + """ + A standalone client for dep is byte-identical to the client app vendors for + it, sees everything a client is allowed to, and builds as a plain Maven + project with a main that uses it. + """ + standaloneClientCheck(ws: Workspace!): Void @check { + let root = clientsWS(ws) + let appChanges = javaSdk.generateAll(root.asWorkspace(cwd: appModulePath)) + let depChanges = javaSdk.generateAll(root.asWorkspace(cwd: depModulePath)) + let withDep = root.withDirectory(depModulePath, depChanges.layer) + + let client = javaSdk.generateClient( + withDep.asWorkspace(cwd: clientsRoot), + module: "/" + depModulePath, + path: clientsRoot + "/dep-client", + ) + assertAdded(client, "dep-client/pom.xml") + assertAdded(client, "dep-client/sdk/src/main/java/io/dagger/sdk/Dagger.java") + assertAdded(client, "dep-client/sdk/src/generated/java/io/dagger/client/dep/Dep.java") + assertAdded(client, "dep-client/sdk/src/generated/java/io/dagger/core/Host.java") + assertContains( + client.layer.file("dep-client/pom.xml").contents, + "dep-client", + "the client pom should be named after its directory", + ) + assert( + sameModes(client.layer.directory("dep-client/" + generatedRoot + "/client/dep")).digest + == sameModes(appChanges.layer.directory(generatedRoot + "/client/dep")).digest, + "the standalone client must be byte-identical to the one app vendors as its dependency", + ) + + let project = client + .layer + .directory("dep-client") + .withFile( + "src/main/java/io/dagger/clients/depclient/Main.java", + currentModule.source.file("fixtures/clients/dep-client/Main.java"), + ) + container + .from(mavenImage) + .withoutEntrypoint + .withMountedCache("/root/.m2", cacheVolume("sdk-java-maven-m2"), sharing: CacheSharingMode.LOCKED) + .withDirectory("/client", project) + .withWorkdir("/client") + .withExec(["mvn", "-q", "package", "-DskipTests", "--no-transfer-progress"]) + .sync + null + } + + """ + A client registered in workspace config is seeded by initClient and + materialized by the generate hook, with the engine resolving the bound module. + """ + registeredClientCheck(ws: Workspace!): Void @check { + let root = clientsWS(ws) + let depChanges = javaSdk.generateAll(root.asWorkspace(cwd: depModulePath)) + let registered = root + .withDirectory(depModulePath, depChanges.layer) + .withNewFile( + "dagger.toml", + currentModule.source.file("fixtures/clients/workspace.toml").contents + + "\n[[modules.java-sdk.as-sdk.clients]]\npath = \"" + + clientsRoot + + "/dep-client\"\nmodule = \"" + + depModulePath + + "\"\n", + ) + + let seeded = javaSdk.initClient(registered.asWorkspace(cwd: clientsRoot), path: clientsRoot + "/dep-client", module: "dep") + assertAdded(seeded, "dep-client/pom.xml") + assert(seeded.addedPaths.length == 1, "initClient should seed the pom and nothing else") + + let withSeed = registered.withDirectory(clientsRoot, seeded.layer) + let all = javaSdk.generateAllClient(withSeed.asWorkspace(cwd: clientsRoot)) + assertAdded(all, "dep-client/sdk/src/generated/java/io/dagger/client/dep/Dep.java") + assertAdded(all, "dep-client/sdk/src/generated/java/io/dagger/core/Client.java") + + let again = javaSdk.generateAllClient( + withSeed.withDirectory(clientsRoot, all.layer).asWorkspace(cwd: clientsRoot), + ) + assert( + again.addedPaths.length == 0 and again.modifiedPaths.length == 0 and again.removedPaths.length == 0, + "regenerating an unchanged registered client should change nothing", + ) + null + } } diff --git a/.dagger/modules/packager/main.dang b/.dagger/modules/packager/main.dang index b1f497a..0fbfa59 100644 --- a/.dagger/modules/packager/main.dang +++ b/.dagger/modules/packager/main.dang @@ -30,7 +30,7 @@ type Packager { let codegenPluginRepo(ws: Workspace!): Directory! { mvn .withoutEntrypoint - .withMountedCache("/root/.m2", cacheVolume("sdk-java-maven-m2")) + .withMountedCache("/root/.m2", cacheVolume("sdk-java-maven-m2"), sharing: CacheSharingMode.LOCKED) .withDirectory("/dagger-io", sdkSource(ws)) .withWorkdir("/dagger-io") .withExec(["mvn", "--projects", "dagger-codegen-maven-plugin", "--also-make", "install", "-T1C", "-Dmaven.test.skip=true", "-Dfmt.skip=true", "-Dproject.build.outputTimestamp=2024-01-01T00:00:00Z", "--no-transfer-progress"]) @@ -67,7 +67,7 @@ type Packager { .introspectionSchemaJSON mvn .withoutEntrypoint - .withMountedCache("/root/.m2", cacheVolume("sdk-java-maven-m2")) + .withMountedCache("/root/.m2", cacheVolume("sdk-java-maven-m2"), sharing: CacheSharingMode.LOCKED) .withMountedFile("/schema.json", introspectionJSON) .withDirectory("/dagger-io", sdkSource(ws)) .withWorkdir("/dagger-io") diff --git a/README.md b/README.md index b677c60..1758c0c 100644 --- a/README.md +++ b/README.md @@ -41,11 +41,80 @@ the generated, committed sources: ``` src/generated/java/io/dagger/gen/entrypoint/Entrypoint.java # generated entrypoint - sdk/src/main/java/... # vendored SDK library + sdk/src/main/java/io/dagger/sdk/... # vendored SDK runtime sdk/src/processor/java/... # vendored annotation processor - sdk/src/generated/java/... # client bindings (from the engine schema) + sdk/src/generated/java/io/dagger/core/... # the core API (Container, Directory, ...) + sdk/src/generated/java/io/dagger/client//... # this module's own client + sdk/src/generated/java/io/dagger/client//... # one client per declared dependency ``` +## Modules have clients, not dependencies + +A dependency declared in `dagger-module.toml` becomes a **generated client**: +a package `io.dagger.client.` holding that module's types and an +entry point on its root type. The module's own API gets the same treatment, so a +self call goes through the engine like any other: + +```java +import static io.dagger.sdk.Dagger.dag; +import static io.dagger.client.hello.Hello.hello; // a dependency named hello +import static io.dagger.client.app.App.app; // this module, named app + +hello(dag()).greet("world"); // the dependency +app(dag()).build(source); // ourselves, through the engine +``` + +`Hello.from(dag())` is the same entry point without the static import. An entry +point serves its module into the session before its first call on a given +client: inside a module the engine has already served it and the serve only +confirms it, in a standalone client it is the bootstrap. Later calls on the same +client skip it. Core types (`io.dagger.core.Container`, `Directory`, ...) are shared by +every client in the tree; a type authored by one module does not cross into +another module's client, which is the rule the engine already enforces for +module APIs. + +## Standalone clients + +The same generator produces a client for any module, for a test or an +application that is not itself a module: + +```sh +dagger call java-sdk generate-client --module= --path= +``` + +``` +/pom.xml # created when absent, yours afterwards +/sdk/src/main/java/io/dagger/sdk/... # SDK runtime +/sdk/src/generated/java/io/dagger/core/... # core API +/sdk/src/generated/java/io/dagger/client//... # the bound module's client +``` + +`io/dagger/client//**` is byte-identical to what a module depending on +`` receives. Clients registered in the workspace config are regenerated by +`dagger generate` like modules are. With no session in its environment the +client starts one with the `dagger` CLI on the `PATH` (or in +`_EXPERIMENTAL_DAGGER_CLI_BIN`): + +```java +try (var dag = Dagger.connect()) { + hello(dag).greet("world"); +} +``` + +## Migrating a module + +Generated types moved: the runtime from `io.dagger.client` to `io.dagger.sdk`, +the core API to `io.dagger.core`. In a module's own sources: + +```sh +sed -i -E 's/io\.dagger\.client\.(Dagger|AutoCloseableClient|Arguments|IDAbleSerializer|IDAble|InputValue|QueryBuilder|ScalarStringDeserializer|ScalarSerializer|Scalar|FieldsStrategy|ModuleBinding|exception|engineconn|graphql|telemetry)/io.dagger.sdk.\1/g; s/io\.dagger\.client\.([A-Z])/io.dagger.core.\1/g' $(git ls-files 'src/main/java/*.java' 'src/main/java/**/*.java') +dagger generate +``` + +The first rule lists the runtime classes a module writes against by name, +because the second one cannot tell them from a core type. `**` does not match +files directly under `src/main/java`, hence the two patterns. + Because everything is committed and the pom defaults `dagger.proc=none`, the module builds with a plain `mvn package` (no annotation processor at build time) — in an IDE or CI, without Dagger. @@ -53,10 +122,13 @@ module builds with a plain `mvn package` (no annotation processor at build time) ## How generation works `generate` runs Maven in containers it controls: it builds the vendored codegen -plugin, generates the client bindings from the engine's introspection schema, -vendors the SDK library and annotation processor as source, and runs the -processor once to produce the entrypoint. It does not delegate code generation -back to the engine. +plugin, generates `io.dagger.core` from the module-facing schema and one +`io.dagger.client.` per declared dependency from that dependency's +client-facing schema, vendors the SDK runtime and annotation processor as +source, and runs the processor once to produce the entrypoint. With the SDK and +the entrypoint staged the module builds, so the engine can load it and hand back +its own client-facing schema; a second codegen pass turns that into the module's +self client. It does not delegate code generation back to the engine. ## The codegen flag diff --git a/client-template/pom.xml b/client-template/pom.xml new file mode 100644 index 0000000..7729fa4 --- /dev/null +++ b/client-template/pom.xml @@ -0,0 +1,126 @@ + + + 4.0.0 + + io.dagger.clients.daggermoduleplaceholder + dagger-module-placeholder + 1.0-SNAPSHOT + dagger-module-placeholder + + + 17 + UTF-8 + + + + + + + io.opentelemetry + opentelemetry-bom + 1.61.0 + pom + import + + + + + + + + jakarta.json + jakarta.json-api + 2.1.3 + + + jakarta.json.bind + jakarta.json.bind-api + 3.0.1 + + + org.slf4j + slf4j-api + 2.0.17 + + + io.opentelemetry + opentelemetry-api + + + io.opentelemetry + opentelemetry-sdk + + + io.opentelemetry + opentelemetry-exporter-otlp + + + io.opentelemetry + opentelemetry-exporter-sender-okhttp + + + + + io.opentelemetry + opentelemetry-exporter-sender-jdk + runtime + + + + + org.slf4j + slf4j-simple + 2.0.17 + runtime + + + org.eclipse + yasson + 3.0.4 + runtime + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-dagger-sdk-sources + + add-source + + + + sdk/src/main/java + sdk/src/generated/java + + + + + + + maven-compiler-plugin + 3.13.0 + + none + + + + + diff --git a/client.dang b/client.dang new file mode 100644 index 0000000..053901c --- /dev/null +++ b/client.dang @@ -0,0 +1,104 @@ +""" +A generated Java client: a plain Maven project bound to one Dagger module. + +The client is byte-for-byte the package a module receives when it declares that +same module as a dependency — the same generator, the same plan entry — with the +core API and the SDK runtime vendored next to it under sdk/ so the project +builds with a plain `mvn package`. +""" +type Client { + """ + Workspace-root-relative directory the client is written to. + """ + pub rootPath: String! + + """ + The bound module's final name. + """ + pub module: String! = "" + + """ + The bound module's source kind, JSON-encoded as the engine reports it. + """ + pub kindJSON: String! = "" + + """ + The bound module's canonical ref (a git module) or workspace-relative root. + """ + pub canonicalRef: String! = "" + + pub rootSubpath: String! = "" + + pub pin: String! = "" + + pub engineVersion: String! = "" + + """ + The bound module's client-facing schema, as introspection JSON. + """ + pub schemaJSON: String! = "" + + let codegen: Codegen! { Codegen() } + + """ + Generate the client under rootPath: the SDK runtime and the generated packages + under sdk/, and the given pom when the directory has none. Both core and the + client come from the bound module's client-facing schema, which hides nothing: + a client is allowed everything the CLI is. + """ + generate(ws: Workspace!, pom: File!): Changeset! { + let plan = codegen.withClientEntry( + codegen.corePlan(schemaJSON), + module, + kindJSON, + canonicalRef, + rootSubpath, + pin, + engineVersion, + schemaJSON, + ) + let built = codegen.sdkBuilt(plan, directory, "", "client-" + module) + let out = seeded(ws, pom) + .withoutDirectory("sdk") + .withDirectory("sdk/src/main/java", built.directory("/dagger-io/dagger-java-sdk/src/main/java")) + .withDirectory("sdk/src/generated/java", built.directory(codegen.generatedSourcesPath)) + # sdk/ is dropped on the workspace as well as in `out`: withNewDirectory + # merges into what the workspace already holds, so a package this + # generation no longer produces would survive the clean directory. + ws + .withoutDirectory(codegen.workspaceRef(joinPath("sdk"))) + .withNewDirectory(codegen.workspaceRef(rootPath), out) + .changes(ws) + } + + """ + Seed the client directory with the given pom, leaving anything already there + alone: init must never remove a user's files. + """ + init(ws: Workspace!, pom: File!): Changeset! { + ws.withNewDirectory(codegen.workspaceRef(rootPath), seeded(ws, pom)).changes(ws) + } + + """ + The client directory as it is, with the pom added when it has none. + """ + let seeded(ws: Workspace!, pom: File!): Directory! { + let existing = existingDir(ws) + if (existing.exists("pom.xml")) { existing } else { existing.withFile("pom.xml", pom) } + } + + """ + Join the client root with a sub-path, handling the root (".") client. + """ + let joinPath(sub: String!): String! { + if (rootPath == ".") { sub } else { rootPath + "/" + sub } + } + + """ + Existing contents of the client directory, empty when it doesn't exist yet. + """ + let existingDir(ws: Workspace!): Directory! { + let filtered = ws.directory("/", include: [rootPath + "/**"]) + if (filtered.exists(rootPath)) { filtered.directory(rootPath) } else { directory } + } +} diff --git a/codegen.dang b/codegen.dang new file mode 100644 index 0000000..bcae3ae --- /dev/null +++ b/codegen.dang @@ -0,0 +1,214 @@ +""" +The Java SDK's code generation: the vendored SDK reactor built against a +generation plan, in containers this module controls. +""" +type Codegen { + """ + Maven container used for codegen (pinned digest, matches the builtin runtime). + """ + mvn: Container! { + container.from("maven:3.9.9-eclipse-temurin-21-alpine@sha256:4cbb8bf76c46b97e028998f2486ed014759a8e932480431039bdb93dffe6813e") + } + + """ + The vendored Java SDK Maven reactor shipped with this module. + """ + let sdkSourceDir: Directory! { currentModule.source.directory("sdk") } + + let prebuiltCodegenRepo: String! { "prebuilt/m2" } + + let prebuiltCodegenPlugin: String! { "prebuilt/dagger-codegen-maven-plugin.jar" } + + """ + Where the codegen plugin writes generated sources inside the SDK reactor. + """ + generatedSourcesPath: String! { "/dagger-io/dagger-java-sdk/target/generated-sources/dagger" } + + """ + The live engine version, without build metadata. + + Codegen only reads the engine version off the CLI when it has to query the + schema itself. Here the schema is handed to it, so without this the version + stays whatever the pom happens to say, and generation cannot tell which shapes + the engine on the other end actually supports. + + The `+` suffix is dropped: it changes on every engine build and would + make every module's SDK rebuild for no reason. + """ + let engineVersion: String! { version.split("+")[0] ?? version } + + """ + A maven container with the codegen plugin available in the local repository. + + Fast path: when the packager module has committed the plugin's local Maven + repository under prebuilt/m2, drop it into ~/.m2/repository with a plain copy — + no maven invocation. Otherwise fall back to installing a committed plugin jar, + and finally to compiling the plugin from the vendored sources. + """ + let codegenBase: Container! { + # Locked: concurrent installs into one local repository corrupt its + # maven-metadata files, and every generate runs two of them. + let base = mvn + .withoutEntrypoint + .withMountedCache("/root/.m2", cacheVolume("sdk-java-maven-m2"), sharing: CacheSharingMode.LOCKED) + .withDirectory("/dagger-io", sdkSourceDir) + .withWorkdir("/dagger-io") + + if (currentModule.source.exists(prebuiltCodegenRepo + "/io/dagger")) { + base + .withDirectory("/prebuilt-m2", currentModule.source.directory(prebuiltCodegenRepo)) + .withExec(["sh", "-c", "mkdir -p /root/.m2/repository && cp -r /prebuilt-m2/. /root/.m2/repository/"]) + } else if (currentModule.source.exists(prebuiltCodegenPlugin)) { + base + .withMountedFile("/codegen-plugin.jar", currentModule.source.file(prebuiltCodegenPlugin)) + .withExec(["mvn", "install:install-file", "-Dfile=/dagger-io/pom.xml", "-Dpackaging=pom", "-DpomFile=/dagger-io/pom.xml", "--no-transfer-progress"]) + .withExec(["mvn", "install:install-file", "-Dfile=/codegen-plugin.jar", "-DpomFile=/dagger-io/dagger-codegen-maven-plugin/pom.xml", "--no-transfer-progress"]) + } else { + base.withExec(["mvn", "--projects", "dagger-codegen-maven-plugin", "--also-make", "install", "-T1C", "-Dmaven.test.skip=true", "-Dfmt.skip=true", "--no-transfer-progress"]) + } + } + + """ + Build and install the SDK + annotation processor jars against a generation + plan, returning the maven container. + + The plan is a directory of entries the codegen plugin turns into packages in one + run: core into io.dagger.core, one io.dagger.client. per client entry. + `seed` is dropped into the generated-sources tree before codegen runs — the + client packages a previous generation produced — and `keep` names the module + whose package survives untouched among them; every other package the plan does + not mention is removed. + + Uses install (rather than generate-sources) so the compiled jars land in the + local Maven repository — the dagger-prebuilt-sdk pom profile compiles the + entrypoint against them instead of recompiling the vendored SDK sources. Jars + are installed under a per-module version so modules that resolve to different + schemas (different dependencies, or a different engine) never share a Maven + coordinate; the codegen plugin keeps its own fixed version and resolves from the + prebuilt repo. Cached across a module's own edits. + """ + sdkBuilt(plan: Directory!, seed: Directory!, keep: String!, name: String!): Container! { + codegenBase + .withDirectory("/plan", plan) + .withDirectory(generatedSourcesPath, seed) + .withExec(["mvn", "versions:set", "-DnewVersion=" + name, "-DgenerateBackupPoms=false", "--no-transfer-progress"]) + .withExec(["mvn", "--projects", "dagger-java-sdk,dagger-java-annotation-processor", "--also-make", "install", "-Ddaggerengine.plan=/plan", "-Ddaggerengine.keep=" + keep, "-Ddaggerengine.version=" + engineVersion, "-Dmaven.test.skip=true", "-Dfmt.skip=true", "--no-transfer-progress"]) + # Directory.digest covers permissions, and the seed of a second pass comes + # back from the engine at 0666 while a fresh file is written at 0644. Left + # alone, the same bytes hash differently depending on how many passes + # produced them, and a client generated in a module stops matching the + # standalone one. + .withExec(["sh", "-c", "find " + generatedSourcesPath + " -type d -exec chmod 0755 {} + ; find " + generatedSourcesPath + " -type f -exec chmod 0644 {} +"]) + } + + """ + Vendor the Java SDK as buildable source out of a built reactor: + src/main/java the hand-written SDK runtime + src/processor/java the annotation processor that generates the entrypoint + src/generated/java io.dagger.core and one io.dagger.client. per client + src/processor/resources/META-INF/services/...Processor + The returned directory is dropped at /sdk. + """ + vendoredSdk(built: Container!): Directory! { + directory + .withDirectory("src/main/java", built.directory("/dagger-io/dagger-java-sdk/src/main/java")) + .withDirectory( + "src/processor/java", + built.directory("/dagger-io/dagger-java-annotation-processor/src/main/java"), + ) + .withDirectory("src/generated/java", built.directory(generatedSourcesPath)) + .withNewFile( + "src/processor/resources/META-INF/services/javax.annotation.processing.Processor", + "io.dagger.annotation.processor.DaggerModuleAnnotationProcessor\n", + ) + } + + """ + Capture the compiled SDK jar as a small local Maven repository to commit under + /sdk/repo. Stored at a fixed version so the module pom's + dagger-vendored-sdk-jar profile resolves it; the runtime build then compiles + only the module's own code against the jar rather than the vendored sources. + """ + vendoredSdkJar(built: Container!, name: String!): Directory! { + let jar = built + .withExec(["sh", "-c", "mkdir -p /out && cp /root/.m2/repository/io/dagger/dagger-java-sdk/" + name + "/dagger-java-sdk-" + name + ".jar /out/dagger-java-sdk-0.21.4.jar"]) + .file("/out/dagger-java-sdk-0.21.4.jar") + directory + .withFile("io/dagger/dagger-java-sdk/0.21.4/dagger-java-sdk-0.21.4.jar", jar) + .withNewFile( + "io/dagger/dagger-java-sdk/0.21.4/dagger-java-sdk-0.21.4.pom", + "4.0.0io.daggerdagger-java-sdk0.21.4jar\n", + ) + } + + """ + A plan holding only a core entry, generated from the given schema. + """ + corePlan(schemaJSON: String!): Directory! { + directory + .withNewFile("core/schema.json", schemaJSON) + .withNewFile("core/meta.json", "{\"mode\":\"core\"}") + } + + """ + The meta.json of a client plan entry: which module the package binds, and + where that module lives so the generated serve preamble can reach it. + """ + clientMeta(module: String!, kindJSON: String!, ref: String!, pin: String!): String! { + "{\"mode\":\"client\",\"module\":" + JSON.encode(module) + + ",\"binding\":{\"kind\":" + kindJSON + ",\"ref\":" + JSON.encode(ref) + ",\"pin\":" + JSON.encode(pin) + "}}" + } + + """ + Add a client entry for a module to a plan, from the module source's fields: + its final name, kind (JSON-encoded), canonical ref, workspace-relative root, + pin, declared engine version and client-facing schema. A git module is reached + by its canonical ref and pin, which resolve from anywhere; a local one by its + workspace-root-relative path through the workspace. + + A pre-1.0 module is refused first (see requireModernEngine). + """ + withClientEntry( + plan: Directory!, + name: String!, + kindJSON: String!, + canonicalRef: String!, + rootSubpath: String!, + pin: String!, + engineVersion: String!, + schemaJSON: String!, + ): Directory! { + requireModernEngine(name, engineVersion) + # Anything that is not git lives in the workspace: LOCAL_SOURCE on the host, + # DIR_SOURCE in a workspace built from a directory value. Both are served by + # workspace path, and baking the same kind for both keeps the generated + # bytes identical wherever the client was generated. + let git = kindJSON.contains("GIT") + let ref = if (git) { canonicalRef } else { workspaceRef(rootSubpath) } + let kind = if (git) { "\"GIT_SOURCE\"" } else { "\"LOCAL_SOURCE\"" } + plan + .withNewFile("client-" + name + "/schema.json", schemaJSON) + .withNewFile("client-" + name + "/meta.json", clientMeta(name, kind, ref, if (git) { pin } else { "" })) + } + + """ + Refuse a module the engine would render core through a pre-1.0 compatibility + view for: that view carries per-type ID scalars io.dagger.core does not have, + so without this it fails in the compiler instead of here. Both spellings the + engine accepts are rejected. + """ + requireModernEngine(name: String!, engineVersion: String!): Void { + if (engineVersion.hasPrefix("v0.") or engineVersion.hasPrefix("0.")) { + raise "module " + name + " declares engineVersion " + engineVersion + "; the Java SDK needs v1.0.0-0 or later" + } + null + } + + """ + A workspace-root-relative path as a workspace-root-absolute one, the form + Workspace resolves from the workspace root rather than from the client's cwd. + """ + workspaceRef(path: String!): String! { + if (path == "." or path == "") { "/" } else { "/" + path.trimPrefix("/") } + } +} diff --git a/hack/designs/done/2026-08-26-modules-have-clients.md b/hack/designs/done/2026-08-26-modules-have-clients.md new file mode 100644 index 0000000..d69518d --- /dev/null +++ b/hack/designs/done/2026-08-26-modules-have-clients.md @@ -0,0 +1,1104 @@ +# Modules have clients, not dependencies + +Status: proposed +Date: 2026-08-26 + +## Problem + +A module's dependency and a standalone generated client are already the same +thing, built twice. + +When a Java module calls a dependency today, it calls generated typed bindings +against a served module. That is the definition of a client. The only thing that +differs from a standalone client — a test, an app, the kind of artifact the Go +and TypeScript SDKs already generate — is *how the session and the target module +are obtained*: inside a module the engine has already served the dependency into +the session schema, while outside the process must open its own connection and +serve the target itself. + +This SDK does not model that. It has one generated package, `io.dagger.client`, +produced from `moduleSource.introspectionSchemaJSON` — the *module-facing* +schema, which loads the module's dependencies and merges them into one flat +schema (`Mod.generateModule`, `mod.dang:231`). Every type from core and from +every dependency lands in that single package, next to the hand-written runtime: + +``` +/sdk/src/main/java/io/dagger/client/** hand-written runtime +/sdk/src/generated/java/io/dagger/client/** core + every dependency, flat +``` + +Three consequences: + +1. **There is no client artifact.** Nothing this SDK produces can be handed to an + application that is not itself a Dagger module. `io.dagger.client.Client` is + reachable only through `Dagger.dag()`, a process-wide singleton + (`Dagger.java:6`), and the bindings it exposes are whatever that one module's + merged schema happened to contain. +2. **Dependency bindings are unattributed.** A dependency's types are + indistinguishable from core's once generated; nothing in the output records + which module contributed `Report` or `Binding.asHello`. Regenerating for a + different dependency set silently changes the meaning of the same package. +3. **The generator only has one mode.** `CodeWriter` hardcodes the target package + (`CodeWriter.java:20`) and every visitor resolves type references with + `ClassName.bestGuess(simpleName)`, which is only correct because everything is + in one package. There is no seam at which a second package could be emitted. + +Meanwhile the engine has already moved. `ModuleSource.clientSchemaIntrospectionJSON` +is the *client-facing* schema. Its implementation +(`core/schema/modulesource.go`, `clientSchemaIntrospectionJSONFile`) starts from +the core-only schema builder and installs exactly one module, namespaced, not as +an entrypoint. Its own doc comment is the specification this design builds on: + +> only the bound module is installed, as a normal namespaced module, so a +> generated client reaches its functions via `dag.` and never through +> a promoted Query root. The module's own dependencies are deliberately excluded +> — a client is generated for a single module plus core, not for its whole +> dependency graph. Unlike the module-facing schema, it hides no core types. + +The Go SDK consumes it already (`go-sdk.dang:360`, `generateClient`). Java does +not consume it at all. + +## Decisions + +These were the fork-in-the-road questions; they are settled and recorded here so +the rest of the document reads as consequences rather than options. + +**D1 — a client is deps-excluded (leaf-shaped).** Confirmed against the engine +source above. A *dependency-authored* type does not cross between two clients. +Core types do, because they are literally the same Java type. + +**This costs nothing, because the engine already forbids it.** An earlier draft +of this document claimed cross-dependency composition works today and that D1 +gives it up. That was wrong. `Module.validateTypeDef` rejects a module whose API +exposes a dependency-authored type, in all three positions — object fields +(`core/module.go:1220`), function return types (`core/module.go:1244`) and +function arguments (`core/module.go:1262`), each with +`cannot reference external type from dependency module %q`. A module can +therefore never hand a dependency's type to another module in the first place, +so a deps-excluded client cannot lose a capability that does not exist. + +Two consequences worth stating, because they fall out of the same fact: + +- The merged, deps-included schema this SDK generates today is *wider than + anything a module is allowed to use*. Flattening it into one package was + always over-generation. +- A client's schema can only reference core types and its own module's types. + There is no "type owned by a third module" case for the partition to handle — + the engine has already made it unrepresentable. + +**D2 — the hand-written runtime moves to `io.dagger.sdk`.** `io.dagger.client` +becomes *exclusively* generated: one package segment per bound module, nothing +else, ever. + +The reason is naming honesty, not collision avoidance. An earlier draft justified +the move by the risk of a module named `telemetry` or `graphql` colliding with an +SDK subpackage; that argument does not hold up, because the collision set is +exactly four names (`engineconn`, `exception`, `graphql`, `telemetry`) and +`io.dagger.clients.` would avoid it while moving zero files. The real reason +is that `io.dagger.client.QueryBuilder` **is not a client**. Once +`io.dagger.client.` means "the generated client for module m", leaving the +transport at that prefix makes the package name a lie. The collision going away +is a welcome side effect, not the justification. + +`io.dagger.runtime` was rejected: this repository already uses "runtime" for the +module runtime (the build/package contract under `runtime/`), so the name would +actively mislead. `io.dagger.sdk` matches both the vendored directory +(`sdk/src/main/java`) and the Maven artifact (`dagger-java-sdk`). + +**D3 — the entry point is a static factory, with a static-import alias.** +`Hello.from(dag())` is primary. The same class also carries a static +`hello(Client)` so a caller who prefers it can +`import static io.dagger.client.hello.Hello.hello;` and write `hello(dag())`. +Both are two lines of generated code delegating to one constructor, so offering +both costs nothing and lets the call site choose. A terse `f()` was considered +and dropped: cryptic abbreviations do not belong in a generated public API. + +**D4 — both entry points, mirroring the Go SDK.** `generateClient`, +`generateAllClient` (`@generate`, driven by workspace config), and `initClient`, +matching `go-sdk.dang:360/384` field-for-field. `currentModule.asSDK.clients` +exists on the pinned engine (verified by introspection: +`CurrentModuleAsSDKClient { id, module, moduleSource, path, pin }`). + +**D5 — a module generates a client for itself, and that is how it calls +itself.** Self calls go through `io.dagger.client.` exactly like calls to +any dependency; there is no separate self-call mechanism. This is a must-have, +not a convenience: a module that cannot reach itself through the engine cannot +benefit from function-level caching on its own calls. + +Reading a module's *own* `clientSchemaIntrospectionJSON` installs the module, +which for Java means building it — and the build needs the very sources this +generation is producing. On a first `init` + `generate` there is no `sdk/` at +all. The circularity is real, and it is broken by **bootstrapping through a +staged workspace**, the same device `generateLocalDependencies` already uses for +local dependencies: + +1. generate `io.dagger.core` and one client per declared dependency (neither + needs the module built — see D7); +2. vendor the runtime plus those packages, **carrying over the previously + committed self client if one exists**, so module code that already references + it still compiles; +3. run the annotation processor to produce the entrypoint, as today; +4. stage all of that onto the workspace (`ws.withNewDirectory(...)`) — the + module is now buildable — and read + `stagedWs.moduleSource(ref).clientSchemaIntrospectionJSON`, which makes the + engine build and introspect the module; +5. generate the self client from that schema with the same `client` mode, and + replace the carried-over one. + +The carried-over self client is only ever a compile-time crutch for step 4; the +committed output always comes from step 5. Adding a function and calling it +through the self client in the same edit fails the bootstrap build, exactly as +it would in any generated-client workflow: generate first, then call. + +The engine already serves a module to itself at call time +(`CallOpts{SkipSelfSchema: false}`, `core/object.go:1436`), so inside the module +the self client's serve is deduplicated by the engine like any other repeat +serve. + +Simple-name overlap is the one ergonomic cost: the authored +`io.dagger.modules.hello.Hello` and the generated `io.dagger.client.hello.Hello` +share a simple name, and inside the module the authored one is in scope. The D3 +static-import alias is the answer, and it is the documented idiom for self calls: + +```java +import static io.dagger.client.hello.Hello.hello; +… +hello(dag()).build(source) // a self call, through the engine +``` + +The static import brings in the *method*, not the type, so nothing clashes. A +module named after a core type (`workspace`, `env`, `secret`, `service`, `cache`) +has the same overlap against `io.dagger.core` and the same answer. + +**D6 — the query transport becomes public SDK API.** Generated code moves out of +`io.dagger.client`, so every runtime symbol it touches has to be reachable across +a package boundary. Today they are package-private: + +| Symbol | Today | Why generated code needs it | +|---|---|---| +| `QueryBuilder` (class, ctor, `chain`, `chainNode`, `execute*`) | package-private | field, ctor param and every field method on every generated type | +| `InputValue` | package-private **interface** | every generated input object has it in `implements` | +| `Arguments.merge` | package-private | optional-argument merging | +| `Scalar.convert()` | package-private | scalar serialization | +| `QueryPart` | package-private | transitively, via `QueryBuilder`'s signature | +| generated `Client` constructors | package-private | `AutoCloseableClient extends Client` becomes cross-package | + +`InputValue` is the one that makes this non-negotiable rather than a preference: +a class cannot implement a non-public interface from another package. Without +D6 the cutover simply does not compile. + +This **reverses a decision already recorded in this repo's design corpus**: +`hack/designs/2026-08-17-nullable-object-returns.md` deliberately kept +`QueryBuilder` package-private and rejected a public transport seam as permanent +API surface. That reasoning was right for that change and does not survive this +one — generated code in another package cannot be served by a package-private +transport. The reversal is deliberate and is called out here rather than made +silently. A public `Client.queryBuilder()` accessor is added too; it does not +exist today. + +**D7 — `io.dagger.core` depends on the engine and the consumer, never on a +bound module.** Core is generated from the schema the *consumer* is entitled to +see, partitioned to strip every module-owned symbol: + +- a **module** gets its core from its own module-facing + `introspectionSchemaJSON`. That schema loads the module's dependencies but + never the module itself (`moduleSourceIntrospectionSchemaJSON` → + `loadDependencyModules` → `SchemaIntrospectionJSONFileForModule`), so there is + no circularity, and it hides `TypesHiddenFromModuleSDKs` — which means + `dag().host()` in module code **stays a compile error**, closing the guard + regression an earlier draft had accepted; +- a **standalone client** gets its core from the bound module's client-facing + `clientSchemaIntrospectionJSON`, which hides nothing, because a client is + allowed everything the CLI is. + +Both are "the engine's core, as this consumer is allowed to see it". The +dependency-owned symbols in the module-facing schema are exactly what the +partition strips, so the result is core-only either way. Because every +`io.dagger.client.` package refers to core types only by name, the +per-module client bytes are identical across both contexts even though the +*core* package legitimately differs (hidden types, compatibility view). That is +the property the byte-identity claim is about. + +The compatibility view is the residual risk: the engine renders core through the +target module's declared `engineVersion` (`core/schema/modulesource.go:3537`), +and a dependency declared at a pre-`v1.0.0` version gets legacy per-type ID +scalars (`Sub1ID`, `loadSub1FromID`) that a `v1.0.0` core does not have. +Generation therefore fails early and clearly when a dependency declares an +`engineVersion` below `v1.0.0-0`, the floor this SDK already requires. Anything +subtler surfaces as a compile error in the vendored SDK build, which is loud if +not pretty. + +**D8 — the SDK can open its own session again.** `Connection.get` regains the +path that `89b80fe` removed as dead code: honour `DAGGER_SESSION_PORT` / +`DAGGER_SESSION_TOKEN` when set (a module runtime, or `dagger run`), otherwise +spawn `dagger session --label dagger.io/sdk.name:java …`, read the +`{port, session_token}` line it prints, and connect. The binary comes from +`_EXPERIMENTAL_DAGGER_CLI_BIN` or `dagger` on `PATH`. This is precisely what the +Go SDK (`dagger.Connect`) and the TypeScript SDK +(`sdk/typescript/src/provisioning/bin.ts:176`) do, minus one thing: both also +auto-download a CLI matching their version when none is found. The Java SDK does +not, in this series — like Testcontainers using whatever Docker the host has, it +uses the `dagger` the host has, and says so clearly when there is none. Download +is a follow-up, not a blocker, and it needs the checksum-verifying downloader +and archive dependencies that were dropped for weight. + +`ProcessBuilder` is enough for the session process; the `fluent-process` +dependency that the old `CLIRunner` used is not reintroduced. `AutoCloseableClient` +closes the session process it started; a connection taken from the environment +owns nothing. + +**D9 — the SDK stages its own local dependencies.** `mod.dang` no longer calls +the engine's `ModuleSource.generateLocalDependencies`. That routes through +`Workspace.generators(include: [])`, and two things make it +unusable here, both verified against engine source and by probing: + +- the engine returns an **empty generator group for any value workspace** + (`isSyntheticWorkspace` → `IsValueWorkspace`: a workspace built from a + `Directory`, which is what every in-memory e2e check runs in), so a Java + module with a local Java dependency can never be generated in a check; +- in this repository the rollup carries exactly four generator nodes + (`dagger-dang-sdk`, `packager`, `sdk-sdk`, `templates`) and none for + `java-sdk`, on `upstream/main` as much as here, whatever the root config + says — so the engine path had never actually worked for this SDK. + +Instead, `generatedOverlay` recurses: for each dependency that is not git and +sits in the modules registered to this SDK in the workspace (`ws.sdk(name: +currentModule.name).modules` — the registry, not the cwd-scoped +`modules(ws)`, since a dependency is usually a sibling), it generates that +module and overlays its `sdk/` and `src/generated/java` onto the workspace with +`withNewDirectory`. Overlays rather than changesets, because a changeset is +measured from the caller's cwd and may not reach a sibling. Dependencies of +other SDKs, remote ones, and skip-marked ones are assumed committed — the same +rule the engine applies. + +Two facts about module sources inside a value workspace follow from the same +probing and are handled explicitly: a local dependency reports +`kind = DIR_SOURCE` (not `LOCAL_SOURCE`) with an empty `asString`, so ownership +is decided on "not git", and the binding baked into a client is normalized to +`LOCAL_SOURCE` by workspace path for both — which also keeps the bytes +identical between a client generated in a value workspace and on the host. + +### Where the Go and TypeScript SDKs actually are + +Worth stating plainly, because it sets expectations for review: + +- **Go SDK**: has `generateClient` / `generateAllClient` / `initClient` exactly as + D4 describes, and this design copies that shape. But Go *module* generation + still delegates to the engine (`generatedContextDirectory`, `mod.dang:72`), + which uses the module-facing merged schema. **Go has not unified + dependencies-as-clients.** +- **TypeScript SDK**: `design/client-gen.md` describes the client schema as + having "deps loaded" and the target module's "own types promoted to `Query` + for self-bindings". **Both statements are stale** against the engine source + quoted above. Do not use that document as the spec for this one. + +So the `generate-a-client` half of this work has a proven reference to copy, and +the `dependencies-become-clients` half does not. Java is first there. That is +where the design risk is concentrated. + +## Goals + +- One generator, one output shape. The package generated for a module is + byte-identical whether it was produced because another module declared that + module as a dependency or because someone asked for a standalone client. +- Core types live in their own package, shared by every generated client. +- A module declares a dependency in `dagger-module.toml` exactly as it does + today; what changes is that the SDK generates a *client* for it. +- Produce a standalone client artifact that an ordinary Maven project can build + and run — structurally the analogue of what the Go and TypeScript SDKs emit, + including opening its own engine session (D8). +- A single engine session shared by every client in a process. +- One idempotent serve preamble: a no-op where the target module is already + served, a real bootstrap where it is not, with no context-dependent branch in + the generated code. + +## Non-goals + +- **No compatibility shim.** `io.dagger.client.Container` and friends move. There + is no alias package, no deprecation window, no dual-mode generator. Breaking + compatibility is in scope and intended. +- **No cross-module type composition between two dependency clients.** See D1. +- **No engine changes.** Everything this needs already exists on + `v1.0.0-beta.10`. If a gap appears, it is a separate `dagger/dagger` proposal, + not a patch in this series. +- **No published Maven artifacts.** The SDK stays self-contained and vendored, as + the README describes. A standalone client vendors what it needs. +- **No CLI auto-download.** The SDK opens a session with the `dagger` binary + the host provides; fetching one is follow-up work (D8). +- No change to module authoring: `@Object`, `@Function`, the entrypoint, and the + two-pass pom stay as they are, and `dag().host()` in module code stays a + compile error (D7). + +## Approach + +### The unification, precisely + +A generated client is **generated bindings plus a serve preamble**, where the +preamble is idempotent. It probes whether its bound module is already present in +the session schema; if it is, the preamble does nothing, and if it is not, it +serves it. Inside a module the dependency has already been served by the engine, +so the probe short-circuits. Outside, the probe misses and the preamble serves. +The generated bytes are the same either way, because the branch is taken at +runtime against session state, not at generation time against context. + +That is the whole feature. Everything below is the mechanics of making the +generator emit one artifact instead of one flat package. + +### Package layout + +```mermaid +graph TD + subgraph handwritten["io.dagger.sdk — hand-written runtime (moved)"] + QB["QueryBuilder, Arguments, IDAble,
Scalar, Dagger, ModuleBinding"] + SUB["…engineconn · …graphql
…exception · …telemetry"] + end + subgraph core["io.dagger.core — generated, one per engine schema"] + CORE["Client (Query root)
Container, Directory, File, Service,
Workspace, TypeDef, …"] + end + subgraph clients["io.dagger.client.<module> — generated, one per bound module"] + C1["io.dagger.client.hello
Hello, HelloReport, …"] + C2["io.dagger.client.builder
Builder, BuilderOptions, …"] + end + CORE --> QB + C1 --> CORE + C2 --> CORE + C1 --> QB + C2 --> QB +``` + +- `io.dagger.sdk` is the hand-written SDK runtime, moved wholesale from + `io.dagger.client`. Its subpackages keep their relative names + (`io.dagger.sdk.engineconn`, `.exception`, `.graphql`, `.telemetry`). One class + is added: `ModuleBinding`, the serve preamble. +- `io.dagger.core` is new and holds the generated core API, including `Client` + (the `Query` root). This is the only package whose contents depend on the + engine version alone. +- `io.dagger.client.` is new, one package per bound module, holding only + the types that module contributes plus its entry point. A module named `hello` + produces `io.dagger.client.hello`. Nothing hand-written lives under + `io.dagger.client` any more, so a module name can never collide (D2). + +`io.dagger.core` referring to `io.dagger.sdk.QueryBuilder` while +`io.dagger.sdk.Dagger` refers to `io.dagger.core.Client` is a package cycle. Java +permits it and both are compiled in the same pass; it is called out here so it is +a decision rather than an accident. Removing it would mean moving `Dagger` into +the generated package, mixing hand-written code into generated output, which is +worse. + +Module names are still normalized to a legal Java package segment (lowercased, +`-` stripped), and a name that cannot be normalized fails at generation time with +a clear error. + +### Type attribution: `@sourceMap` is the partition + +The introspection JSON already says which module contributed each type and each +field: the engine emits `@sourceMap(module: "", …)` on both. Core types and +core fields carry no `module`. `dagger/dagger`'s own codegen partitions on +exactly this (`cmd/codegen/introspection/filters.go`, `isOwnedByModules`), and +this SDK already parses directives on `Type` and `Field` +(`Type.java:86`, `Field.java:84`) — only the accessor is missing. + +So the partition is exact, needs no second schema, and needs no name-prefix +heuristics: + +- a type whose `@sourceMap.module` is empty belongs to `io.dagger.core`; +- a type whose `@sourceMap.module` is `M` belongs to `io.dagger.client.`; +- a **field** whose `@sourceMap.module` is `M`, on a type that belongs to core, + is a module extension of a core type (`Query.hello`, `Binding.asHello`) and + belongs to `M`, not to core. + +This is verified against a real schema, not assumed. Dumping +`clientSchemaIntrospectionJSON` for `.dagger/modules/e2e` (which declares a +`java-sdk` dependency) on the pinned engine gives 124 types, of which: + +- exactly one type carries `@sourceMap(module: "e2e")` — `E2E`; +- eight *fields* carry it — the seven `@check` functions on `E2E`, plus + **`Query.e2E`**, which is precisely the "module extension on a core type" case + the partition has to handle; +- `JavaSdk` and `Mod` — the dependency's types — are **absent**, confirming D1 + empirically as well as from the source; +- `Container`, `Directory`, `File`, `Service`, `Workspace` and `Host` are all + present and unhidden. + +Two implementation details fall out of that dump and are easy to get wrong: + +- the directive argument is a **JSON-quoted** string (`"\"e2e\""`), so + `getSourceMapModule` must strip the surrounding quotes exactly as the existing + `Directive.getExpectedType` already does; +- **the module's root type name cannot be derived by capitalizing the module + name.** Module `e2e` has root type `E2E`, not `E2e`. The root type must be read + off the schema — it is the return type of the `Query` field owned by that + module (`Query.e2E` → `E2E`). Generating the name by string manipulation + produces a type that does not exist. + +**Module-owned fields on core types need somewhere to go.** `Query.e2E` is the +entry point and is handled by the factory, but the engine also lets a module +extend `Binding` and `Env` — `cmd/codegen/introspection/filters.go:5` lists the +extendable types as `Query`, `Binding` **and** `Env` (the earlier draft of this +document said "just Query", which was wrong). Java has no extension methods, so +`Binding.asHello()` has no home in `io.dagger.client.hello` and no business in +`io.dagger.core`. + +These are emitted as **static shims on the module's entry-point class**: + +```java +public static Hello asHello(io.dagger.core.Binding binding) { + return new Hello(binding.queryBuilder().chain("asHello")); +} +``` + +Without this rule the partition silently deletes the whole LLM/agent surface for +module types (`Binding.asHello`, `Env.withHelloInput`, `LLM.hello`). Dropping +them would be a capability loss disguised as a partition detail, so it is made +an explicit emission rule with its own test. + +Field-level attribution is applied to *every* type, not only to `Query`. +`dagger/dagger`'s Go filter restricts field filtering to an `ExtendableTypes` +list containing just `Query`, which leaves a module-contributed `Binding.asHello` +in the core partition while its return type is filtered out of it. Java cannot +tolerate that — it is a compile error, not a soft inconsistency — so the stricter +rule is used here. + +### Generation modes + +`DaggerCodegenMojo` gains a mode and a target package. Both modes read one +`clientSchemaIntrospectionJSON` — core plus exactly one module: + +| mode | emits | into | +|---|---|---| +| `core` | every type and field with no owning module, plus the non-schema emissions `Version` and `JsonConverter` | `io.dagger.core` | +| `client` | every type and field owned by module `M`, plus `M`'s entry point and its core-type shims | `io.dagger.client.` | + +`Version` (`VersionVisitor`) and `JsonConverter` (`IDAbleVisitor`) are not schema +types, so they do not fall out of the partition and would otherwise be emitted +into *every* package. `JsonConverter` in particular is imported by name by the +annotation processor, so a duplicate in a client package is an ambiguous import. +Both are core-mode only. + +In `client` mode, references to non-owned types resolve to `io.dagger.core` +rather than to the local package. A `TypeRegistry`, built once from the schema +partition, replaces every `ClassName.bestGuess(simpleName)` in the visitors and — +critically — in `TypeRef`, which is the actual type-reference resolver. That +substitution is the bulk of the codegen change and is mechanical. + +Because `core` mode drops everything with an owning module, the core package it +emits is identical no matter which module's client schema it was derived from. +That is what makes `io.dagger.core` shareable, and it is asserted by a test +rather than assumed. + +### The serve preamble + +The entry point generated into `io.dagger.client.` is a static factory on the +module's root type, plus the D3 alias: + +```java +package io.dagger.client.hello; + +public class Hello { + public static Hello from(io.dagger.core.Client dag) { + QueryBuilder qb = dag.queryBuilder(); + ModuleBinding.ensureServed(qb, "hello", "Hello", "LOCAL_SOURCE", "dagger/modules/hello", ""); + return new Hello(qb.chain("hello")); + } + + /** Alias for {@link #from}, for use with a static import. */ + public static Hello hello(io.dagger.core.Client dag) { + return from(dag); + } + … +} +``` + +Java has no extension methods, so `dag().hello()` would require regenerating the +core `Client` per bound module — which would make core non-shareable and the +client non-identical. The static factory is the cost of the language. + +The five baked values are the bound module's identity, and they depend only on +that module — never on the consumer. That is why the emitted bytes are identical +in every context. They come off the module source exactly as the Go SDK reads +them (`moduleOriginalName`, `kind`, the ref, `asString`, `pin`). + +`ModuleBinding.ensureServed` is hand-written runtime, so the generated code +carries data and no logic. It **serves on the first call and remembers the exact +tuple it served, per session** — there is no probe: + +```mermaid +sequenceDiagram + autonumber + participant App as caller + participant MB as ModuleBinding + participant E as engine session + App->>MB: ensureServed(name, kind, ref, pin) + alt kind = GIT_SOURCE + MB->>E: moduleSource(ref, refPin: pin).withName(name).asModule().serve() + else local + MB->>E: currentWorkspace().moduleSource(ref).withName(name).asModule().serve() + end + E-->>MB: ok (same identity already served -> dedup) + E-->>MB: error (same name, different source) +``` + +An earlier draft probed `{ __type(name: rootType) { name } }` first and skipped +the serve when the type was present. That is removed, because the engine already +does the right thing and does it atomically. `Server.serveModule` +(`engine/server/session.go:1960`) looks the module up by name and: + +- if it is **not** served, serves it; +- if it **is** served from the same source and pin, `isSameModuleReference` + matches and the call succeeds — `With` "handles deduplication and promotion + internally"; +- if it is served from a *different* source, it returns + `module %s ... already exists with different source %s`. + +So unconditional serving is idempotent for free, and the probe was strictly +worse than useless: `__type` only proves a type *name* exists, so it would skip +serving when a **different** module of the same name was already present — +silently binding the caller to the wrong module and suppressing exactly the +conflict the engine is there to report. The `Module.serve` doc comment saying +"once per session" is stale relative to this implementation. + +Dropping the probe also removes the need for `QueryBuilder` to express a raw +`__type` query (it cannot — it only builds `{field{field}}` chains). The +preamble is now pure data plus one engine call. + +**The guard that stays is a cache of successfully served tuples, keyed on the +session.** `ModuleBinding` holds a weak map from `GraphQLClient` to the set of +`(name, kind, ref, pin)` tuples that have been served on it, and skips a repeat +of an exact tuple. That is not the probe wearing another hat, and the difference +is the reason it is safe: the probe would have skipped a serve *before* the +engine had ever been asked about that name, so a different module already served +under it went unreported. The cache only ever skips a serve the engine has +already accepted for that exact source and pin — a conflict would have errored +on the first call — so nothing it suppresses could have failed. Without it every +entry-point call in a module pays a round trip on a serve the engine has already +deduplicated, which is the common case, not the rare one. + +The bound module's **final** name — after any `withName` alias — is what gets +baked and what the serve applies. The engine applies dependency aliases with +`withName` (`core/modulesource.go:1978`) and namespaces the schema by the final +name (`core/gqlformat.go:36`), so a client generated for a dependency aliased to +`alias` chains `alias` and must serve under `alias` too. Using +`moduleOriginalName` here would generate a client that chains one name while +serving another — a runtime wrong answer, not merely different bytes. + +Local bindings bake the module's **workspace-relative** path, resolved through +`currentWorkspace().moduleSource(path)` — never a cwd-relative or absolute host +path. A local binding does not survive being shipped away from the workspace; a +git binding does. That limitation is the engine's and is repeated in the +generated javadoc rather than papered over. + +### Where the schemas and identities come from + +All of it is reachable from dang today, with no engine change: + +```mermaid +graph LR + MS["ws.moduleSource(modPath)"] -->|introspectionSchemaJSON| CORESCHEMA["module-facing:
core + deps, self absent"] + MS -->|dependencies| DEPS["[ModuleSource!]!"] + DEPS -->|clientSchemaIntrospectionJSON| DEPSCHEMA["core + dep"] + DEPS -->|moduleName, kind, sourceRootSubpath, asString, pin| IDENT["baked binding identity"] + CORESCHEMA -->|mode=core| P1["io.dagger.core"] + DEPSCHEMA -->|mode=client| P2["io.dagger.client.<dep>"] + IDENT --> P2 + P1 --> STAGE["staged workspace:
runtime + core + deps + entrypoint"] + P2 --> STAGE + STAGE -->|"moduleSource(modPath).clientSchemaIntrospectionJSON"| SELFSCHEMA["core + self"] + SELFSCHEMA -->|mode=client| P3["io.dagger.client.<self>"] +``` + +`ModuleSource.dependencies` returns `[ModuleSource!]!` with dependency aliases +already applied, so each dependency's client schema, its **final** name +(`moduleName`), and its identity (`kind`, `sourceRootSubpath` for a local +module, `asString` and `pin` for git) come straight off the graph. The self +client comes from the staged workspace per D5. + +One codegen invocation handles every package. `DaggerCodegenMojo` reads a +**plan directory** — `//schema.json` plus +`//meta.json` carrying `mode`, `module`, and the binding identity — +and emits all entries into one output tree in a single Maven run, so the number +of dependencies does not multiply Maven invocations. The self client is a second, +codegen-only run over a one-entry plan after the bootstrap build. The Mojo cleans +the SDK-owned generated package roots it is about to write before writing, so a +removed dependency or a renamed alias does not leave a stale package behind. + +`mod.dang:226` already stages `generateLocalDependencies(ws)` before resolving +the module source, so a local dependency's own generated output is up to date +before its client schema is read. That staging is kept. + +### What a module's tree looks like + +``` +/sdk/src/main/java/io/dagger/sdk/** runtime (vendored, moved) +/sdk/src/processor/java/** processor (vendored) +/sdk/src/generated/java/io/dagger/core/** core API +/sdk/src/generated/java/io/dagger/client//** the module's own client (D5) +/sdk/src/generated/java/io/dagger/client//** one package per declared dependency +/src/generated/java/io/dagger/gen/entrypoint/** entrypoint (unchanged) +``` + +Everything stays under the existing `sdk/src/generated/java` source root, so the +module pom needs no change. + +### What a standalone client looks like + +`generateClient(ws, module, path)` produces a plain Maven project: + +``` +/pom.xml seeded when absent, then the user's +/sdk/src/main/java/io/dagger/sdk/** runtime (vendored) +/sdk/src/generated/java/io/dagger/core/** core API +/sdk/src/generated/java/io/dagger/client//** the bound module's client +``` + +Everything generated sits under `sdk/`, exactly as in a module, so the user's +own `src/main/java` is never touched and the whole of `sdk/` can be dropped and +rewritten on every run. The pom is rendered from `client-template/` by the same +helper that renders module templates — it sits outside `templates/`, which is +the list of *module* init templates — so there is one source of truth for the +dependency list. + +`sdk/src/generated/java/io/dagger/client//**` is byte-identical to the +`sdk/src/generated/java/io/dagger/client//**` that a module depending on +`` receives. This is the feature's central claim, and it is checked +directly (see Testing). + +Stated precisely, because the unqualified version is false: the emitted bytes are +identical **for a fixed binding tuple** — final module name, source kind, +canonical ref or workspace-relative path, pin, compatibility view, schema bytes, +and generator revision. The same module resolved locally and from git is *not* +byte-identical, and should not be: a local binding bakes a workspace-relative +path and no pin, a git binding bakes a canonical ref and a pin +(`core/modulesource.go:978,991`). What the claim rules out is the *context* — who +is generating, and whether the consumer is a module or a standalone project — +mattering. That is the property worth having, and it is the one tested. + +The claim is about **content**, not about file modes, and the check normalizes +modes before comparing digests. Codegen emits one mode everywhere +(`Codegen.sdkBuilt` chmods its output, so the number of passes cannot change +it), but a `Changeset.layer` does not carry that mode through: the engine writes +a module's generated tree into the workspace at 0666/0777 while a standalone +client's lands at 0644/0755, from byte-identical 0644 input. Measured by +exporting both trees. The mode is the workspace's to decide, so the check +levels it and still compares every byte and the whole shape. + +`generateAllClient(ws)` is the `@generate` rollup over +`currentModule.asSDK(workspace: ws).clients`, the same API `generateAll` reads +the module list from (`CurrentModuleAsSDKClient{path, module, moduleSource, +pin}`). Cwd-scoped exactly as `generateAll` already is for modules: the engine +owns the list, the cwd policy, and the resolution of each bound module, so the +local-vs-git branch `go-sdk.dang:384` writes by hand is not needed here — the +`moduleSource` the engine hands back is already resolved. `initClient` seeds the +SDK-owned files for a newly registered client — for Java that is `pom.xml` (the +Go SDK needs none, so its `initClient` is empty). + +## Alternatives considered + +**Keep the merged, deps-included schema and just split packages.** Split +`io.dagger.client` into core plus one package per dependency, still generated +from `introspectionSchemaJSON`. This preserves cross-module type composition and +is less work. Rejected: the resulting packages are not clients — they cannot be +generated outside a module, because the module-facing schema exists only for a +module. It would produce a nicer version of today's problem, not the unification. + +**A per-module wider schema (core + all of that module's client-deps together).** +Keeps today's ergonomics — dependency-authored types interoperate across a +module's own clients — at the cost of a schema-sharing mechanism the engine does +not expose, and of clients whose bytes differ between the module and standalone +cases, which contradicts the central goal. Rejected per D1; it remains the escape +hatch if dependency-type crossing turns out to matter in practice, and it would +begin as a `dagger/dagger` proposal. + +What is actually lost is narrower than it looks. Core types cross freely: a +`Container` returned by client A and passed to client B is +`io.dagger.core.Container` on both sides — the same Java type, no conversion. +Only a *dependency-authored* type crossing between two different clients is +unsupported. + +**Keep the runtime in `io.dagger.client`.** Fewer files move. Rejected per D2: +it leaves transport code under a prefix that means "generated client". + +**`io.dagger.clients.` for the generated clients instead.** Reserves no names, +moves no files, and removes the collision just as completely — on the +collision criterion alone it dominates D2. Rejected anyway: `io.dagger.client` +and `io.dagger.clients` differing by one letter, with completely different +contents, is worse to read and to import than moving the transport once. + +**Move the runtime into `io.dagger.core` as well.** Would remove the package +cycle. Rejected: it conflates "the schema-derived API" with "the transport", +which are versioned by different things, and puts hand-written files inside a +generated package. + +**Serve unconditionally and ignore an "already served" error.** Fewer round +trips. Rejected: it depends on matching an engine error string, which is not a +contract. + +**Publish `io.dagger:dagger-java-core` to Maven and depend on it.** What the +TypeScript SDK does with `@dagger.io/dagger`. Rejected: it contradicts this +repository's self-contained, no-published-artifact design, and it would make +generation depend on release infrastructure that does not exist yet. + +## Affected components + +| Component | Change | +|---|---| +| `sdk/dagger-codegen-maven-plugin` | `CodeWriter` takes a package; new `TypeRegistry` and schema partition; all visitors **and `TypeRef`** resolve through the registry; `Directive.getSourceMapModule`; new entry-point emission; `DaggerCodegenMojo` gains `mode`, `package`, `module`, and binding parameters | +| `sdk/dagger-java-sdk` | package move `io.dagger.client` → `io.dagger.sdk`; new `io.dagger.sdk.ModuleBinding`; `Dagger` returns `io.dagger.core.Client`; generated-type constructors widened to public so cross-package construction works | +| `sdk/dagger-java-annotation-processor` | imports and hardcoded type names move to `io.dagger.sdk.*` / `io.dagger.core.*` | +| `mod.dang` | `generateModule` drives core generation off the module-facing schema, one client per declared dependency off each dependency's `clientSchemaIntrospectionJSON`, and the self client through the staged-workspace bootstrap; rejects dependencies declared below `v1.0.0-0` | +| `sdk/dagger-java-sdk` (`engineconn`) | `Connection.get` regains `dagger session` provisioning behind the environment path (D8) | +| `prebuilt/m2` | regenerated — `mod.dang` prefers the committed codegen plugin whenever `prebuilt/m2/io/dagger` exists, so codegen changes are inert until the plugin jar is rebuilt and committed | +| `client.dang` (new) | client generation: schema, identity, vendoring, pom | +| `main.dang` | `generateClient`, `generateAllClient` (`@generate`), `initClient` | +| `templates/{default,empty,legacy}` | imports move | +| `sdk/dagger-java-samples` | imports move | +| `.dagger/modules/e2e` | new fixtures and checks (below) | +| `README.md` | the layout section and the generation description | + +## Testing + +What exists, exactly. + +Unit, in `dagger-codegen-maven-plugin` (`mvn -Ptests --projects +dagger-codegen-maven-plugin test`): + +- `SchemaPartitionTest` — a fixture schema with `@sourceMap` on types and on + fields splits into the expected core and module sets, including a + module-contributed field on a core type (`Binding.asHello` goes to the module, + `Binding` stays in core); core is the same whichever module the schema was + bound to; narrowing does not mutate the schema it came from; `Version` and the + IDAble helpers are core-only. +- `SourceMapAttributionTest` — the directive accessor, including the + JSON-quoted value and the field-on-a-core-type case. +- `GeneratorTest` — a plan emits core and one package per client into one tree; + core's *bytes* do not depend on which module's schema they came from; a full + plan drops the client packages it does not mention except the kept one, and a + plan without core touches nothing else; two modules naming one package are + rejected; a plan holds at most one core. +- `ModuleClientCodegenTest` — the `from(Client)` factory with the module's + constructor arguments, the static-import alias, a git binding's ref and pin, a + shim on `Binding` (with its preamble starting at the session root), the root + type read off the schema (`e2e`/`E2E`), a module named after a core type + rejected, two shims of one field name getting helper classes of their own, + schema arguments escaped where they would shadow a generated local, package + segments, and that the emitted client compiles against stubs of core and the + runtime. +- `NullableObjectCodegenTest`, `SchemaTest`, `DaggerCLIUtilsTest` — the + pre-existing nullable-object surface, the version gate, and `dagger version` + parsing. + +Unit, in `dagger-java-sdk` (needs `-Ddaggerengine.schema`): + +- `ModuleBindingTest` — over a fake engine: a local module served by workspace + path under its final name, a git module by canonical ref and pin, an unpinned + git module, a binding served once per client and again for a different name or + a second client, and a source kind a client cannot serve rejected before any + request. +- `QueryBuilderTest` — `root()` drops the selection and keeps the session, plus + the nullable-object query shapes. +- `CLISessionTest` — the announcement is parsed, the process is stopped by + `close()` and by a failure to read the announcement, a CLI that exits without + announcing and a missing CLI are explained. + +e2e, as `@check` functions in `.dagger/modules/e2e` — three, each running real +generation in the engine: + +- `clients-generate-check` — generating fixture module `app` (which declares + `dep`, and `dep` again aliased to `greeter`) from nothing produces + `io.dagger.core`, both dependency clients and app's own client; `Host` is + absent, so it stays hidden from module code; the dependency client serves + `dep` by workspace path and the aliased one serves `greeter`; core types + returned by a dependency resolve to `io.dagger.core`; with the self client + vendored the module calls itself and a second generate picks the new function + up; a committed client package the plan no longer mentions is removed; and a + third generate with no edits changes nothing. +- `standalone-client-check` — a standalone client for `dep` is + **byte-identical** (`Directory.digest`, over trees levelled to one file mode — + see the byte-identity note above) to the client `app` vendors for it, + sees `Host`, is named after its directory, and builds with a plain `mvn + package` together with a `main` that uses it. +- `registered-client-check` — `initClient` seeds the pom and nothing else, the + `@generate` rollup materializes the registered client from workspace config, + and a second rollup on the applied result is an empty changeset. + +What stays untested, plainly: + +- **No generated client is invoked at runtime through the engine.** The e2e + checks compile and build; nothing calls `dep(dag()).greet("x")` and asserts + the answer. That needs committed generated fixtures or a git-bound module. + The unit tests cover the request shapes the preamble sends. +- **Git-bound dependencies and clients.** Every fixture is local. The git branch + of the binding is covered by unit tests on the emitted code and on + `ModuleBinding`, not end to end. +- **Session provisioning end to end.** `CLISessionTest` drives a fake CLI; no + check opens a real `dagger session` from a standalone client and calls + through it. + +Regression surface that must stay green: the existing e2e checks, the `sdk-sdk` +contract suite (`seeds-files`, `does-not-write-config`, `honors-custom-path`, the +`chain` generation checks), `packager:unit-tests`, and `templates:generate`. + +## Risks + +- **Blast radius.** Every generated import in every Java module changes, and D2 + moves every hand-written runtime file too. This is intended and unavoidable + given the no-shim decision, but it means a broken intermediate patch is very + visible. Mitigated by ordering the series so the tree builds at every patch and + by the two-pass pom being unchanged. +- **Java is first at deps-as-clients.** The Go SDK's module generation still uses + the engine's merged schema, so there is no reference implementation for the + half of this design that turns dependencies into clients — only for the + standalone-client half. Expect the dependency path to need more iteration. +- **`Host` and `Engine*` stay hidden from module code, and are visible to a + standalone client.** An earlier draft of this bullet had them leaking into + modules; D7 is what closes it. A module's core comes from its own + module-facing `introspectionSchemaJSON`, which hides + `TypesToIgnoreForModuleIntrospection` and `TypesHiddenFromModuleSDKs`, so + `dag().host()` in module code stays a compile error. Only a standalone + client's core comes from the client-facing schema, which hides nothing — which + is correct, because a client is allowed everything the CLI is. The e2e + generate check asserts `io/dagger/core/Host.java` is absent from a module and + present in a standalone client. +- **`serve` once-per-session.** Serving is idempotent in the engine for an exact + source and pin; the per-client cache of served tuples keeps the repeat off the + wire. `ModuleBindingTest` pins both the request shapes and the call counts. +- **Simple-name overlap.** The authored `io.dagger.modules..M` and the + generated `io.dagger.client..M`, and a module named after a core type + against `io.dagger.core`. Compiles — JavaPoet emits fully-qualified names — + and the static-import alias is the idiom that keeps call sites clean (D5). +- **Bootstrap cost.** The self client adds one engine-driven module build per + `generate`. It is the same class of cost `generateLocalDependencies` already + pays for each local dependency, and it is cached by the engine across + unchanged inputs. +- **Self serve identity.** Inside a module, the self client serves + `currentWorkspace().moduleSource()`; the engine deduplicates only + if that resolves to the same canonical reference it served the module under. + The self-client e2e check is what proves it; if it does not match, the fix is + in how the path is baked, not in the design. +- **Dependency-authored types cannot cross clients.** Accepted per D1. The + generator fails loudly at generation time when a client's schema references a + type it cannot resolve, rather than emitting code that does not compile. +- **Constructor visibility widening.** Generated types need public `QueryBuilder` + constructors for cross-package construction, which enlarges the public surface + of generated classes. Documented as internal in the generated javadoc; no + better option exists without sealing, which Java 17 does not offer across + packages. +- **A standalone client cannot deserialize IDs against its own session.** The + generated `Deserializer` nested classes resolve through `Dagger.dag()`, the + process-wide singleton, so a client opened with `Dagger.connect()` has + `JsonConverter` talking to a different session than the one it holds. This is + the pre-existing singleton model, not something this series introduces; + threading the session through deserialization is follow-up work. +- **No startup timeout on a spawned session.** `CLISession.start` reads the + CLI's stdout until it announces a port and token or exits, so a `dagger + session` that hangs before announcing hangs the caller. The CLI's own + behaviour is the bound; a deadline is follow-up work. + +# Implementation plan + +Stacked Git series on `unified-clients-lead-c699e437`, based on `upstream/main` +@ `d806484`. Every patch carries +`Signed-off-by: Yves Brissaud `. + +Ordering constraint discovered in review: the transport must become public +(D6) **before** anything is generated outside `io.dagger.client`, and the +package move must land before the generator starts emitting multiple packages. +Each patch below compiles on its own; the codegen learns the new shape while +still driven in a single-package configuration, and the switch-over is one +patch with every consumer. + +### Patch 1 — `hack/designs`: this document ✅ + +### Patch 2 — `codegen`: read `@sourceMap` module attribution ✅ + +`Directive.getSourceMapModule`, stripping the JSON quotes around the value as +`getExpectedType` does, plus `Type.getOwningModule()` and +`Field.getOwningModule()`. Tests cover the `Query.e2E` field-on-a-core-type case. + +### Patch 3 — `codegen`: partition a schema into core and one module + +`SchemaPartition`: given a schema and a module name, the core type set (types +with no owner, with owned fields removed) and the module type set (owned types, +plus owned fields on core types, which become the shims). Unit tests including +the `Binding.asHello` case and core stability across two modules. + +### Patch 4 — `codegen`: resolve type references through a registry + +`TypeRegistry` maps a GraphQL type name to a `ClassName`. `CodeWriter` takes a +target package. Every `ClassName.bestGuess(...)` goes through the registry — in +`ObjectVisitor`, `InterfaceVisitor`, `InputVisitor`, `ScalarVisitor`, +`IDAbleVisitor`, `Helpers`, and **`TypeRef`** (the actual resolver). Behaviour +unchanged: the registry maps everything to one package. + +### Patch 5 — `sdk`: widen the query transport to public API (D6) + +`QueryBuilder` (class, constructor, `chain`, `chainNode`, `execute*`), +`InputValue`, `Arguments.merge`, `Scalar.convert()`, `QueryPart`, and a new +public `Client.queryBuilder()` accessor. Generated `Client` constructors widen so +`AutoCloseableClient` can extend across packages. No package has moved yet, so +this patch is pure visibility plus one accessor, and the existing tests pin the +behaviour. + +### Patch 6 — `sdk`: move the runtime to `io.dagger.sdk` (D2) + +The package move, mechanical, with every in-repo consumer updated in the same +patch (SDK, processor, samples, templates). No behaviour change; the generated +package is still `io.dagger.client`. + +### Patch 7 — `sdk`: `ModuleBinding`, the unconditional serve preamble, and session provisioning (D8) + +Landed before the entry-point codegen (they are swapped relative to the first +draft) so that the generated code never references a runtime class that does +not exist yet. + +### Patch 8 — `codegen`: emit the entry point, the alias, and the core-type shims + +The static `from(Client)` factory, the D3 `(Client)` alias, the static +shims for module-owned fields on `Query`/`Binding`/`Env`, and module-name +normalization. The root type is resolved **from the schema** — the return type of +the `Query` field owned by the module — never by capitalizing the module name, +which gives `E2e` where the real type is `E2E`. Binding identity uses the +module's **final** name, so an aliased dependency chains and serves the same +name. Unit tests pin the local binding, the git binding, the `e2e`/`E2E` case, +and an aliased binding. + +Hand-written `io.dagger.sdk.ModuleBinding`: no probe — serve the exact binding, +let the engine deduplicate, and remember the tuple per session so a repeat costs +nothing. Applies `withName(finalName)`. Unit tests over a faked engine cover +local, git, alias and repeat calls, and assert the request count so the +once-per-client rule is observed rather than inferred. + +In the same patch, `Connection.get` regains `dagger session` provisioning +(`ProcessBuilder`, no new dependency) behind the environment path, and the +connection shuts the session process down. A unit test drives it with a fake +`dagger` script that prints the announcement line. + +### Patch 9 — the cutover + +One patch, because the tree cannot build between halves: + +- `DaggerCodegenMojo` reads a plan directory (one entry per package, each with + its schema and `meta.json`) and cleans the package roots it writes; a plan + that carries core also drops every client package it does not mention, + except the one named by `keep` — the module's own previous self client, + carried through the first pass so module code that already calls it still + compiles; the single-schema form it accepts today becomes a one-entry core + plan; +- `io.dagger.core` is generated from the consumer's schema (D7); +- the annotation processor's hardcoded type names move to `io.dagger.core`; +- templates and samples move. + +### Patch 10 — `prebuilt`: regenerate the committed codegen plugin + +`mod.dang` prefers `prebuilt/m2` whenever it exists, so every codegen change +above is inert in module generation until the plugin jar is rebuilt and +committed. `packager:generate` produces it; this patch commits the result. + +### Patch 11 — `mod.dang`: generate core, one client per dependency, and the self client + +`generateModule` builds a plan (core from the module-facing schema, one +`client` entry per `modSource.dependencies`), vendors the result with the +previous self client carried over, produces the entrypoint, stages everything +onto the workspace, reads the module's own client schema off the staged +workspace, and generates the self client from a one-entry plan (D5). Rejects a +dependency declared below `v1.0.0-0`. Keeps the existing +`generateLocalDependencies` staging. + +### Patch 12 — `client.dang` + `main.dang`: standalone clients + +`generateClient(ws, module, path)`, `initClient` and the `@generate` rollup +`generateAllClient`, mirroring the Go SDK's surface. The rollup reads +`currentModule.asSDK(workspace: ws).clients` with the bound module's identity +and schema selected as data — the engine owns the list, the cwd policy and the +resolution of each bound module, exactly as it does for modules. `initClient` +seeds the pom, rendered from `client-template/` so there is one source of +truth. A second `@generate` function alongside `generateAll` is accepted by the +engine: `dagger generate java-sdk` runs both. + +### Patch 13 — `README`: layout, entry points, and a migration recipe + +With no shim, every existing Java module breaks on the next `dagger generate`. +The README carries the `sed` recipe for the import moves. + +### Patch 14 — `e2e`: fixtures and checks + +Two real Java modules under `fixtures/clients/` — `dep`, and `app` depending on +it — with a workspace config of their own that the checks place at the +workspace root: the engine scopes a local dependency's generation to +`Workspace.generators(include: [])` read from the root, so it has to be +the same config the SDK's module list comes from. Three checks: generation from +nothing (core, the dependency client, the self client, then a self call added +and picked up, then an idempotent run); the standalone client (byte-identical +to the vendored dependency client, sees `Host`, builds with `mvn package` and a +main that uses it); and a registered client (`initClient`, then the rollup). + +### Patch 15 — lock the shared Maven cache + +Every generate now runs two installs into the shared `~/.m2` volume, and +concurrent installs corrupt `maven-metadata-local.xml`. All mounts of that +volume use `CacheSharingMode.LOCKED`. + +### Verification + +Local, before hand-off: + +- `mvn -Ptests --projects dagger-codegen-maven-plugin test` for the codegen unit + tests. The bare `mvn -f sdk/pom.xml test` in an earlier draft **does not work**: + JUnit and AssertJ live behind the `tests` profile (`sdk/pom.xml:205`), and the + full reactor additionally needs an explicit `-Ddaggerengine.schema` and an + installed plugin, as `packager:unit-tests` does. +- `dagger check` for the e2e and packager checks; +- `dagger generate` on this repository's own modules, with an empty changeset + expected on a second run. + +CI must be green on: the existing e2e checks, `sdk-sdk` contract and chain +checks, `packager:unit-tests`, `packager:generate`, and the new e2e checks. + +## Progress + +- **Phase 0 — orientation: done.** Repository `dagger/java-sdk`, base + `upstream/main` @ `d806484` (the fork's `origin/main` is strictly behind). + Worktree + `/home/yves/.tailcall/worktrees/dagger-java-sdk-577e555c72f0/unified-clients-lead-c699e437-1cb176ae`, + branch `unified-clients-lead-c699e437`. Design home `hack/designs/`, archive + `hack/designs/done/`. VCS: StGit. Host: GitHub, fork remote `origin` = + `eunomie/java-sdk`, upstream `dagger/java-sdk`. CI: Dagger Cloud checks + (`dagger check`), no GitHub Actions workflows. Provenance: + `Signed-off-by: Yves Brissaud `, no AI attribution. +- **Phase 1/2 — feature doc and plan: done.** +- **Phase 3 — adversarial plan review: done, one round.** A Codex skeptic and a + Claude design reviewer both rejected the first draft. Verified findings folded + in above: D1 is not a regression (the engine already forbids it), the serve + probe removed in favour of unconditional serving, D6 (public transport) added + as a blocker that would otherwise not compile, D7 (core from the engine + schema) added to break a bootstrap circularity, the self client cut, alias + identity corrected, byte-identity narrowed to a stated tuple, `prebuilt/m2` + regeneration added, and the verification command fixed. +- **Kickoff answers, round 2 (Yves):** one PR, organized as an stg series; the + self client is a must-have (D5 restored with the staged-workspace bootstrap); + sessions modelled on the Go and TypeScript SDKs (D8); D7 kept. +- **Phase 5 — code review and fix: done, one round.** A Claude reviewer and a + Codex reviewer on the implemented diff; 18 curated findings (A–R) applied by + a fixer and folded into the owning patches — among them: `main.dang` is + rendered from `main.dang.tmpl` (the entry points now live in the template); + the byte-identity check compares digests after levelling file modes, which + the workspace layer rewrites differently on the two paths; static shims start + their serve from a root builder; `withoutDirectory` before every overlay, + since `Workspace.withNewDirectory` merges; a module named after a core type + and two modules that name the same package fail loudly; schema arguments + that shadow generated locals are escaped; the serve preamble caches exact + tuples per client after success; the session and connection lifecycle no + longer leaks a `dagger session` process. Full suite before the fixes: 34 of + 36 green; after: the four affected checks green, unit tests 49 / 14 / 3. +- **Phase 4 — implementation: done.** Patches 1–13 landed. The whole + pipeline has run in the real engine: the e2e generate check exercised pass 1 + (core + dependency clients), the entrypoint, the engine's bootstrap load of + the module from the staged workspace, and pass 2 (the self client). Found on + the way and fixed: generated scalar constructors were package-private and + `QueryBuilder` instantiates them reflectively from `io.dagger.sdk`; Dang folds + over engine object lists only through `{{...}}` record selections; and + concurrent Maven installs corrupt the shared m2 cache (now `LOCKED`). Codegen + 42 tests, SDK 10, processor 3, all green. +- **Known limit, to state in the PR:** no e2e invokes a generated client at + runtime through the engine — that needs committed generated fixtures or a + git-bound module. The module build and bootstrap, the compile-time API, the + identical-bytes property and the serve preamble's requests are covered. diff --git a/main.dang b/main.dang index 0ecc12e..f5fd54b 100644 --- a/main.dang +++ b/main.dang @@ -61,6 +61,14 @@ type JavaSdk { Render a Java init template, substituting the requested module name. """ let renderedTemplate(name: String!, template: String!): Directory! { + renderedTemplateDir(name, "templates/" + template) + } + + """ + Render a template directory of the module source, substituting the requested + module name into every path and file it holds. + """ + let renderedTemplateDir(name: String!, dir: String!): Directory! { container .from("golang:1.25-alpine") .withoutEntrypoint @@ -76,7 +84,7 @@ type JavaSdk { # that then breaks the scaffolded module's build. .withDirectory( "/template", - currentModule.source.directory("templates/" + template), + currentModule.source.directory(dir), exclude: ["**/target/**"], ) .withWorkdir("/helper") @@ -122,4 +130,100 @@ type JavaSdk { .map { mod => mod.generate(ws) }, ) } + + """ + Generate a typed Java client for the module at `module`, written to `path`. + `module` is a workspace path (leading "/", "./" or a bare path) or a git ref. + """ + generateClient(ws: Workspace!, module: String!, path: String!): Changeset! { + let clientPath = cleanClientPath(path) + # A git ref opens on its host, so only a dot in the first segment makes one: + # `libs/my.mod` is a workspace path, `github.com/dagger/hello` is not. + let firstSegment = module.split("/")[0] ?? module + let local = module.hasPrefix("/") + or module.hasPrefix("./") + or module == "." + or (firstSegment.contains(".") == false) + let src = if (local) { + ws.moduleSource("/" + module.trimPrefix("./").trimPrefix("/")) + } else { + moduleSource(refString: module) + } + Client( + rootPath: clientPath, + module: src.moduleName, + kindJSON: JSON.encode(src.kind), + canonicalRef: src.asString, + rootSubpath: src.sourceRootSubpath, + pin: src.pin, + engineVersion: src.engineVersion, + schemaJSON: src.clientSchemaIntrospectionJSON.contents, + ).generate(ws, clientPom(clientPath)) + } + + """ + Seed the SDK-owned files a new Java client needs at `path`: a pom that builds + the generated sources. The engine records the managed client in workspace + config and materializes the client itself through the `@generate` hook. + """ + initClient( + ws: Workspace!, + path: String!, + module: String!, + dev: Boolean! = false, + ): Changeset! { + let clientPath = cleanClientPath(path) + Client(rootPath: clientPath).init(ws, clientPom(clientPath)) + } + + """ + Regenerate every Java client registered on this SDK that is visible from the + client's current location (runs at `dagger generate`). As with modules, the + engine owns the list, the cwd policy, and the resolution of each bound module. + """ + generateAllClient(ws: Workspace!): Changeset! @generate { + changeset.withChangesets( + currentModule + .asSDK(workspace: ws) + .clients + .{{path, moduleSource.{{moduleName, kind, asString, sourceRootSubpath, pin, engineVersion, clientSchemaIntrospectionJSON.{{contents}}}}}} + .map { client => + Client( + rootPath: client.path, + module: client.moduleSource.moduleName, + kindJSON: JSON.encode(client.moduleSource.kind), + canonicalRef: client.moduleSource.asString, + rootSubpath: client.moduleSource.sourceRootSubpath, + pin: client.moduleSource.pin, + engineVersion: client.moduleSource.engineVersion, + schemaJSON: client.moduleSource.clientSchemaIntrospectionJSON.contents, + ).generate(ws, clientPom(client.path)) + }, + ) + } + + """ + The pom of a client at `path`, named after its directory. + """ + let clientPom(path: String!): File! { + let name = if (path == ".") { "client" } else { path.split("/").reduce("client") { acc, seg => seg } } + renderedTemplateDir(name, "client-template").file("pom.xml") + } + + """ + A client path as workspace-root-relative, "." for the root. + """ + let cleanClientPath(path: String!): String! { + let rawPath = path.trimPrefix("./").trimPrefix("/") + if (rawPath == "" or rawPath == ".") { + "." + } else if (rawPath == ".." + or rawPath.trimPrefix("../") != rawPath + or rawPath.contains("/../") + or rawPath.trimSuffix("/..") != rawPath) { + raise "path escapes workspace: " + rawPath + } else { + rawPath.trimSuffix("/") + } + } } diff --git a/main.dang.tmpl b/main.dang.tmpl index 228a5e2..4a6a5f6 100644 --- a/main.dang.tmpl +++ b/main.dang.tmpl @@ -61,6 +61,14 @@ type JavaSdk { Render a Java init template, substituting the requested module name. """ let renderedTemplate(name: String!, template: String!): Directory! { + renderedTemplateDir(name, "templates/" + template) + } + + """ + Render a template directory of the module source, substituting the requested + module name into every path and file it holds. + """ + let renderedTemplateDir(name: String!, dir: String!): Directory! { container .from("golang:1.25-alpine") .withoutEntrypoint @@ -76,7 +84,7 @@ type JavaSdk { # that then breaks the scaffolded module's build. .withDirectory( "/template", - currentModule.source.directory("templates/" + template), + currentModule.source.directory(dir), exclude: ["**/target/**"], ) .withWorkdir("/helper") @@ -122,4 +130,100 @@ type JavaSdk { .map { mod => mod.generate(ws) }, ) } + + """ + Generate a typed Java client for the module at `module`, written to `path`. + `module` is a workspace path (leading "/", "./" or a bare path) or a git ref. + """ + generateClient(ws: Workspace!, module: String!, path: String!): Changeset! { + let clientPath = cleanClientPath(path) + # A git ref opens on its host, so only a dot in the first segment makes one: + # `libs/my.mod` is a workspace path, `github.com/dagger/hello` is not. + let firstSegment = module.split("/")[0] ?? module + let local = module.hasPrefix("/") + or module.hasPrefix("./") + or module == "." + or (firstSegment.contains(".") == false) + let src = if (local) { + ws.moduleSource("/" + module.trimPrefix("./").trimPrefix("/")) + } else { + moduleSource(refString: module) + } + Client( + rootPath: clientPath, + module: src.moduleName, + kindJSON: JSON.encode(src.kind), + canonicalRef: src.asString, + rootSubpath: src.sourceRootSubpath, + pin: src.pin, + engineVersion: src.engineVersion, + schemaJSON: src.clientSchemaIntrospectionJSON.contents, + ).generate(ws, clientPom(clientPath)) + } + + """ + Seed the SDK-owned files a new Java client needs at `path`: a pom that builds + the generated sources. The engine records the managed client in workspace + config and materializes the client itself through the `@generate` hook. + """ + initClient( + ws: Workspace!, + path: String!, + module: String!, + dev: Boolean! = false, + ): Changeset! { + let clientPath = cleanClientPath(path) + Client(rootPath: clientPath).init(ws, clientPom(clientPath)) + } + + """ + Regenerate every Java client registered on this SDK that is visible from the + client's current location (runs at `dagger generate`). As with modules, the + engine owns the list, the cwd policy, and the resolution of each bound module. + """ + generateAllClient(ws: Workspace!): Changeset! @generate { + changeset.withChangesets( + currentModule + .asSDK(workspace: ws) + .clients + .{{path, moduleSource.{{moduleName, kind, asString, sourceRootSubpath, pin, engineVersion, clientSchemaIntrospectionJSON.{{contents}}}}}} + .map { client => + Client( + rootPath: client.path, + module: client.moduleSource.moduleName, + kindJSON: JSON.encode(client.moduleSource.kind), + canonicalRef: client.moduleSource.asString, + rootSubpath: client.moduleSource.sourceRootSubpath, + pin: client.moduleSource.pin, + engineVersion: client.moduleSource.engineVersion, + schemaJSON: client.moduleSource.clientSchemaIntrospectionJSON.contents, + ).generate(ws, clientPom(client.path)) + }, + ) + } + + """ + The pom of a client at `path`, named after its directory. + """ + let clientPom(path: String!): File! { + let name = if (path == ".") { "client" } else { path.split("/").reduce("client") { acc, seg => seg } } + renderedTemplateDir(name, "client-template").file("pom.xml") + } + + """ + A client path as workspace-root-relative, "." for the root. + """ + let cleanClientPath(path: String!): String! { + let rawPath = path.trimPrefix("./").trimPrefix("/") + if (rawPath == "" or rawPath == ".") { + "." + } else if (rawPath == ".." + or rawPath.trimPrefix("../") != rawPath + or rawPath.contains("/../") + or rawPath.trimSuffix("/..") != rawPath) { + raise "path escapes workspace: " + rawPath + } else { + rawPath.trimSuffix("/") + } + } } diff --git a/mod.dang b/mod.dang index 5d2da06..536441f 100644 --- a/mod.dang +++ b/mod.dang @@ -23,6 +23,8 @@ type Mod { """ let vendorSdkJar: Boolean! + let codegen: Codegen! { Codegen() } + """ This module root relative to the client's current location: "." when the cwd is the module itself, "sub/mod" for a module beneath it, ".." for an enclosing one. @@ -72,140 +74,115 @@ type Mod { } """ - Maven container used for codegen (pinned digest, matches the builtin runtime). - """ - let mvn: Container! { - container.from("maven:3.9.9-eclipse-temurin-21-alpine@sha256:4cbb8bf76c46b97e028998f2486ed014759a8e932480431039bdb93dffe6813e") - } - - """ - The vendored Java SDK Maven reactor shipped with this module. - """ - let sdkSourceDir: Directory! { currentModule.source.directory("sdk") } - - """ - Vendor the Java SDK as buildable source for a module: - src/main/java the hand-written SDK library - src/processor/java the annotation processor that generates the entrypoint - src/generated/java the client bindings generated from the engine schema - src/processor/resources/META-INF/services/...Processor - The returned directory is dropped at /sdk. - """ - let prebuiltCodegenRepo: String! { "prebuilt/m2" } - - let prebuiltCodegenPlugin: String! { "prebuilt/dagger-codegen-maven-plugin.jar" } - - """ - A maven container with the codegen plugin available in the local repository. - - Fast path: when the packager module has committed the plugin's local Maven - repository under prebuilt/m2, drop it into ~/.m2/repository with a plain copy — - no maven invocation. Otherwise fall back to installing a committed plugin jar, - and finally to compiling the plugin from the vendored sources. + Compile the module with the annotation processor enabled (proc=full) so it + emits io.dagger.gen.entrypoint.Entrypoint, and return the generated-sources + directory (the entrypoint) for vendoring under /src/generated/java. """ - let codegenBase(introspectionJSON: File!): Container! { - let base = mvn + let generatedEntrypoint(moduleDir: Directory!, name: String!): Directory! { + codegen.mvn .withoutEntrypoint - .withMountedCache("/root/.m2", cacheVolume("sdk-java-maven-m2")) - .withMountedFile("/schema.json", introspectionJSON) - .withDirectory("/dagger-io", sdkSourceDir) - .withWorkdir("/dagger-io") - - if (currentModule.source.exists(prebuiltCodegenRepo + "/io/dagger")) { - base - .withDirectory("/prebuilt-m2", currentModule.source.directory(prebuiltCodegenRepo)) - .withExec(["sh", "-c", "mkdir -p /root/.m2/repository && cp -r /prebuilt-m2/. /root/.m2/repository/"]) - } else if (currentModule.source.exists(prebuiltCodegenPlugin)) { - base - .withMountedFile("/codegen-plugin.jar", currentModule.source.file(prebuiltCodegenPlugin)) - .withExec(["mvn", "install:install-file", "-Dfile=/dagger-io/pom.xml", "-Dpackaging=pom", "-DpomFile=/dagger-io/pom.xml", "--no-transfer-progress"]) - .withExec(["mvn", "install:install-file", "-Dfile=/codegen-plugin.jar", "-DpomFile=/dagger-io/dagger-codegen-maven-plugin/pom.xml", "--no-transfer-progress"]) - } else { - base.withExec(["mvn", "--projects", "dagger-codegen-maven-plugin", "--also-make", "install", "-T1C", "-Dmaven.test.skip=true", "-Dfmt.skip=true", "--no-transfer-progress"]) - } + .withMountedCache("/root/.m2", cacheVolume("sdk-java-maven-m2"), sharing: CacheSharingMode.LOCKED) + .withDirectory("/module", moduleDir) + .withWorkdir("/module") + .withEnvVariable("_DAGGER_JAVA_SDK_MODULE_NAME", name) + .withExec(["mvn", "compile", "-Ddagger.proc=full", "-Ddagger.sdk=prebuilt", "-Ddagger.sdk.version=" + name, "--no-transfer-progress"]) + .directory("/module/target/generated-sources/annotations") } """ - Build and install the SDK + annotation processor jars, returning the maven - container. - - Uses install (rather than generate-sources) so the compiled jars land in the - local Maven repository — the dagger-prebuilt-sdk pom profile compiles the - entrypoint against them instead of recompiling the vendored SDK sources. Jars - are installed under a per-module version so modules that resolve to different - schemas (different dependencies, or a different engine) never share a Maven - coordinate; the codegen plugin keeps its own fixed version and resolves from the - prebuilt repo. Cached across a module's own edits. - """ - let sdkBuilt(introspectionJSON: File!, name: String!): Container! { - codegenBase(introspectionJSON) - .withExec(["mvn", "versions:set", "-DnewVersion=" + name, "-DgenerateBackupPoms=false", "--no-transfer-progress"]) - .withExec(["mvn", "--projects", "dagger-java-sdk,dagger-java-annotation-processor", "--also-make", "install", "-Ddaggerengine.schema=/schema.json", "-Ddaggerengine.version=" + engineVersion, "-Dmaven.test.skip=true", "-Dfmt.skip=true", "--no-transfer-progress"]) + The workspace with every local dependency this SDK manages generated and + staged, recursively, so their client-facing schemas can be read. + + Done here rather than through the engine's generateLocalDependencies: that + routes through the workspace's generator rollup, which the engine leaves + empty for a value workspace — where every in-memory check runs — and which + does not carry this SDK's own generators in this repository. Ownership is the + engine's rule: a dependency is ours when it sits in the modules registered to + this SDK; a dependency of another SDK, or a remote one, is assumed committed. + """ + withLocalDependenciesGenerated(ws: Workspace!): Workspace! { + # The registry, not modules(ws): that list is scoped to the caller's cwd, + # and a dependency is usually a sibling, outside it. A workspace that + # registers this SDK under another key has no such record, and the engine + # errors rather than returning nothing, which would fail every generate; + # an empty registry means "every dependency is assumed committed". + let managed = ws.sdk(name: currentModule.name).modules.{{source}}.map { m => normalizePath(m.source) } rescue [] + ws.moduleSource(workspaceRef(rootPath)) + .dependencies + .{{kind, sourceRootSubpath}} + # A dependency inside the workspace is LOCAL_SOURCE on the host and + # DIR_SOURCE in a workspace built from a directory value; both live at + # sourceRootSubpath. Only a git dependency is somewhere else. + .filter { dep => JSON.encode(dep.kind).contains("GIT") == false } + .filter { dep => managed.contains(normalizePath(dep.sourceRootSubpath)) } + .reduce(ws) { staged, dep => + # Overlaid as directories, not applied as a changeset: a changeset is + # measured from the caller's cwd and may not reach a sibling. + let mod = Mod( + rootPath: dep.sourceRootSubpath, + ws: staged, + skipGenerateFilename: skipGenerateFilename, + vendorSdkJar: vendorSdkJar, + ) + if (mod.skipGenerate(staged)) { + staged + } else { + let depStaged = mod.withLocalDependenciesGenerated(staged) + withOverlay(depStaged, dep.sourceRootSubpath, mod.generatedOverlay(depStaged)) + } + } } """ - The live engine version, without build metadata. - - Codegen only reads the engine version off the CLI when it has to query the - schema itself. Here the schema is handed to it, so without this the version - stays whatever the pom happens to say, and generation cannot tell which shapes - the engine on the other end actually supports. + A workspace with a module's generated overlay in place of what it had. - The `+` suffix is dropped: it changes on every engine build and would - make every module's SDK rebuild for no reason. + Dropped before it is written: withNewDirectory merges into what is already + there, so a client package a dependency no longer produces — one dropped from + dagger-module.toml, or renamed by an alias — would survive every generate. """ - let engineVersion: String! { version.split("+")[0] ?? version } - - let vendoredSdk(introspectionJSON: File!, name: String!): Directory! { - let built = sdkBuilt(introspectionJSON, name) - directory - .withDirectory("src/main/java", built.directory("/dagger-io/dagger-java-sdk/src/main/java")) - .withDirectory( - "src/processor/java", - built.directory("/dagger-io/dagger-java-annotation-processor/src/main/java"), - ) - .withDirectory( - "src/generated/java", - built.directory("/dagger-io/dagger-java-sdk/target/generated-sources/dagger"), - ) - .withNewFile( - "src/processor/resources/META-INF/services/javax.annotation.processing.Processor", - "io.dagger.annotation.processor.DaggerModuleAnnotationProcessor\n", - ) + let withOverlay(ws: Workspace!, modPath: String!, overlay: Directory!): Workspace! { + let sdkPath = workspaceRef(joinPath(modPath, "sdk")) + let entrypointPath = workspaceRef(joinPath(modPath, "src/generated/java")) + ws + .withoutDirectory(sdkPath) + .withNewDirectory(sdkPath, overlay.directory("sdk")) + .withoutDirectory(entrypointPath) + .withNewDirectory(entrypointPath, overlay.directory("src/generated/java")) } """ - Capture the compiled SDK jar as a small local Maven repository to commit under - /sdk/repo. Stored at a fixed version so the module pom's - dagger-vendored-sdk-jar profile resolves it; the runtime build then compiles - only the module's own code against the jar rather than the vendored sources. - """ - let vendoredSdkJar(introspectionJSON: File!, name: String!): Directory! { - let jar = sdkBuilt(introspectionJSON, name) - .withExec(["sh", "-c", "mkdir -p /out && cp /root/.m2/repository/io/dagger/dagger-java-sdk/" + name + "/dagger-java-sdk-" + name + ".jar /out/dagger-java-sdk-0.21.4.jar"]) - .file("/out/dagger-java-sdk-0.21.4.jar") - directory - .withFile("io/dagger/dagger-java-sdk/0.21.4/dagger-java-sdk-0.21.4.jar", jar) - .withNewFile( - "io/dagger/dagger-java-sdk/0.21.4/dagger-java-sdk-0.21.4.pom", - "4.0.0io.daggerdagger-java-sdk0.21.4jar\n", - ) + The plan for this module: core from its own module-facing schema — which + leaves the types hidden from module code hidden — plus one client per + declared dependency, each from that dependency's client-facing schema. + """ + let modulePlan(modSource: ModuleSource!): Directory! { + # A list of engine objects cannot be folded over directly: select what each + # entry needs — its schema as text included — and fold over the records. + modSource + .dependencies + .{{moduleName, kind, asString, sourceRootSubpath, pin, engineVersion, clientSchemaIntrospectionJSON.{{contents}}}} + .reduce(codegen.corePlan(modSource.introspectionSchemaJSON.contents)) { plan, dep => + codegen.withClientEntry( + plan, + dep.moduleName, + JSON.encode(dep.kind), + dep.asString, + dep.sourceRootSubpath, + dep.pin, + dep.engineVersion, + dep.clientSchemaIntrospectionJSON.contents, + ) + } } """ - Compile the module with the annotation processor enabled (proc=full) so it - emits io.dagger.gen.entrypoint.Entrypoint, and return the generated-sources - directory (the entrypoint) for vendoring under /src/generated/java. + A one-entry plan for this module's own client, bound to the module by its + workspace path. """ - let generatedEntrypoint(moduleDir: Directory!, name: String!): Directory! { - mvn - .withoutEntrypoint - .withMountedCache("/root/.m2", cacheVolume("sdk-java-maven-m2")) - .withDirectory("/module", moduleDir) - .withWorkdir("/module") - .withEnvVariable("_DAGGER_JAVA_SDK_MODULE_NAME", name) - .withExec(["mvn", "compile", "-Ddagger.proc=full", "-Ddagger.sdk=prebuilt", "-Ddagger.sdk.version=" + name, "--no-transfer-progress"]) - .directory("/module/target/generated-sources/annotations") + let selfPlan(schema: File!, name: String!): Directory! { + directory + .withFile("client-" + name + "/schema.json", schema) + .withNewFile("client-" + name + "/meta.json", codegen.clientMeta(name, "\"LOCAL_SOURCE\"", workspaceRef(rootPath), "")) } """ @@ -217,45 +194,75 @@ type Mod { and diffing it against the baseline the caller already holds is all this needs. """ let generateModule(ws: Workspace!): Changeset! { - # A module that depends on another by local path cannot resolve its own - # schema until that dependency has been generated, so overlay the generated - # closure onto the workspace first. The engine skips remote dependencies — - # assumed committed, as it does itself — and returns an empty changeset when - # there is nothing to stage. - let wsWithDeps = ws.withChanges( - ws.moduleSource(workspaceRef(rootPath)).generateLocalDependencies(ws), - ) + let wsWithDeps = withLocalDependenciesGenerated(ws) + # Baseline on the staged-dependency workspace, not the one handed in: the + # dependencies' generated code belongs to their own SDKs, not this changeset. + withOverlay(wsWithDeps, rootPath, generatedOverlay(wsWithDeps)).changes(wsWithDeps) + } + + """ + Everything generation writes for this module, rooted at the module: sdk/ — + the vendored runtime and processor, io.dagger.core, one io.dagger.client + package per dependency and the module's own, plus the committed jar when + enabled — and src/generated/java, the entrypoint. - let modSource = wsWithDeps.moduleSource(workspaceRef(rootPath)) + `ws` is expected to carry this module's local dependencies generated already + (withLocalDependenciesGenerated): a module cannot read a local dependency's + schema before that dependency has been generated, and staging the closure + here as well would walk the whole dependency graph twice per generate. + """ + generatedOverlay(ws: Workspace!): Directory! { + let modSource = ws.moduleSource(workspaceRef(rootPath)) let name = modSource.moduleName - let introspectionJSON = modSource.introspectionSchemaJSON - let vendored = vendoredSdk(introspectionJSON, name) + # Before the first pass: the self client comes off this module's own schema, + # which the engine renders through this very version, and the plan that + # carries it is only built after an expensive build. + codegen.requireModernEngine(name, modSource.engineVersion) + let committed = moduleDir(ws, rootPath) + + # First pass: core and one client per dependency. The client packages the + # previous generation produced are carried in, and this module's own kept: + # module code that already calls its self client has to keep compiling + # until that client is regenerated below. + let clientsPath = "sdk/src/generated/java/io/dagger/client" + let carried = if (committed.exists(clientsPath)) { committed.directory(clientsPath) } else { directory } + let firstBuild = codegen.sdkBuilt(modulePlan(modSource), directory.withDirectory("io/dagger/client", carried), name, name) + let firstVendored = codegen.vendoredSdk(firstBuild) # the module as committed, with the whole committed sdk/ dropped (source and # any previously vendored jar) and the freshly vendored SDK sources overlaid, # plus any stale generated entrypoint dropped so the processor regenerates it - let baseDir = moduleDir(wsWithDeps, rootPath) + let baseDir = committed .withoutDirectory("sdk") - .withDirectory("sdk", vendored) + .withDirectory("sdk", firstVendored) .withoutDirectory("src/generated") let entrypoint = generatedEntrypoint(baseDir, name) - let staged = wsWithDeps - .withNewDirectory(workspaceRef(joinPath(rootPath, "sdk")), vendored) - .withNewDirectory(workspaceRef(joinPath(rootPath, "src/generated/java")), entrypoint) + # With the SDK and the entrypoint staged the module builds, so the engine can + # load it and hand back its client-facing schema: this is what breaks the + # circularity of a module needing its own client to be generated. + # Overlaid with withOverlay, which drops the committed tree first: a stale + # committed package would otherwise be compiled into this bootstrap build. + let bootstrapped = withOverlay( + ws, + rootPath, + directory.withDirectory("sdk", firstVendored).withDirectory("src/generated/java", entrypoint), + ) + let selfSchema = bootstrapped.moduleSource(workspaceRef(rootPath)).clientSchemaIntrospectionJSON + + # Second pass: the self client, generated over the first pass's output so the + # vendored tree is exactly core, the dependency clients and this client. + let secondBuild = codegen.sdkBuilt(selfPlan(selfSchema, name), firstBuild.directory(codegen.generatedSourcesPath), "", name) + let vendored = codegen.vendoredSdk(secondBuild) - let after = if (vendorSdkJar) { - staged.withNewDirectory( - workspaceRef(joinPath(rootPath, "sdk/repo")), - vendoredSdkJar(introspectionJSON, name), - ) + let overlay = directory + .withDirectory("sdk", vendored) + .withDirectory("src/generated/java", entrypoint) + if (vendorSdkJar) { + overlay.withDirectory("sdk/repo", codegen.vendoredSdkJar(secondBuild, name)) } else { - staged + overlay } - - # Baseline on the staged-dependency workspace, not the one handed in: the - # dependencies' generated code belongs to their own SDKs, not this changeset. - after.changes(wsWithDeps) } """ @@ -284,6 +291,15 @@ type Mod { if (modPathArg == ".") { "/" } else { "/" + modPathArg } } + """ + A workspace path in the form the registry and the engine agree on: no + leading "./" or "/", no trailing "/", "." for the root. + """ + let normalizePath(path: String!): String! { + let normalized = path.trimPrefix("./").trimPrefix("/").trimSuffix("/") + if (normalized == "") { "." } else { normalized } + } + """ Join a module path with a sub-path, handling the root (".") module. """ diff --git a/prebuilt/m2/io/dagger/dagger-codegen-maven-plugin/0.21.4/dagger-codegen-maven-plugin-0.21.4.jar b/prebuilt/m2/io/dagger/dagger-codegen-maven-plugin/0.21.4/dagger-codegen-maven-plugin-0.21.4.jar index a68c20d..21c1094 100644 Binary files a/prebuilt/m2/io/dagger/dagger-codegen-maven-plugin/0.21.4/dagger-codegen-maven-plugin-0.21.4.jar and b/prebuilt/m2/io/dagger/dagger-codegen-maven-plugin/0.21.4/dagger-codegen-maven-plugin-0.21.4.jar differ diff --git a/prebuilt/m2/io/dagger/dagger-sdk-parent/0.21.4/dagger-sdk-parent-0.21.4.pom b/prebuilt/m2/io/dagger/dagger-sdk-parent/0.21.4/dagger-sdk-parent-0.21.4.pom index bb7b189..ae80fb4 100644 --- a/prebuilt/m2/io/dagger/dagger-sdk-parent/0.21.4/dagger-sdk-parent-0.21.4.pom +++ b/prebuilt/m2/io/dagger/dagger-sdk-parent/0.21.4/dagger-sdk-parent-0.21.4.pom @@ -252,6 +252,8 @@ UTF-8 0.21.4 + + diff --git a/templates/default/src/main/java/io/dagger/modules/daggermoduleplaceholder/DaggerModule.java b/templates/default/src/main/java/io/dagger/modules/daggermoduleplaceholder/DaggerModule.java index d18b7ad..5823d92 100644 --- a/templates/default/src/main/java/io/dagger/modules/daggermoduleplaceholder/DaggerModule.java +++ b/templates/default/src/main/java/io/dagger/modules/daggermoduleplaceholder/DaggerModule.java @@ -1,11 +1,11 @@ package io.dagger.modules.daggermoduleplaceholder; -import static io.dagger.client.Dagger.dag; +import static io.dagger.sdk.Dagger.dag; -import io.dagger.client.Container; -import io.dagger.client.exception.DaggerQueryException; -import io.dagger.client.Directory; -import io.dagger.client.Workspace; +import io.dagger.core.Container; +import io.dagger.sdk.exception.DaggerQueryException; +import io.dagger.core.Directory; +import io.dagger.core.Workspace; import io.dagger.module.annotation.Default; import io.dagger.module.annotation.Function; import io.dagger.module.annotation.Object; diff --git a/templates/empty/pom.xml b/templates/empty/pom.xml index 85da6da..ef9b401 100644 --- a/templates/empty/pom.xml +++ b/templates/empty/pom.xml @@ -168,7 +168,8 @@ entrypoint as compilation roots: - sdk/src/main/java the hand-written SDK library - sdk/src/processor/java the annotation processor (generates the entrypoint) - - sdk/src/generated/java the client bindings generated from the engine schema + - sdk/src/generated/java io.dagger.core, the engine API, plus one + io.dagger.client. per bound module - src/generated/java io.dagger.gen.entrypoint.Entrypoint, generated from the module code (see maven-compiler-plugin) --> diff --git a/templates/legacy/pom.xml b/templates/legacy/pom.xml index 85da6da..ef9b401 100644 --- a/templates/legacy/pom.xml +++ b/templates/legacy/pom.xml @@ -168,7 +168,8 @@ entrypoint as compilation roots: - sdk/src/main/java the hand-written SDK library - sdk/src/processor/java the annotation processor (generates the entrypoint) - - sdk/src/generated/java the client bindings generated from the engine schema + - sdk/src/generated/java io.dagger.core, the engine API, plus one + io.dagger.client. per bound module - src/generated/java io.dagger.gen.entrypoint.Entrypoint, generated from the module code (see maven-compiler-plugin) --> diff --git a/templates/legacy/src/main/java/io/dagger/modules/daggermoduleplaceholder/DaggerModule.java b/templates/legacy/src/main/java/io/dagger/modules/daggermoduleplaceholder/DaggerModule.java index 7853b96..77942ac 100644 --- a/templates/legacy/src/main/java/io/dagger/modules/daggermoduleplaceholder/DaggerModule.java +++ b/templates/legacy/src/main/java/io/dagger/modules/daggermoduleplaceholder/DaggerModule.java @@ -1,10 +1,10 @@ package io.dagger.modules.daggermoduleplaceholder; -import static io.dagger.client.Dagger.dag; +import static io.dagger.sdk.Dagger.dag; -import io.dagger.client.Container; -import io.dagger.client.exception.DaggerQueryException; -import io.dagger.client.Directory; +import io.dagger.core.Container; +import io.dagger.sdk.exception.DaggerQueryException; +import io.dagger.core.Directory; import io.dagger.module.annotation.Function; import io.dagger.module.annotation.Object; import java.util.List;