diff --git a/.dagger/modules/e2e/fixtures/clients/app/dagger-module.toml b/.dagger/modules/e2e/fixtures/clients/app/dagger-module.toml new file mode 100644 index 0000000..5a0ce89 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/app/dagger-module.toml @@ -0,0 +1,12 @@ +name = "app" +engineVersion = "v1.0.0-0" + +[runtime] +source = "python" + +[[dependencies]] +source = "../dep" + +[[dependencies]] +name = "greeter" +source = "../dep" diff --git a/.dagger/modules/e2e/fixtures/clients/app/pyproject.toml b/.dagger/modules/e2e/fixtures/clients/app/pyproject.toml new file mode 100644 index 0000000..af46be1 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/app/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "app" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.12.0"] +build-backend = "uv_build" + +[tool.uv.sources] +dagger-io = { path = "sdk", editable = true } diff --git a/.dagger/modules/e2e/fixtures/clients/app/src/app/__init__.py b/.dagger/modules/e2e/fixtures/clients/app/src/app/__init__.py new file mode 100644 index 0000000..ec7b462 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/app/src/app/__init__.py @@ -0,0 +1,23 @@ +from dagger import function, object_type +from dagger.clients.app import app +from dagger.clients.dep import dep +from dagger.clients.greeter import greeter + + +@object_type +class App: + @function + async def greet_via_dep(self) -> str: + return await dep().greet("dep") + + @function + async def greet_via_alias(self) -> str: + return await greeter().greet("alias") + + @function + def local(self) -> str: + return "local" + + @function + async def greet_self(self) -> str: + return (await app().local()).upper() diff --git a/.dagger/modules/e2e/fixtures/clients/dep/dagger-module.toml b/.dagger/modules/e2e/fixtures/clients/dep/dagger-module.toml new file mode 100644 index 0000000..57732c6 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/dep/dagger-module.toml @@ -0,0 +1,5 @@ +name = "dep" +engineVersion = "v1.0.0-0" + +[runtime] +source = "python" diff --git a/.dagger/modules/e2e/fixtures/clients/dep/pyproject.toml b/.dagger/modules/e2e/fixtures/clients/dep/pyproject.toml new file mode 100644 index 0000000..d325b45 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/dep/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "dep" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.12.0"] +build-backend = "uv_build" + +[tool.uv.sources] +dagger-io = { path = "sdk", editable = true } diff --git a/.dagger/modules/e2e/fixtures/clients/dep/src/dep/__init__.py b/.dagger/modules/e2e/fixtures/clients/dep/src/dep/__init__.py new file mode 100644 index 0000000..1772f30 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/dep/src/dep/__init__.py @@ -0,0 +1,13 @@ +import dagger +from dagger import dag, function, object_type + + +@object_type +class Dep: + @function + def greet(self, name: str) -> str: + return f"hello, {name}" + + @function + def container(self) -> dagger.Container: + return dag.container().from_("alpine:3.22") diff --git a/.dagger/modules/e2e/fixtures/clients/foreign-app/dagger-module.toml b/.dagger/modules/e2e/fixtures/clients/foreign-app/dagger-module.toml new file mode 100644 index 0000000..3da452d --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/foreign-app/dagger-module.toml @@ -0,0 +1,8 @@ +name = "foreign-app" +engineVersion = "v1.0.0-0" + +[runtime] +source = "python" + +[[dependencies]] +source = "../../../../../../runtime" diff --git a/.dagger/modules/e2e/fixtures/clients/foreign-app/pyproject.toml b/.dagger/modules/e2e/fixtures/clients/foreign-app/pyproject.toml new file mode 100644 index 0000000..93f3b45 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/foreign-app/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "foreign-app" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.12.0"] +build-backend = "uv_build" + +[tool.uv.sources] +dagger-io = { path = "sdk", editable = true } diff --git a/.dagger/modules/e2e/fixtures/clients/foreign-app/src/foreign_app/__init__.py b/.dagger/modules/e2e/fixtures/clients/foreign-app/src/foreign_app/__init__.py new file mode 100644 index 0000000..66c7e47 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/clients/foreign-app/src/foreign_app/__init__.py @@ -0,0 +1,6 @@ +from dagger import object_type + + +@object_type +class ForeignApp: + pass diff --git a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_binding.py b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_binding.py new file mode 100644 index 0000000..13d6d02 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_binding.py @@ -0,0 +1,105 @@ +"""Serve a generated client's bound module before its first query. + +A generated client under ``dagger.clients`` carries a :class:`ModuleBinding`: +the identity of the one module it is bound to. The binding rides on the query +context of every object the client builds, and is served into the session the +first time a query through that client executes. +""" + +import dataclasses +import json +import logging +import typing + +import gql + +if typing.TYPE_CHECKING: + from dagger.client._session import BaseConnection + +logger = logging.getLogger(__name__) + +GIT_SOURCE = "GIT_SOURCE" +LOCAL_SOURCE = "LOCAL_SOURCE" + +_module_runtime = False + + +def mark_module_runtime() -> None: + """Record that this process is a Dagger module runtime. + + The engine builds a module's session with the module's dependencies and the + module itself already served, before the module runs a single query. A + module only vendors clients for those, so there is nothing a client could + serve that is not already there — and a local binding baked into a module + consumed from git would resolve against the caller's workspace, not the + module's. Serving is therefore skipped for the whole process. + """ + global _module_runtime # noqa: PLW0603 + _module_runtime = True + + +@dataclasses.dataclass(frozen=True, slots=True) +class ModuleBinding: + """The identity a generated client serves its module under. + + Parameters + ---------- + name: + The module's final name, after any dependency alias. It is the name + the client chains on ``Query`` and the name the module is served as. + kind: + ``LOCAL_SOURCE`` for a module in the workspace, ``GIT_SOURCE`` for a + remote one. + ref: + A workspace-root-relative path with a leading slash for a local + module; the canonical git ref for a remote one. + pin: + The resolved commit of a remote module, empty for a local one. + """ + + name: str + kind: str + ref: str + pin: str = "" + + def __post_init__(self): + if self.kind not in (LOCAL_SOURCE, GIT_SOURCE): + msg = f"unsupported module source kind for a client: {self.kind!r}" + raise ValueError(msg) + + def query(self) -> str: + """The document that serves the module, as raw GraphQL. + + Raw rather than built through the DSL: ``currentWorkspace`` is hidden + from a module's codegen schema but present in every live session, and + the query builder only knows the schema the session fetched on connect. + """ + serve = f"withName(name: {json.dumps(self.name)}) {{ asModule {{ serve }} }}" + if self.kind == GIT_SOURCE: + source = ( + f"moduleSource(refString: {json.dumps(self.ref)}, " + f"refPin: {json.dumps(self.pin)})" + ) + return f"{{ {source} {{ {serve} }} }}" + source = f"currentWorkspace {{ moduleSource(path: {json.dumps(self.ref)})" + return f"{{ {source} {{ {serve} }} }} }}" + + async def ensure_served(self, conn: "BaseConnection") -> None: + """Serve the module into the connection's session, once. + + Unconditional on the first use per session — the engine deduplicates a + repeat of the same source and reports a different source under the same + name — then remembered on the session so later uses cost nothing. The + schema the session caches predates the serve, so it is fetched again + before the binding is marked served. + """ + if _module_runtime: + return + session = conn.session + async with session.serve_lock: + if self in session.served: + return + logger.debug("Serving module %s from %s", self.name, self.ref) + await session.execute(gql.gql(self.query())) + await session.refetch_schema() + session.served.add(self) diff --git a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_core.py b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_core.py index 7e0fd38..97a3281 100644 --- a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_core.py +++ b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_core.py @@ -43,6 +43,7 @@ TransportError, ) from dagger._exceptions import _query_error_from_transport +from dagger.client._binding import ModuleBinding from dagger.client._session import BaseConnection, SharedConnection from dagger.client.base import Scalar, Type @@ -102,6 +103,7 @@ class Context: selections: collections.deque[Field] = dataclasses.field( default_factory=collections.deque ) + bindings: tuple[ModuleBinding, ...] = () converter: cattrs.Converter = dataclasses.field( init=False, compare=False, @@ -138,6 +140,12 @@ def select_multiple(self, type_name: str, **fields: str) -> "Context": selections.append(field_) return dataclasses.replace(self, selections=selections) + def with_binding(self, binding: ModuleBinding) -> "Context": + """Attach a bound module to serve before any query on this context runs.""" + if binding in self.bindings: + return self + return dataclasses.replace(self, bindings=(*self.bindings, binding)) + def root_select( self, field_name: str, @@ -197,6 +205,8 @@ async def execute(self, return_type: TypeForm[T] | type[T]) -> T: ... async def execute( self, return_type: TypeForm[T] | type[T] | None = None ) -> T | None: + for binding in self.bindings: + await binding.ensure_served(self.conn) await self.resolve_ids() request = await self.request() diff --git a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_session.py b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_session.py index b32b074..9d8c9c6 100644 --- a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_session.py +++ b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/_session.py @@ -2,8 +2,9 @@ import logging import os from dataclasses import dataclass, field -from typing import Any +from typing import TYPE_CHECKING, Any +import anyio import gql import graphql import httpx @@ -23,6 +24,9 @@ from dagger._managers import ResourceManager from dagger.client._config import ConnectConfig, Retry +if TYPE_CHECKING: + from dagger.client._binding import ModuleBinding + logger = logging.getLogger(__name__) @@ -98,6 +102,11 @@ def __init__(self, conn: ConnectParams, cfg: ConnectConfig | None = None): self.client = retrying_client(client, cfg.retry) if cfg.retry else client self._session: AsyncClientSession | None = None + # Modules a generated client has served into this session, and the lock + # that keeps serve, schema refetch and the mark as one step. + self.served: set[ModuleBinding] = set() + self.serve_lock = anyio.Lock() + async def __aenter__(self) -> Self: await self.start() return self @@ -145,6 +154,10 @@ async def get_schema(self) -> graphql.GraphQLSchema: async def execute(self, query: gql.GraphQLRequest) -> Any: return await (await self.get_session()).execute(query) + async def refetch_schema(self) -> None: + """Fetch the schema again, after a module was served into the session.""" + await (await self.get_session()).fetch_schema() + async def close(self) -> None: logger.debug("Closing client session to GraphQL server") await super().close() diff --git a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/gen.py b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/gen.py index debb1e9..7f45048 100644 --- a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/gen.py +++ b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/client/gen.py @@ -1909,7 +1909,10 @@ def from_(self, address: str, *, registry_service: "Service | None" = None, prot ---------- address: Address of the container image to download, in standard OCI ref - format. Example:"registry.dagger.io/engine:latest" + format. Example: "registry.dagger.io/engine:latest". + An address without a tag or digest selects the greatest stable + release tag, falling back to the literal "latest" tag when no + eligible release exists. registry_service: Service to use as the registry endpoint for the image address. The service will be started only for this pull. @@ -7688,10 +7691,15 @@ async def id(self) -> str: _ctx = self._select("id", _args) return await _ctx.execute(str) - def latest_version(self) -> GitRef: - """Returns details for the latest semver tag.""" + def latest(self) -> GitRef: + """Return the latest stable release tag, falling back to HEAD when no + release exists. + + Release selection accepts an optional "v" prefix, incomplete versions, + and zero-padded numeric components. This operation is pinned. + """ _args: list[Arg] = [] - _ctx = self._select("latestVersion", _args) + _ctx = self._select("latest", _args) return GitRef(_ctx) def ref(self, name: str) -> GitRef: @@ -14022,6 +14030,28 @@ def with_config_value(self, key: str, value: str, *, values: list[str] | None = _ctx = self._select("withConfigValue", _args) return Workspace(_ctx) + def with_directory(self, path: str, source: Directory) -> Self: + """Return this workspace with a directory merged into the given path, + without mutating the source. + + Anything already at the path stays, and files the source carries win, + as with Directory.withDirectory. Use withNewDirectory to replace the + path instead. + + Parameters + ---------- + path: + Path to merge into. Relative paths resolve from the workspace cwd. + source: + Directory to merge there. + """ + _args = [ + Arg("path", path), + Arg("source", source), + ] + _ctx = self._select("withDirectory", _args) + return Workspace(_ctx) + def with_init_client(self, path: str, sdk: str, module: str, *, args: JSON | None = None, here: bool | None = False, no_generate: bool | None = False,) -> Self: """Return this workspace with a generated API client initialized. @@ -14168,16 +14198,19 @@ def with_mounted_file(self, path: str, source: File) -> Self: return Workspace(_ctx) def with_new_directory(self, path: str, source: Directory) -> Self: - """Return this workspace with a directory added, without mutating the - source. + """Return this workspace with the given path replaced by a directory, + without mutating the source. + + The source becomes the entire contents of the path: anything already + there that the source does not carry is removed. Use withDirectory to + keep it instead. Parameters ---------- path: - Path of the added directory. Relative paths resolve from the - workspace cwd. + Path to replace. Relative paths resolve from the workspace cwd. source: - Directory to add. + Directory to write there. """ _args = [ Arg("path", path), diff --git a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/clients/__init__.py b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/clients/__init__.py new file mode 100644 index 0000000..c763c65 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/clients/__init__.py @@ -0,0 +1,5 @@ +"""Generated clients for the modules this library is bound to. + +Every module in this package is generated by the Python SDK: one per bound +module, named after it. Nothing here is written by hand. +""" diff --git a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/clients/runtime_app.py b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/clients/runtime_app.py new file mode 100644 index 0000000..dc6937a --- /dev/null +++ b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/clients/runtime_app.py @@ -0,0 +1,91 @@ +# Code generated by dagger. DO NOT EDIT. + +import warnings # noqa: F401 +from collections.abc import Callable # noqa: F401 +from dataclasses import dataclass # noqa: F401 +from typing import Protocol, runtime_checkable # noqa: F401 + +from typing_extensions import Self # noqa: F401 + +from dagger.client import gen as _core +from dagger.client._binding import ModuleBinding +from dagger.client._core import Arg +from dagger.client._guards import typecheck +from dagger.client.base import Enum, Input, Scalar, Type # noqa: F401 + + +_BINDING = ModuleBinding( + name='runtime-app', + kind='LOCAL_SOURCE', + ref='/.dagger/modules/e2e/fixtures/runtime/app', + pin='', +) + +@typecheck +class RuntimeApp(Type): + + async def greeting(self) -> str: + """Returns + ------- + str + The `String` scalar type represents textual data, represented as + UTF-8 character sequences. The String type is most often used by + GraphQL to represent free-form human-readable text. + + Raises + ------ + ExecuteTimeoutError + If the time to execute the query exceeds the configured timeout. + QueryError + If the API returns an error. + """ + _args: list[Arg] = [] + _ctx = self._select("greeting", _args) + return await _ctx.execute(str) + + async def id(self) -> str: + """A unique identifier for this RuntimeApp. + + Note + ---- + This is lazily evaluated, no operation is actually run. + + Returns + ------- + str + The `ID` scalar type represents a unique identifier, often used to + refetch an object or as key for a cache. The ID type appears in a + JSON response as a String; however, it is not intended to be + human-readable. When expected as an input type, any string (such + as `"4"`) or integer (such as `4`) input value will be accepted as + an ID. + + Raises + ------ + ExecuteTimeoutError + If the time to execute the query exceeds the configured timeout. + QueryError + If the API returns an error. + """ + _args: list[Arg] = [] + _ctx = self._select("id", _args) + return await _ctx.execute(str) + + + +def runtime_app(*, client: _core.Client | None = None) -> RuntimeApp: + """Client for the `runtime-app` module. + + Executing any query through the returned object serves the module into + the session first; a module bound by workspace path needs a workspace, + a module bound by git ref resolves anywhere. Pass `client` to use a + connection other than the global one. + """ + _args: list[Arg] = [] + _ctx = (_core.dag if client is None else client)._ctx.with_binding(_BINDING).root_select("runtimeApp", _args) # noqa: SLF001 + return RuntimeApp(_ctx) + +__all__ = [ + "RuntimeApp", + "runtime_app", +] \ No newline at end of file diff --git a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/mod/_converter.py b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/mod/_converter.py index 272ce7a..c00f92a 100644 --- a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/mod/_converter.py +++ b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/mod/_converter.py @@ -164,7 +164,7 @@ async def exec_method(self, *args, **kwargs): @functools.cache -def to_typedef(annotation: typing.Any, context: str = "type") -> "TypeDef": # noqa: C901, PLR0911 +def to_typedef(annotation: typing.Any, context: str = "type") -> "TypeDef": # noqa: C901, PLR0911, PLR0912 """Convert Python object to API type.""" if is_initvar(annotation): return to_typedef(annotation.type, context) @@ -203,6 +203,17 @@ def to_typedef(annotation: typing.Any, context: str = "type") -> "TypeDef": # n if inspect.isclass(cls := typ.hint): name = cls.__name__ + # A generated client's types, including its bootstrap placeholder, are + # the API of another module (or of this one, seen from outside): a + # module's own API can only use its own types and the core API. + if cls.__module__.startswith("dagger.clients."): + msg = ( + f"unsupported {context}: {typ.hint!r} is a generated client " + "type; a module's API can only use its own object types " + "and the core API" + ) + raise TypeError(msg) + if is_subclass(cls, enum.Enum): return td.with_enum(name, description=get_doc(cls)) diff --git a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/mod/cli.py b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/mod/cli.py index 86d0866..615e40a 100644 --- a/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/mod/cli.py +++ b/.dagger/modules/e2e/fixtures/runtime/app/sdk/src/dagger/mod/cli.py @@ -11,6 +11,7 @@ import dagger from dagger import telemetry +from dagger.client._binding import mark_module_runtime from dagger.mod._exceptions import ModuleError, ModuleLoadError, record_exception from dagger.mod._module import MAIN_OBJECT, Module @@ -19,6 +20,7 @@ ENTRY_POINT_NAME: typing.Final[str] = "main_object" ENTRY_POINT_GROUP: typing.Final[str] = typing.cast(str, __package__) IMPORT_PKG: typing.Final[str] = os.getenv("DAGGER_DEFAULT_PYTHON_PACKAGE", "main") +CLIENTS_PKG: typing.Final[str] = "dagger.clients" def app(mod: Module | None = None, register: bool = False) -> int | None: @@ -32,6 +34,7 @@ def app(mod: Module | None = None, register: bool = False) -> int | None: async def main(mod: Module | None = None, register: bool = False) -> int | None: """Async entrypoint for a Dagger module.""" + mark_module_runtime() # Establishing connection early on to allow returning dag.error(). # Note: if there's a connection error dag.error() won't be sent but # should be logged and the traceback shown on the function's stderr output. @@ -56,6 +59,18 @@ def load_module() -> Module: ep = get_entry_point() try: cls = ep.load() + except ModuleNotFoundError as e: + if e.name and (e.name == CLIENTS_PKG or e.name.startswith(f"{CLIENTS_PKG}.")): + msg = ( + f"generated client '{e.name}' is missing; run `dagger generate` " + "and commit the generated files" + ) + raise ModuleLoadError(msg) from e + logger.exception( + "Error while importing Python module '%s' with Dagger functions", + ep.module, + ) + raise ModuleLoadError(str(e)) from e except Exception as e: logger.exception( "Error while importing Python module '%s' with Dagger functions", diff --git a/.dagger/modules/e2e/main.dang b/.dagger/modules/e2e/main.dang index 3e27eab..51ee728 100644 --- a/.dagger/modules/e2e/main.dang +++ b/.dagger/modules/e2e/main.dang @@ -19,6 +19,14 @@ type E2e { let runtimeGreeting: String! = "served by the python-sdk runtime" let mixedDiscoveryModulePath: String! = fixtureRoot + "/mixed-discovery/ancestor/work/app" let mixedDiscoveryNestedPath: String! = mixedDiscoveryModulePath + "/nested/deeper" + let clientsDepPath: String! = fixtureRoot + "/clients/dep" + let clientsAppPath: String! = fixtureRoot + "/clients/app" + let foreignAppPath: String! = fixtureRoot + "/clients/foreign-app" + let clientsDir: String! = "sdk/src/dagger/clients" + let standaloneClientPath: String! = outputRoot + "/client-dep" + let gitClientPath: String! = outputRoot + "/client-sdk-sdk" + let gitModuleRef: String! = "github.com/dagger/sdk-sdk" + let gitModulePin: String! = "334448911a8292fba0d677e5f31926c79ad80ad3" let generatedMarkerPath: String! = "sdk/src/dagger/client/gen.py" let generatedMarkerContents: String! = "Code generated by dagger." @@ -472,7 +480,7 @@ type E2e { .withMountedCache("/root/.cache/uv", cacheVolume("python-sdk-uv")) .withDirectory("/sdk", sdkSource(ws)) .withWorkdir("/sdk") - .withExec(["uv", "run", "--frozen", "pytest", "-q", "tests/codegen", "tests/mod"], experimentalPrivilegedNesting: true) + .withExec(["uv", "run", "--frozen", "pytest", "-q", "tests/codegen", "tests/mod", "tests/client"], experimentalPrivilegedNesting: true) .sync null @@ -547,4 +555,224 @@ type E2e { null } + + """ + A module gets the core API from its own schema, one client per declared + dependency under the dependency's final name, and a client for itself — + generated from nothing, with the module's own code importing its self client + from the start, so the bootstrap stub is what carries the first build. + """ + pub clientsGenerateCheck(ws: Workspace!): Void @check { + let changes = pythonSdk.mod(ws, path: clientsAppPath).generate + let core = changes.layer.file(clientsAppPath + "/" + generatedMarkerPath).contents + assertContains(core, generatedMarkerContents, "generate did not produce the core API") + assertContainsNone(core, ["class Host(", "def current_workspace(", "class Dep(", "def dep("]) + assertAdded(changes, clientsAppPath + "/" + clientsDir + "/__init__.py") + + let depClient = changes.layer.file(clientsAppPath + "/" + clientsDir + "/dep.py").contents + assertContainsAll(depClient, [ + "name='dep'", + "kind='LOCAL_SOURCE'", + "ref='/" + clientsDepPath + "'", + "class Dep(Type):", + "def dep(*, client: _core.Client | None = None) -> Dep:", + "-> _core.Container:", + "from dagger.client import gen as _core", + ]) + assertContainsNone(depClient, ["class Container(", "class Client(", "dag = "]) + + let aliasClient = changes.layer.file(clientsAppPath + "/" + clientsDir + "/greeter.py").contents + assertContainsAll(aliasClient, ["name='greeter'", "ref='/" + clientsDepPath + "'", "class Greeter(Type):", "def greeter(", "root_select(\"greeter\""]) + + let selfClient = changes.layer.file(clientsAppPath + "/" + clientsDir + "/app.py").contents + assertContainsAll(selfClient, ["name='app'", "ref='/" + clientsAppPath + "'", "class App(Type):", "def app(", "async def greet_self(self)"]) + + assert(contains(changes.addedPaths, clientsDepPath + "/" + generatedMarkerPath) == false, "the dependency's generated files rode along with the module's") + assert(contains(changes.addedPaths, clientsAppPath + "/sdk/src/dagger/provisioning/__init__.py") == false, "a module vendored engine provisioning") + + # A second generate on the applied result changes nothing. + let generated = applied(ws, changes) + assert(pythonSdk.mod(generated.asWorkspace(cwd: "."), path: clientsAppPath).generate.isEmpty, "a second generate was not empty") + + # A function added after the first generate shows up in the regenerated + # self client: the committed one is carried through the bootstrap. + let sourcePath = clientsAppPath + "/src/app/__init__.py" + let edited = generated + .withNewFile(sourcePath, generated.file(sourcePath).contents + "\n @function\n def extra(self) -> str:\n return \"extra\"\n") + .asWorkspace(cwd: ".") + let regenerated = pythonSdk.mod(edited, path: clientsAppPath).generate + assertContains(regenerated.layer.file(clientsAppPath + "/" + clientsDir + "/app.py").contents, "async def extra(self)", "the regenerated self client missed the new function") + assert(contains(regenerated.modifiedPaths, clientsAppPath + "/" + clientsDir + "/app.py"), "the self client was not reported as modified") + + # Dropping the alias drops its client, as a removal. + let withoutAlias = generated + .withNewFile(clientsAppPath + "/dagger-module.toml", "name = \"app\"\nengineVersion = \"v1.0.0-0\"\n\n[runtime]\nsource = \"python\"\n\n[[dependencies]]\nsource = \"../dep\"\n") + .withNewFile(sourcePath, "from dagger import function, object_type\nfrom dagger.clients.app import app\nfrom dagger.clients.dep import dep\n\n\n@object_type\nclass App:\n @function\n async def greet_via_dep(self) -> str:\n return await dep().greet(\"dep\")\n\n @function\n def local(self) -> str:\n return \"local\"\n\n @function\n async def greet_self(self) -> str:\n return (await app().local()).upper()\n") + .asWorkspace(cwd: ".") + let pruned = pythonSdk.mod(withoutAlias, path: clientsAppPath).generate + assert(contains(pruned.removedPaths, clientsAppPath + "/" + clientsDir + "/greeter.py"), "a dropped alias left its client behind") + assert(contains(pruned.removedPaths, sourcePath) == false, "the module's own source was reported removed") + + null + } + + """ + The generated clients work at runtime, inside a module: a module calls its + dependency, the aliased dependency, and itself through them, driven by a + real CLI. Inside a module runtime the serve preamble is a no-op, so this pins + that the engine's own serving is what the clients chain onto. + """ + pub clientsRuntimeCheck(ws: Workspace!): Void @check { + let withApp = applied(ws, pythonSdk.mod(ws, path: clientsAppPath).generate) + let appWs = withApp.asWorkspace(cwd: ".") + let withBoth = applied(appWs, pythonSdk.mod(appWs, path: clientsDepPath).generate) + let target = sdkSdk.target(withBoth, ".") + let fixture = "vendor/sdk-workspace/" + clientsAppPath + + let viaDep = target.runInstalled(["call", "-m", fixture, "greet-via-dep"]) + viaDep.assertSuccess + assertContains(viaDep.stdout, "hello, dep", "the module did not reach its dependency through the generated client") + + let viaAlias = target.runInstalled(["call", "-m", fixture, "greet-via-alias"]) + viaAlias.assertSuccess + assertContains(viaAlias.stdout, "hello, alias", "the module did not reach the aliased dependency through the generated client") + + let viaSelf = target.runInstalled(["call", "-m", fixture, "greet-self"]) + viaSelf.assertSuccess + assertContains(viaSelf.stdout, "LOCAL", "the module did not reach itself through the generated client") + + null + } + + """ + A standalone client is the same artifact a module vendors for a dependency, + plus what running outside a module needs: provisioning, the core API as a + client sees it, and the CLI version of the engine it was generated against. + """ + pub standaloneClientCheck(ws: Workspace!): Void @check { + let changes = pythonSdk.generateClient(ws, module: clientsDepPath, path: standaloneClientPath) + let clientFile = standaloneClientPath + "/" + clientsDir + "/dep.py" + assertAdded(changes, clientFile) + assertAdded(changes, standaloneClientPath + "/sdk/src/dagger/provisioning/__init__.py") + assertAdded(changes, standaloneClientPath + "/" + clientsDir + "/__init__.py") + assertContains(changes.layer.file(standaloneClientPath + "/" + generatedMarkerPath).contents, "class Host(", "a standalone client's core must see everything the CLI does") + assertContains( + changes.layer.file(standaloneClientPath + "/sdk/src/dagger/_engine/_version.py").contents, + "CLI_VERSION = " + JSON.encode(releaseTag(version)), + "the client's CLI version is not the engine it was generated against", + ) + + # Byte-identical to what a module depending on dep vendors for it. + let vendored = pythonSdk.mod(ws, path: clientsAppPath).generate.layer.file(clientsAppPath + "/" + clientsDir + "/dep.py").contents + assert(changes.layer.file(clientFile).contents == vendored, "the standalone client differs from the one a module vendors") + + # initClient seeds the project file and nothing else. + let seeded = pythonSdk.initClient(ws, path: outputRoot + "/client-init", module: clientsDepPath) + let seededFiles = seeded.addedPaths.filter { added => added.trimSuffix("/") == added } + assert(seededFiles.length == 1, "initClient wrote more than the project file: " + seededFiles.join(", ")) + assertAdded(seeded, outputRoot + "/client-init/pyproject.toml") + assertContains(seeded.layer.file(outputRoot + "/client-init/pyproject.toml").contents, "name = \"client_init\"", "the project file is not named after its directory") + + # A path that leaves the workspace is refused, not written somewhere else. + # A backslash is a separator to the engine, so it is one here too. + let escaping = ["../escape", "..\\escape", "nested/../escape"].map { path => + pythonSdk.generateClient(ws, module: clientsDepPath, path: path).addedPaths.join(", ") rescue "raised" + } + assert(escaping.join("|") == "raised|raised|raised", "generateClient wrote outside the workspace: " + escaping.join("|")) + + # A git module binds by canonical ref and pin, which resolve anywhere. + let gitClient = pythonSdk.generateClient(ws, module: gitModuleRef + "@" + gitModulePin, path: gitClientPath) + let gitFile = gitClient.layer.file(gitClientPath + "/" + clientsDir + "/sdk_sdk.py").contents + assertContainsAll(gitFile, ["name='sdk-sdk'", "kind='GIT_SOURCE'", "ref='" + gitModuleRef, "pin='" + gitModulePin + "'", "def sdk_sdk("]) + + # A registered client is materialized by the rollup, which is empty once applied. + let config = ws.directory("/", include: ["dagger.toml"]).file("dagger.toml").contents + + "\n[[modules.python-sdk.as-sdk.clients]]\npath = \"" + standaloneClientPath + "\"\nmodule = \"" + clientsDepPath + "\"\n" + let registeredTree = ws.directory("/").withNewFile("dagger.toml", config) + let registered = registeredTree.asWorkspace(cwd: ".") + let rollup = pythonSdk.generateAllClient(registered) + assertAdded(rollup, clientFile) + let rolledUp = registeredTree.withDirectory(".", rollup.layer).asWorkspace(cwd: ".") + assert(pythonSdk.generateAllClient(rolledUp).isEmpty, "a second rollup was not empty") + + null + } + + """ + A standalone client runs from a plain Python process: bound to a git module + by ref and pin, it serves that module into its session on the first query + and calls it. The process attaches to the session nesting hands it, which + is the `dagger run` shape; a workspace-bound module would resolve against + that session's workspace, so the git binding is what this pins end to end. + """ + pub standaloneRuntimeCheck(ws: Workspace!): Void @check { + let withClient = applied(ws, pythonSdk.generateClient(ws, module: gitModuleRef + "@" + gitModulePin, path: gitClientPath)) + let tree = withClient + .withNewFile(gitClientPath + "/pyproject.toml", "[project]\nname = \"client-sdk-sdk\"\nversion = \"0.1.0\"\nrequires-python = \">=3.14\"\ndependencies = [\"dagger-io\"]\n\n[tool.uv.sources]\ndagger-io = { path = \"sdk\", editable = true }\n") + .withNewFile(gitClientPath + "/main.py", "import sys\n\nimport anyio\n\nimport dagger\nfrom dagger.clients.sdk_sdk import sdk_sdk\n\n\nasync def main() -> None:\n async with dagger.connection(dagger.Config(log_output=sys.stderr)):\n print(\"cli:\", await sdk_sdk().dagger_cli_version())\n\n\nanyio.run(main)\n") + + let run = pythonRunner + .withDirectory("/work", tree) + .withWorkdir("/work/" + gitClientPath) + .withExec(["uv", "run", "--quiet", "python", "main.py"], experimentalPrivilegedNesting: true) + # The CLI version the module at gitModulePin reports; bump both together. + assertContains(run.stdout, "cli: 1.0.0-beta.10", "the standalone client did not reach the git-bound module") + + null + } + + """ + A local dependency another SDK manages is handed to the engine, which knows + its generator, and gets a client like any other dependency. Here that is + this repository's Go runtime module. Where the engine can stage it — a + checkout it can diff, as in CI — generation succeeds with the Go module's + client vendored; where it cannot (a git worktree, whose `.git` file breaks + the engine's context diff), generation fails loudly rather than silently + skipping the dependency. The same fixture without the dependency generates + either way, which pins the dependency as the cause. + """ + pub foreignDependencyCheck(ws: Workspace!): Void @check { + let foreignClient = foreignAppPath + "/" + clientsDir + "/python_sdk_runtime.py" + let added = pythonSdk.mod(ws, path: foreignAppPath, findUp: false).generate.addedPaths rescue [] + assert( + added.length == 0 or contains(added, foreignClient), + "a dependency managed by another SDK was generated around rather than handed to the engine", + ) + + let alone = ws.directory("/") + .withNewFile(foreignAppPath + "/dagger-module.toml", "name = \"foreign-app\"\nengineVersion = \"v1.0.0-0\"\n\n[runtime]\nsource = \"python\"\n") + .asWorkspace(cwd: ".") + let addedAlone = pythonSdk.mod(alone, path: foreignAppPath, findUp: false).generate.addedPaths + assert(addedAlone.length > 0, "the fixture fails for a reason unrelated to its foreign dependency") + assert(contains(addedAlone, foreignClient) == false, "a client was generated for a dependency the module does not declare") + + null + } + + """ + A workspace's tree with a changeset applied: what the caller would have on + disk after accepting it. + """ + let applied(ws: Workspace!, changes: Changeset!): Directory! { + ws.directory("/").withDirectory(".", changes.layer) + } + + """ + The engine's version as the bare release tag provisioning downloads. + """ + let releaseTag(engineVersion: String!): String! { + engineVersion.trimPrefix("v").split("+").reduce("") { acc, part => if (acc == "") { part } else { acc } } + } + + """ + A Python-capable runner for running a standalone client the way a user + would. + """ + let pythonRunner: Container! { + container + .from("ghcr.io/astral-sh/uv:python3.14-alpine") + .withoutEntrypoint + .withMountedCache("/root/.cache/uv", cacheVolume("python-sdk-uv")) + .withEnvVariable("UV_LINK_MODE", "copy") + } } diff --git a/README.md b/README.md index 132a9d4..c16b172 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ It uses the engine's native `Workspace` and `ModuleSource` APIs directly. | Path | What it is | | --- | --- | | `python-sdk.dang`, `mod.dang`, `templates/` | authoring: `initModule`, `generate`, config, discovery | +| `client.dang`, `client-template/` | standalone clients: `generateClient`, `initClient` | +| `codegen.dang` | how both drive the code generator and vendor the library | | `sdk/` | the `dagger-io` client library and code generator | | `runtime/` | the module runtime the engine calls to run a module | @@ -157,6 +159,17 @@ For a single module: dagger call python-sdk mod --path my-module generate ``` +Generation vendors the client library into `/sdk/` and, next to it, +everything the module can call: + +``` +/sdk/src/dagger/client/gen.py the core API, as a module sees it +/sdk/src/dagger/clients/.py a client for the module itself +/sdk/src/dagger/clients/.py one client per declared dependency +``` + +See [Clients](#clients) for what a client is and how module code uses one. + For every Python SDK module in the workspace (skipping any with a `.dagger-python-sdk-skip-generate` marker at or above the module root): @@ -164,6 +177,117 @@ For every Python SDK module in the workspace (skipping any with a dagger call python-sdk generate-all ``` +## Clients + +A module's dependency and a standalone client are the same thing: generated +bindings for one module, plus a serve preamble that makes sure the module is +served in the session before the first query goes through them. Every module +under `sdk/src/dagger/clients/` is one such client, named after the module it +binds to — its final name, so a dependency declared with an alias gets a client +under the alias. Core types are shared: a `dagger.Container` returned by one +client is the same class everywhere. + +A module calls a dependency, and itself, through the client's entry point — a +function named after the module, taking the module's constructor arguments and +an optional `client`: + +```python +from dagger import function, object_type +from dagger.clients.app import app # the module's own client +from dagger.clients.builder import builder # a declared dependency + + +@object_type +class App: + @function + async def build(self) -> str: + return await builder().build("main") + + @function + async def twice(self) -> str: + return (await app().build()) * 2 # a self call, through the engine +``` + +Inside a module the engine has already served the module's dependencies and +the module itself, so the preamble does nothing there. A self call goes +through the engine like any other call, so it gets function-level caching. + +Generate first, then call: a symbol imported from the module's own client has +to exist in the last generated client, so add the function, generate, then +call it. A client's types are not a module's own types — a function returns +the module's `@object_type`, never `dagger.clients.app.App`. + +### Standalone clients + +The same client works from a Python project that is not a module. Register +one and let `dagger generate` produce it: + +```sh +dagger api client init python clients/builder ./builder +``` + +This records the client in the workspace config, seeds `clients/builder/pyproject.toml` +(declaring the vendored library, so `uv sync` there just works) and generates +`clients/builder/sdk/`. Regenerate with `dagger generate`, or directly: + +```sh +dagger call python-sdk generate-client --module ./builder --path clients/builder +``` + +`module` is a workspace path or a git ref. A local module this SDK manages is +generated first, so the client is never read off an ungenerated module. +Everything generated sits under `sdk/`; files next to it are yours. + +```python +import dagger +from dagger.clients.builder import builder + + +async def main() -> None: + async with dagger.connection(): + print(await builder().build("main")) +``` + +Outside a module the preamble serves the bound module the first time a query +runs, then remembers it for the session. A module bound by workspace path +needs a workspace — run from inside one, or pass `dagger.Config(workdir=...)`; +a module bound by git ref resolves anywhere. The vendored library carries +engine provisioning and the CLI version of the engine the client was +generated against. + +### Migrating a module + +Modules on the `dagger-module.toml` config get clients on their next +`dagger generate`; `dag.()` is gone. For each dependency: + +```python +# before +from dagger import dag +await dag.builder().build("main") + +# after +from dagger.clients.builder import builder +await builder().build("main") +``` + +A dependency that adds a function to a core type moves the same way: the +function is no longer a method on the core object, it is a module-level +function taking that object first. Its name is always its parent's name and +the field's — only the entry point is bare — so `Directory.asHello` is +`directory_as_hello`, and adding `File.asHello` next to it renames nothing: + +```python +# before +await directory.as_hello().greet("world") + +# after +from dagger.clients.hello import directory_as_hello +await directory_as_hello(directory).greet("world") +``` + +Legacy `dagger.json` modules are untouched: they keep the merged `gen.py` the +engine's builtin Python SDK generates. + ## Manage dependencies and the engine version Editing a module's dependencies or its required engine version is identical diff --git a/client-template/pyproject.toml.tmpl b/client-template/pyproject.toml.tmpl new file mode 100644 index 0000000..15da1f3 --- /dev/null +++ b/client-template/pyproject.toml.tmpl @@ -0,0 +1,8 @@ +[project] +name = "{{ .ModulePackage }}" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io"] + +[tool.uv.sources] +dagger-io = { path = "sdk", editable = true } diff --git a/client.dang b/client.dang new file mode 100644 index 0000000..9386d29 --- /dev/null +++ b/client.dang @@ -0,0 +1,50 @@ +""" +A generated client for one module, for use outside any module: a Python +project that vendors this SDK's library, the core API as a client sees it, and +the bound module's client. +""" +type GeneratedClient { + """ + Workspace-root-relative path of the client directory. + """ + pub path: String! + + """ + The workspace the client lives in. + """ + let ws: Workspace! + + """ + The module the client is bound to, resolved. + """ + let source: ModuleSource! + + """ + Generate this client. + + Everything generated sits under `sdk/` inside the client directory, so the + user's own files next to it survive every run. The generated-clients + subtree is swept, the rest of `sdk/` is merged onto what is there. + """ + pub generate: Changeset! { + codegen + .overlay(ws, path, codegen.vendorDirName + "/" + codegen.clientsDirName, generatedTree) + .changes(ws) + } + + """ + The client's generated files, rooted at the client directory: a directory + holding only `sdk/`. + """ + pub generatedTree: Directory! { + let clientFile = codegen.clientsDirName + "/" + codegen.clientName(source.moduleName) + ".py" + directory.withDirectory( + codegen.vendorDirName, + codegen.library(true) + .withFile(codegen.generatedBindingsPath, codegen.core(source.clientSchemaIntrospectionJSON)) + .withFile(clientFile, codegen.clientOf(source)), + ) + } + + let codegen: Codegen! { Codegen() } +} diff --git a/codegen.dang b/codegen.dang new file mode 100644 index 0000000..ce341fe --- /dev/null +++ b/codegen.dang @@ -0,0 +1,229 @@ +""" +This SDK's code generator and the client library it vendors, shared by module +generation and standalone client generation. +""" +type Codegen { + """ + Where a module or a client vendors this SDK's library. + """ + pub vendorDirName: String! = "sdk" + + """ + The core API file inside the vendored library. + """ + pub generatedBindingsPath: String! = "src/dagger/client/gen.py" + + """ + The generated-clients package inside the vendored library, one file per + bound module. Exclusively generated. + """ + pub clientsDirName: String! = "src/dagger/clients" + + """ + The core API generated from a schema: every type no module owns, plus the + Client root and the dag singleton. + """ + pub core(schemaJSON: File!): File! { + base + .withMountedFile(schemaPath, schemaJSON) + .withExec(command + ["generate", "-i", schemaPath, "-o", "/gen.py"]) + .file("/gen.py") + } + + """ + One module's client, generated from its own client-facing schema under its + final name: its types, its entry point, and the identity it is served under, + all read off the resolved source. + + A module loaded from the workspace binds by its workspace-root-relative path, + which resolves against the workspace the client runs in; a git module binds + by its canonical ref and pin, which resolve anywhere. + """ + pub clientOf(src: ModuleSource!): File! { + let name = src.moduleName + let binding = if (isGitSource(src.kind)) { + bindingJSON(name, "GIT_SOURCE", src.asString, src.pin) + } else { + bindingJSON(name, "LOCAL_SOURCE", "/" + normalizePath(src.sourceRootSubpath), "") + } + + base + .withMountedFile(schemaPath, src.clientSchemaIntrospectionJSON) + .withExec(command + [ + "generate", "-i", schemaPath, "-o", "/client.py", + "--mode", "client", "--module", name, "--binding", binding, "--engine-version", src.engineVersion, + ]) + .file("/client.py") + } + + """ + The Python module names of several bound modules' clients, as the code + generator normalizes them, so Dang and Python agree on the file names. Two + names that would share one file are rejected there, in one place. + """ + pub clientNames(moduleNames: [String!]!): [String!]! { + base.withExec(command + ["client-name"] + moduleNames).stdout.trimSuffix("\n").split("\n") + } + + """ + The Python module name of one bound module's client. + """ + pub clientName(moduleName: String!): String! { + clientNames([moduleName]).reduce("") { acc, name => name } + } + + """ + A placeholder standing in for a module's own client while the module is + built for the first time: every name resolves to a class whose use says the + client is not generated yet. + """ + pub stubClient(moduleName: String!): File! { + let contents = "# Bootstrap placeholder for the `" + moduleName + "` client; `dagger generate` replaces it.\n" + + "\n" + + "__all__ = []\n" + + "\n" + + "\n" + + "class _NotGenerated:\n" + + " def __init__(self, *args, **kwargs):\n" + + " msg = \"client for module '" + moduleName + "' is not generated yet: run `dagger generate`\"\n" + + " raise RuntimeError(msg)\n" + + "\n" + + "\n" + + "def __getattr__(name):\n" + + " if name.startswith(\"__\"):\n" + + " raise AttributeError(name)\n" + + " return _NotGenerated\n" + directory.withNewFile("stub.py", contents).file("stub.py") + } + + """ + What gets vendored: the importable client library, its license, and a + project file describing just that. + + A module gets the library without engine provisioning: it runs inside a + session the engine opened. A standalone client gets provisioning too, with + the CLI version it downloads stamped from the engine the client was + generated against, so it does not start an older engine than its bindings. + """ + pub library(standalone: Boolean!): Directory! { + let excludes = if (standalone) { [] } else { ["!src/dagger/provisioning/**"] } + let files = currentModule.source + .directory("sdk") + .filter(include: ["LICENSE", "README.md", "src/**/*.py", "src/**/*.typed"] + excludes) + .withFile("pyproject.toml", libraryPyproject) + + if (standalone) { + files.withNewFile("src/dagger/_engine/_version.py", versionStamp) + } else { + files + } + } + + """ + Write generated files at `path` in a workspace: `tree` layered onto what is + there, with the `swept` subtree (relative to `path`) replaced rather than + merged, so a generated file that no longer exists is reported as a removal. + + The whole directory is composed first and written in one go. The engine + merges `Workspace.withDirectory` onto an untouched path but replaces it once + a `withoutDirectory` below it has run, so writing only the generated tree + after the sweep would drop everything else at `path`. + """ + pub overlay(ws: Workspace!, path: String!, swept: String!, tree: Directory!): Workspace! { + let composed = existingDir(ws, path).withoutDirectory(swept).withDirectory(".", tree) + ws + .withoutDirectory("/" + subpath(path, swept)) + .withDirectory("/" + path, composed) + } + + """ + Existing contents of a workspace directory, empty when it doesn't exist yet. + """ + pub existingDir(ws: Workspace!, path: String!): Directory! { + let root = ws.directory("/") + if (path == ".") { + root + } else if (root.exists(path)) { + root.directory(path) + } else { + directory + } + } + + pub subpath(path: String!, rel: String!): String! { + if (path == ".") { rel } else { path + "/" + rel } + } + + pub isGitSource(kind: ModuleSourceKind!): Boolean! { + JSON.encode(kind) == "\"GIT_SOURCE\"" + } + + pub normalizePath(path: String!): String! { + let normalized = path.trimPrefix("./").trimPrefix("/").trimSuffix("/") + if (normalized == "") { "." } else { normalized } + } + + let bindingJSON(name: String!, kind: String!, ref: String!, pin: String!): String! { + "{\"name\":" + JSON.encode(name) + ",\"kind\":" + JSON.encode(kind) + ",\"ref\":" + JSON.encode(ref) + ",\"pin\":" + JSON.encode(pin) + "}" + } + + """ + The `_version.py` a standalone client provisions from: the running engine's + version as the bare release tag the downloader expects. + """ + let versionStamp: String! { + let tag = base.withExec(command + ["cli-version", version]).stdout.trimSuffix("\n") + "# Code generated by dagger. DO NOT EDIT.\n\nCLI_VERSION = " + JSON.encode(tag) + "\n" + } + + """ + The library's project file with its development sections removed. + + As published, it declares the code generator as a uv workspace member and a + dev dependency. Vendoring that verbatim without the generator makes `uv` + refuse to install the library at all, so the sections that only describe + developing this SDK are dropped. + """ + let libraryPyproject: File! { + base + .withFile(stripScriptPath, currentModule.source.file("helpers/vendor-pyproject/strip_dev_sections.py")) + .withExec(["python", stripScriptPath, "pyproject.toml", "/library-pyproject.toml"]) + .file("/library-pyproject.toml") + } + + """ + Container with this SDK's client library and code generator mounted. + """ + let base: Container! { + container + .from(image) + .withoutEntrypoint + .withMountedCache("/root/.cache/uv", cacheVolume("python-sdk-uv")) + .withEnvVariable("UV_LINK_MODE", "copy") + .withDirectory("/sdk", source) + .withWorkdir("/sdk") + } + + """ + What the code generator needs to run: the library plus the generator itself, + and the lock that pins the generator's own dependencies. + """ + let source: Directory! { + currentModule.source.directory("sdk").filter(include: [ + "pyproject.toml", + "uv.lock", + "src/**/*.py", + "src/**/*.typed", + "codegen/pyproject.toml", + "codegen/**/*.py", + ]) + } + + let command: [String!]! = [ + "uv", "run", "--isolated", "--frozen", "--package", "codegen", "python", "-m", "codegen", + ] + + let stripScriptPath: String! = "/strip-dev-sections.py" + let schemaPath: String! = "/schema.json" + let image: String! = "ghcr.io/astral-sh/uv:python3.14-alpine" +} diff --git a/dagger.toml b/dagger.toml index 01998c7..a884e0f 100644 --- a/dagger.toml +++ b/dagger.toml @@ -54,3 +54,9 @@ path = ".dagger/modules/e2e/fixtures/config/configured" [[modules.python-sdk.as-sdk.modules]] path = ".dagger/modules/e2e/fixtures/toml-generate/app" + +[[modules.python-sdk.as-sdk.modules]] +path = ".dagger/modules/e2e/fixtures/clients/dep" + +[[modules.python-sdk.as-sdk.modules]] +path = ".dagger/modules/e2e/fixtures/clients/app" diff --git a/future/done/modules-have-clients.md b/future/done/modules-have-clients.md new file mode 100644 index 0000000..22c5b3f --- /dev/null +++ b/future/done/modules-have-clients.md @@ -0,0 +1,1232 @@ +# Modules have clients, not dependencies + +author: yves +created: 2026-08-27 +status: done (draft PR dagger/python-sdk#22, CI green at 2dc64c7) +related: `github.com/dagger/java-sdk` PR #17 +(`hack/designs/done/2026-08-26-modules-have-clients.md`, the source design this +ports); `future/done/self-contained-python-sdk.md` (the layout this builds on); +`github.com/dagger/go-sdk` `go-sdk.dang` (`generateClient` shape) + +Every engine claim below was re-verified against `dagger/dagger` +`v1.0.0-beta.11` (`a4e1e4ff`) — by reading the source and, where it decides +the design, by probing a live beta.11 engine from inside a Python module. Line +numbers are beta.11's. + +## Problem + +A module's dependency and a standalone generated client are the same thing, +built twice — and in Python, only one of the two is built at all. + +When a Python module calls a dependency today it calls typed bindings against +a served module: `dag.hello().greet("x")`. That is a client. The only thing a +standalone client — a script, an app, a test — would add is *how the session +and the target module are obtained*: inside a module the engine has served the +dependency into the session before the function runs; outside, the process +opens its own session and has to serve the target itself. + +This SDK does not model that. `Mod.generate` reads the module-facing +`ModuleSource.introspectionSchemaJSON` — core plus every dependency, merged — +and renders one flat `sdk/src/dagger/client/gen.py` (`mod.dang:79`, +`vendoredDir`). Three consequences: + +1. **There is no client artifact.** Nothing this SDK produces can be handed to + a Python project that is not a Dagger module. The library has full session + provisioning (`dagger.connection()`, `sdk/src/dagger/provisioning/`), but + the bindings it ships are the engine's core API only; a module's functions + are reachable solely from inside another module. +2. **Dependency bindings are unattributed.** `Hello` and `Container` land in + the same generated file, indistinguishable. Regenerating for a different + dependency set silently changes what `gen.py` means, and nothing in the + output records which module contributed what. +3. **The generator has one mode.** `codegen.generator.generate` renders every + type in the schema it is given and hard-codes `class Client(Query)` plus the + `dag` singleton at the end. There is no seam at which a second output could + be emitted. + +Meanwhile the engine already draws the line this design needs. +`ModuleSource.clientSchemaIntrospectionJSON` is the *client-facing* schema +(`core/schema/modulesource.go:3934`, `clientSchemaIntrospectionJSONFile`): it +starts from the core-only schema builder and, when the module has a runtime +SDK, installs exactly that one module, namespaced, never as an entrypoint, and +hides no core types. Its own comment is the specification: + +> only the bound module is installed, as a normal namespaced module, so a +> generated client reaches its functions via `dag.` and never +> through a promoted Query root. The module's own dependencies are deliberately +> excluded -- a client is generated for a single module plus core, not for its +> whole dependency graph. Unlike the module-facing schema, it hides no core +> types. + +The Go SDK consumes it (`go-sdk.dang`, `generateClient`); this SDK does not. + +## Decisions + +The java-sdk design settled nine questions (D1–D9). Each is re-examined here: +the engine-level ones are re-verified on beta.11 and kept; the Java-specific +ones are redesigned for Python or dropped when Python does not have the +problem. One (the serve preamble, P3) deviates from the Java design on +evidence, and says so. + +**P1 — a client is deps-excluded (leaf-shaped) — engine fact, kept (D1).** +A dependency-authored type never crosses between two clients; core types do, +because they are literally the same Python class (`dagger.Container`). This +costs nothing because the engine forbids the alternative: `Module.validateTypeDef` +rejects a module whose API exposes a dependency-authored type in an object +field (`core/module.go:1362`, "cannot reference external type"), a function +return (`:1395`, "cannot return external type") or a function argument +(`:1412`). A capability that cannot exist cannot be lost. Confirmed +empirically too: the client schema of a probe module has exactly one owned +type and one owned `Query` field, and none of the dependency's types. + +**P2 — the hand-written runtime stays where it is; generated clients get a +package of their own — replaces D2.** Java moved its runtime out of +`io.dagger.client` because generated clients were going *into* that package. +Python never needs that: the runtime lives in `dagger.client` (`_core`, +`_session`, `base`, `gen`) and generated clients go into a new, purely +generated subpackage, **`dagger.clients`**, one module file per bound module: +`dagger/clients/hello.py`. Nothing hand-written lives under `dagger.clients`, +so a module name can never collide with runtime code, and no file moves. The +one-letter difference between `dagger.client` and `dagger.clients` is accepted: +`dagger.client` is the transport a generated file imports from and +`dagger.clients` is what user code imports from, and the two never appear on +the same line. A module named `client` or `clients` is fine +(`dagger/clients/client.py`). Names are normalized and checked for collisions; +see *Naming*. + +**P3 — the entry point is a module-level function; the serve rides on the +query context; inside a module runtime the serve is a no-op — replaces D3 +and deviates from the Java serve preamble.** Java needed a static factory plus +an alias because it has no free functions. Python has them: + +```python +from dagger.clients.hello import hello + +await hello().greet("x") # global client, like `dag` +await hello(client=my_client).greet("x") # an explicit client +``` + +`hello(...)` takes the module's constructor arguments and returns the +lazily-built `Hello` object; it does *not* serve. The Python query builder is +lazy — `dag.container()` executes nothing until an `await` — so the serve is +attached to the query context and runs at execution time, once per session per +binding. That keeps the natural chaining idiom and keeps the entry point +synchronous. + +The deviation: **inside a module runtime the preamble does not serve at +all.** The Java design serves unconditionally everywhere and relies on the +engine deduplicating. Review of this design found that to be wrong for a +module consumed from git: a module client inherits its *caller's* workspace +(`engine/server/session_workspaces.go:210`, `inheritWorkspaceBinding`), so a +self client or local-dependency client that bakes `/path/in/the/authoring/workspace` +would resolve that path against the consumer's workspace and fail — and the +engine rewrites a git module's local dependencies into git sources +(`core/modulesource.go:2049-2077`), so no baked local identity can be right +there. What makes the no-op correct rather than a shortcut is an engine +guarantee, not a schema probe: a module's session is constructed with the +module's dependencies and the module itself already served, before any query +runs (`engine/server/session.go:936-938`: `client.servedMods = +client.mod.Self().Deps...; ...Append(core.NewUserMod(client.mod))`). A module +only ever vendors clients for its declared dependencies and for itself, so +there is nothing a client could serve in that session that the engine has not. +The no-op asks the schema nothing and chooses nothing; it defers to the +engine's own dependency resolution — the same resolution that produced the +client — which is why it does not reopen the wrong-module risk a `__type` +probe has. + +The signal is **a process-local flag the Python runtime entrypoint sets**: +`dagger.mod.cli.app` — what `runtime.py` runs, under both this repository's +runtime and the engine-baked one (the two `runtime.py` files are identical) — +marks the process as a module runtime in `main`, before it connects, and the +preamble reads the flag at call time. It is not an environment variable: +`DAGGER_MODULE`, the obvious candidate, is a user-facing CLI selector +(`internal/cmd/dagger/module.go:168`) that CI setups export, and a standalone +client run under it would silently never serve. A module process has exactly +one session to consider — a module vendors no `dagger.provisioning`, so it +cannot open another — which is why a process-level flag and a session-level +flag are the same thing there. + +Outside a module runtime — a script under `dagger run`, an application that +provisions its own session — the preamble serves **unconditionally**, with no +`__type` probe, and caches the served tuple per session. See *The serve +preamble*. The generated bytes carry data only; the branch is in hand-written +runtime, exactly as `SharedConnection` already branches on +`DAGGER_SESSION_PORT`. + +**P4 — both entry points, mirroring the Go SDK — engine fact, kept (D4).** +`generateClient(ws, module, path)`, `generateAllClient(ws)` and +`initClient(ws, path, module)`. The engine dispatches `initClient` with +`{path, module}` plus any SDK args (`core/sdk/module_init.go:41`), records the +client in `[modules..as-sdk.clients]` with `{path, module, pin}` +(`core/schema/workspace_client.go:95`), and hands the list back through +`currentModule.asSDK(workspace:).clients` as `{path, module, pin, moduleSource}` +(`core/schema/module_as_sdk.go:89`, `core/current_module_as_sdk.go:104`). The +pinned go-sdk snapshot (`902440e`) predates that API — it reads +`ws.sdk(name:).clients.{{name, source}}` and declares an unused `dev:` argument +the engine never passes — so this design follows beta.11's API, not the +snapshot. Clients are **not** cwd-filtered by the engine (only `modules` are, +`module_as_sdk.go:78` vs `:81-95`); the rollup filters by path containment +itself, as go-sdk does. + +One thing is not mirrored: go-sdk registers two `@generate` hooks. The engine +runs a workspace's generators **concurrently** (`core/generators.go:165-178`, +`GeneratorGroup.Run`), so a client rollup could read a local module's client +schema while the module rollup is still producing that module's generated +files. This SDK keeps **one** `@generate` hook, `generateAll`, which generates +the managed modules and then the registered clients on top of the +module-generated state, and returns one changeset. `generateAllClient` stays +a public function for direct use. Independently of ordering, client generation +for a local module this SDK manages stages that module first (P9), so it is +never read ungenerated. + +`generateAll` generates the managed modules **at or below the caller's cwd** +only. The engine's cwd policy for `asSDK.modules` also returns the nearest +*enclosing* module when the cwd sits inside one, but a changeset that reaches +outside the cwd is rejected by the engine (`core/schema/workspace.go:2047`, +"changes fall outside the current directory"), so that entry could never be +applied from a rollup anyway — and `dagger api client init` runs every +generator of the SDK scoped to the **client's** path +(`core/schema/workspace_builders.go:495-516`, `withScopedGeneration`), which, +for a client inside a registered module, would otherwise fail on the enclosing +module. Scoped that way, a client init generates exactly the new client. + +**P5 — a module generates a client for itself, and that is how it calls +itself — kept (D5).** Self calls go through `dagger.clients.` like calls +to any dependency; a module that cannot reach itself through the engine cannot +benefit from function-level caching on its own calls. Reading a module's own +`clientSchemaIntrospectionJSON` installs the module, which means the engine +builds it — and for a fresh `init` + `generate` there is no `sdk/` yet. The +circularity is broken by **bootstrapping through a staged workspace**, as the +Java design does; Python differs in what "build" means — the runtime installs +the vendored `sdk/` and imports the module's package — and therefore in what +pass 1 must carry so that import succeeds. See *The bootstrap* below. + +Verified live: from inside a module function on beta.11, serving the module +itself by workspace path returns success, and `currentWorkspace` resolves +(cwd `/`). + +**P6 — no visibility change is needed — drops D6.** Java had to make its +query transport public because a class cannot implement a non-public interface +across packages. Python has no compile-time visibility: the generated client +imports `dagger.client._core.Arg`, `dagger.client.base.Type` and the new +`dagger.client._binding.ModuleBinding` exactly as `gen.py` imports its runtime +today. Nothing changes. + +**P7 — core lives in `dagger.client.gen`, generated per consumer, never +coupled to a bound module — kept (D7).** `gen.py` becomes core-only: + +- a **module** gets its core from its own module-facing + `introspectionSchemaJSON`, narrowed to symbols with no owning module. That + schema hides `TypesToIgnoreForModuleIntrospection` (`Host`), + `TypesHiddenFromModuleSDKs` (`Engine*`) and + `FieldsToIgnoreForModuleIntrospection` (`Query.currentWorkspace`, + `Query.engineVolume`, …) (`core/moddeps.go:17-24`, `core/env.go:46`), so + `dag.host()` in module code stays an `AttributeError`, as it is today. + Measured on the probe: 118 types, no `Host`, no `currentWorkspace`; +- a **standalone client** gets its core from the bound module's + `clientSchemaIntrospectionJSON`, which hides nothing (124 types, `Host`, + `Engine`, `currentWorkspace` all present) — a client is allowed everything + the CLI is. + +Both are "the engine's core, as this consumer is allowed to see it". A +per-module client file refers to core symbols only through a module alias +(`_core.Container`, where `_core` is `dagger.client.gen`), so its bytes are +identical in both contexts even though the core file legitimately differs. + +The residual risk is the compatibility view: the engine renders a module's +schema through that module's declared `engineVersion`, and a module declared +below `v1.0.0` gets the legacy per-type ID surface a modern core does not +have. Generation therefore fails early and clearly when a **bound** module's +`engineVersion` has a numeric core below `1.0.0` — the generator's existing +`parse_version` reads `major.minor.patch` and ignores the prerelease, so +`v1.0.0-beta.11` passes and `v0.20.8` fails; an empty or unparsable value +(`latest`, or a legacy `dagger.json` with no field) resolves to the running +engine and is allowed. The check is the code generator's +(`--engine-version`) because Dang has no version comparison; it is not passed +in `core` mode, where the consumer's own view still comes from the schema's +`__schemaVersion` as today. + +**P8 — the SDK already opens its own session — drops D8.** Java had removed +that code path; Python never did. `dagger.connection()` / +`dagger.Connection()` honour `DAGGER_SESSION_PORT`/`DAGGER_SESSION_TOKEN`, +otherwise spawn `dagger session` from `_EXPERIMENTAL_DAGGER_CLI_BIN`, a +downloaded CLI matching `dagger._engine._version.CLI_VERSION`, or `dagger` on +`PATH` (`sdk/src/dagger/provisioning/_engine.py`). Two things change in what +gets vendored for a standalone client: `dagger.provisioning` is included (a +module still excludes it), and `_engine/_version.py` is stamped. The committed +file pins `1.0.0-beta.10` and nothing in this repository regenerates it, so an +unstamped client would provision the wrong engine. The stamp is +`Query.version` **normalized to the bare release tag** — `v1.0.0-beta.11+a4e1e4ff` +becomes `1.0.0-beta.11`, leading `v` and build metadata dropped — because the +downloader adds its own `v` and builds +`/dagger/releases/{version}/dagger_v{version}_…` and caches as +`dagger-{version}` (`provisioning/_download.py:125-140,174`). An engine +whose version has no published release yields a stamp nothing can download; +provisioning then falls back to `dagger` on `PATH` with a warning +(`_engine.py`, `fallback_to_local_cli`), which is the existing behaviour for +an unavailable release. + +**P9 — the SDK stages its own local dependencies; the engine stages the +ones it cannot — replaces D9's mechanism, keeps its finding.** Re-verified +on beta.11: `Workspace.generators` returns an empty group for a value +workspace (`core/schema/workspace.go:3278`, `isSyntheticWorkspace`), and +`generateLocalDependencies` then **fails** — `validateDependencyGeneratorGroup` +rejects an empty group with `owning SDK %q exposes no generators` +(`core/schema/modulesource.go:3523`). Nor is there a clean way to ask "is this +a value workspace": `Workspace.generators` seen from inside a module client +reports only the calling client's served modules (`workspace.go:3650`, +`currentWorkspacePrimaryModules`), so any predicate built on it measures the +SDK's own presence, and is flipped by `[modules.] generate.skip`. + +So the split is by **ownership**, not by workspace kind: + +- every local dependency **registered to this SDK** — read from the workspace + re-anchored at the root, `currentModule.asSDK(workspace: ws.withWorkdir(".")).modules` + (`"."`, not `"/"`: the engine rejects an absolute workdir), because + `asSDK.modules` is cwd-scoped (`core/schema/module_as_sdk.go:78`) + and a dependency is usually a sibling — is generated by the SDK itself, + recursively, and its generated output overlaid onto the staging workspace. + Overlays, not changesets, because a changeset is cwd-measured and may not + reach a sibling. A dependency carrying this SDK's skip marker is assumed + committed, as today; +- the engine's `generateLocalDependencies` is called **only when the module + has a local dependency registered to another SDK** — read from + `ws.sdks` and each `ws.sdk(name).modules`, the same registry the engine's + owner lookup uses. On a host workspace that is what stages a Go module a + Python module depends on; on a value workspace it errors, which is correct — + nothing there can generate it — though, because the engine walks the whole + local closure in parallel (`modulesource.go:3405`), its message may name a + Python sibling rather than the foreign dependency; accepted, since that + combination has no supported in-memory path anyway; +- a local dependency **registered to no SDK at all** is assumed committed and + is neither generated nor handed to the engine — the engine would only warn + and skip it (`modulesource.go:3468-3476`), and calling it for that would + drag every Python sibling through the value-workspace failure above. + +A graph whose local dependencies are all Python-owned or unregistered +therefore never touches the engine call, and nothing is generated twice; a +graph with a foreign-SDK dependency pays one engine pass that also covers the +Python dependencies, which the engine caches. The SDK recursion carries an +explicit list of the modules on the active path and fails with a clear +message on a cycle; a diamond (`B` and `C` both depending on `D`) generates +`D` twice from identical inputs, which the engine's cache absorbs. The +engine's own `StagedGeneration` marks are internal (`core/workspace.go:154-162`) +and are not relied on. Recursion is new to this repository's Dang — nothing +here or in go-sdk recurses today — so it is the implementation unknown of +patch 5 and gets a spike before the pipeline is built on it. + +### Where the Go and TypeScript SDKs actually are + +Stated plainly, because it sets expectations for review: the Go SDK has +`generateClient` / `generateAllClient` / `initClient`, and this design copies +that shape. But Go *module* generation still delegates to the engine's merged +module-facing schema (`go-sdk` `mod.dang:generate`), so Go has not unified +dependencies-as-clients. Java is doing it now (PR #17, open). Python is +second, and the first with a dynamic language: no compile step means the +bootstrap needs no compiler, but also that a stale or missing client only +fails at import time. + +## Goals + +- One generator, two modes, one output shape. `dagger/clients/.py` is + byte-identical whether it was produced because another module declared `m` + as a dependency or because someone asked for a standalone client of `m`. +- Core types live in `dagger.client.gen` (re-exported as `dagger.*`), shared by + every client in the process. +- A module declares a dependency in `dagger-module.toml` exactly as today; the + SDK generates a *client* for it instead of merging its schema in. +- A real standalone client artifact: a Python project that `pip install`s (or + `uv add`s) its vendored `sdk/`, imports `dagger.clients.`, and opens its + own engine session. +- One shared engine session per process across every client (the existing + `SharedConnection` singleton, or an explicit client passed to the entry + point). +- One idempotent serve preamble: a no-op where the engine has already served + everything a module can bind (inside a module runtime), an unconditional + serve the engine deduplicates everywhere else, cached per session per exact + tuple, never a schema probe. + +## Non-goals (YAGNI) + +- **No compatibility shim.** `dag.hello()` goes away for modern modules; there + is no dual-mode generator and no deprecation window. Breaking changes are in + scope. +- **No change to legacy `dagger.json` modules.** They keep being generated by + the engine's builtin Python SDK, with the merged `gen.py` shape they have + today. Two shapes coexist, by config format, exactly as two runtimes already + do (`README.md`, "Two runtimes, one name"). +- No cross-module type composition between two dependency clients (P1), and + no client type in a module's own API surface: a module returns its own + `@object_type`, never `dagger.clients..M` — the runtime rejects any + class whose `__module__` starts with `dagger.clients.` at registration, + before classifying it as object, enum or scalar, with a message that says + so (`dagger/__init__.py` rewrites `__module__` only for its own re-exports, + never for client classes, so the prefix is reliable). +- No engine changes. Everything needed exists on `v1.0.0-beta.11`. One engine + limitation is recorded rather than worked around: see *Local-name + collisions* under Risks. +- No published `dagger-io` on PyPI; a client vendors what it needs. +- No change to module authoring (`@object_type`, `@function`, `runtime.py`). + `runtime/` changes by one line (the generated-files check). +- No client for a module that is not a declared dependency of the module it + is vendored into. Inside a module runtime the preamble relies on the engine + having served the module's declared dependencies; a copied-in client for + anything else is unsupported. +- No self-call in the default starter. The starter's constructor takes a + `Workspace`, which a self call would have to thread through; demonstrating + the idiom there is not worth a required argument on every new module. The + README shows it. +- No `dagger run`-free session for local bindings outside a workspace: a local + binding needs `currentWorkspace`, which needs a workspace. That limit is the + engine's and is repeated in the generated docstring. + +## Approach + +### The unification, precisely + +A generated client is **generated bindings plus a serve preamble**. The +bindings are the module's types and its entry point. The preamble makes sure +the bound module is served in the session the first time any query built +through that client executes. Inside a module runtime the engine has already +done that for every module the client could bind, so the preamble does +nothing; outside, it serves. The generated bytes are the same either way. + +### Package layout + +```mermaid +graph TD + subgraph runtime["dagger.client — hand-written runtime (unchanged location)"] + RT["_core.Context (+ bindings)
_session.ClientSession (+ served set, refetch)
base · _guards
_binding.ModuleBinding (new)"] + end + subgraph core["dagger.client.gen — generated core, one per consumer"] + CORE["Query, Client, dag
Container, Directory, File, …"] + end + subgraph clients["dagger.clients.<module> — generated, one file per bound module"] + C1["dagger/clients/hello.py
Hello, HelloReport, hello()"] + C2["dagger/clients/builder.py
Builder, builder()"] + end + CORE --> RT + C1 --> CORE + C2 --> CORE + C1 --> RT + C2 --> RT +``` + +- `dagger.client` is the runtime, where it is today. One module is added: + `_binding.py`, the serve preamble. `Context` in `_core.py` learns a + `bindings` tuple; `ClientSession` in `_session.py` learns a served set, a + lock and `refetch_schema()`; `dagger.mod.cli.main` marks the process as a + module runtime. +- `dagger.client.gen` stays the core API, still ending in `class Client(Query)` + and `dag = Client()`, re-exported by `dagger/__init__.py` as today. It is the + only file whose contents depend on the consumer's schema view alone. +- `dagger.clients` is new: an empty `__init__.py` plus one generated file per + bound module. **This subtree is exclusively generated** and is swept on every + generate; that narrows, deliberately, the "a stray file under `sdk/` survives" + contract `python-sdk#21` chose for the rest of `sdk/`. The runtime's + `requireGeneratedFiles` learns the package marker in its vendored-library + branch (the only layout that has one), so a module missing it fails with + the same "run `dagger generate` and commit" error as a missing `gen.py`; + and `dagger.mod.cli.load_module` translates a `ModuleNotFoundError` for + `dagger.clients.` — a self client deleted or never committed — into that + same actionable error instead of a bare import traceback. + +### Naming + +A bound module's **final** name (after any dependency alias) is normalized to +a Python identifier the way the generator already normalizes field names +(`format_name`: initialisms grouped, `camel_to_snake`, keywords and reserved +builtins suffixed with `_`): `hello-world` → `hello_world`, `helloWorld` → +`hello_world`, `import` → `import_`. The file is `dagger/clients/.py`; +the entry point is the same identifier. Generation fails loudly when: + +- the normalized name is not an identifier, or starts with `__` (so `__init__` + and friends can never be overwritten); +- two clients of one consumer normalize to the same file (`foo-bar` and + `foo_bar`, or `HelloWorld` and `hello_world`); +- the module's root type name is a core type name (`container` → `Container`). + +Module-owned fields on core types other than `Query` are emitted as +module-level functions named `format_name(parent)_format_name(field)`, always: +a name that depended on which other parents currently carry the same field +would turn adding `File.asHello` next to `Directory.asHello` into a rename of +the exported `as_hello`. Only the entry point, on `Query`, is bare. A function +name that still collides with the entry point or an owned type fails +generation. + +A schema argument that would shadow a name the entry point's body uses +(`client`, `_ctx`, `_args`, `_core`, `_BINDING`) is suffixed with `_`, exactly +as keywords are today (`from` → `from_`), and its `Arg(...)` keeps the GraphQL +name. Two arguments of one entry point that normalize to the same Python name +(`client` and `client_`) fail generation with a message naming both, rather +than emitting a signature Python would reject. + +### Type attribution: `@sourceMap` is the partition + +The introspection JSON says which module contributed each type and field: the +engine emits `@sourceMap(module: "", filename:, line:, column:, url:)` +(`dagql/server.go:412`) on both. Core symbols carry no `module`. +`codegen.ast.insert_stubs` already parses every directive into the graphql-core +AST for object, interface, input and enum types, and `parse_const_value` +unquotes the JSON-quoted argument; it is extended to scalars so a module-owned +scalar attributes correctly, and the partition fails loudly on a type kind it +cannot attribute (unions — none exist in the engine schema today). + +- a type whose owner is empty belongs to core; +- a type whose owner is `M` belongs to `dagger.clients.`, **with all its + fields** — `id` on an owned type carries no `@sourceMap` (probe: `Probe.id` + has no directives) and must stay, since `Type.id()`, implicit ID resolution + and `execute_object_list` depend on it; +- a **field** on an **unowned** type whose owner is `M` (`Query.hello`) belongs + to `M`. + +Verified on the probe's client schema: one owned type (`Probe`), its five +owned fields plus `id`, and `Query.probe`. The root type is read off the +schema — the return type of the `Query` field owned by the module — never +derived by capitalizing the name (`e2e` → `E2E`, not `E2e`). + +Core narrowing (`core` mode) is done on the introspection result before +`build_client_schema`: owned types are dropped, owned fields on unowned types +are dropped, and `possibleTypes` entries naming a dropped type are pruned. An +owned type is referenced only by owned fields or by other owned types (P1), +and an owned object implementing a core interface is reached through +`possibleTypes` only, so the result is closed; `build_client_schema` validates +it and raises on anything dangling. Tested with an owned object implementing a +core interface and with owned input, enum and scalar types. + +### Generation modes + +`python -m codegen generate` gains `--mode core|client` (default `core`, so the +existing invocation in `mod.dang` keeps producing today's bytes for a schema +with no owned symbols), `--module `, `--binding ` and +`--engine-version `: + +| mode | input | emits | into | +|---|---|---|---| +| `core` | any schema, narrowed as above | every core type, `Client`, `dag` — today's `gen.py` shape | `sdk/src/dagger/client/gen.py` | +| `client` | a client schema (core + one module); the full schema is kept so references resolve | the types owned by `--module`, the entry point, functions for owned fields on other core types, `_BINDING` | `sdk/src/dagger/clients/.py` | + +In `client` mode a reference to an unowned type renders through the alias +`_core` (`from dagger.client import gen as _core`), and a reference to an owned +type renders bare, quoted as a forward reference where needed, as today. The +qualification is one `Context.type_ref(name)` applied at every site that emits +a type name: parameter and return annotations, `execute(T)` and +`execute_object_list(T)` arguments, the `return T(_ctx)` construction, the +`_Client` concrete class for interface returns (reachable as +`_core._FooClient` because `_core` is the module, not its `__all__`), enum +default values, and the `expectedType` ID conversion. `dag` in the entry point +is `_core.dag`. Because `core` mode drops everything owned, the core file is +identical no matter which module's client schema it came from — asserted by a +test. + +### The serve preamble + +```python +# dagger/clients/hello.py +_BINDING = ModuleBinding( + name="hello", kind="LOCAL_SOURCE", ref="/.dagger/modules/hello", pin="" +) + +def hello(name: str, *, greeting: str = "hi", client: _core.Client | None = None) -> Hello: + """Client for the `hello` module. ...""" + _ctx = (_core.dag if client is None else client)._ctx.with_binding(_BINDING) + _args = [Arg("name", name), Arg("greeting", greeting, "hi")] + return Hello(_ctx.root_select("hello", _args)) +``` + +`ModuleBinding` is hand-written runtime; the generated file carries data and no +logic. `Context.execute` runs `await binding.ensure_served(conn)` for each +binding on the context before building the request. `select`, `root_select`, +`select_id`, `execute_object_list` and `execute_sync` all derive contexts with +`dataclasses.replace`, so bindings travel with the chain and with objects +loaded back by ID. + +```mermaid +sequenceDiagram + autonumber + participant Q as query execution + participant B as ModuleBinding + participant S as ClientSession + participant E as engine session + Q->>B: ensure_served(conn) + alt process marked as a module runtime + B-->>Q: return — the engine served deps and self before any query + else + B->>S: acquire serve lock + alt tuple in session.served + B-->>Q: return + else + alt kind = GIT_SOURCE + B->>E: moduleSource(refString: ref, refPin: pin).withName(name).asModule.serve + else local + B->>E: currentWorkspace.moduleSource(path: ref).withName(name).asModule.serve + end + E-->>B: ok (same identity already served → dedup) / error + B->>S: refetch_schema() + B->>S: served.add(tuple) + end + end +``` + +State lives on the live `ClientSession` (`_session.py`): the served set, and +an `anyio.Lock` that serializes serve → refetch → mark, so two clients racing +on one session cannot interleave two introspections and cannot mark a tuple +served before its refetch succeeded; a failure propagates and marks nothing. +`SharedConnection.close()` drops the session, so a reconnect starts with +nothing served, as it must. + +Why no probe: for a git binding, `Server.serveModule` +(`engine/server/session.go:2143`) looks the name up and, if served, compares +`canonicalModuleReference` and pin (`session_workspaces.go:103`) — same source +succeeds, different source fails with `module %s ... already exists with +different source`. A `__type` probe would skip serving when a *different* +module of the same name was present, binding the caller silently to the wrong +module and suppressing exactly that error. For a **local** binding the engine +cannot detect the conflict on beta.11: a workspace-path source has an empty +`AsString` (probe: `kind: DIR_SOURCE`, `asString: ""`) and +`isSameModuleReference` treats an empty side as "same" (`session.go:2164`), +so a same-name collision keeps whatever was served first. The unconditional +serve is still the right call — it is the engine's decision to make, and a +probe would make it the client's — but the conflict guarantee holds for git +bindings only. Recorded under Risks. + +Why the schema refetch: the Python query builder renders through gql's DSL +against the schema the session fetched when it connected. Outside a module +that schema predates the serve and lacks `Query.hello`; after a real serve the +session refetches (`ClientSession.refetch_schema()` → +`AsyncClientSession.fetch_schema()`, which rebuilds `client.schema` in place — +verified against gql 4.0.0 and live on beta.11). Refetching after every real +serve, rather than after inspecting the cached schema, keeps the preamble free +of any schema-presence check; it costs one introspection per binding per +standalone session. + +The preamble sends a raw GraphQL document (`gql.gql(...)`), not a DSL query: +`currentWorkspace` is hidden from a module's codegen schema but present in +the live session, and the raw form depends on neither. + +**The identity tuple.** The bound module's **final** name (after any +dependency alias) is what gets baked and served under: the engine aliases with +`withName` when it loads dependencies (`core/modulesource.go:2034`), and the +synthesized `@sourceMap` carries `mod.Name()` (`core/module.go:1927-1955`), so +an aliased dependency chains and serves the same name. A local binding bakes +the module's workspace-root-relative path with a leading `/` — +`Workspace.moduleSource` resolves a leading slash against the root and a bare +path against the cwd (`core/schema/workspace.go:2818`), and the cwd is not the +client's to assume. A git binding bakes `asString` and `pin`, which resolve +from anywhere. A workspace-loaded module reports `kind = DIR_SOURCE` with an +empty `asString`; it is normalized to `LOCAL_SOURCE` by path, which also keeps +the bytes identical between a client generated in a value workspace and on the +host. + +### Where the schemas and identities come from + +```mermaid +graph LR + MS["stagedWs.moduleSource(/mod)"] -->|introspectionSchemaJSON| MODSCHEMA["module-facing:
core + deps, self absent, Host hidden"] + MS -->|"dependencies.{{moduleName, kind,
asString, pin, sourceRootSubpath, engineVersion}}"| DEPS + DEPS -->|"moduleSource(...).withName(final).clientSchemaIntrospectionJSON"| DEPSCHEMA["core + dep"] + MODSCHEMA -->|mode=core| CORE["dagger/client/gen.py"] + DEPSCHEMA -->|mode=client| DEPC["dagger/clients/<dep>.py"] + CORE --> STAGE["bootstrap workspace:
library + core + dep clients
+ carried-over or stub self client"] + DEPC --> STAGE + STAGE -->|"moduleSource(/mod).clientSchemaIntrospectionJSON"| SELFSCHEMA["core + self"] + SELFSCHEMA -->|mode=client| SELFC["dagger/clients/<self>.py"] +``` + +Dang reads engine object lists through `{{...}}` record selections, so each +dependency is re-resolved from its identity: a local one as +`stagedWs.moduleSource("/" + sourceRootSubpath).withName(moduleName)`, a git +one as `moduleSource(refString: asString, refPin: pin).withName(moduleName)`. +`withName` is what the engine itself uses to alias, so the client schema comes +out namespaced under the final name. + +### The bootstrap + +Pass 1 vendors the library, the core and the dependency clients, plus a +`dagger/clients/.py` that lets the module's own code import during the +bootstrap build: + +- the **committed** self client when the workspace has one — module code that + already calls itself keeps importing the symbols it imported before; +- otherwise a **stub**: `__all__ = []` and a module-level `__getattr__` that + returns, for any name, one placeholder class whose instantiation raises + `RuntimeError("client for module 'app' is not generated yet: run dagger + generate")`. A placeholder *class* rather than a function so that a name + used in an annotation is at least a type, and the error a module author sees + is the one that names the fix. + +The supported model is the Java design's: **generate first, then call.** A +symbol added to the module and imported from its own client in the same edit +is not in the carried-over client, so the bootstrap build fails on that +import; the next `generate` after removing the call produces it. A module +that uses one of its client's types in its *own* API — `-> dagger.clients.app.App` +— is rejected by the runtime at registration ("client types cannot appear in +a module's API; return the module's own object type"), whether against the +stub (whose placeholder class lives in the same `dagger.clients.` +module and is caught by the same prefix rule), a stale client, or a current +one. + +### Generation pipeline (a module) + +``` +Mod.generate: # public, no arguments, as documented in the README + ws.withoutDirectory(mod/sdk/src/dagger/clients).withDirectory(mod, generatedTree(ws, [])).changes(ws) + +generatedTree(ws, active: [String!]! = []): # a module-rooted directory holding sdk/ + 1. deps = moduleSource(/mod).dependencies.{{moduleName, kind, sourceRootSubpath, asString, pin, engineVersion}} + local = deps where kind != GIT_SOURCE + owned = local ∩ this SDK's registry (asSDK on ws.withWorkdir(".")) + foreign = local ∩ another SDK's registry (ws.sdks, ws.sdk(name).modules) + # anything else local is unregistered: assumed committed + 2. stagedWs = if foreign is non-empty: ws.withChanges(moduleSource(/mod).generateLocalDependencies(ws)) else ws + 3. stagedWs = for dep in owned, not skip-marked: + raise if dep in active + [mod] # cycle + stagedWs.withoutDirectory(dep/sdk).withDirectory(dep, Mod(dep).generatedTree(stagedWs, active + [mod])) + 4. src = stagedWs.moduleSource(/mod) + 5. core = codegen --mode core <- src.introspectionSchemaJSON + clients = for each dep: codegen --mode client <- dep client schema, identity, engineVersion + 6. pass1 = library(no provisioning) + core + clients + (committed self client | stub) + 7. bootWs = stagedWs.withoutDirectory(mod/sdk).withDirectory(mod, {sdk: pass1}) + self = codegen --mode client <- bootWs.moduleSource(/mod).clientSchemaIntrospectionJSON, + LOCAL_SOURCE "/mod", src.moduleName + 8. final = pass1 with .py replaced → {sdk: final} +``` + +`generatedTree` is the module-rooted generated tree for either kind of +module: a `dagger-module.toml` module produces steps 4–8; a legacy +`dagger.json` module produces the engine's `generatedContextDirectory` exactly +as today — that path is untouched, it works around `dagger/dagger#13947`, +still open on beta.11. Step 7 is the only place the module itself is built. +`Mod.generate` sweeps the generated-clients subtree before merging so a +dropped dependency or a renamed alias is reported as a removal, while the rest +of `sdk/` keeps the merge semantics `python-sdk#21` chose. + +Found while implementing, and load-bearing for every overlay above: +`Workspace.withDirectory` merges onto an untouched path but **replaces** the +path once a `withoutDirectory` below it has run (reproduced with a raw query +on beta.11: `withoutDirectory("/m/sdk")` then `withDirectory("/m", {sdk})` +leaves `/m` holding only `sdk/`; without the sweep it merges). Writing only +the generated tree after sweeping the clients subtree would therefore drop the +module's own sources on the second generate of any module with a committed +client. So every write goes through one helper, `Codegen.overlay`, that +composes the full directory first — the existing contents, the swept subtree +removed, the generated tree layered on — and writes that in one go, which is +right under either semantics. A `dagger/dagger` issue is the follow-up. + +### What a module's tree looks like + +``` +/sdk/pyproject.toml, LICENSE, README.md +/sdk/src/dagger/** runtime (vendored, unchanged) +/sdk/src/dagger/client/gen.py core API, module view (no Host) +/sdk/src/dagger/clients/__init__.py +/sdk/src/dagger/clients/.py the module's own client (P5) +/sdk/src/dagger/clients/.py one per declared dependency, final name +``` + +Module code changes from `dag.dep().fn()` to: + +```python +from dagger.clients.dep import dep +from dagger.clients.app import app # self + + return await dep().greet("x") + return await app().greet("y") # a self call, through the engine +``` + +### What a standalone client looks like + +`generateClient(ws, module, path)`: + +``` +/pyproject.toml seeded by initClient when absent, then the user's +/sdk/pyproject.toml, LICENSE, README.md +/sdk/src/dagger/** runtime, including dagger.provisioning +/sdk/src/dagger/_engine/_version.py CLI_VERSION stamped from the engine +/sdk/src/dagger/client/gen.py core API, client view (Host present) +/sdk/src/dagger/clients/.py the bound module's client +``` + +`sdk/src/dagger/clients/.py` is byte-identical to the file a module +depending on `` receives. Stated precisely: identical **for a fixed +binding tuple** — final name, kind, ref or workspace-root path, pin, +compatibility view, schema bytes, generator revision. The same module bound +locally and from git is not identical and should not be. What the claim rules +out is the *context* mattering, and that is what is tested. + +A local bound module this SDK manages is staged first, exactly as a dependency +is in P9, so a client is never generated against an ungenerated module; a git +or foreign-SDK module is read as is. Writing mirrors step 9: +`/sdk/src/dagger/clients` is swept, the rest of `` is merged onto +whatever the user has there, so a `main.py` next to `sdk/` survives every run. + +`initClient` seeds `/pyproject.toml` from `client-template/`, rendered +by the existing `render-template` helper (project name = the directory's +basename), declaring `dagger-io` with `[tool.uv.sources] dagger-io = { path = +"sdk", editable = true }` — the same shape a module's `pyproject.toml` has, so +`uv sync` in the client directory just works. Nothing else is written; the +engine creates the directory and registers the client. + +`generateAllClient(ws)` regenerates every registered client under the +caller's cwd from `currentModule.asSDK(workspace: ws).clients`, using the +`moduleSource` the engine already resolved (pin applied, local or git). +`generateAll(ws)`, the one `@generate` hook, generates the managed modules, +applies that to the workspace, generates the clients on the result, and +returns the whole difference against the caller's workspace as one changeset. + +Usage, outside a module: + +```python +import dagger +from dagger.clients.hello import hello + +async with dagger.connection(): + print(await hello().greet("world")) +``` + +## Alternatives considered + +**Keep the merged schema; just split `gen.py` into core plus one file per +dependency.** Less work, and it preserves `dag.dep()` via monkeypatching +`Query` at import. Rejected: those files are not clients — they cannot be +generated outside a module, because the module-facing schema exists only for +a module — and import-time monkeypatching is invisible to type checkers. + +**Monkeypatch `Query.hello` from the client file so `dag.hello()` keeps +working.** Tempting in Python. Rejected for the same reason the java design +has a static factory: it would make the entry point depend on import order and +on a shared mutable core, and no static analyzer would see it. + +**Make the entry point async (`await hello()`) and serve eagerly.** Explicit, +but it breaks the lazy chaining idiom every other binding follows +(`hello().greet("x")` would need two awaits). Attaching the serve to the query +context keeps one idiom. + +**Serve unconditionally inside modules too (the Java preamble).** Rejected on +the git-consumed-module evidence in P3. Deriving a portable identity at +runtime (`currentModule.source`) exists for the module itself but not for its +dependencies, so it would fix half the problem; the engine already serves +both halves. + +**Detect the module runtime from `DAGGER_MODULE`.** Rejected: it is a CLI +selector users export, so a standalone client under it would silently never +serve. The runtime entrypoint sets the flag instead. + +**Put generated clients at the top of the `dagger` package +(`dagger/hello.py`).** Shortest import. Rejected: `dagger/log.py`, +`dagger/mod/`, `dagger/telemetry.py` exist, so a module named `log`, `mod` or +`telemetry` would overwrite runtime code. + +**Probe `__type` before serving.** Rejected; see *The serve preamble*. + +**Always recurse in the SDK and drop the engine's staging (Java's D9).** +Rejected: on a host workspace it is what stages a local dependency owned by +another SDK. **Always call the engine and also recurse.** Rejected: the engine +call errors on a value workspace, and every Python dependency would be +generated twice. **Branch on `Workspace.generators` being empty.** Rejected: +seen from a module it measures the SDK's own generators, not the engine's +ability to stage, and `generate.skip` flips it silently. + +**Two `@generate` hooks like go-sdk.** Rejected: the engine runs generators +concurrently, so the client rollup could race the module rollup on a local +target. + +**Serve unconditionally and ignore an "already served" error.** Depends on +matching an engine error string. Rejected. + +## Affected components + +| Component | Change | +|---|---| +| `sdk/codegen/src/codegen/partition.py` (new) | `@sourceMap` ownership of types and fields; core narrowing of an introspection result; entry field / root-type lookup; client naming | +| `sdk/codegen/src/codegen/ast.py` | directive stubs for scalar types | +| `sdk/codegen/src/codegen/generator.py` | `Context` gains `mode`, `module`, `binding`, owned sets and `type_ref()`; handlers skip unowned symbols in client mode; entry point and core-type functions; `Client`/`dag` only in core mode | +| `sdk/codegen/src/codegen/cli.py` | `--mode`, `--module`, `--binding`, `--engine-version`; `client-name`; `cli-version` (the `_version.py` stamp) | +| `sdk/src/dagger/client/_binding.py` (new) | `ModuleBinding`: module-runtime no-op, unconditional serve, per-session cache; the runtime flag | +| `sdk/src/dagger/client/_session.py` | served set, serve lock, `refetch_schema()` | +| `sdk/src/dagger/client/_core.py` | `Context.bindings`, `with_binding`, serve before execute | +| `sdk/src/dagger/mod/cli.py` | marks the process as a module runtime; reports a missing `dagger.clients.` as the generate-and-commit error | +| `sdk/src/dagger/mod/_converter.py` | rejects a `dagger.clients.*` class in a module's API, before classification | +| `sdk/src/dagger/clients/__init__.py` (new) | package marker, vendored with the library | +| `runtime/main.go` | `requireGeneratedFiles` also requires `sdk/src/dagger/clients/__init__.py` in the vendored layout | +| `mod.dang` | the pipeline above: ownership-split staging with cycle detection, core + dependency clients, bootstrap self client with the stub, stale-client sweep, engine-version floor | +| `client.dang` (new) | shared client generation: identity, schema, naming, library with provisioning and the stamped version, output tree | +| `python-sdk.dang` | `generateClient`, `generateAllClient`, `initClient`; `generateAll` covers clients and only modules at or below the cwd | +| `client-template/pyproject.toml.tmpl` (new) | the seeded client project file | +| `.dagger/modules/e2e` | fixtures `clients/{dep,app,foreign-app}`, a Python-capable runner for the standalone check, the checks below, `sdkTestCheck` running `tests/client`, the runtime fixture's vendored `sdk/` regenerated | +| `README.md` | layout, the client entry points, a migration recipe | + +## Testing + +Unit, `sdk/tests/codegen` (`uv run pytest tests/codegen`): + +- partition: owned types and fields split as expected, including + `Query.hello` on core and `id` kept on an owned type; core narrowing is + identical whichever module the schema was bound to and does not mutate its + input; an owned object implementing a core interface, and owned input, enum + and scalar types, narrow to a schema `build_client_schema` accepts; the root + type is read from the schema (`e2e` → `E2E`); a union fails loudly; client + names normalize and reject. +- client rendering: entry point with the module's constructor arguments, + keyword-only `client`, an argument named `client` escaped, `client` and + `client_` together rejected; `_BINDING` for a + local and a git binding, aliased name; core references as `_core.X` + including a `_core._FooClient` interface return and `execute(_core.T)`; a + `list[Hello]` return and a `Hello` argument; a function for an owned field on + a non-`Query` core type, always qualified by its parent; + duplicate client names and a root type named after a core type rejected; + the emitted file compiles and imports against the real `dagger` package; + the engine-version floor accepts `v1.0.0-beta.11`, `latest` and `""` and + rejects `v0.20.8`; the `CLI_VERSION` stamp normalizes + `v1.0.0-beta.11+a4e1e4ff` to `1.0.0-beta.11` and `Downloader(version=…).archive_url` + is the release URL. +- core rendering: owned symbols absent, `Client` and `dag` present, unchanged + bytes for a schema with no modules and no flags. + +Unit, `sdk/tests/client` (new; `sdkTestCheck` gains it in the same patch): + +- `ModuleBinding`: no request at all when the process is marked as a module + runtime, and a request regardless of `DAGGER_MODULE` when it is not; the + exact GraphQL document for a local and a git binding; served once per + session and again on a second session; refetch after every real serve; + concurrent first uses of one tuple serve once; a failed serve or refetch + marks nothing and the next use retries. +- `Context.with_binding` survives `select`, `root_select`, `select_id`, and + `execute` serves before requesting — over a fake session. +- `tests/mod`: a function returning or taking a `dagger.clients.*` object, + enum, scalar or the bootstrap placeholder is rejected at registration; a + missing `dagger.clients.` import is reported as the generate-and-commit + error by `load_module`. + +e2e, `@check` functions in `.dagger/modules/e2e`, all real generation in the +engine against `fixtures/clients/dep` and `fixtures/clients/app` (modern +modules on the engine's `python` runtime; `app` declares `dep` twice, once +aliased `greeter`, and its source imports its own client from the start). They +cannot run on this repository's `runtime/` by relative path: the engine cannot +load a Go module through `Workspace.moduleSource` (`Directory.asModuleSource` +fails to load the runtime SDK from a workspace snapshot, on the host as much as +in a value workspace), which is also why the runtime fixture has never been +registered for generation. What the checks exercise is the vendored library, +which is the same under either runtime; `runtimeCallCheck` keeps covering the +Go runtime itself, with its fixture's `sdk/` regenerated through a temporary +`python` runtime: + +- `clientsGenerateCheck` — from nothing: `gen.py` without `Host`, + `clients/dep.py`, `clients/greeter.py`, `clients/app.py` (so the stub + carried the self import through the bootstrap); the dependency client binds + `/…/dep` under `dep`, the alias under `greeter`; a core type returned by + `dep` renders as `_core.Container`; a second generate on the applied result + is empty; a function added to `app` after the first generate appears in the + regenerated self client; removing the alias from the config removes + `greeter.py` (a `removedPaths` entry). +- `clientsRuntimeCheck` — the applied result is loaded through the sdk-sdk + harness's real CLI (`runInstalled(["call", "-m", …])`) and + `greet-via-dep` / `greet-via-alias` / `greet-self` return the dependency's + and the module's own answers. This is the runtime coverage the Java series + could not get: it proves the preamble is a correct no-op inside a module and + that the clients chain the right names. +- `standaloneClientCheck` — `generateClient` for `dep` produces a + `clients/dep.py` byte-identical to the one `app` vendors (compared as file + contents), a core with `Host`, provisioning, and `CLI_VERSION` equal to the + normalized engine version; a client for `github.com/dagger/sdk-sdk` at its + pinned commit bakes a `GIT_SOURCE` binding with that pin; `initClient` + seeds only `pyproject.toml`; the registered-client rollup materializes a + client and is empty on a second run. +- `standaloneRuntimeCheck` — a script using a client bound to + `github.com/dagger/sdk-sdk` by ref and pin runs under `uv run` in a + Python-capable runner and calls the module. Under nesting the only session + a process can reach is the check's own (no runner host is exposed, and a + spawned `dagger session` would try to start an engine), so the script + attaches to it — the `dagger run` shape — and that session's workspace is + the outer checkout, where no fixture has committed generated files; a git + binding resolves regardless, which is what the check pins: the serve by ref + and pin, the schema refetch, and the call. The local-binding standalone + flow was verified by hand on the host (a script with + `dagger.Config(workdir=…)` serving `dep` by workspace path) and is listed + as untested in CI below. The stamp itself is + covered by `standaloneClientCheck` (literal equals the normalized engine + version) and the `Downloader.archive_url` unit test; the download path is + not exercised end to end. +- `foreignDependencyCheck` — a fixture depending on this repository's Go + `runtime` module (registered to go-sdk in `dagger.toml`) is handed to the + engine's staging. Where the engine can stage it — a checkout it can diff, + as in CI — generation succeeds with a client for the Go module vendored; + where it cannot (a git worktree, whose `.git` file breaks the engine's + context diff), it fails loudly rather than silently skipping the + dependency. The same fixture without the dependency generates either way, + which pins the dependency as the cause. The fixture is reached with + `findUp: false` so it stays unregistered and out of `generateAll`. + +Regression net that must stay green: every existing e2e check (in particular +`tomlGenerateCheck`, `runtimeCallCheck`, `sdkTestCheck`), and the sdk-sdk +suite — `chain:*` is a Python-only local-dependency graph through a real CLI +on a host workspace (the SDK-recursion branch), `generation:*` and `module:*` +the scaffold-then-generate-then-load contract. + +What stays untested, plainly: a standalone client with a **local** binding +serving by workspace path from its own session (verified by hand on the host; +in CI every nested process attaches to the check's session, whose workspace +has no generated fixture); a module consumed from git (needs a published +module; covered by design — no serve inside a module — and by the unit test on +the runtime flag); a foreign-SDK local dependency generating *successfully* +(needs a host workspace with two SDKs installed; the branch is pinned, its +success is the engine's pre-existing mechanism); a baked local ref resolving +*inside* a module (by design it never does — `standaloneRuntimeCheck` is the +only place a baked local ref is resolved); `pip install` of a client (`uv run` +installs the same `pyproject.toml`); the CLI download path end to end. + +## Risks + +- **Every modern module breaks on its next `dagger generate`.** Intended, no + shim. The README carries the two-line migration (`from dagger.clients.dep + import dep`; `dag.dep()` → `dep()`). +- **Bootstrap cost.** Each generate now builds the module once in the engine + (self client) and each dependency once (its client schema). The build is + `uv sync` of the vendored library, cached by the engine across unchanged + inputs. Not measured yet; recorded when the e2e checks run. +- **Local-name collisions are undetectable on beta.11.** Two different local + modules served under one name in one standalone session bind to whichever + came first (`isSameModuleReference`, empty `AsString`). Git bindings are + conflict-checked. A `dagger/dagger` issue is the follow-up, not a + client-side check. +- **The module-runtime no-op is load-bearing.** It rests on + `engine/server/session.go:936-938` and on `dagger.mod.cli.main` being every + Python runtime's entrypoint (`runtime/runtime.py`, and the engine-baked + runtime's `runtime.py`, both call `dagger.mod.cli.app`). Pinned by + `clientsRuntimeCheck` and a unit test. +- **Generate first, then call.** A self-client symbol imported in the same + edit that adds it fails the bootstrap build; the error names the import. +- **Local bindings need a workspace.** A client with a local binding used + outside any workspace fails at `currentWorkspace`. Documented in the + generated docstring; git bindings resolve anywhere. +- **The schema refetch is a second introspection round trip** per real serve + in a standalone session. Bounded by the tuple cache; absent inside modules. +- **`dagger.client` / `dagger.clients`** differ by one letter. Accepted (P2). +- **Bound modules below `1.0.0` are rejected** rather than served through a + legacy view. Loud and early. +- **Legacy `dagger.json` modules keep the old shape.** Two idioms coexist by + config format until legacy modules are gone; nothing here changes them. +- **A mixed-SDK dependency graph generates its Python dependencies twice** + (once by the engine, once by the SDK); the engine's cache absorbs it. + +# Implementation plan + +StGit series on `python-unified-clients-lead-afe256c1`, based on `main` @ +`71c445f`. Every patch carries `Signed-off-by: Yves Brissaud ` +and no AI attribution. Each patch leaves the tree green for what it touches: +the runtime seam lands before any generated code references it, the generator +learns its modes while `mod.dang` still drives it in the default (core) mode, +and the cutover in `mod.dang` is one patch because a module cannot +half-generate. + +1. **`future: modules have clients, not dependencies`** — this document. +2. **`codegen: attribute schema symbols to their module`** — `ast.py` scalar + stubs; `partition.py`: `owner_of`, `ownership`, `narrow_to_core`, + `entry_field` / `root_type_for`, `client_module_name`. Tests including + `Query.e2E` → `E2E`, `id` on an owned type, the interface/input/enum/scalar + narrowing cases, a union. +3. **`sdk: serve a bound module before its first query`** — `_binding.py` + with the runtime flag set by `dagger.mod.cli.main`, `ClientSession.served` + / lock / `refetch_schema()`, `Context.bindings` / `with_binding`, + serve-before-execute; the `_converter` rejection and the `load_module` + translation; `dagger/clients/__init__.py`; `tests/client`, and + `sdkTestCheck` running it. Nothing generated references any of it yet. +4. **`codegen: render core and per-module client modes`** — `Context` + modes, `type_ref` at every emission site, handler filtering, `_BINDING`, + the entry point and core-type functions, naming rules, CLI flags, the + `cli-version` stamp. Default mode `core` produces today's bytes for a + module-free schema (pinned). +5. **`python-sdk: generate a client for every dependency and for the module + itself`** — `mod.dang` pipeline: ownership-split staging with cycle + detection (spiked first: the first recursive Dang field in this + repository), core + dependency clients, bootstrap self client with the + stub, stale-client sweep, engine-version floor; `client.dang` shared + pieces. +6. **`python-sdk: standalone clients`** — `generateClient`, + `generateAllClient`, `initClient`, `generateAll` covering clients, + `client-template/`, provisioning and the stamped version in the vendored + library. +7. **`e2e: dependency clients, self clients and standalone clients`** — + fixtures, the Python-capable runner, the five checks, and — together, + because the second needs the first — `runtime/main.go` requiring the + clients package marker and the runtime fixture's `sdk/` regenerated. +8. **`docs: clients, entry points and the migration recipe`** — README. + +### Verification + +- `cd sdk && uv run --frozen pytest -q tests/codegen tests/mod tests/client`, + `uv run --frozen ruff check`. +- `dagger check` for every e2e check, and the sdk-sdk suite. +- `dagger generate` on the scratch workspace used for the probe: a fresh + module generates from nothing, a second run is empty, a self call added + after the first run is picked up by the next. + +## Progress + +- **Phase 0 — orientation: done.** Repository `dagger/python-sdk` + (`upstream`), fork `origin` = `eunomie/python-sdk`, base `main` @ `71c445f` + (upstream and origin agree). Worktree + `…/python-unified-clients-lead-afe256c1-6e9e5050`, branch + `python-unified-clients-lead-afe256c1`. Design home `future/`, archive + `future/done/`. VCS: StGit. Host: GitHub. CI: Dagger Cloud checks + (`dagger check`, no `.github/`). Provenance: `Signed-off-by: Yves Brissaud + `, no AI attribution. Local: dagger CLI `v1.0.0-beta.11`, + `uv`, Python 3.14; Go only in containers. `dagger/dagger#13947` still open. +- **Phase 1/2 — feature doc and plan: this document.** Engine claims + re-verified on beta.11 by source and by a live probe module (serve-by-path + inside a module, `currentWorkspace` inside a module, the real client and + module schema partitions). +- **Phase 3 — adversarial plan review, round 1: 40 findings from a Codex + skeptic and a Claude design reviewer, folded in.** The ones that changed + the design: `asSDK.modules` is cwd-scoped (registry now read from a + root-anchored workspace); `generateLocalDependencies` errors, not no-ops, + on a value workspace; baked local paths cannot work for a module consumed + from git (P3 became a module-runtime no-op, on the engine's served-set + guarantee); the first generate of a self-calling module needs a bootstrap + stub; `ClientSession` has no `fetch_schema` (added `refetch_schema`, state + and lock moved onto the session); `id` on owned types is unowned; + construction sites and `_FooClient` need the `_core` alias; the standalone + client would provision beta.10; scalars need directive stubs; shim, + file-name and argument collisions get explicit rules; the patch order puts + the runtime seam first. +- **Round 2: 36 of 40 confirmed resolved; 13 new findings, folded in.** The + ones that changed the design again: `DAGGER_MODULE` is a CLI selector, not + a runtime marker (the runtime entrypoint now sets a process flag); + `Query.version` is `v…+build` (normalized before stamping, and the + standalone runtime check no longer seeds the cache); a `Workspace.generators` + predicate measures the wrong thing from inside a module (P9 now splits by + dependency ownership, with explicit cycle detection); go-sdk's two + `@generate` hooks race under the engine's concurrent rollup (one hook, + modules then clients, and client generation stages its local target); the + stub returns a placeholder class and the runtime rejects client types in a + module's API; reserved names extended; the runtime's generated-files check + covers the clients package; the default-starter self-call was dropped. +- **Phase 4 — implementation: patches 1–8 landed as an StGit series.** Every + piece was run against a live beta.11 engine before its patch was cut: the + `app` fixture generates from nothing with `dep`, `greeter` and its own + client; `greet-via-dep`, `greet-via-alias` and `greet-self` answer at + runtime through the generated clients; the standalone `dep` client is + byte-identical to the vendored one and runs from a plain Python process; + `dagger api client init` registers, seeds and generates a client; a second + generate is empty; the six pre-existing generate/init/template checks stay + green; core mode is byte-identical to the previous generator on the real + module-facing schema. Two things the plan did not know: the Go runtime + cannot be loaded through `Workspace.moduleSource`, so the clients fixtures + run on the engine's `python` runtime; and `Workspace.withoutDirectory` + followed by `withDirectory` on an ancestor replaces rather than merges, so + every overlay composes the full directory first (`Codegen.overlay`). The + Dang recursion (`Mod.generatedTreeIn`) works as written; `self` is a + reserved name in Dang, and octal literals are not a thing. +- **Phase 5 — code review and fix: done, one round.** A Claude reviewer + (approve with changes, 14 findings) and a Codex reviewer (reject pending + fixes, 5 findings) on the implemented diff; every finding was curated into + one brief and applied by a fixer into the owning patches: reserved-name + collisions in a client file and a root type named after a core type are + rejected loudly; a local module ref is decided by the workspace (a config + file at the path) rather than by string shape, so a dotted directory is no + longer taken for a git ref; the foreign-dependency check carries a + counterfactual so its failure is attributable; duplicate client names are + rejected by the code generator and tested; the `load_module` translation + covers the bare `dagger.clients` package; the scalar directive stubs leave + graphql-core's shared scalars alone; bindings are pinned through + `execute_object_list` and `execute_sync`; an owned enum argument on a core + field is pinned; `initClient` at the workspace root names the project after + the module; plus the nits. The full `dagger check` also caught a regression + the reviewers did not: `generateAll` applied the modules' cwd-relative + changeset with `withChanges`, which reads root-relative paths, and broke + `dagger module init` through the sdk-sdk harness — the modules' output is + now overlaid directly. Two pre-existing issues surfaced and are left alone: + `initModule` renders `src/probe_2/` for a module named `probe2` + (`strcase.ToSnake`) while `pyproject.toml` names `probe2`, so a name with a + digit cannot build; and the `sdk-sdk:contract:*` checks fail in this + environment on `main` itself (`.dagger/lock` is a version-1 lock the local + engine refuses to parse) — bisected with `stg pop -a`, so not this series'. + Unit suite: 261 tests. +- **Phases 6–8 — draft PR, CI, archive: done.** Draft PR + https://github.com/dagger/python-sdk/pull/22 on `eunomie:python-unified-clients-lead-afe256c1`, + base `main` @ `71c445f`. Two CI fix rounds: `foreignDependencyCheck` had + encoded a git-worktree-only failure (in a real checkout the engine stages + the Go dependency and the module generates with a client for it — the + check now asserts that contract in both environments); and the one-line + `runtime/main.go` edit shifted the source-map line numbers embedded in the + committed `runtime/dagger.gen.go`, which `go-sdk:generate` guards + (regenerated). CI green at `2dc64c7`: 56 checks, including the five new + e2e checks, the sdk-sdk chain/contract suites, and `go-sdk:generate`. + Follow-ups, not in this series: a `dagger/dagger` issue for + `withoutDirectory` turning a later `withDirectory` on an ancestor into a + replace; one for the undetectable same-name collision of two local + bindings; the `initModule` template rendering `src/probe_2/` for a module + named `probe2`. +- **Round 3 (the cap): the design reviewer passes; the skeptic still fails + on one blocker and five majors, every one of which is folded in above:** + "foreign" now means registered to *another* SDK, so an unregistered + dependency never drags Python siblings through the engine call; the + `_converter` rejection runs before classification and covers the + placeholder; the runtime change moves into the e2e patch with the fixture + it needs; escaped argument names that collide fail generation; a missing + self client import is translated into the actionable error; a + foreign-dependency check pins that branch. From the design reviewer: + `generateAll` filters modules to the cwd (client init would otherwise fail + under a registered module), the standalone runtime check is made + deterministic, the runtime check is scoped to the vendored layout, and the + Dang recursion is flagged as the spike. With every finding adopted and the + cap reached, the plan proceeds to implementation on the lead's call; the + dissent is recorded here rather than resolved by a fourth round. + +- **Post-landing review: six findings, folded into their owning patches.** + - Correctness (generated code), major: a description or deprecation reason + carrying `\` or `"""` rendered a client Python cannot parse, and a + deprecation reason with a newline broke its one-line literal. `doc()` + escapes both, the deprecation message escapes backslash first, then the + quote, then the newlines; tested from the client renderer. Patch + `codegen-modes`. + - API stability, major: a function on a core type was qualified by its + parent only when two parents shared the field name, so adding a field + elsewhere renamed an exported one. Every non-`Query` function is now + parent-qualified. Patch `codegen-modes`, with the rule in the README. + - Correctness (paths), major: `generateClient` and `initClient` only + normalized their `path`, so `../x` was canonicalized by the engine into a + root-level directory the caller never named. One `workspacePath` helper + rejects any `..` segment, and `initModule` uses it instead of its own + inline check. Patch `python-sdk-standalone`, with an e2e assertion in + `e2e-clients`. + - Correctness (templates), minor: the client's `pyproject.toml` took the + directory basename verbatim as its distribution name. It takes + `ModulePackage` now, and the template helper accepts only a package name + PEP 508 and TOML both take — which also protects `initModule`. Patch + `python-sdk-standalone`. + - Simplicity, minor: `stagedDependencies` staged an aliased dependency + twice; the local dependency paths are deduplicated. Patch + `python-sdk-clients`. + - Docs, minor: the migration recipe only covered `dag.()`. It + covers a module-owned function on a core type too. Patches + `docs-clients` and `future-archive`. + + Re-review of those fixes, five more findings, folded into the same patches: + + - Correctness (generated code), major: `textwrap.wrap` is applied to the + complete docstring literal, and its default `break_long_words` splits a + long word anywhere — including inside an escape pair or the closing + delimiter. `wrap` breaks on whitespace only now, which glues every + delimiter and escape to its word. Patch `codegen-modes`. + - Correctness (generated code), major: a NUL made the client unimportable, + a carriage return was rewritten as a newline, and a trailing quote got a + space appended instead of round-tripping. `doc()` escapes `\r` and + `\x00`, and escapes a trailing quote instead of padding it; the + deprecation message escapes `\x00` too. Patch `codegen-modes`. + - Correctness (paths), major: the engine reads `\` as a separator + (`pathutil.SandboxedRelativePath`), so `..\escape` walked straight past + the `..` check. `workspacePath` converts backslashes to slashes first. + Patch `python-sdk-standalone`, with the e2e assertion extended in + `e2e-clients`. + - Correctness (templates), minor: the template helper's guard was a + blacklist, and `strcase.ToSnake` keeps `@` and newlines, so `foo@bar` + rendered a project `uv lock` rejects. One positive rule on the derived + name instead. Patch `python-sdk-standalone`. + - Docs, minor: this ledger. Patch `future-archive`. + + Follow-ups, not in this series, both for the next refresh of the committed + `sdk/src/dagger/client/gen.py` (not regenerated here): `Directory.withPatch` + carries an unescaped `\n` in its docstring, rendered before the escaping + fix; and a core description ending in a quote will render as `\"` where it + is `" ` today. diff --git a/helpers/render-template/main.go b/helpers/render-template/main.go index b9afba5..4feac39 100644 --- a/helpers/render-template/main.go +++ b/helpers/render-template/main.go @@ -5,12 +5,17 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" "text/template" "github.com/iancoleman/strcase" ) +// What the templates render as both a distribution name and a package +// directory: valid where PEP 508 and TOML are, with nothing to escape. +var packageName = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9_]*[A-Za-z0-9])?$`) + func main() { if err := run(os.Args[1:]); err != nil { fmt.Fprintln(os.Stderr, err) @@ -32,6 +37,9 @@ func run(args []string) error { "ModuleImport": "dagger/" + strcase.ToKebab(moduleName), "ModulePackage": strcase.ToSnake(moduleName), } + if pkg := data["ModulePackage"]; !packageName.MatchString(pkg) { + return fmt.Errorf("cannot name a Python package after %q: derived %q is not a valid package name", moduleName, pkg) + } return filepath.WalkDir(templateDir, func(path string, entry os.DirEntry, err error) error { if err != nil { diff --git a/mod.dang b/mod.dang index e2b2d4a..fbe2ef4 100644 --- a/mod.dang +++ b/mod.dang @@ -58,141 +58,211 @@ type Mod { if (skipGenerate) { ws.changes(ws) } else { - # Stage the local dependency closure so this module's codegen sees - # up-to-date dependency bindings. It is only an input to resolving the - # module source, never the diff baseline, so the dependencies' codegen - # does not ride along. - let stagedWs = ws.withChanges( - ws.moduleSource("/" + rootPath).generateLocalDependencies(ws), - ) - let generated = if (isModern) { - # A dagger-module.toml module is generated here, by this SDK's own code - # generator. Its runtime generates nothing, so asking the engine for a - # generated context would come back empty. - vendoredDir(stagedWs) - } else { - # A pre-1.0 module keeps being generated by the runtime its dagger.json - # names, which is the engine's builtin Python SDK. - stagedWs - .moduleSource("/" + rootPath) - .generatedContextDirectory - .directory(rootPath) - } - - # The generated context holds only generated files, so merge it onto the - # module rather than replacing it. Workspace.changes then reports just - # the difference, rooted at the caller's cwd. - ws.withDirectory("/" + rootPath, generated).changes(ws) + # The generated-clients subtree is exclusively generated: sweep it so a + # dropped dependency or a renamed alias is reported as a removal. The + # rest of sdk/ is merged onto, so a stray file there survives. + codegen + .overlay(ws, rootPath, vendorDirName + "/" + clientsDirName, generatedTree(ws)) + .changes(ws) } } """ - Whether this module uses the 1.0 dagger-module.toml config. + This module's generated files, rooted at the module, for either config + format: a directory holding only what generation produces. """ - let isModern: Boolean! { - let configPath = if (rootPath == ".") { "dagger-module.toml" } else { rootPath + "/dagger-module.toml" } - ws.directory("/", include: [configPath]).exists(configPath) + pub generatedTree(base: Workspace!): Directory! { + generatedTreeIn(base, []) } """ - This SDK's client library, with bindings generated against the module's own - schema, laid out the way it is vendored into a module. + `active` names the modules whose generation is in progress up the dependency + chain, so a cycle fails with a message instead of recursing without end. + """ + let generatedTreeIn(base: Workspace!, active: [String!]!): Directory! { + let stagedWs = stagedDependencies(base, active + [rootPath]) + if (isModern) { + # A dagger-module.toml module is generated here, by this SDK's own code + # generator. Its runtime generates nothing, so asking the engine for a + # generated context would come back empty. + vendoredDir(stagedWs) + } else { + # A pre-1.0 module keeps being generated by the runtime its dagger.json + # names, which is the engine's builtin Python SDK. + stagedWs + .moduleSource("/" + rootPath) + .generatedContextDirectory + .directory(rootPath) + } + } - A module-rooted directory holding only `sdk/`, so the caller can merge it onto - the module without touching anything else the module owns. """ - let vendoredDir(stagedWs: Workspace!): Directory! { - let schemaJSON = stagedWs.moduleSource("/" + rootPath).introspectionSchemaJSON + The workspace with this module's local dependencies generated, as an input + to resolving its module source — never the diff baseline, so the + dependencies' generated files do not ride along. + + Dependencies this SDK manages are generated here, recursively, and overlaid + onto the workspace; a changeset would be measured from the caller's cwd and + might not reach a sibling. Dependencies another SDK manages are handed to + the engine, which knows their generator. Anything else local is assumed + committed, as the engine assumes too. A dependency declared twice, once + under an alias, is one path, and is staged once. + """ + let stagedDependencies(base: Workspace!, active: [String!]!): Workspace! { + let localDeps = base + .moduleSource("/" + rootPath) + .dependencies.{{kind, sourceRootSubpath}} + .filter { dep => isGitSource(dep.kind) == false } + .map { dep => normalizePath(dep.sourceRootSubpath) } + .reduce([]) { acc, depPath => if (contains(acc, depPath)) { acc } else { acc + [depPath] } } + let mine = managedPaths(base) + let theirs = foreignPaths(base, mine) + let foreign = localDeps.filter { depPath => contains(theirs, depPath) } + let engineStaged = if (foreign.length > 0) { + base.withChanges(base.moduleSource("/" + rootPath).generateLocalDependencies(base)) + } else { + base + } + + localDeps + .filter { depPath => contains(mine, depPath) } + .reduce(engineStaged) { staged, depPath => + if (contains(active, depPath)) { + raise "circular dependency between Python SDK modules: " + active.join(" -> ") + " -> " + depPath + } else { + let dep = Mod(rootPath: depPath, ws: staged, skipGenerateFilename: skipGenerateFilename) + if (dep.skipGenerate) { + staged + } else { + codegen.overlay(staged, depPath, vendorDirName, dep.generatedTreeIn(staged, active)) + } + } + } + } - directory.withDirectory( - vendorDirName, - library.withFile(generatedBindingsPath, bindings(schemaJSON)), - ) + """ + Workspace-root-relative paths of the modules this SDK manages, wherever the + caller's cwd is: the engine scopes `asSDK.modules` to the cwd, and a + dependency is usually a sibling. + """ + let managedPaths(base: Workspace!): [String!]! { + currentModule + .asSDK(workspace: base.withWorkdir(".")) + .modules.{{path}} + .map { module => normalizePath(module.path) } } """ - Client bindings generated from a module's schema by this SDK's code generator. + Workspace-root-relative paths of the modules registered to any SDK other + than this one. """ - let bindings(schemaJSON: File!): File! { - codegenBase - .withMountedFile(schemaPath, schemaJSON) - .withExec([ - "uv", "run", "--isolated", "--frozen", "--package", "codegen", - "python", "-m", "codegen", "generate", "-i", schemaPath, "-o", "/gen.py", - ]) - .file("/gen.py") + let foreignPaths(base: Workspace!, mine: [String!]!): [String!]! { + base.sdks.{{name}} + .reduce([]) { acc, sdk => + acc + base.sdk(name: sdk.name).modules.{{source}}.map { module => normalizePath(module.source) } + } + .filter { modPath => contains(mine, modPath) == false } + } + + let isGitSource(kind: ModuleSourceKind!): Boolean! { + codegen.isGitSource(kind) + } + + let contains(values: [String!]!, want: String!): Boolean! { + values.filter { value => value == want }.length > 0 + } + + let normalizePath(path: String!): String! { + let normalized = path.trimPrefix("./").trimPrefix("/").trimSuffix("/") + if (normalized == "") { "." } else { normalized } } """ - Container with this SDK's client library and code generator mounted. + A path inside this module, workspace-root-relative without a leading slash. """ - let codegenBase: Container! { - container - .from(codegenImage) - .withoutEntrypoint - .withMountedCache("/root/.cache/uv", cacheVolume("python-sdk-uv")) - .withEnvVariable("UV_LINK_MODE", "copy") - .withDirectory("/sdk", codegenSource) - .withWorkdir("/sdk") + let modulePath(rel: String!): String! { + if (rootPath == ".") { rel } else { rootPath + "/" + rel } } """ - What the code generator needs to run: the library plus the generator itself, - and the lock that pins the generator's own dependencies. + Whether this module uses the 1.0 dagger-module.toml config. """ - let codegenSource: Directory! { - currentModule.source.directory("sdk").filter(include: [ - "pyproject.toml", - "uv.lock", - "src/**/*.py", - "src/**/*.typed", - "codegen/pyproject.toml", - "codegen/**/*.py", - ]) + let isModern: Boolean! { + let configPath = if (rootPath == ".") { "dagger-module.toml" } else { rootPath + "/dagger-module.toml" } + ws.directory("/", include: [configPath]).exists(configPath) } """ - What a module actually needs vendored: the importable client library, its - license, and a project file describing just that. + This SDK's client library laid out the way it is vendored into a module: the + core API from the module's own schema, one client per dependency, and the + module's own client. - The code generator runs in this SDK, never in a module, and the library's own - lock pins the generator's development environment rather than the module's. + A module-rooted directory holding only `sdk/`, so the caller can merge it onto + the module without touching anything else the module owns. """ - let library: Directory! { - currentModule.source - .directory("sdk") - .filter(include: [ - "LICENSE", - "README.md", - "src/**/*.py", - "src/**/*.typed", - # The library imports this under suppress(ModuleNotFoundError) and says - # it "doesn't make sense in modules": it provisions an engine for a - # standalone script, which a module already has. - "!src/dagger/provisioning/**", - ]) - .withFile("pyproject.toml", libraryPyproject) + let vendoredDir(stagedWs: Workspace!): Directory! { + let src = stagedWs.moduleSource("/" + rootPath) + let deps = src.dependencies.{{moduleName, kind, sourceRootSubpath, asString, pin}} + # One call: the generator normalizes every name and rejects two clients of + # this module that would share a file under dagger/clients. The module's + # own name goes last, so the self client's name is the last one back. + let names = codegen.clientNames(deps.map { dep => dep.moduleName } + [src.moduleName]) + let selfName = names.reduce("") { acc, name => name } + + let clients = deps.reduce(directory) { dir, dep => + dir.withFile( + codegen.clientName(dep.moduleName) + ".py", + codegen.clientOf(dependencySource(stagedWs, dep.moduleName, dep.kind, dep.sourceRootSubpath, dep.asString, dep.pin)), + ) + } + let selfClientPath = codegen.clientsDirName + "/" + selfName + ".py" + + # Pass 1 carries the committed self client, or a stub, so the module's + # own code imports during the bootstrap build below. + let pass1 = codegen.library(false) + .withFile(codegen.generatedBindingsPath, codegen.core(src.introspectionSchemaJSON)) + .withDirectory(codegen.clientsDirName, clients) + .withFile(selfClientPath, carriedSelfClient(selfName, src.moduleName)) + + # Reading the module's own client-facing schema installs the module, + # which builds it from the staged sdk/. + let bootWs = codegen.overlay(stagedWs, rootPath, vendorDirName, directory.withDirectory(vendorDirName, pass1)) + let selfClient = codegen.clientOf(bootWs.moduleSource("/" + rootPath)) + + directory.withDirectory(vendorDirName, pass1.withFile(selfClientPath, selfClient)) } """ - The library's project file with its development sections removed. + One declared dependency, re-resolved from its identity under its final name: + a local one from the staged workspace, a git one from its ref and pin. + `withName` is what the engine itself uses to alias a dependency, so the + client schema comes out namespaced under that name. + """ + let dependencySource(stagedWs: Workspace!, name: String!, kind: ModuleSourceKind!, subpath: String!, ref: String!, pin: String!): ModuleSource! { + if (codegen.isGitSource(kind)) { + moduleSource(refString: ref, refPin: pin).withName(name) + } else { + stagedWs.moduleSource("/" + normalizePath(subpath)).withName(name) + } + } - As published, it declares the code generator as a uv workspace member and a - dev dependency. Vendoring that verbatim without the generator makes `uv` - refuse to install the library at all, so the sections that only describe - developing this SDK are dropped. """ - let libraryPyproject: File! { - codegenBase - .withFile(stripScriptPath, currentModule.source.file("helpers/vendor-pyproject/strip_dev_sections.py")) - .withExec(["python", stripScriptPath, "pyproject.toml", "/library-pyproject.toml"]) - .file("/library-pyproject.toml") + The module's committed self client when it has one, so code that already + imports it keeps importing during the bootstrap; otherwise a stub whose + every name is a placeholder that says the client is not generated yet. + """ + let carriedSelfClient(selfName: String!, moduleName: String!): File! { + let committed = modulePath(vendorDirName + "/" + codegen.clientsDirName + "/" + selfName + ".py") + let view = ws.directory("/", include: [committed]) + if (view.exists(committed)) { + view.file(committed) + } else { + codegen.stubClient(moduleName) + } } - let stripScriptPath: String! = "/strip-dev-sections.py" + let codegen: Codegen! { Codegen() } - let vendorDirName: String! = "sdk" - let generatedBindingsPath: String! = "src/dagger/client/gen.py" - let schemaPath: String! = "/schema.json" - let codegenImage: String! = "ghcr.io/astral-sh/uv:python3.14-alpine" + let vendorDirName: String! { codegen.vendorDirName } + let clientsDirName: String! { codegen.clientsDirName } } diff --git a/python-sdk.dang b/python-sdk.dang index e34dcbb..2641bc1 100644 --- a/python-sdk.dang +++ b/python-sdk.dang @@ -30,6 +30,23 @@ type PythonSdk { if (normalized == "") { "." } else { normalized } } + """ + Normalize a workspace path a caller named, rejecting one that leaves the + workspace: a `..` segment reaches a directory the caller did not name, which + the engine would canonicalize into a root-level path of its own. Backslashes + are separators to the engine (`pathutil.SandboxedRelativePath`), so they are + separators here too, before the check. + """ + let workspacePath(path: String!): String! { + let normalized = normalizePath(path.split("\\").join("/")) + let escapes = normalized.split("/").reduce(false) { acc, part => acc or part == ".." } + if (escapes) { + raise "path escapes workspace: " + path + } else { + normalized + } + } + let pathDepth(path: String): Int! { if (path == null) { -1 } else if (path == ".") { 0 } else { path.split("/").length } } @@ -127,15 +144,7 @@ type PythonSdk { version is used, uv is enabled, and no base image override is written. """ pub initModule(ws: Workspace!, name: String!, path: String!, template: String! = "default", pythonVersion: String! = "", useUv: Boolean! = true, baseImage: String! = ""): Changeset! { - let rawPath = path.trimPrefix("./").trimPrefix("/") - - let modPath = if (rawPath == "" or rawPath == ".") { - "." - } else if (rawPath == ".." or rawPath.trimPrefix("../") != rawPath) { - raise "path escapes workspace: " + rawPath - } else { - rawPath.trimSuffix("/") - } + let modPath = workspacePath(path) let selectedTemplate = if (template == "") { "default" } else { template } @@ -154,13 +163,20 @@ type PythonSdk { Render a Python template with the requested module name. """ let renderedTemplate(name: String!, templateName: String!): Directory! { + renderTemplate(name, currentModule.source.directory("templates/" + templateName)) + } + + """ + Render a template directory with the requested name. + """ + let renderTemplate(name: String!, template: Directory!): Directory! { container .from("golang:1.25-alpine") .withoutEntrypoint .withMountedCache("/go/pkg/mod", cacheVolume("go-mod")) .withMountedCache("/root/.cache/go-build", cacheVolume("go-build")) .withDirectory("/helper", currentModule.source.directory("helpers/render-template")) - .withDirectory("/template", currentModule.source.directory("templates/" + templateName)) + .withDirectory("/template", template) .withWorkdir("/helper") .withExec(["go", "build", "-o", "/usr/local/bin/render-template", "."]) .withExec(["render-template", name, "/template", "/rendered"]) @@ -196,15 +212,146 @@ type PythonSdk { } """ - Generate every managed Python SDK module visible from the client's cwd. + Generate every managed Python SDK module at or below the client's cwd, then + every registered client there, on top of the modules' generated state. - Modules with the generate skip marker are skipped. + One hook rather than two: the engine runs a workspace's generators + concurrently, and a client bound to a local module must be generated after + that module. Modules with the generate skip marker are skipped. The nearest + enclosing module of a cwd inside one is left alone: a changeset reaching + outside the cwd cannot be applied, and a client initialized inside a module + runs this scoped to the client's own directory. """ pub generateAll(ws: Workspace!): Changeset! @generate { - changeset.withChangesets( - modules(ws) - .filter { mod => mod.skipGenerate == false } - .map { mod => mod.generate }, - ) + let cwd = normalizePath(ws.cwd) + # Overlaid directly rather than applied as a changeset: Workspace.changes + # measures paths from the cwd and withChanges reads them from the root. + let withModules = modules(ws) + .filter { mod => mod.skipGenerate == false and pathContains(cwd, mod.rootPath) } + .reduce(ws) { staged, mod => + codegen.overlay(staged, mod.rootPath, codegen.vendorDirName + "/" + codegen.clientsDirName, mod.generatedTree(staged)) + } + + clients(withModules) + .reduce(withModules) { staged, client => + codegen.overlay(staged, client.path, codegen.vendorDirName + "/" + codegen.clientsDirName, client.generatedTree) + } + .changes(ws) + } + + """ + Generate a typed Python client for the module at `module`, written to `path`. + + `module` is a workspace-root-relative path to a local module, or a git ref. + A local module this SDK manages is generated first, so the client is never + read off an ungenerated module. Everything generated sits under `path/sdk/`; + the user's own files next to it survive every run. + """ + pub generateClient(ws: Workspace!, module: String!, path: String!): Changeset! { + client(ws, workspacePath(path), module, "").generate + } + + """ + Regenerate every client registered to this SDK at or below the client's cwd. + """ + pub generateAllClient(ws: Workspace!): Changeset! { + changeset.withChangesets(clients(ws).map { client => client.generate }) + } + + """ + Seed the SDK-owned files of a new Python client at `path`: a project file + declaring the vendored library, so `uv sync` there just works. + + The engine records the client in the workspace config and generates it + through the `@generate` hook; an existing project file is left alone. + """ + pub initClient(ws: Workspace!, path: String!, module: String!): Changeset! { + let clientPath = workspacePath(path) + let pyproject = clientSubpath(clientPath, "pyproject.toml") + if (ws.directory("/", include: [pyproject]).exists(pyproject)) { + ws.changes(ws) + } else { + let dirName = clientPath.split("/").reduce("") { acc, part => part } + # A client generated at the workspace root has no directory to name it + # after; the module it is bound to does. + let name = if (dirName == ".") { refBaseName(module) } else { dirName } + ws.withDirectory("/" + clientPath, renderTemplate(name, currentModule.source.directory("client-template"))).changes(ws) + } + } + + """ + The clients registered to this SDK at or below the workspace's cwd. The + engine does not scope its client list to the cwd, so this does. + """ + let clients(ws: Workspace!): [GeneratedClient!]! { + let cwd = normalizePath(ws.cwd) + currentModule + .asSDK(workspace: ws) + .clients.{{path, module, pin}} + .filter { entry => pathContains(cwd, normalizePath(entry.path)) } + .map { entry => client(ws, normalizePath(entry.path), entry.module, entry.pin) } } + + let client(ws: Workspace!, path: String!, module: String!, pin: String!): GeneratedClient! { + GeneratedClient(path: path, ws: ws, source: boundModule(ws, module, pin)) + } + + """ + The module a client binds to, resolved. A local module this SDK manages is + generated onto a staging workspace first — the same staging a dependency + gets — so its client-facing schema can be read; any other local module is + assumed committed, and a git module resolves from its ref and pin. + """ + let boundModule(ws: Workspace!, module: String!, pin: String!): ModuleSource! { + if (isLocalModuleRef(ws, module)) { + let modPath = normalizePath(module) + let managed = currentModule + .asSDK(workspace: ws.withWorkdir(".")) + .modules.{{path}} + .filter { entry => normalizePath(entry.path) == modPath } + .length > 0 + let mod = Mod(rootPath: modPath, ws: ws, skipGenerateFilename: skipGenerateFilename) + let staged = if (managed and mod.skipGenerate == false) { + codegen.overlay(ws, modPath, codegen.vendorDirName, mod.generatedTree(ws)) + } else { + ws + } + staged.moduleSource("/" + modPath) + } else { + moduleSource(refString: module, refPin: pin) + } + } + + """ + Whether a client's bound-module ref points into the workspace rather than + at a remote module. + + A ref that leads to a module config in the workspace is a path; anything + else is a git ref. Asked of the workspace rather than of the string, so a + path with a dot in a directory name (`my.app/dep`) is not read as a git ref. + """ + let isLocalModuleRef(ws: Workspace!, ref: String!): Boolean! { + if (ref.hasPrefix("/") or ref.hasPrefix(".")) { + true + } else { + let root = ws.directory("/") + let modPath = normalizePath(ref) + root.exists(modPath + "/dagger-module.toml") or root.exists(modPath + "/dagger.json") + } + } + + """ + Last path segment of a module ref, without any version suffix: + `github.com/x/hello@v1` and `.dagger/modules/hello` both name `hello`. + """ + let refBaseName(ref: String!): String! { + let unpinned = ref.split("@").reduce("") { acc, part => if (acc == "") { part } else { acc } } + normalizePath(unpinned).split("/").reduce("") { acc, part => part } + } + + let clientSubpath(clientPath: String!, rel: String!): String! { + codegen.subpath(clientPath, rel) + } + + let codegen: Codegen! { Codegen() } } diff --git a/runtime/dagger.gen.go b/runtime/dagger.gen.go index c762970..d1d871c 100644 --- a/runtime/dagger.gen.go +++ b/runtime/dagger.gen.go @@ -281,18 +281,18 @@ func invoke(ctx context.Context, parentJSON []byte, parentName string, fnName st return dag.Module(). WithDescription("Runtime module for the Python SDK\n"). WithObject( - dag.TypeDef().WithObject("PythonSdkRuntime", dagger.TypeDefWithObjectOpts{Description: "State threaded through the steps that build a module's runtime container.\n\nModuleRuntime is the only thing the engine calls; everything else here is\ninternal to it.", SourceMap: dag.SourceMap("main.go", 65, 6)}). + dag.TypeDef().WithObject("PythonSdkRuntime", dagger.TypeDefWithObjectOpts{Description: "State threaded through the steps that build a module's runtime container.\n\nModuleRuntime is the only thing the engine calls; everything else here is\ninternal to it.", SourceMap: dag.SourceMap("main.go", 66, 6)}). WithFunction( dag.Function("ModuleRuntime", dag.TypeDef().WithObject("Container")). WithDescription("Container for executing the Python module runtime\n\nThe container is built from the module's committed generated files. This\nruntime generates nothing: code generation belongs to `dagger generate`,\nwhich the Python SDK module owns. Dependencies are still installed — the\nlanguage-level assemble step, like the Go SDK still running go build.\n\nintrospectionJSON is declared, and never read, on purpose: its optionality is\nthe signal the engine reads (RuntimeTrustsCommittedFiles) to decide it may\nskip runtime codegen and omit the argument altogether. Dropping the argument\nwould tell the engine the opposite. It is the opt-out, not a code path."). - WithSourceMap(dag.SourceMap("main.go", 143, 1)). - WithArg("modSource", dag.TypeDef().WithObject("ModuleSource"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 145, 2)}). - WithArg("introspectionJSON", dag.TypeDef().WithObject("File").WithOptional(true), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 147, 2)})). + WithSourceMap(dag.SourceMap("main.go", 144, 1)). + WithArg("modSource", dag.TypeDef().WithObject("ModuleSource"), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 146, 2)}). + WithArg("introspectionJSON", dag.TypeDef().WithObject("File").WithOptional(true), dagger.FunctionWithArgOpts{SourceMap: dag.SourceMap("main.go", 148, 2)})). WithConstructor( dag.Function("New", dag.TypeDef().WithObject("PythonSdkRuntime")). - WithSourceMap(dag.SourceMap("main.go", 45, 1)))), nil + WithSourceMap(dag.SourceMap("main.go", 46, 1)))), nil default: return nil, fmt.Errorf("unknown object %s", parentName) } diff --git a/runtime/main.go b/runtime/main.go index e2aa052..95c5e89 100644 --- a/runtime/main.go +++ b/runtime/main.go @@ -15,6 +15,7 @@ const ( RuntimeExecutablePath = "/runtime" GenDir = "sdk" SDKGenPath = "src/dagger/client/gen.py" + ClientsPkgPath = "src/dagger/clients/__init__.py" UserGenPath = "src/dagger_gen.py" VenvPath = "/opt/venv" ProjectCfg = "pyproject.toml" @@ -180,6 +181,7 @@ func (m *PythonSdkRuntime) requireGeneratedFiles(ctx context.Context) error { required = []string{ path.Join(m.VendorPath, ProjectCfg), path.Join(m.VendorPath, SDKGenPath), + path.Join(m.VendorPath, ClientsPkgPath), } } for _, rel := range required { diff --git a/sdk/codegen/src/codegen/ast.py b/sdk/codegen/src/codegen/ast.py index 17016bb..bffb48f 100644 --- a/sdk/codegen/src/codegen/ast.py +++ b/sdk/codegen/src/codegen/ast.py @@ -3,7 +3,7 @@ import graphql -def insert_stubs(introspection: Any, schema: graphql.GraphQLSchema): +def insert_stubs(introspection: Any, schema: graphql.GraphQLSchema): # noqa: C901 """Insert ast node stubs into the parsed schema.""" for tp in introspection["types"]: tp_schema = schema.get_type(tp["name"]) @@ -41,6 +41,16 @@ def insert_stubs(introspection: Any, schema: graphql.GraphQLSchema): directives=parse_directives(tp["directives"]), ) + elif isinstance(tp_schema, graphql.GraphQLScalarType) and ( + not graphql.is_specified_scalar_type(tp_schema) + ): + # The specified scalars are process-wide singletons in graphql-core; + # stamping an ast node on them leaks into every other schema built. + tp_schema.ast_node = graphql.ScalarTypeDefinitionNode( + name=graphql.NameNode(value=tp["name"]), + directives=parse_directives(tp.get("directives") or []), + ) + elif isinstance(tp_schema, graphql.GraphQLEnumType): if values := tp.get("enumValues"): value_defs = [] diff --git a/sdk/codegen/src/codegen/cli.py b/sdk/codegen/src/codegen/cli.py index f61de09..8ae6d00 100644 --- a/sdk/codegen/src/codegen/cli.py +++ b/sdk/codegen/src/codegen/cli.py @@ -2,10 +2,11 @@ import json import pathlib import sys +from typing import Any import graphql -from codegen import ast, generator +from codegen import ast, generator, partition, version parser = argparse.ArgumentParser( prog="python -m codegen", description="Dagger Python SDK" @@ -17,9 +18,10 @@ def main(): title="additional commands", required=True, ) + gen_parser = subparsers.add_parser( "generate", - help="generate a Python client for the API", + help="generate the core API, or one module's client, from a schema", ) gen_parser.add_argument( "-i", @@ -37,18 +39,131 @@ def main(): "(defaults to printing it to stdout)" ), ) + gen_parser.add_argument( + "--mode", + choices=[generator.CORE_MODE, generator.CLIENT_MODE], + default=generator.CORE_MODE, + help=( + "core: every type no module owns, plus Client and dag; " + "client: the types and functions one module owns (default: core)" + ), + ) + gen_parser.add_argument( + "--module", + default="", + help="client mode: the bound module's final name", + ) + gen_parser.add_argument( + "--binding", + default="", + help=( + "client mode: the bound module's identity as JSON " + '({"name", "kind", "ref", "pin"})' + ), + ) + gen_parser.add_argument( + "--engine-version", + default="", + help="client mode: the bound module's declared engine version", + ) + gen_parser.set_defaults(func=run_generate) + + name_parser = subparsers.add_parser( + "client-name", + help="print the Python module name of each bound module's client", + ) + name_parser.add_argument( + "module", nargs="+", help="the bound modules' final names, one per line out" + ) + name_parser.set_defaults(func=run_client_name) + + version_parser = subparsers.add_parser( + "cli-version", + help="print the CLI release a client should provision for an engine", + ) + version_parser.add_argument("engine_version", help="as reported by Query.version") + version_parser.set_defaults(func=run_cli_version) + args = parser.parse_args() + args.func(args) - # TODO: Add argument for module init. - codegen(args.introspection, args.output) +def run_generate(args: argparse.Namespace): + result = json.loads(args.introspection.read_text()) + binding = json.loads(args.binding) if args.binding else None + code = render( + result, + mode=args.mode, + module=args.module, + binding=binding, + engine_version=args.engine_version, + ) -def codegen(introspection: pathlib.Path, output: pathlib.Path | None): - result = json.loads(introspection.read_text()) + if args.output: + args.output.write_text(code) + sys.stdout.write(f"Client generated successfully to {args.output}\n") + else: + sys.stdout.write(f"{code}\n") + + +def run_client_name(args: argparse.Namespace): + try: + names = partition.client_module_names(args.module) + except partition.PartitionError as e: + parser.exit(2, f"{parser.prog}: error: {e}\n") + sys.stdout.writelines(f"{name}\n" for name in names) + + +def run_cli_version(args: argparse.Namespace): + sys.stdout.write(f"{version.cli_version(args.engine_version)}\n") + + +def render( + result: dict[str, Any], + *, + mode: str = generator.CORE_MODE, + module: str = "", + binding: dict[str, str] | None = None, + engine_version: str = "", +) -> str: + """Generate one Python module from an introspection result.""" + schema_version = result.get("__schemaVersion", "") + + if mode == generator.CORE_MODE: + narrowed = partition.narrow_to_core(result["__schema"]) + schema = graphql.build_client_schema({"__schema": narrowed}) + ast.insert_stubs(narrowed, schema) + return generator.generate(schema, schema_version=schema_version) + + if not module or binding is None: + msg = "client mode needs --module and --binding" + raise ValueError(msg) + version.check_bound_module_version(engine_version) + + owned = partition.ownership(result["__schema"]) + root = partition.root_type_for(result["__schema"], module) + if root not in owned.types: + msg = ( + f"module {module!r} has root type {root!r}, which is a core type: " + "its client cannot both define it and refer to core's" + ) + raise partition.PartitionError(msg) schema = graphql.build_client_schema(result) ast.insert_stubs(result["__schema"], schema) - code = generator.generate(schema, schema_version=result.get("__schemaVersion", "")) + return generator.generate( + schema, + schema_version=schema_version, + mode=generator.CLIENT_MODE, + module=module, + binding=generator.Binding(**binding), + owned_types=owned.owned_types(module), + owned_fields=owned.owned_fields(module), + ) + +def codegen(introspection: pathlib.Path, output: pathlib.Path | None): + """Generate the core API from an introspection file (kept for callers).""" + code = render(json.loads(introspection.read_text())) if output: output.write_text(code) sys.stdout.write(f"Client generated successfully to {output}\n") diff --git a/sdk/codegen/src/codegen/generator.py b/sdk/codegen/src/codegen/generator.py index dc9f0f1..efbe4fb 100644 --- a/sdk/codegen/src/codegen/generator.py +++ b/sdk/codegen/src/codegen/generator.py @@ -53,6 +53,8 @@ from graphql.pyutils import camel_to_snake from graphql.type.schema import TypeMap +from codegen.partition import PartitionError, client_module_name + ACRONYM_RE = re.compile(r"([A-Z\d]+)(?=[A-Z\d]|$)") """Pattern for grouping initialisms.""" @@ -62,7 +64,7 @@ logger = logging.getLogger(__name__) indent = partial(textwrap.indent, prefix=" " * 4) -wrap = textwrap.wrap +wrap = partial(textwrap.wrap, break_long_words=False) wrap_indent = partial(wrap, initial_indent=" " * 4, subsequent_indent=" " * 4) @@ -107,6 +109,54 @@ def from_type(cls, t: GraphQLScalarType) -> str: return t.name +CORE_MODE = "core" +CLIENT_MODE = "client" + +CORE_ALIAS = "_core" +"""How a client module refers to the core API package.""" + +BINDING_NAME = "_BINDING" +"""The module-level name of a client's ModuleBinding.""" + +ENTRY_CLIENT_PARAM = "client" +"""The entry point's keyword-only parameter selecting the client to use.""" + +_body_reserved = frozenset(["_ctx", "_args", CORE_ALIAS, BINDING_NAME]) +"""Names a client function's body uses, which a schema argument may not shadow.""" + +_module_reserved = frozenset( + [ + "Arg", + "Callable", + "Client", + "Enum", + "Input", + "ModuleBinding", + "Protocol", + "Scalar", + "Self", + "Type", + "dataclass", + "runtime_checkable", + "typecheck", + "warnings", + CORE_ALIAS, + BINDING_NAME, + ] +) +"""Module-level names a client file already binds, which nothing rendered may take.""" + + +@dataclass(frozen=True) +class Binding: + """The identity a client is bound to; rendered verbatim into the client.""" + + name: str + kind: str + ref: str + pin: str = "" + + @dataclass class Context: """Shared state during execution.""" @@ -126,11 +176,53 @@ class Context: remaining: set[str] = field(default_factory=set) """Remaining type names that haven't been defined yet.""" + mode: str = CORE_MODE + """What is being rendered: the core API, or one module's client.""" + + module: str = "" + """In client mode, the bound module's final name.""" + + binding: Binding | None = None + """In client mode, the identity the client serves its module under.""" + + owned_types: frozenset[TypeName] = field(default_factory=frozenset) + """In client mode, the types the bound module contributes.""" + + owned_fields: frozenset[tuple[TypeName, FieldName]] = field( + default_factory=frozenset + ) + """In client mode, the fields the bound module contributes to core types.""" + @property def legacy_sdk_compat(self) -> bool: """Generate the pre-v0.21 ID/load helper source facade.""" return legacy_sdk_compat(self.schema_version) + @property + def is_client(self) -> bool: + return self.mode == CLIENT_MODE + + def renders_type(self, name: TypeName) -> bool: + """Whether this file defines the named type.""" + return not self.is_client or name in self.owned_types + + def type_ref(self, name: TypeName) -> str: + """How this file refers to a named type. + + Bare when this file defines it, through the core alias otherwise. + """ + if not self.is_client: + return name + if name in self.owned_types: + return name + if ( + name.startswith("_") + and name.endswith("Client") + and name[1:-6] in self.owned_types + ): + return name + return f"{CORE_ALIAS}.{name}" + def process_type(self, name: str): # This is only needed to keep track of remaining types because # of forward references. @@ -191,31 +283,89 @@ def render_body(self, t: _H) -> Iterator[str]: yield from wrap(doc(t.description)) +CORE_HEADER = """\ +# Code generated by dagger. DO NOT EDIT. + +import warnings # noqa: F401 +from collections.abc import Callable +from dataclasses import dataclass +from typing import Protocol, runtime_checkable + +from typing_extensions import Self + +from dagger.client._core import Arg +from dagger.client._guards import typecheck +from dagger.client.base import Enum, Input, Root, Scalar, Type +""" + +CLIENT_HEADER = """\ +# Code generated by dagger. DO NOT EDIT. + +import warnings # noqa: F401 +from collections.abc import Callable # noqa: F401 +from dataclasses import dataclass # noqa: F401 +from typing import Protocol, runtime_checkable # noqa: F401 + +from typing_extensions import Self # noqa: F401 + +from dagger.client import gen as _core +from dagger.client._binding import ModuleBinding +from dagger.client._core import Arg +from dagger.client._guards import typecheck +from dagger.client.base import Enum, Input, Scalar, Type # noqa: F401 +""" + + @joiner -def generate(schema: GraphQLSchema, schema_version: str = "") -> Iterator[str]: - """Code generation main function.""" - yield textwrap.dedent( - """\ - # Code generated by dagger. DO NOT EDIT. - - import warnings # noqa: F401 - from collections.abc import Callable - from dataclasses import dataclass - from typing import Protocol, runtime_checkable - - from typing_extensions import Self - - from dagger.client._core import Arg - from dagger.client._guards import typecheck - from dagger.client.base import Enum, Input, Root, Scalar, Type - """, - ) +def generate( # noqa: PLR0913 + schema: GraphQLSchema, + schema_version: str = "", + *, + mode: str = CORE_MODE, + module: str = "", + binding: Binding | None = None, + owned_types: frozenset[TypeName] = frozenset(), + owned_fields: frozenset[tuple[TypeName, FieldName]] = frozenset(), +) -> Iterator[str]: + """Code generation main function. + + In core mode the whole schema is rendered as the core API, ending in the + ``Client`` root and the ``dag`` singleton. In client mode only what the + bound module owns is rendered — its types, its entry point on ``Query`` + and its functions on other core types — and every other type is referred + to through the core alias. + """ + if mode == CLIENT_MODE and (not module or binding is None): + msg = "client mode needs the bound module's name and binding" + raise ValueError(msg) + + yield textwrap.dedent(CLIENT_HEADER if mode == CLIENT_MODE else CORE_HEADER) # Pre-create handy maps to make handler code simpler. ids = frozenset(n for n, t in schema.type_map.items() if is_id_type(t)) # shared state between all handler instances - ctx = Context(ids=ids, schema=schema, schema_version=schema_version) + ctx = Context( + ids=ids, + schema=schema, + schema_version=schema_version, + mode=mode, + module=module, + binding=binding, + owned_types=owned_types, + owned_fields=owned_fields, + ) + + if ctx.is_client: + assert ctx.binding is not None + yield "" + yield f"{BINDING_NAME} = ModuleBinding(" + yield indent(f"name={ctx.binding.name!r},") + yield indent(f"kind={ctx.binding.kind!r},") + yield indent(f"ref={ctx.binding.ref!r},") + yield indent(f"pin={ctx.binding.pin!r},") + yield ")" + yield "" handlers: tuple[Handler, ...] = ( Scalar(ctx), @@ -225,13 +375,15 @@ def generate(schema: GraphQLSchema, schema_version: str = "") -> Iterator[str]: Object(ctx), ) - if ctx.legacy_sdk_compat: + if ctx.legacy_sdk_compat and not ctx.is_client: for type_name in legacy_id_names(schema): yield legacy_id_class(type_name) ctx.defined.add(type_name) # Split into two iterators to update ctx.remaining. - types_n, types_g = itertools.tee(get_grouped_types(handlers, schema.type_map)) + types_n, types_g = itertools.tee( + get_grouped_types(handlers, schema.type_map, ctx.renders_type) + ) # Track types that haven't been defined yet, to format as a forward reference. ctx.remaining.update(name for _, name, _ in types_n) @@ -240,21 +392,26 @@ def generate(schema: GraphQLSchema, schema_version: str = "") -> Iterator[str]: yield handler.render(named_type) ctx.process_type(type_name) - yield "" - yield "" - yield "class Client(Query):" - yield indent( - '"""The Dagger client.\n' - "\n" - "Inherits all Query API methods and adds connection management.\n" - '"""' - ) - ctx.defined.add("Client") + if ctx.is_client: + for shim in client_functions(ctx): + yield shim.render() + ctx.defined.add(shim.name) + else: + yield "" + yield "" + yield "class Client(Query):" + yield indent( + '"""The Dagger client.\n' + "\n" + "Inherits all Query API methods and adds connection management.\n" + '"""' + ) + ctx.defined.add("Client") - yield "" - yield "dag = Client()" - yield '"""The global client instance."""' - ctx.defined.add("dag") + yield "" + yield "dag = Client()" + yield '"""The global client instance."""' + ctx.defined.add("dag") yield "" yield "__all__ = [" @@ -262,12 +419,16 @@ def generate(schema: GraphQLSchema, schema_version: str = "") -> Iterator[str]: yield "]" -def get_grouped_types(handlers: tuple[Handler, ...], type_map: TypeMap): +def get_grouped_types( + handlers: tuple[Handler, ...], + type_map: TypeMap, + include: Callable[[TypeName], bool] = lambda _: True, +): """Group types by handler and sorted by their name.""" def _filtered(): for n, t in type_map.items(): - if n.startswith("_") or is_builtin_scalar_type(t): + if n.startswith("_") or is_builtin_scalar_type(t) or not include(n): continue for i, handler in enumerate(handlers): if handler.predicate(t): @@ -479,13 +640,25 @@ def format_name(s: str) -> str: return s +TypeRef: TypeAlias = Callable[[TypeName], str] + + +def _bare(name: TypeName) -> str: + return name + + def format_input_type( t: GraphQLInputType, convert_id=True, expected_type: TypeName | None = None, legacy_ids: bool = False, + ref: TypeRef = _bare, ) -> str: - """May be used in an input object field or an object field parameter.""" + """May be used in an input object field or an object field parameter. + + ``ref`` renders a named type the way the file being generated refers to + it; builtin scalars and ``Type`` are always bare. + """ if is_required_type(t): t = t.of_type fmt = "%s" @@ -493,32 +666,37 @@ def format_input_type( fmt = "%s | None" if is_list_type(t): - inner = format_input_type(t.of_type, convert_id, expected_type, legacy_ids) + inner = format_input_type(t.of_type, convert_id, expected_type, legacy_ids, ref) return fmt % f"list[{inner}]" if is_id_type(t): if convert_id: if expected_type is not None: - return fmt % expected_type + return fmt % ref(expected_type) # Generic ID scalar — accept any Type (Dagger object) return fmt % "Type" if legacy_ids and expected_type is not None: - return fmt % legacy_id_name(expected_type) + return fmt % ref(legacy_id_name(expected_type)) - return fmt % (Scalars.from_type(t) if is_scalar_type(t) else get_named_type(t).name) + if is_scalar_type(t): + return fmt % ( + Scalars.from_type(t) if is_builtin_scalar_type(t) else ref(t.name) + ) + return fmt % ref(get_named_type(t).name) def format_output_type( t: GraphQLOutputType, expected_type: TypeName | None = None, legacy_ids: bool = False, + ref: TypeRef = _bare, ) -> str: """May be used as the output type of an object field.""" # When returning objects we're in query building mode, so don't return # None even if the field's return is optional. if not is_output_leaf_type(t) and not is_required_type(t): t = GraphQLNonNull(t) - return format_input_type(t, False, expected_type, legacy_ids) + return format_input_type(t, False, expected_type, legacy_ids, ref) def output_type_description(t: GraphQLOutputType) -> str: @@ -531,10 +709,14 @@ def output_type_description(t: GraphQLOutputType) -> str: def doc(s: str) -> str: """Wrap string in docstring quotes.""" + s = s.replace("\\", "\\\\") + # Escaped before the block quotes, while no quote in the string is escaped + # yet, so a trailing one is unambiguously a delimiter in the making. + if s.endswith('"'): + s = f'{s[:-1]}\\"' + s = s.replace('"""', '\\"\\"\\"').replace("\r", "\\r").replace("\x00", "\\x00") if "\n" in s: s = f"{s}\n" - elif s.endswith('"'): - s += " " return f'"""{s}"""' @@ -591,8 +773,12 @@ def __init__( convert_id, self.expected_type, ctx.legacy_sdk_compat, + ctx.type_ref, + ) + self.is_self = ( + self.parent_object_name is not None + and self.type == ctx.type_ref(self.parent_object_name) ) - self.is_self = self.type == self.parent_object_name self.description = graphql.description self.has_default = graphql.default_value is not Undefined reason = getattr(graphql, "deprecation_reason", None) @@ -606,7 +792,8 @@ def __init__( self.has_default = True if default_value and is_enum_type(self.named_type): - self.default_value = f"{self.named_type.name}.{default_value}" + enum_ref = ctx.type_ref(self.named_type.name) + self.default_value = f"{enum_ref}.{default_value}" else: self.default_value = repr(default_value) @@ -707,6 +894,7 @@ def __init__( field.type, legacy_output_id_type, ctx.legacy_sdk_compat, + ctx.type_ref, ) # Any field in the API that returns an ID for its parent object should @@ -726,7 +914,7 @@ def __init__( and self.expected_type and self.parent_name == self.expected_type ): - self.type = self.expected_type + self.type = ctx.type_ref(self.expected_type) self.convert_id = True self.is_sync = self.convert_id and self.name == "sync" @@ -761,7 +949,9 @@ def func_signature(self) -> str: if len(params) > 40: # noqa: PLR2004 params = f"{params}," - ret_type = "Self" if self.type == self.parent_name else self.type + ret_type = ( + "Self" if self.type == self.ctx.type_ref(self.parent_name) else self.type + ) sig = self.ctx.render_types(f"def {self.name}({params}) -> {ret_type}:") if self.is_exec: sig = f"async {sig}" @@ -773,9 +963,15 @@ def func_body(self) -> Iterator[str]: yield doc(docstring) if deprecated := self.deprecated(): - msg = f'Method "{self.name}" is deprecated: {deprecated}'.replace( - '"', '\\"' - ) + msg = f'Method "{self.name}" is deprecated: {deprecated}' + for char, escaped in ( + ("\\", "\\\\"), + ('"', '\\"'), + ("\n", "\\n"), + ("\r", "\\r"), + ("\x00", "\\x00"), + ): + msg = msg.replace(char, escaped) yield textwrap.dedent( f"""\ warnings.warn( @@ -798,26 +994,30 @@ def func_body(self) -> Iterator[str]: yield f"return await self._ctx.execute_sync({', '.join(args)})" return - yield f'_ctx = self._select("{self.graphql_name}", _args)' + yield self.select_line() if not self.is_exec: # Use the concrete client class for interface types - t = self._iface_client_name(self.type) - yield f"return {t}(_ctx)" + yield f"return {self.object_class()}(_ctx)" elif self.is_list: - n = self.named_type.name - t = self._iface_client_name(n) - yield f"return await _ctx.execute_object_list({t})" + yield f"return await _ctx.execute_object_list({self.object_class()})" elif self.is_void: yield "await _ctx.execute()" else: yield f"return await _ctx.execute({self.type})" - def _iface_client_name(self, name: str) -> str: - """Return concrete client class name for interface types.""" + def select_line(self) -> str: + return f'_ctx = self._select("{self.graphql_name}", _args)' + + def object_class(self) -> str: + """The class to build for an object return. + + The concrete client class for an interface, the type itself otherwise. + """ + name = self.named_type.name if is_interface_type(self.named_type): - return f"_{name}Client" - return name + name = f"_{name}Client" + return self.ctx.type_ref(name) def func_doc(self) -> str: def _out(): @@ -1089,3 +1289,162 @@ def with_(self, cb: Callable[["{self_name}"], "{self_name}"]) -> "{self_name}": return cb(self) ''' # noqa: E501 ) + + +class _ClientFunction(_ObjectField): + """A module-owned field on a core type, rendered as a module-level function. + + ``Query.`` is the client's entry point: it takes the module's + constructor arguments plus a keyword-only ``client`` and starts a new + chain with the binding attached. Any other core parent (``Binding``, + ``Env``, …) becomes a function taking that parent as its first argument. + """ + + def __init__( + self, + ctx: Context, + name: str, + field: GraphQLField, + parent: GraphQLObjectType, + python_name: str, + ) -> None: + super().__init__(ctx, name, field, parent) + self.name = python_name + self.is_entry = self.parent_name == "Query" + self.parent_param = format_name(self.parent_name) + if self.convert_id: + msg = ( + f"function {self.parent_name}.{self.graphql_name!r} of module " + f"{ctx.module!r} returns its parent's ID: a module-level function " + "has no object to chain the result from" + ) + raise PartitionError(msg) + reserved = _body_reserved | { + ENTRY_CLIENT_PARAM if self.is_entry else self.parent_param + } + for arg in self.args: + if arg.name in reserved: + arg.name = f"{arg.name}_" + names = [arg.name for arg in self.args] + if len(set(names)) != len(names): + dupes = sorted({n for n in names if names.count(n) > 1}) + msg = ( + f"function {self.parent_name}.{self.graphql_name!r} of module " + f"{ctx.module!r} has arguments that normalize to the same Python " + f"name: {', '.join(dupes)}" + ) + raise PartitionError(msg) + + @joiner + def render(self) -> Iterator[str]: + yield "" + yield self.func_signature() + yield indent(self.func_body()) + yield "" + + def func_signature(self) -> str: + params: list[str] = [] + if not self.is_entry: + params.append(f"{self.parent_param}: {self.ctx.type_ref(self.parent_name)}") + params.extend(a.as_param() for a in self.required_args) + params.append("*") + params.extend(a.as_param() for a in self.default_args) + if self.is_entry: + client = ( + f"{ENTRY_CLIENT_PARAM}: {self.ctx.type_ref('Client')} | None = None" + ) + params.append(client) + if params[-1] == "*": + params.pop() + joined = ", ".join(params) + # arbitrary heuristic to force trailing comma in long signatures + if len(joined) > 40: # noqa: PLR2004 + joined = f"{joined}," + sig = self.ctx.render_types(f"def {self.name}({joined}) -> {self.type}:") + if self.is_exec: + sig = f"async {sig}" + return sig + + def select_line(self) -> str: + if self.is_entry: + root = ( + f"{CORE_ALIAS}.dag if {ENTRY_CLIENT_PARAM} is None " + f"else {ENTRY_CLIENT_PARAM}" + ) + chain = f"({root})._ctx.with_binding({BINDING_NAME})" + select = f'root_select("{self.graphql_name}", _args)' + else: + chain = f"{self.parent_param}._ctx.with_binding({BINDING_NAME})" + select = f'select("{self.parent_name}", "{self.graphql_name}", _args)' + return f"_ctx = {chain}.{select} # noqa: SLF001" + + def func_doc(self) -> str: + if not self.is_entry: + return super().func_doc() + intro = "\n".join( + wrap( + "Executing any query through the returned object serves the " + "module into the session first; a module bound by workspace path " + "needs a workspace, a module bound by git ref resolves anywhere. " + f"Pass `{ENTRY_CLIENT_PARAM}` to use a connection other than the " + "global one." + ) + ) + head = f"Client for the `{self.ctx.module}` module.\n\n{intro}" + doc = super().func_doc() + return f"{head}\n\n{doc}" if doc else head + + +def client_functions(ctx: Context) -> list[_ClientFunction]: + """The module-level functions of a client, entry point first. + + A function on a core type is always named after its parent and field, so + that adding a field to another parent never renames an exported one. The + entry point is named after the module, like the file it lives in. + """ + functions: list[_ClientFunction] = [] + entry_name = client_module_name(ctx.module) + for parent, field_name in sorted(ctx.owned_fields): + parent_type = cast(GraphQLObjectType, ctx.schema.get_type(parent)) + graphql_field = parent_type.fields[field_name] + name = ( + entry_name + if parent == "Query" + else f"{format_name(parent)}_{format_name(field_name)}" + ) + functions.append( + _ClientFunction(ctx, field_name, graphql_field, parent_type, name) + ) + + entries = [f for f in functions if f.is_entry] + if len(entries) != 1: + msg = ( + f"module {ctx.module!r} contributes {len(entries)} Query fields, " + "expected exactly 1" + ) + raise PartitionError(msg) + + _reject_name_collisions(ctx, functions) + functions.sort(key=lambda f: (not f.is_entry, f.name)) + return functions + + +def _reject_name_collisions(ctx: Context, functions: list[_ClientFunction]): + """Every name a client file binds is its own: nothing shadows anything.""" + for name in sorted(ctx.owned_types | {f.name for f in functions}): + if name in _module_reserved: + msg = ( + f"module {ctx.module!r}: {name!r} is a name its client file " + "already binds" + ) + raise PartitionError(msg) + + taken: set[str] = set(ctx.owned_types) + for function in functions: + if function.name in taken: + msg = ( + f"module {ctx.module!r}: function {function.name!r} collides " + "with another name in its client" + ) + raise PartitionError(msg) + taken.add(function.name) diff --git a/sdk/codegen/src/codegen/partition.py b/sdk/codegen/src/codegen/partition.py new file mode 100644 index 0000000..d552ea6 --- /dev/null +++ b/sdk/codegen/src/codegen/partition.py @@ -0,0 +1,179 @@ +"""Attribute schema symbols to the module that contributed them. + +The engine annotates every type and field it installs on behalf of a module +with ``@sourceMap(module: "")``; core symbols carry no ``module``. That +directive is the partition between the core API and each module's client. +""" + +import copy +import keyword +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Any, TypeAlias + +import graphql + +Introspection: TypeAlias = dict[str, Any] +TypeName: TypeAlias = str +FieldName: TypeAlias = str +ModuleName: TypeAlias = str + +SOURCE_MAP_DIRECTIVE = "sourceMap" +QUERY_TYPE = "Query" + +# Directives are only read off kinds the engine can attribute to a module. +_ATTRIBUTABLE_KINDS = frozenset( + {"OBJECT", "INTERFACE", "INPUT_OBJECT", "ENUM", "SCALAR"} +) + + +class PartitionError(ValueError): + """The schema cannot be split into core and module-owned symbols.""" + + +def owner_of(directives: list[dict[str, Any]] | None) -> ModuleName: + """The module named by a ``@sourceMap`` directive, or "" for core.""" + for directive in directives or (): + if directive["name"] != SOURCE_MAP_DIRECTIVE: + continue + for arg in directive["args"]: + if arg["name"] == "module": + value = graphql.value_from_ast_untyped( + graphql.parse_const_value(arg["value"]) + ) + return str(value or "") + return "" + + +@dataclass(frozen=True) +class Ownership: + """Which module owns each non-core type and each non-core field on a core type.""" + + types: dict[TypeName, ModuleName] = field(default_factory=dict) + fields: dict[tuple[TypeName, FieldName], ModuleName] = field(default_factory=dict) + + @property + def modules(self) -> frozenset[ModuleName]: + return frozenset(self.types.values()) | frozenset(self.fields.values()) + + def owned_types(self, module: ModuleName) -> frozenset[TypeName]: + return frozenset(n for n, m in self.types.items() if m == module) + + def owned_fields(self, module: ModuleName) -> frozenset[tuple[TypeName, FieldName]]: + return frozenset(k for k, m in self.fields.items() if m == module) + + +def ownership(schema: Introspection) -> Ownership: + """Read the partition off an introspection result's ``__schema``.""" + types: dict[TypeName, ModuleName] = {} + fields: dict[tuple[TypeName, FieldName], ModuleName] = {} + for type_ in schema["types"]: + name = type_["name"] + if type_["kind"] == "UNION": + msg = f"cannot attribute union type {name!r}: unions are not supported" + raise PartitionError(msg) + if type_["kind"] not in _ATTRIBUTABLE_KINDS: + continue + if owner := owner_of(type_.get("directives")): + types[name] = owner + continue + for field_ in type_.get("fields") or (): + if owner := owner_of(field_.get("directives")): + fields[(name, field_["name"])] = owner + return Ownership(types, fields) + + +def narrow_to_core(schema: Introspection) -> Introspection: + """A copy of the schema holding only what no module owns. + + Owned types go, owned fields on core types go, and references to a dropped + type from an interface's ``possibleTypes`` go with them. A module-owned type + is only ever referenced by module-owned symbols (the engine rejects a module + that exposes another module's types), so the result is closed; building a + client schema from it is what proves that. + """ + owned = ownership(schema) + narrowed = copy.deepcopy(schema) + kept: list[Introspection] = [] + for type_ in narrowed["types"]: + if type_["name"] in owned.types: + continue + if type_.get("fields") is not None: + type_["fields"] = [ + f + for f in type_["fields"] + if (type_["name"], f["name"]) not in owned.fields + ] + if type_.get("possibleTypes") is not None: + type_["possibleTypes"] = [ + t for t in type_["possibleTypes"] if t["name"] not in owned.types + ] + kept.append(type_) + narrowed["types"] = kept + return narrowed + + +def entry_field(schema: Introspection, module: ModuleName) -> Introspection: + """The ``Query`` field the module contributes: its entry point. + + The module's root type is the return type of that field. It is read off the + schema rather than derived from the module name, because the engine's name + for it is not a capitalization rule (module ``e2e`` has root type ``E2E``). + """ + query = next(t for t in schema["types"] if t["name"] == QUERY_TYPE) + owned = [ + f + for f in query.get("fields") or () + if owner_of(f.get("directives")) == module + and _named_type(f["type"])["kind"] == "OBJECT" + ] + if len(owned) != 1: + msg = ( + f"module {module!r} contributes {len(owned)} Query fields, " + "expected exactly 1" + ) + raise PartitionError(msg) + return owned[0] + + +def root_type_for(schema: Introspection, module: ModuleName) -> TypeName: + """Name of the module's root object type.""" + return _named_type(entry_field(schema, module)["type"])["name"] + + +def _named_type(type_ref: Introspection) -> Introspection: + while type_ref.get("ofType"): + type_ref = type_ref["ofType"] + return type_ref + + +def client_module_names(modules: Sequence[ModuleName]) -> list[str]: + """The client module names of several bound modules, rejecting a collision. + + Two clients of one consumer that normalize to the same file (``foo-bar`` + and ``foo_bar``) would overwrite each other, so they fail here. + """ + names = [client_module_name(module) for module in modules] + taken: dict[str, ModuleName] = {} + for module, name in zip(modules, names, strict=True): + if name in taken: + msg = ( + f"modules {taken[name]!r} and {module!r} would share the same " + f"client file: {name}.py" + ) + raise PartitionError(msg) + taken[name] = module + return names + + +def client_module_name(module: ModuleName) -> str: + """The Python module name of a bound module's client, under ``dagger.clients``.""" + from codegen.generator import format_name + + name = format_name(module.replace("-", "_").replace(".", "_")) + if not name.isidentifier() or keyword.iskeyword(name) or name.startswith("__"): + msg = ( + f"module name {module!r} does not normalize to a usable Python module name" + ) + raise PartitionError(msg) + return name diff --git a/sdk/codegen/src/codegen/version.py b/sdk/codegen/src/codegen/version.py new file mode 100644 index 0000000..9add210 --- /dev/null +++ b/sdk/codegen/src/codegen/version.py @@ -0,0 +1,40 @@ +"""Engine version handling for generated clients.""" + +import re + +from codegen.generator import parse_version + +BOUND_MODULE_FLOOR = (1, 0, 0) +"""Bound modules declared below this version render a legacy core view.""" + +_BUILD_METADATA_RE = re.compile(r"\+.*$") + + +class UnsupportedEngineVersionError(ValueError): + """A bound module declares an engine version this SDK cannot bind to.""" + + +def check_bound_module_version(engine_version: str) -> None: + """Reject a bound module whose declared engine version is below the floor. + + Only the numeric core is compared: a `v1.0.0-beta.11` prerelease is fine, a + `v0.20.8` is not. An empty or unparsable value (`latest`) resolves to the + running engine and is allowed. + """ + version = parse_version(engine_version) + if version is not None and version < BOUND_MODULE_FLOOR: + msg = ( + f"cannot generate a client for a module declaring engine version " + f"{engine_version!r}: it is rendered through a legacy core view; " + "the module needs engineVersion v1.0.0 or later" + ) + raise UnsupportedEngineVersionError(msg) + + +def cli_version(engine_version: str) -> str: + """The CLI release tag provisioning downloads for a given engine version. + + `Query.version` reports `v1.0.0-beta.11+a4e1e4ff`; the downloader adds its + own `v` and builds URLs and cache names from the bare tag. + """ + return _BUILD_METADATA_RE.sub("", engine_version.strip()).removeprefix("v") diff --git a/sdk/ruff.toml b/sdk/ruff.toml index 4dd24d2..02d0c38 100644 --- a/sdk/ruff.toml +++ b/sdk/ruff.toml @@ -82,6 +82,13 @@ ignore = [ "SLF001", "PLR0913", ] +"**/src/dagger/clients/*.py" = [ + "A", + "D", + "E501", + "SLF001", + "PLR0913", +] # Same as above, for dev module "src/dagger_gen.py" = ["A", "D", "E501", "SLF001", "PLR0913"] # Ignore built-in shadowing in test mocks. diff --git a/sdk/src/dagger/client/_binding.py b/sdk/src/dagger/client/_binding.py new file mode 100644 index 0000000..13d6d02 --- /dev/null +++ b/sdk/src/dagger/client/_binding.py @@ -0,0 +1,105 @@ +"""Serve a generated client's bound module before its first query. + +A generated client under ``dagger.clients`` carries a :class:`ModuleBinding`: +the identity of the one module it is bound to. The binding rides on the query +context of every object the client builds, and is served into the session the +first time a query through that client executes. +""" + +import dataclasses +import json +import logging +import typing + +import gql + +if typing.TYPE_CHECKING: + from dagger.client._session import BaseConnection + +logger = logging.getLogger(__name__) + +GIT_SOURCE = "GIT_SOURCE" +LOCAL_SOURCE = "LOCAL_SOURCE" + +_module_runtime = False + + +def mark_module_runtime() -> None: + """Record that this process is a Dagger module runtime. + + The engine builds a module's session with the module's dependencies and the + module itself already served, before the module runs a single query. A + module only vendors clients for those, so there is nothing a client could + serve that is not already there — and a local binding baked into a module + consumed from git would resolve against the caller's workspace, not the + module's. Serving is therefore skipped for the whole process. + """ + global _module_runtime # noqa: PLW0603 + _module_runtime = True + + +@dataclasses.dataclass(frozen=True, slots=True) +class ModuleBinding: + """The identity a generated client serves its module under. + + Parameters + ---------- + name: + The module's final name, after any dependency alias. It is the name + the client chains on ``Query`` and the name the module is served as. + kind: + ``LOCAL_SOURCE`` for a module in the workspace, ``GIT_SOURCE`` for a + remote one. + ref: + A workspace-root-relative path with a leading slash for a local + module; the canonical git ref for a remote one. + pin: + The resolved commit of a remote module, empty for a local one. + """ + + name: str + kind: str + ref: str + pin: str = "" + + def __post_init__(self): + if self.kind not in (LOCAL_SOURCE, GIT_SOURCE): + msg = f"unsupported module source kind for a client: {self.kind!r}" + raise ValueError(msg) + + def query(self) -> str: + """The document that serves the module, as raw GraphQL. + + Raw rather than built through the DSL: ``currentWorkspace`` is hidden + from a module's codegen schema but present in every live session, and + the query builder only knows the schema the session fetched on connect. + """ + serve = f"withName(name: {json.dumps(self.name)}) {{ asModule {{ serve }} }}" + if self.kind == GIT_SOURCE: + source = ( + f"moduleSource(refString: {json.dumps(self.ref)}, " + f"refPin: {json.dumps(self.pin)})" + ) + return f"{{ {source} {{ {serve} }} }}" + source = f"currentWorkspace {{ moduleSource(path: {json.dumps(self.ref)})" + return f"{{ {source} {{ {serve} }} }} }}" + + async def ensure_served(self, conn: "BaseConnection") -> None: + """Serve the module into the connection's session, once. + + Unconditional on the first use per session — the engine deduplicates a + repeat of the same source and reports a different source under the same + name — then remembered on the session so later uses cost nothing. The + schema the session caches predates the serve, so it is fetched again + before the binding is marked served. + """ + if _module_runtime: + return + session = conn.session + async with session.serve_lock: + if self in session.served: + return + logger.debug("Serving module %s from %s", self.name, self.ref) + await session.execute(gql.gql(self.query())) + await session.refetch_schema() + session.served.add(self) diff --git a/sdk/src/dagger/client/_core.py b/sdk/src/dagger/client/_core.py index 7e0fd38..97a3281 100644 --- a/sdk/src/dagger/client/_core.py +++ b/sdk/src/dagger/client/_core.py @@ -43,6 +43,7 @@ TransportError, ) from dagger._exceptions import _query_error_from_transport +from dagger.client._binding import ModuleBinding from dagger.client._session import BaseConnection, SharedConnection from dagger.client.base import Scalar, Type @@ -102,6 +103,7 @@ class Context: selections: collections.deque[Field] = dataclasses.field( default_factory=collections.deque ) + bindings: tuple[ModuleBinding, ...] = () converter: cattrs.Converter = dataclasses.field( init=False, compare=False, @@ -138,6 +140,12 @@ def select_multiple(self, type_name: str, **fields: str) -> "Context": selections.append(field_) return dataclasses.replace(self, selections=selections) + def with_binding(self, binding: ModuleBinding) -> "Context": + """Attach a bound module to serve before any query on this context runs.""" + if binding in self.bindings: + return self + return dataclasses.replace(self, bindings=(*self.bindings, binding)) + def root_select( self, field_name: str, @@ -197,6 +205,8 @@ async def execute(self, return_type: TypeForm[T] | type[T]) -> T: ... async def execute( self, return_type: TypeForm[T] | type[T] | None = None ) -> T | None: + for binding in self.bindings: + await binding.ensure_served(self.conn) await self.resolve_ids() request = await self.request() diff --git a/sdk/src/dagger/client/_session.py b/sdk/src/dagger/client/_session.py index b32b074..9d8c9c6 100644 --- a/sdk/src/dagger/client/_session.py +++ b/sdk/src/dagger/client/_session.py @@ -2,8 +2,9 @@ import logging import os from dataclasses import dataclass, field -from typing import Any +from typing import TYPE_CHECKING, Any +import anyio import gql import graphql import httpx @@ -23,6 +24,9 @@ from dagger._managers import ResourceManager from dagger.client._config import ConnectConfig, Retry +if TYPE_CHECKING: + from dagger.client._binding import ModuleBinding + logger = logging.getLogger(__name__) @@ -98,6 +102,11 @@ def __init__(self, conn: ConnectParams, cfg: ConnectConfig | None = None): self.client = retrying_client(client, cfg.retry) if cfg.retry else client self._session: AsyncClientSession | None = None + # Modules a generated client has served into this session, and the lock + # that keeps serve, schema refetch and the mark as one step. + self.served: set[ModuleBinding] = set() + self.serve_lock = anyio.Lock() + async def __aenter__(self) -> Self: await self.start() return self @@ -145,6 +154,10 @@ async def get_schema(self) -> graphql.GraphQLSchema: async def execute(self, query: gql.GraphQLRequest) -> Any: return await (await self.get_session()).execute(query) + async def refetch_schema(self) -> None: + """Fetch the schema again, after a module was served into the session.""" + await (await self.get_session()).fetch_schema() + async def close(self) -> None: logger.debug("Closing client session to GraphQL server") await super().close() diff --git a/sdk/src/dagger/clients/__init__.py b/sdk/src/dagger/clients/__init__.py new file mode 100644 index 0000000..c763c65 --- /dev/null +++ b/sdk/src/dagger/clients/__init__.py @@ -0,0 +1,5 @@ +"""Generated clients for the modules this library is bound to. + +Every module in this package is generated by the Python SDK: one per bound +module, named after it. Nothing here is written by hand. +""" diff --git a/sdk/src/dagger/mod/_converter.py b/sdk/src/dagger/mod/_converter.py index 272ce7a..c00f92a 100644 --- a/sdk/src/dagger/mod/_converter.py +++ b/sdk/src/dagger/mod/_converter.py @@ -164,7 +164,7 @@ async def exec_method(self, *args, **kwargs): @functools.cache -def to_typedef(annotation: typing.Any, context: str = "type") -> "TypeDef": # noqa: C901, PLR0911 +def to_typedef(annotation: typing.Any, context: str = "type") -> "TypeDef": # noqa: C901, PLR0911, PLR0912 """Convert Python object to API type.""" if is_initvar(annotation): return to_typedef(annotation.type, context) @@ -203,6 +203,17 @@ def to_typedef(annotation: typing.Any, context: str = "type") -> "TypeDef": # n if inspect.isclass(cls := typ.hint): name = cls.__name__ + # A generated client's types, including its bootstrap placeholder, are + # the API of another module (or of this one, seen from outside): a + # module's own API can only use its own types and the core API. + if cls.__module__.startswith("dagger.clients."): + msg = ( + f"unsupported {context}: {typ.hint!r} is a generated client " + "type; a module's API can only use its own object types " + "and the core API" + ) + raise TypeError(msg) + if is_subclass(cls, enum.Enum): return td.with_enum(name, description=get_doc(cls)) diff --git a/sdk/src/dagger/mod/cli.py b/sdk/src/dagger/mod/cli.py index 86d0866..615e40a 100644 --- a/sdk/src/dagger/mod/cli.py +++ b/sdk/src/dagger/mod/cli.py @@ -11,6 +11,7 @@ import dagger from dagger import telemetry +from dagger.client._binding import mark_module_runtime from dagger.mod._exceptions import ModuleError, ModuleLoadError, record_exception from dagger.mod._module import MAIN_OBJECT, Module @@ -19,6 +20,7 @@ ENTRY_POINT_NAME: typing.Final[str] = "main_object" ENTRY_POINT_GROUP: typing.Final[str] = typing.cast(str, __package__) IMPORT_PKG: typing.Final[str] = os.getenv("DAGGER_DEFAULT_PYTHON_PACKAGE", "main") +CLIENTS_PKG: typing.Final[str] = "dagger.clients" def app(mod: Module | None = None, register: bool = False) -> int | None: @@ -32,6 +34,7 @@ def app(mod: Module | None = None, register: bool = False) -> int | None: async def main(mod: Module | None = None, register: bool = False) -> int | None: """Async entrypoint for a Dagger module.""" + mark_module_runtime() # Establishing connection early on to allow returning dag.error(). # Note: if there's a connection error dag.error() won't be sent but # should be logged and the traceback shown on the function's stderr output. @@ -56,6 +59,18 @@ def load_module() -> Module: ep = get_entry_point() try: cls = ep.load() + except ModuleNotFoundError as e: + if e.name and (e.name == CLIENTS_PKG or e.name.startswith(f"{CLIENTS_PKG}.")): + msg = ( + f"generated client '{e.name}' is missing; run `dagger generate` " + "and commit the generated files" + ) + raise ModuleLoadError(msg) from e + logger.exception( + "Error while importing Python module '%s' with Dagger functions", + ep.module, + ) + raise ModuleLoadError(str(e)) from e except Exception as e: logger.exception( "Error while importing Python module '%s' with Dagger functions", diff --git a/sdk/tests/client/test_binding.py b/sdk/tests/client/test_binding.py new file mode 100644 index 0000000..f963a80 --- /dev/null +++ b/sdk/tests/client/test_binding.py @@ -0,0 +1,258 @@ +import dataclasses +from typing import Any + +import anyio +import graphql +import pytest + +from dagger.client import _binding +from dagger.client._binding import ModuleBinding +from dagger.client._core import Context +from dagger.client.base import Type + +pytestmark = pytest.mark.anyio + +LOCAL = ModuleBinding(name="hello", kind="LOCAL_SOURCE", ref="/mods/hello") +GIT = ModuleBinding( + name="hello", + kind="GIT_SOURCE", + ref="github.com/acme/hello@v1", + pin="abc123", +) + + +class FakeSession: + """Just enough of ClientSession for the preamble and the query builder.""" + + def __init__(self, schema: graphql.GraphQLSchema | None = None): + self.schema = schema or graphql.build_schema("type Query { hello: String }") + self.response: Any = {"hello": "hi"} + self.served: set[ModuleBinding] = set() + self.serve_lock = anyio.Lock() + self.executed: list[str] = [] + self.refetches = 0 + self.fail_serve = False + self.fail_refetch = False + self.delay = 0.0 + + async def execute(self, request) -> Any: + self.executed.append(graphql.print_ast(request.document)) + if self.delay: + await anyio.sleep(self.delay) + if self.fail_serve and "serve" in self.executed[-1]: + msg = "serve failed" + raise RuntimeError(msg) + return self.response + + async def refetch_schema(self) -> None: + self.refetches += 1 + if self.fail_refetch: + msg = "refetch failed" + raise RuntimeError(msg) + + async def get_schema(self) -> graphql.GraphQLSchema: + return self.schema + + +@dataclasses.dataclass +class FakeConnection: + session: FakeSession + + +@pytest.fixture +def session(): + return FakeSession() + + +@pytest.fixture +def conn(session): + return FakeConnection(session) + + +@pytest.fixture +def standalone(monkeypatch): + monkeypatch.setattr(_binding, "_module_runtime", False) + + +@pytest.fixture +def module_runtime(monkeypatch): + monkeypatch.setattr(_binding, "_module_runtime", True) + + +def test_rejects_a_kind_a_client_cannot_serve(): + with pytest.raises(ValueError, match="DIR_SOURCE"): + ModuleBinding(name="x", kind="DIR_SOURCE", ref="/x") + + +def test_local_query_serves_by_workspace_path_under_the_final_name(): + doc = graphql.parse(LOCAL.query()) + assert graphql.print_ast(doc) == graphql.print_ast( + graphql.parse( + """ + { + currentWorkspace { + moduleSource(path: "/mods/hello") { + withName(name: "hello") { asModule { serve } } + } + } + } + """ + ) + ) + + +def test_git_query_serves_by_ref_and_pin(): + doc = graphql.parse(GIT.query()) + assert graphql.print_ast(doc) == graphql.print_ast( + graphql.parse( + """ + { + moduleSource(refString: "github.com/acme/hello@v1", refPin: "abc123") { + withName(name: "hello") { asModule { serve } } + } + } + """ + ) + ) + + +@pytest.mark.usefixtures("standalone") +async def test_serves_once_per_session_then_refetches(conn, session): + await LOCAL.ensure_served(conn) + await LOCAL.ensure_served(conn) + + assert len(session.executed) == 1 + assert "serve" in session.executed[0] + assert session.refetches == 1 + assert session.served == {LOCAL} + + +@pytest.mark.usefixtures("standalone") +async def test_a_second_session_serves_again(conn): + await LOCAL.ensure_served(conn) + other = FakeConnection(FakeSession()) + await LOCAL.ensure_served(other) + + assert len(other.session.executed) == 1 + + +@pytest.mark.usefixtures("standalone") +async def test_different_bindings_each_serve(conn, session): + await LOCAL.ensure_served(conn) + await GIT.ensure_served(conn) + + assert len(session.executed) == 2 + assert session.served == {LOCAL, GIT} + + +@pytest.mark.usefixtures("module_runtime") +async def test_module_runtime_never_serves(conn, session, monkeypatch): + # The variable a user might export is irrelevant: only the runtime's own + # mark counts. + monkeypatch.delenv("DAGGER_MODULE", raising=False) + await LOCAL.ensure_served(conn) + + assert session.executed == [] + assert session.refetches == 0 + + +@pytest.mark.usefixtures("standalone") +async def test_dagger_module_variable_does_not_mark_a_runtime( + conn, session, monkeypatch +): + monkeypatch.setenv("DAGGER_MODULE", "hello") + await LOCAL.ensure_served(conn) + + assert len(session.executed) == 1 + + +@pytest.mark.usefixtures("standalone") +async def test_concurrent_first_uses_serve_once(conn, session): + session.delay = 0.01 + async with anyio.create_task_group() as tg: + for _ in range(5): + tg.start_soon(LOCAL.ensure_served, conn) + + assert len(session.executed) == 1 + assert session.refetches == 1 + + +@pytest.mark.usefixtures("standalone") +async def test_failed_serve_marks_nothing_and_retries(conn, session): + session.fail_serve = True + with pytest.raises(RuntimeError, match="serve failed"): + await LOCAL.ensure_served(conn) + assert session.served == set() + assert session.refetches == 0 + + session.fail_serve = False + await LOCAL.ensure_served(conn) + assert session.served == {LOCAL} + assert len(session.executed) == 2 + + +@pytest.mark.usefixtures("standalone") +async def test_failed_refetch_marks_nothing(conn, session): + session.fail_refetch = True + with pytest.raises(RuntimeError, match="refetch failed"): + await LOCAL.ensure_served(conn) + + assert session.served == set() + + +def test_binding_travels_with_the_query_context(conn): + ctx = Context(conn).with_binding(LOCAL) + + assert ctx.with_binding(LOCAL) is ctx + assert ctx.root_select("hello", []).bindings == (LOCAL,) + assert ctx.select("Query", "hello", []).bindings == (LOCAL,) + assert ctx.select_id("Hello", "id").bindings == (LOCAL,) + + +@pytest.mark.usefixtures("standalone") +async def test_execute_serves_before_the_query(conn, session): + ctx = Context(conn).with_binding(LOCAL).root_select("hello", []) + + assert await ctx.execute(str) == "hi" + assert len(session.executed) == 2 + assert "serve" in session.executed[0] + assert "hello" in session.executed[1] + assert "serve" not in session.executed[1] + + +OBJECT_SDL = """ +interface Node { id: ID! } +type Hello implements Node { id: ID! sync: ID! } +type Query { hello: Hello! hellos: [Hello!]! node(id: ID!): Node } +""" + + +class Hello(Type): + __slots__ = () + + +@pytest.fixture +def object_conn(): + return FakeConnection(FakeSession(graphql.build_schema(OBJECT_SDL))) + + +@pytest.mark.usefixtures("standalone") +async def test_execute_object_list_serves_and_keeps_the_binding(object_conn): + object_conn.session.response = {"hellos": [{"id": "hello-1"}]} + ctx = Context(object_conn).with_binding(LOCAL).root_select("hellos", []) + + [hello] = await ctx.execute_object_list(Hello) + + assert "serve" in object_conn.session.executed[0] + assert hello._ctx.bindings == (LOCAL,) + + +@pytest.mark.usefixtures("standalone") +async def test_execute_sync_serves_and_keeps_the_binding(object_conn): + object_conn.session.response = {"hello": {"sync": "hello-1"}} + ctx = Context(object_conn).with_binding(LOCAL).root_select("hello", []) + + synced = await ctx.execute_sync(Hello(ctx)) + + assert "serve" in object_conn.session.executed[0] + assert synced._ctx.bindings == (LOCAL,) diff --git a/sdk/tests/codegen/conftest.py b/sdk/tests/codegen/conftest.py new file mode 100644 index 0000000..c9e4fdd --- /dev/null +++ b/sdk/tests/codegen/conftest.py @@ -0,0 +1,58 @@ +"""Build engine-shaped introspection results from SDL for the codegen tests. + +The engine's introspection is not the standard one: every type, field, +argument, input field and enum value carries a ``directives`` list, and the +symbols a module contributes are marked ``@sourceMap(module: "")``. +""" + +import json +from collections.abc import Callable +from typing import Any + +import graphql +import pytest + +Owners = dict[str | tuple[str, str], str] + + +def _source_map(module: str) -> dict[str, Any]: + return { + "name": "sourceMap", + "args": [{"name": "module", "value": json.dumps(module)}], + } + + +def _with_directives(node: dict[str, Any], directives: list[dict[str, Any]]) -> None: + node["directives"] = directives + + +def introspect(sdl: str, owners: Owners | None = None) -> dict[str, Any]: + """Introspect an SDL schema the way the engine does. + + ``owners`` maps a type name, or a ``(type, field)`` pair, to the module that + contributes it. Everything else is core. + """ + owners = owners or {} + schema = graphql.build_schema(sdl) + result: dict[str, Any] = dict(graphql.introspection_from_schema(schema)) + for type_ in result["__schema"]["types"]: + name = type_["name"] + _with_directives(type_, [_source_map(owners[name])] if name in owners else []) + for field_ in type_.get("fields") or (): + key = (name, field_["name"]) + _with_directives( + field_, [_source_map(owners[key])] if key in owners else [] + ) + for arg in field_.get("args") or (): + _with_directives(arg, []) + for input_field in type_.get("inputFields") or (): + _with_directives(input_field, []) + for value in type_.get("enumValues") or (): + _with_directives(value, []) + result["__schemaVersion"] = "v1.0.0" + return result + + +@pytest.fixture +def introspection() -> Callable[..., dict[str, Any]]: + return introspect diff --git a/sdk/tests/codegen/test_client_mode.py b/sdk/tests/codegen/test_client_mode.py new file mode 100644 index 0000000..e843585 --- /dev/null +++ b/sdk/tests/codegen/test_client_mode.py @@ -0,0 +1,504 @@ +import argparse +import ast +import importlib.util +import json +import re +import sys +import types + +import pytest + +import dagger +from codegen.cli import render, run_client_name +from codegen.partition import PartitionError +from dagger.client._binding import ModuleBinding + +SDL = """ +interface Node { id: ID! } +interface Exportable { id: ID! export(path: String!): String! } + +type Query { + container: Container! + e2E: E2E! + hello(name: String!, greeting: String = "hi"): Hello! + node(id: ID!): Node + version: String! +} + +type Container implements Node & Exportable { + id: ID! + stdout: String! + export(path: String!): String! +} + +type Directory implements Node { + id: ID! + entries: [String!]! + asHello: Hello! + tag(kind: HelloKind!): Directory! +} + +type File implements Node { + id: ID! + name: String! + asHello: Hello! + withHelloInput(name: String!, value: String!): File! +} + +scalar JSON +scalar HelloToken +enum CacheSharingMode { SHARED LOCKED } +enum HelloKind { FULL SHORT } +input HelloOpts { verbose: Boolean } + +type Hello implements Node { + id: ID! + greet(name: String!): String! + report(kind: HelloKind = FULL): HelloReport! + reports: [HelloReport!]! + build(opts: HelloOpts, cache: CacheSharingMode = SHARED): Container! + exportable: Exportable! + token: HelloToken! + json: JSON! + withGreeting(greeting: String!): Hello! + sync: ID! +} + +type HelloReport implements Node { + id: ID! + text: String! + hello: Hello! +} + +type E2E implements Node { + id: ID! + run: String! +} +""" + +HELLO_OWNERS = { + "Hello": "hello", + "HelloReport": "hello", + "HelloKind": "hello", + "HelloOpts": "hello", + "HelloToken": "hello", + ("Hello", "greet"): "hello", + ("Hello", "report"): "hello", + ("Hello", "reports"): "hello", + ("Hello", "build"): "hello", + ("Hello", "exportable"): "hello", + ("Hello", "token"): "hello", + ("Hello", "json"): "hello", + ("Hello", "withGreeting"): "hello", + ("Hello", "sync"): "hello", + ("HelloReport", "text"): "hello", + ("HelloReport", "hello"): "hello", + ("Query", "hello"): "hello", + ("Directory", "asHello"): "hello", + ("Directory", "tag"): "hello", + ("File", "asHello"): "hello", + ("File", "withHelloInput"): "hello", +} + +E2E_OWNERS = { + "E2E": "e2e", + ("E2E", "run"): "e2e", + ("Query", "e2E"): "e2e", +} + +LOCAL_BINDING = { + "name": "hello", + "kind": "LOCAL_SOURCE", + "ref": "/.dagger/modules/hello", + "pin": "", +} +GIT_BINDING = { + "name": "hello", + "kind": "GIT_SOURCE", + "ref": "github.com/acme/hello", + "pin": "abc123", +} + + +@pytest.fixture +def result(introspection): + return introspection(SDL, HELLO_OWNERS | E2E_OWNERS) + + +@pytest.fixture +def hello_client(result) -> str: + return render(result, mode="client", module="hello", binding=LOCAL_BINDING) + + +def load(source: str, name: str = "dagger.clients.hello") -> types.ModuleType: + """Import generated source as a module, against the real dagger package.""" + spec = importlib.util.spec_from_loader(name, loader=None) + assert spec is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + exec(compile(source, f"{name}.py", "exec"), module.__dict__) + finally: + sys.modules.pop(name, None) + return module + + +def signature(source: str, name: str) -> str: + match = re.search( + rf"^(?:async )?def {name}\((.*?)\) -> (.*?):$", + source, + re.MULTILINE | re.DOTALL, + ) + assert match, f"no function {name} in generated source" + return " ".join(match.group(0).split()).replace("( ", "(").replace(",)", ")") + + +def describe(result: dict, description: str, *, reason: str | None = None) -> None: + """Give every symbol the hello module owns the same description.""" + for type_ in result["__schema"]["types"]: + if not type_["name"].startswith("Hello"): + continue + type_["description"] = description + for field in type_.get("fields") or (): + field["description"] = description + for arg in field.get("args") or (): + arg["description"] = description + if reason is not None and field["name"] == "greet": + field["isDeprecated"] = True + field["deprecationReason"] = reason + for input_field in type_.get("inputFields") or (): + input_field["description"] = description + for value in type_.get("enumValues") or (): + value["description"] = description + + +class TestCoreMode: + def test_drops_owned_symbols_and_keeps_the_root(self, result): + core = render(result) + + assert "class Hello(" not in core + assert "class E2E(" not in core + assert "def hello(" not in core + assert "def as_hello(" not in core + assert "class Container(Type):" in core + assert "class Client(Query):" in core + assert "dag = Client()" in core + assert "class File(Type):" in core + assert "def as_hello(" not in core + + def test_is_identical_whichever_module_the_schema_was_bound_to(self, introspection): + core = "type Container { id: ID! stdout: String! }" + bound_to_hello = introspection( + core + " type Query { container: Container! hello: Hello! }" + " type Hello { id: ID! greet: String! }", + { + "Hello": "hello", + ("Hello", "greet"): "hello", + ("Query", "hello"): "hello", + }, + ) + bound_to_e2e = introspection( + core + " type Query { container: Container! e2E: E2E! }" + " type E2E { id: ID! run: String! }", + {"E2E": "e2e", ("E2E", "run"): "e2e", ("Query", "e2E"): "e2e"}, + ) + assert render(bound_to_hello) == render(bound_to_e2e) + + def test_default_mode_matches_todays_output_for_a_module_free_schema( + self, introspection + ): + # No owners: nothing to narrow, so the historical single-mode output. + core = render(introspection(SDL)) + assert "class Hello(Type):" in core + assert "def hello(self, name: str, *, greeting: str | None = " in core + + +class TestClientMode: + def test_binding_is_rendered_verbatim(self, result): + local = render(result, mode="client", module="hello", binding=LOCAL_BINDING) + assert ( + "_BINDING = ModuleBinding(\n" + " name='hello',\n" + " kind='LOCAL_SOURCE',\n" + " ref='/.dagger/modules/hello',\n" + " pin='',\n" + ")" + ) in local + + git = render(result, mode="client", module="hello", binding=GIT_BINDING) + assert "kind='GIT_SOURCE'" in git + assert "ref='github.com/acme/hello'" in git + assert "pin='abc123'" in git + + # Only the binding differs between the two. + assert local.replace("LOCAL_SOURCE", "").replace( + "/.dagger/modules/hello", "" + ) == git.replace("GIT_SOURCE", "").replace("github.com/acme/hello", "").replace( + "abc123", "" + ) + + def test_renders_only_owned_types(self, hello_client): + assert "class Hello(Type):" in hello_client + assert "class HelloReport(Type):" in hello_client + assert "class HelloKind(Enum):" in hello_client + assert "class HelloOpts(Input):" in hello_client + assert "class HelloToken(Scalar):" in hello_client + assert "class E2E(" not in hello_client + assert "class Container(" not in hello_client + assert "class Query(" not in hello_client + assert "class Client(" not in hello_client + assert "dag = " not in hello_client + + def test_entry_point_takes_constructor_args_and_a_keyword_only_client( + self, hello_client + ): + assert signature(hello_client, "hello") == ( + "def hello(name: str, *, greeting: str | None = 'hi', " + "client: _core.Client | None = None) -> Hello:" + ) + assert ( + "_ctx = (_core.dag if client is None else client)._ctx" + '.with_binding(_BINDING).root_select("hello", _args) # noqa: SLF001' + ) in hello_client + assert "Arg(\"greeting\", greeting, 'hi')," in hello_client + assert "return Hello(_ctx)" in hello_client + assert "Client for the `hello` module." in hello_client + + def test_core_references_go_through_the_alias(self, hello_client): + # return annotation, construction, list element and interface impl + assert "-> _core.Container:" in hello_client + assert "return _core.Container(_ctx)" in hello_client + assert "-> _core.Exportable:" in hello_client + assert "return _core._ExportableClient(_ctx)" in hello_client + assert "return await _ctx.execute(_core.JSON)" in hello_client + # enum defaults and parameter annotations + assert ( + "cache: _core.CacheSharingMode | None = _core.CacheSharingMode.SHARED" + in (hello_client) + ) + # owned symbols stay bare + assert "return await _ctx.execute(HelloToken)" in hello_client + assert "kind: HelloKind | None = HelloKind.FULL" in hello_client + assert "return await _ctx.execute_object_list(HelloReport)" in hello_client + assert "-> Self:" in hello_client # withGreeting + + def test_owned_fields_on_other_core_types_become_functions(self, hello_client): + # every function on a core type is qualified by its parent. + assert signature(hello_client, "directory_as_hello") == ( + "def directory_as_hello(directory: _core.Directory) -> Hello:" + ) + assert signature(hello_client, "file_as_hello") == ( + "def file_as_hello(file: _core.File) -> Hello:" + ) + assert ( + "_ctx = file._ctx.with_binding(_BINDING)" + '.select("File", "asHello", _args) # noqa: SLF001' + ) in hello_client + assert signature(hello_client, "file_with_hello_input") == ( + "def file_with_hello_input(file: _core.File, name: str, value: str)" + " -> _core.File:" + ) + + def test_an_owned_type_is_bare_even_on_a_core_parent(self, hello_client): + assert signature(hello_client, "directory_tag") == ( + "def directory_tag(directory: _core.Directory, kind: HelloKind)" + " -> _core.Directory:" + ) + assert 'Arg("kind", kind),' in hello_client + assert ( + "_ctx = directory._ctx.with_binding(_BINDING)" + '.select("Directory", "tag", _args) # noqa: SLF001' + ) in hello_client + assert "return _core.Directory(_ctx)" in hello_client + + @pytest.mark.parametrize( + "description", + [ + 'a """block""" quote', + "a windows path C:\\Users\\x", + "a trailing backslash \\", + 'a trailing quote "', + 'ends with a block quote """', + 'multi\nline """ with C:\\x\nand a trailing \\', + "a" * 33 + "\\" + "b" * 100, + "z" * 200, + ], + ) + def test_a_tricky_description_renders_valid_python(self, result, description): + describe(result, description) + + client = render(result, mode="client", module="hello", binding=LOCAL_BINDING) + + compile(client, "dagger/clients/hello.py", "exec") + + @pytest.mark.parametrize( + "description", + [ + 'a trailing quote "', + '\\"""', + "carriage\rreturn", + "nul\x00byte", + 'C:\\Users\\x "q" \\', + "a" * 33 + "\\" + "b" * 100, + ], + ) + def test_descriptions_and_deprecation_reasons_are_escaped( + self, result, description + ): + reason = 'use "greet"\nor C:\\x\x00 instead\\' + describe(result, description, reason=reason) + + client = render(result, mode="client", module="hello", binding=LOCAL_BINDING) + module = load(client) + + assert module.Hello.__doc__ == description + warning = re.search(r'warnings\.warn\(\s*("(?:\\.|[^"\\])*")', client) + assert warning + assert ast.literal_eval(warning.group(1)) == ( + f'Method "greet" is deprecated: {reason}' + ) + + def test_exports(self, hello_client): + exported = re.search(r"__all__ = \[\n(.*?)\]", hello_client, re.DOTALL) + assert exported + names = {line.strip().strip('",') for line in exported.group(1).splitlines()} + assert names == { + "Hello", + "HelloReport", + "HelloKind", + "HelloOpts", + "HelloToken", + "hello", + "directory_as_hello", + "file_as_hello", + "directory_tag", + "file_with_hello_input", + } + assert "_BINDING" not in names + + def test_root_type_is_read_off_the_schema(self, result): + e2e = render( + result, + mode="client", + module="e2e", + binding={"name": "e2e", "kind": "LOCAL_SOURCE", "ref": "/e2e", "pin": ""}, + ) + assert "class E2E(Type):" in e2e + assert signature(e2e, "e2e") == ( + "def e2e(*, client: _core.Client | None = None) -> E2E:" + ) + assert "class Hello(" not in e2e + + def test_imports_and_runs_against_the_real_package(self, hello_client): + module = load(hello_client) + + assert ( + ModuleBinding( + name="hello", kind="LOCAL_SOURCE", ref="/.dagger/modules/hello", pin="" + ) + == module._BINDING + ) + hello = module.hello("world") + assert isinstance(hello, module.Hello) + assert hello._ctx.bindings == (module._BINDING,) + assert [f.name for f in hello._ctx.selections] == ["hello"] + assert hello._ctx.selections[0].args == {"name": "world"} + + # an explicit client is honoured, and core types are the real ones + other = dagger.Client() + assert module.hello("x", client=other)._ctx.conn is other._ctx.conn + assert module.file_as_hello(dagger.File(other._ctx)) + + def test_argument_named_client_is_escaped(self, introspection): + result = introspection( + "type Query { hello(client: String!): Hello! }" + " type Hello { greet: String! }", + { + "Hello": "hello", + ("Query", "hello"): "hello", + ("Hello", "greet"): "hello", + }, + ) + client = render(result, mode="client", module="hello", binding=LOCAL_BINDING) + assert signature(client, "hello") == ( + "def hello(client_: str, *, client: _core.Client | None = None) -> Hello:" + ) + assert 'Arg("client", client_),' in client + + def test_colliding_argument_names_are_rejected(self, introspection): + result = introspection( + "type Query { hello(client: String!, client_: String!): Hello! } " + "type Hello { greet: String! }", + { + "Hello": "hello", + ("Query", "hello"): "hello", + ("Hello", "greet"): "hello", + }, + ) + with pytest.raises(PartitionError, match="client_"): + render(result, mode="client", module="hello", binding=LOCAL_BINDING) + + @pytest.mark.parametrize( + ("module", "root"), [("arg", "Arg"), ("client", "Client"), ("type", "Type")] + ) + def test_a_module_whose_names_the_client_file_binds_is_rejected( + self, introspection, module, root + ): + result = introspection( + f"type Query {{ {module}: {root}! }} type {root} {{ greet: String! }}", + {root: module, ("Query", module): module, (root, "greet"): module}, + ) + with pytest.raises(PartitionError, match=root): + render(result, mode="client", module=module, binding=LOCAL_BINDING) + + def test_a_root_type_named_after_a_core_type_is_rejected(self, introspection): + result = introspection( + "type Query { container: Container! } type Container { id: ID! }", + {("Query", "container"): "container"}, + ) + with pytest.raises(PartitionError, match="core type"): + render(result, mode="client", module="container", binding=LOCAL_BINDING) + + def test_module_without_an_entry_point_is_rejected(self, result): + with pytest.raises(PartitionError, match="expected exactly 1"): + render(result, mode="client", module="nobody", binding=LOCAL_BINDING) + + def test_client_mode_requires_module_and_binding(self, result): + with pytest.raises(ValueError, match="--module and --binding"): + render(result, mode="client") + + def test_bound_module_below_the_floor_is_rejected(self, result): + with pytest.raises(ValueError, match=re.escape("v0.20.8")): + render( + result, + mode="client", + module="hello", + binding=LOCAL_BINDING, + engine_version="v0.20.8", + ) + for ok in ("v1.0.0-beta.11", "v1.0.0-0", "latest", ""): + render( + result, + mode="client", + module="hello", + binding=LOCAL_BINDING, + engine_version=ok, + ) + + +def test_client_name_prints_one_name_per_line(capsys): + run_client_name(argparse.Namespace(module=["foo-bar", "e2e"])) + + assert capsys.readouterr().out == "foo_bar\ne2e\n" + + +def test_client_name_rejects_two_modules_that_share_a_file(capsys): + with pytest.raises(SystemExit) as exc: + run_client_name(argparse.Namespace(module=["foo-bar", "foo_bar"])) + + assert exc.value.code == 2 + assert "foo_bar.py" in capsys.readouterr().err + + +def test_binding_round_trips_through_json(): + assert json.loads(json.dumps(LOCAL_BINDING)) == LOCAL_BINDING diff --git a/sdk/tests/codegen/test_generator.py b/sdk/tests/codegen/test_generator.py index e83b20a..2c9737b 100644 --- a/sdk/tests/codegen/test_generator.py +++ b/sdk/tests/codegen/test_generator.py @@ -560,7 +560,7 @@ def test_enum_render(type_, expected, ctx: Context): ), ( 'Example: "foobar"', - r'"""Example: "foobar" """', + '"""Example: "foobar\\""""', ), ( 'Lorem ipsum dolores est.\n\nExample: "foobar"', @@ -568,7 +568,7 @@ def test_enum_render(type_, expected, ctx: Context): '''\ """Lorem ipsum dolores est. - Example: "foobar" + Example: "foobar\\" """''', ), ), diff --git a/sdk/tests/codegen/test_partition.py b/sdk/tests/codegen/test_partition.py new file mode 100644 index 0000000..62fc35a --- /dev/null +++ b/sdk/tests/codegen/test_partition.py @@ -0,0 +1,247 @@ +import copy +import re + +import graphql +import pytest + +from codegen import ast +from codegen.partition import ( + PartitionError, + client_module_name, + client_module_names, + entry_field, + narrow_to_core, + owner_of, + ownership, + root_type_for, +) + +SDL = """ +interface Node { id: ID! } + +type Query { + container: Container! + e2E: E2E! + hello(name: String!): Hello! + node(id: ID!): Node +} + +type Container implements Node { + id: ID! + stdout: String! +} + +type Binding { + name: String! + asHello: Hello! +} + +type Hello implements Node { + id: ID! + greet(name: String!): String! + report(kind: HelloKind = FULL): HelloReport! + build(opts: HelloOpts): Container! +} + +type HelloReport implements Node { + id: ID! + text: String! +} + +enum HelloKind { FULL SHORT } + +input HelloOpts { verbose: Boolean } + +scalar HelloToken + +type E2E implements Node { + id: ID! + run: String! +} +""" + +OWNERS = { + "Hello": "hello", + "HelloReport": "hello", + "HelloKind": "hello", + "HelloOpts": "hello", + "HelloToken": "hello", + ("Hello", "greet"): "hello", + ("Hello", "report"): "hello", + ("Hello", "build"): "hello", + ("Query", "hello"): "hello", + ("Binding", "asHello"): "hello", + "E2E": "e2e", + ("E2E", "run"): "e2e", + ("Query", "e2E"): "e2e", +} + + +@pytest.fixture +def schema(introspection): + return introspection(SDL, OWNERS)["__schema"] + + +def type_names(schema) -> set[str]: + return {t["name"] for t in schema["types"]} + + +def field_names(schema, type_name: str) -> set[str]: + type_ = next(t for t in schema["types"] if t["name"] == type_name) + return {f["name"] for f in type_["fields"]} + + +def test_owner_of_reads_the_json_quoted_module(): + directives = [{"name": "sourceMap", "args": [{"name": "module", "value": '"e2e"'}]}] + assert owner_of(directives) == "e2e" + assert owner_of([]) == "" + assert owner_of(None) == "" + + +def test_ownership_splits_types_and_fields(schema): + owned = ownership(schema) + assert owned.owned_types("hello") == { + "Hello", + "HelloReport", + "HelloKind", + "HelloOpts", + "HelloToken", + } + # Fields on owned types are the type's; only fields on core types are listed. + assert owned.owned_fields("hello") == {("Query", "hello"), ("Binding", "asHello")} + assert owned.owned_types("e2e") == {"E2E"} + assert owned.owned_fields("e2e") == {("Query", "e2E")} + assert owned.modules == {"hello", "e2e"} + + +def test_scalar_ownership_is_read(schema): + assert ownership(schema).types["HelloToken"] == "hello" + + +def test_insert_stubs_leaves_the_shared_specified_scalars_alone(schema): + built = graphql.build_client_schema({"__schema": schema}) + ast.insert_stubs(schema, built) + + assert built.get_type("HelloToken").ast_node is not None + assert graphql.GraphQLString.ast_node is None + assert graphql.GraphQLID.ast_node is None + + +def test_narrow_to_core_drops_owned_symbols_and_prunes_references(schema): + core = narrow_to_core(schema) + + assert type_names(core) == { + "Query", + "Node", + "Container", + "Binding", + "String", + "Boolean", + "ID", + } | {t["name"] for t in schema["types"] if t["name"].startswith("__")} + assert field_names(core, "Query") == {"container", "node"} + assert field_names(core, "Binding") == {"name"} + node = next(t for t in core["types"] if t["name"] == "Node") + assert [t["name"] for t in node["possibleTypes"]] == ["Container"] + + +def test_narrow_to_core_builds_a_valid_client_schema(schema): + built = graphql.build_client_schema({"__schema": narrow_to_core(schema)}) + assert built.get_type("Container") is not None + assert built.get_type("Hello") is None + assert "hello" not in built.query_type.fields + + +def test_narrow_to_core_does_not_mutate_its_input(schema): + before = copy.deepcopy(schema) + narrow_to_core(schema) + assert schema == before + + +CORE_SDL = """ +interface Node { id: ID! } +type Container implements Node { id: ID! stdout: String! } +""" + + +def test_core_is_the_same_whichever_module_the_schema_was_bound_to(introspection): + bound_to_hello = introspection( + CORE_SDL + + """ + type Query { container: Container! hello: Hello! } + type Hello implements Node { id: ID! greet: String! } + """, + {"Hello": "hello", ("Hello", "greet"): "hello", ("Query", "hello"): "hello"}, + )["__schema"] + bound_to_e2e = introspection( + CORE_SDL + + """ + type Query { container: Container! e2E: E2E! } + type E2E implements Node { id: ID! run: String! } + """, + {"E2E": "e2e", ("E2E", "run"): "e2e", ("Query", "e2E"): "e2e"}, + )["__schema"] + + assert narrow_to_core(bound_to_hello) == narrow_to_core(bound_to_e2e) + + +def test_entry_field_and_root_type_are_read_off_the_schema(schema): + assert entry_field(schema, "hello")["name"] == "hello" + assert root_type_for(schema, "hello") == "Hello" + # Not a capitalization rule: module e2e has root type E2E. + assert entry_field(schema, "e2e")["name"] == "e2E" + assert root_type_for(schema, "e2e") == "E2E" + + +def test_entry_field_requires_exactly_one_owned_query_field(schema): + with pytest.raises(PartitionError, match="expected exactly 1"): + entry_field(schema, "nobody") + + +def test_unions_are_rejected(introspection): + result = introspection( + """ + type Query { pick: Pick! } + type A { a: String! } + type B { b: String! } + union Pick = A | B + """ + ) + with pytest.raises(PartitionError, match="union"): + ownership(result["__schema"]) + + +@pytest.mark.parametrize( + ("module", "expected"), + [ + ("hello", "hello"), + ("hello-world", "hello_world"), + ("helloWorld", "hello_world"), + ("HelloWorld", "hello_world"), + ("hello.world", "hello_world"), + ("import", "import_"), + ("e2e", "e2e"), + ("my-sdk2", "my_sdk2"), + ], +) +def test_client_module_name(module, expected): + assert client_module_name(module) == expected + + +@pytest.mark.parametrize("module", ["__init__", "__main__", "2fa", "", "a b"]) +def test_client_module_name_rejects_unusable_names(module): + with pytest.raises(PartitionError): + client_module_name(module) + + +def test_client_module_names_normalizes_each_in_order(): + assert client_module_names(["hello-world", "e2e", "import"]) == [ + "hello_world", + "e2e", + "import_", + ] + + +def test_client_module_names_rejects_two_that_share_a_file(): + with pytest.raises(PartitionError, match=re.escape("foo_bar.py")): + client_module_names(["foo-bar", "dep", "foo_bar"]) diff --git a/sdk/tests/codegen/test_version.py b/sdk/tests/codegen/test_version.py new file mode 100644 index 0000000..ed6cf3b --- /dev/null +++ b/sdk/tests/codegen/test_version.py @@ -0,0 +1,49 @@ +import pytest + +from codegen.version import ( + UnsupportedEngineVersionError, + check_bound_module_version, + cli_version, +) +from dagger.provisioning._download import Downloader, Platform + + +@pytest.mark.parametrize( + ("engine", "expected"), + [ + ("v1.0.0-beta.11+a4e1e4ff", "1.0.0-beta.11"), + ("v1.0.0-beta.11", "1.0.0-beta.11"), + ("1.0.0", "1.0.0"), + ("v1.2.3+abc", "1.2.3"), + (" v1.0.0-rc.1\n", "1.0.0-rc.1"), + ], +) +def test_cli_version_is_the_bare_release_tag(engine, expected): + assert cli_version(engine) == expected + + +def test_stamped_version_builds_the_release_url(): + downloader = Downloader( + cli_version("v1.0.0-beta.11+a4e1e4ff"), + platform=Platform("linux", "amd64"), + ) + assert str(downloader.archive_url) == ( + "https://dl.dagger.io/dagger/releases/1.0.0-beta.11/" + "dagger_v1.0.0-beta.11_linux_amd64.tar.gz" + ) + + +@pytest.mark.parametrize("engine", ["v1.0.0-beta.11", "v1.0.0-0", "v1.0.0", "2.3.4"]) +def test_modern_bound_modules_pass_the_floor(engine): + check_bound_module_version(engine) + + +@pytest.mark.parametrize("engine", ["", "latest", "main", "dev"]) +def test_unparsable_versions_resolve_to_the_engine_and_pass(engine): + check_bound_module_version(engine) + + +@pytest.mark.parametrize("engine", ["v0.20.8", "0.21.0", "v0.99.99"]) +def test_legacy_bound_modules_are_rejected(engine): + with pytest.raises(UnsupportedEngineVersionError, match=engine): + check_bound_module_version(engine) diff --git a/sdk/tests/mod/test_client_types.py b/sdk/tests/mod/test_client_types.py new file mode 100644 index 0000000..0b89b30 --- /dev/null +++ b/sdk/tests/mod/test_client_types.py @@ -0,0 +1,60 @@ +import enum +import sys +import types + +import pytest + +from dagger.client.base import Scalar, Type +from dagger.mod._converter import to_typedef +from dagger.mod._exceptions import ModuleLoadError +from dagger.mod.cli import load_module + + +@pytest.fixture +def client_module(monkeypatch): + """Types the way a generated client under dagger.clients defines them.""" + module = types.ModuleType("dagger.clients.hello") + monkeypatch.setitem(sys.modules, "dagger.clients.hello", module) + + class Hello(Type): + __slots__ = () + + async def id(self) -> str: + return "" + + class HelloKind(enum.Enum): + FULL = "FULL" + + class HelloToken(Scalar): + __slots__ = () + + class _NotGenerated: + """The bootstrap stub's placeholder.""" + + for cls in (Hello, HelloKind, HelloToken, _NotGenerated): + cls.__module__ = "dagger.clients.hello" + setattr(module, cls.__name__, cls) + return module + + +@pytest.mark.parametrize("name", ["Hello", "HelloKind", "HelloToken", "_NotGenerated"]) +@pytest.mark.parametrize("context", ["return type", "argument"]) +def test_client_types_are_not_module_api_types(client_module, name, context): + to_typedef.cache_clear() + with pytest.raises(TypeError, match="generated client type"): + to_typedef(getattr(client_module, name), context) + + +@pytest.mark.parametrize("missing", ["dagger.clients.hello", "dagger.clients"]) +def test_missing_client_is_a_generate_and_commit_error(monkeypatch, missing): + class Broken: + module = "hello" + + @staticmethod + def load(): + msg = f"No module named {missing!r}" + raise ModuleNotFoundError(msg, name=missing) + + monkeypatch.setattr("dagger.mod.cli.get_entry_point", Broken) + with pytest.raises(ModuleLoadError, match="run `dagger generate` and commit"): + load_module()