chore(cli): declare and verify command output (CLI-2961)

apps/cli/src/command-internal/command-facts/command-catalog.unit.test.ts

@@ -43,6 +43,23 @@ describe("commandCatalog", () => {
     }),
   );
 
+  it.effect("reports a command's declared output schema as test-checked", () =>
+    Effect.gen(function* () {
+      const entry = yield* entryAt("branches delete");
+
+      expect(entry?.facts["supabase/CommandOutput"]).toMatchObject({
+        checked: ["test"],
+        value: {
+          schema: {
+            type: "object",
+            properties: { project_ref: { type: "string" } },
+            required: expect.arrayContaining(["project_ref", "message"]),
+          },
+        },
+      });
+    }),
+  );
+
   it.effect("keeps a permission entry's conditions and request details", () =>
     Effect.gen(function* () {
       const entry = yield* entryAt("projects api-keys");

apps/cli/src/command-internal/command-facts/command-fact-adoption.unit.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it } from "@effect/vitest";
+import { Context, Effect } from "effect";
+import { Command } from "effect/cli";
+
+import { rootCommand } from "../../cli/root.ts";
+import { flattenSubcommands } from "../../docs/docs-introspection.ts";
+import { hasOptionalSubcommand } from "../../shared/output/optional-subcommand-usage.ts";
+import { CommandPermissionsAnnotation } from "../command-permissions/command-permissions.annotation.ts";
+import { TargetPolicyAnnotation } from "../target/target-policy.ts";
+import { CommandOutputAnnotation } from "./command-output.ts";
+import { OUTPUT_EXCEPTIONS } from "./command-output.exceptions.ts";
+
+const REQUIRED_FACTS = [
+  CommandOutputAnnotation.key,
+  TargetPolicyAnnotation.key,
+  CommandPermissionsAnnotation.key,
+] as const;
+
+type Exceptions = Readonly<Record<string, ReadonlyMap<string, string>>>;
+
+const executableCommands = (root: Command.Command.Any) => {
+  const found: Array<{ readonly path: string; readonly command: Command.Command.Any }> = [];
+  const visit = (command: Command.Command.Any, path: ReadonlyArray<string>) => {
+    const children = flattenSubcommands(command);
+    if (path.length > 0 && (children.length === 0 || hasOptionalSubcommand(command))) {
+      found.push({ path: path.join(" "), command });
+    }
+    children.forEach((child) => visit(child, [...path, child.name]));
+  };
+  visit(root, []);
+  return found;
+};
+
+/** Missing facts without an exception, and exceptions for facts the command now declares. */
+const adoptionGaps = (root: Command.Command.Any, exceptions: Exceptions) => {
+  const missing: Array<string> = [];
+  const stale: Array<string> = [];
+  const executable = executableCommands(root);
+  for (const { path, command } of executable) {
+    for (const fact of REQUIRED_FACTS) {
+      const present = command.annotations.mapUnsafe.has(fact);
+      const excepted = exceptions[fact]?.has(path) ?? false;
+      if (!present && !excepted) missing.push(`${path}: ${fact}`);
+      if (present && excepted) stale.push(`${path}: ${fact}`);
+    }
+  }
+  const paths = new Set(executable.map((entry) => entry.path));
+  for (const [fact, entries] of Object.entries(exceptions)) {
+    for (const path of entries.keys()) {
+      if (!paths.has(path)) stale.push(`${path}: ${fact}`);
+    }
+  }
+  return { missing, stale };
+};
+
+const PROJECT_EXCEPTIONS: Exceptions = { [CommandOutputAnnotation.key]: OUTPUT_EXCEPTIONS };
+
+const withFacts = (
+  command: Command.Command.Any,
+  facts: ReadonlyArray<(typeof REQUIRED_FACTS)[number]>,
+): Command.Command.Any =>
+  facts.reduce(
+    (current, fact) => Command.annotate(Context.Service<never, unknown>()(fact), true)(current),
+    command,
+  );
+
+const fixture = (facts: ReadonlyArray<(typeof REQUIRED_FACTS)[number]>) =>
+  Command.make("supabase").pipe(
+    Command.withSubcommands([
+      withFacts(
+        Command.make("leaf", {}, () => Effect.void),
+        facts,
+      ),
+    ]),
+  );
+
+describe("command fact adoption", () => {
+  it("requires output, target policy and permissions on every executable command", () => {
+    expect(adoptionGaps(rootCommand, PROJECT_EXCEPTIONS)).toEqual({ missing: [], stale: [] });
+  });
+
+  it("only excepts streaming commands for a reason other than pending adoption", () => {
+    const streaming = [...OUTPUT_EXCEPTIONS].filter(([, reason]) => reason === "streaming");
+    expect(streaming.map(([path]) => path).toSorted()).toEqual(["functions serve", "logs"]);
+  });
+
+  it("does not except branches delete", () => {
+    expect(OUTPUT_EXCEPTIONS.has("branches delete")).toBe(false);
+  });
+
+  it.each(REQUIRED_FACTS)("fails a command that drops %s", (dropped) => {
+    const root = fixture(REQUIRED_FACTS.filter((fact) => fact !== dropped));
+    expect(adoptionGaps(root, {}).missing).toEqual([`leaf: ${dropped}`]);
+  });
+
+  it("does not let an output exception excuse a missing target policy", () => {
+    const root = fixture([CommandPermissionsAnnotation.key]);
+    const exceptions = { [CommandOutputAnnotation.key]: new Map([["leaf", "streaming"]]) };
+    expect(adoptionGaps(root, exceptions).missing).toEqual([`leaf: ${TargetPolicyAnnotation.key}`]);
+  });
+
+  it("fails an exception for a command that now declares the fact", () => {
+    const root = fixture(REQUIRED_FACTS);
+    const exceptions = { [CommandOutputAnnotation.key]: new Map([["leaf", "not-yet-declared"]]) };
+    expect(adoptionGaps(root, exceptions).stale).toEqual([`leaf: ${CommandOutputAnnotation.key}`]);
+  });
+
+  it("fails an exception for a command that no longer exists", () => {
+    const root = fixture(REQUIRED_FACTS);
+    const exceptions = { [CommandOutputAnnotation.key]: new Map([["gone", "not-yet-declared"]]) };
+    expect(adoptionGaps(root, exceptions).stale).toEqual([`gone: ${CommandOutputAnnotation.key}`]);
+  });
+});

apps/cli/src/command-internal/command-facts/command-fact.ts

@@ -1,8 +1,11 @@
 import { Context, Effect, Option, Schema } from "effect";
 import type { Command } from "effect/cli";
 
-/** How a fact is enforced: by the compiler, by a render-time check, by a runtime gate, or not at all. */
-export type FactCheck = "compiler" | "render" | "runtime" | "none";
+/**
+ * How a fact is checked: by the compiler, a render-time check, a runtime gate, opted-in tests,
+ * a production observer that reports without enforcing, or not at all.
+ */
+export type FactCheck = "compiler" | "render" | "runtime" | "test" | "report-only" | "none";
 
 /** A command fact stored under one `Context` key on `Command.annotations`, serialized for the catalog. */
 export interface CommandFact {
@@ -26,3 +29,19 @@ export const registerFact = <I, A extends T, T, E>(
       ),
   };
 };
+
+/** Registers an annotation key whose catalog value `serialize` derives, such as a schema's JSON Schema. */
+export const registerDerivedFact = <I, A>(
+  key: Context.Key<I, A>,
+  options: {
+    readonly serialize: (value: A) => unknown;
+    readonly checked: ReadonlyArray<FactCheck>;
+  },
+): CommandFact => ({
+  id: key.key,
+  checked: options.checked,
+  read: (command) =>
+    Option.map(Context.getOption(command.annotations, key), (value) =>
+      Effect.sync(() => options.serialize(value)),
+    ),
+});

apps/cli/src/command-internal/command-facts/command-facts.registry.ts

@@ -7,7 +7,8 @@ import type * as Permissions from "../command-permissions/model.ts";
 import { CommandFlagsAnnotation } from "../command-permissions/permission-flags.ts";
 import { TargetPolicyAnnotation } from "../target/target-policy.ts";
 import type { TargetPolicyShape } from "../target/target-policy.ts";
-import { registerFact } from "./command-fact.ts";
+import { registerDerivedFact, registerFact } from "./command-fact.ts";
+import { CommandOutputAnnotation } from "./command-output.ts";
 
 type AllKeys<T> = T extends unknown ? keyof T : never;
 type SameKeys<A, B> = [AllKeys<A>] extends [AllKeys<B>]
@@ -91,6 +92,10 @@ const TargetPolicy = covering<TargetPolicyShape>()(
 export const COMMAND_FACTS = [
   registerFact(CommandPermissionsAnnotation, { schema: CommandPermissions, checked: ["none"] }),
   registerFact(TargetPolicyAnnotation, { schema: TargetPolicy, checked: ["runtime", "compiler"] }),
+  registerDerivedFact(CommandOutputAnnotation, {
+    serialize: (contract) => Schema.toJsonSchemaDocument(contract.schema),
+    checked: ["test"],
+  }),
 ] as const;
 
 /**

apps/cli/src/command-internal/command-facts/command-output.exceptions.ts

@@ -0,0 +1,158 @@
+/** Why a command may lack an output declaration. */
+export type OutputException = "streaming" | "not-yet-declared";
+
+/**
+ * Executable commands allowed to omit `withOutput`. Entries may only be removed: a new command
+ * declares its output, and the adoption test fails on entries that no longer apply.
+ */
+export const OUTPUT_EXCEPTIONS: ReadonlyMap<string, OutputException> = new Map([
+  ["backups list", "not-yet-declared"],
+  ["backups restore", "not-yet-declared"],
+  ["bootstrap", "not-yet-declared"],
+  ["branches list", "not-yet-declared"],
+  ["branches create", "not-yet-declared"],
+  ["branches get", "not-yet-declared"],
+  ["branches update", "not-yet-declared"],
+  ["branches pause", "not-yet-declared"],
+  ["branches unpause", "not-yet-declared"],
+  ["branches disable", "not-yet-declared"],
+  ["completion bash", "not-yet-declared"],
+  ["completion fish", "not-yet-declared"],
+  ["completion powershell", "not-yet-declared"],
+  ["completion zsh", "not-yet-declared"],
+  ["config convert", "not-yet-declared"],
+  ["config diff", "not-yet-declared"],
+  ["config pull", "not-yet-declared"],
+  ["config push", "not-yet-declared"],
+  ["db diff", "not-yet-declared"],
+  ["db dump", "not-yet-declared"],
+  ["db push", "not-yet-declared"],
+  ["db pull", "not-yet-declared"],
+  ["db reset", "not-yet-declared"],
+  ["db lint", "not-yet-declared"],
+  ["db start", "not-yet-declared"],
+  ["db query", "not-yet-declared"],
+  ["db advisors", "not-yet-declared"],
+  ["db test", "not-yet-declared"],
+  ["db branch create", "not-yet-declared"],
+  ["db branch delete", "not-yet-declared"],
+  ["db branch list", "not-yet-declared"],
+  ["db branch switch", "not-yet-declared"],
+  ["db remote changes", "not-yet-declared"],
+  ["db remote commit", "not-yet-declared"],
+  ["db schema declarative sync", "not-yet-declared"],
+  ["db schema declarative generate", "not-yet-declared"],
+  ["domains create", "not-yet-declared"],
+  ["domains get", "not-yet-declared"],
+  ["domains reverify", "not-yet-declared"],
+  ["domains activate", "not-yet-declared"],
+  ["domains delete", "not-yet-declared"],
+  ["encryption get-root-key", "not-yet-declared"],
+  ["encryption update-root-key", "not-yet-declared"],
+  ["feedback add", "not-yet-declared"],
+  ["feedback delete", "not-yet-declared"],
+  ["functions list", "not-yet-declared"],
+  ["functions delete", "not-yet-declared"],
+  ["functions download", "not-yet-declared"],
+  ["functions deploy", "not-yet-declared"],
+  ["functions new", "not-yet-declared"],
+  ["functions serve", "streaming"],
+  ["gen types", "not-yet-declared"],
+  ["gen signing-key", "not-yet-declared"],
+  ["gen bearer-jwt", "not-yet-declared"],
+  ["gen keys", "not-yet-declared"],
+  ["init", "not-yet-declared"],
+  ["inspect report", "not-yet-declared"],
+  ["inspect db db-stats", "not-yet-declared"],
+  ["inspect db replication-slots", "not-yet-declared"],
+  ["inspect db locks", "not-yet-declared"],
+  ["inspect db blocking", "not-yet-declared"],
+  ["inspect db outliers", "not-yet-declared"],
+  ["inspect db calls", "not-yet-declared"],
+  ["inspect db index-stats", "not-yet-declared"],
+  ["inspect db collation-drift", "not-yet-declared"],
+  ["inspect db long-running-queries", "not-yet-declared"],
+  ["inspect db bloat", "not-yet-declared"],
+  ["inspect db role-stats", "not-yet-declared"],
+  ["inspect db vacuum-stats", "not-yet-declared"],
+  ["inspect db table-stats", "not-yet-declared"],
+  ["inspect db traffic-profile", "not-yet-declared"],
+  ["inspect db toast-sizes", "not-yet-declared"],
+  ["inspect db xid-age", "not-yet-declared"],
+  ["inspect db cache-hit", "not-yet-declared"],
+  ["inspect db index-usage", "not-yet-declared"],
+  ["inspect db total-index-size", "not-yet-declared"],
+  ["inspect db index-sizes", "not-yet-declared"],
+  ["inspect db table-sizes", "not-yet-declared"],
+  ["inspect db table-index-sizes", "not-yet-declared"],
+  ["inspect db total-table-sizes", "not-yet-declared"],
+  ["inspect db unused-indexes", "not-yet-declared"],
+  ["inspect db table-record-counts", "not-yet-declared"],
+  ["inspect db seq-scans", "not-yet-declared"],
+  ["inspect db role-configs", "not-yet-declared"],
+  ["inspect db role-connections", "not-yet-declared"],
+  ["issue bug", "not-yet-declared"],
+  ["issue feature", "not-yet-declared"],
+  ["issue docs", "not-yet-declared"],
+  ["link", "not-yet-declared"],
+  ["login", "not-yet-declared"],
+  ["logout", "not-yet-declared"],
+  ["logs", "streaming"],
+  ["logs sources", "not-yet-declared"],
+  ["migration list", "not-yet-declared"],
+  ["migration new", "not-yet-declared"],
+  ["migration repair", "not-yet-declared"],
+  ["migration squash", "not-yet-declared"],
+  ["migration up", "not-yet-declared"],
+  ["migration down", "not-yet-declared"],
+  ["migration fetch", "not-yet-declared"],
+  ["network-bans get", "not-yet-declared"],
+  ["network-bans remove", "not-yet-declared"],
+  ["network-restrictions get", "not-yet-declared"],
+  ["network-restrictions update", "not-yet-declared"],
+  ["notebooks push", "not-yet-declared"],
+  ["notebooks pull", "not-yet-declared"],
+  ["orgs list", "not-yet-declared"],
+  ["orgs create", "not-yet-declared"],
+  ["postgres-config get", "not-yet-declared"],
+  ["postgres-config update", "not-yet-declared"],
+  ["postgres-config delete", "not-yet-declared"],
+  ["projects list", "not-yet-declared"],
+  ["projects create", "not-yet-declared"],
+  ["projects api-keys", "not-yet-declared"],
+  ["projects delete", "not-yet-declared"],
+  ["pull", "not-yet-declared"],
+  ["secrets list", "not-yet-declared"],
+  ["secrets set", "not-yet-declared"],
+  ["secrets unset", "not-yet-declared"],
+  ["seed buckets", "not-yet-declared"],
+  ["services", "not-yet-declared"],
+  ["snippets list", "not-yet-declared"],
+  ["snippets download", "not-yet-declared"],
+  ["ssl-enforcement get", "not-yet-declared"],
+  ["ssl-enforcement update", "not-yet-declared"],
+  ["sso list", "not-yet-declared"],
+  ["sso add", "not-yet-declared"],
+  ["sso remove", "not-yet-declared"],
+  ["sso update", "not-yet-declared"],
+  ["sso show", "not-yet-declared"],
+  ["sso info", "not-yet-declared"],
+  ["start", "not-yet-declared"],
+  ["status", "not-yet-declared"],
+  ["stop", "not-yet-declared"],
+  ["storage ls", "not-yet-declared"],
+  ["storage cp", "not-yet-declared"],
+  ["storage mv", "not-yet-declared"],
+  ["storage rm", "not-yet-declared"],
+  ["telemetry enable", "not-yet-declared"],
+  ["telemetry disable", "not-yet-declared"],
+  ["telemetry status", "not-yet-declared"],
+  ["test db", "not-yet-declared"],
+  ["test new", "not-yet-declared"],
+  ["unlink", "not-yet-declared"],
+  ["vanity-subdomains get", "not-yet-declared"],
+  ["vanity-subdomains check-availability", "not-yet-declared"],
+  ["vanity-subdomains activate", "not-yet-declared"],
+  ["vanity-subdomains delete", "not-yet-declared"],
+  ["whoami", "not-yet-declared"],
+]);

apps/cli/src/command-internal/command-facts/command-output.ts

@@ -0,0 +1,34 @@
+import { Context, Exit, Option, Schema } from "effect";
+import { Command } from "effect/cli";
+
+/** A command's declared machine result: the JSON value its `Output.success` or `Output.result` writes. */
+export interface CommandOutputContract {
+  readonly schema: Schema.Decoder<unknown>;
+}
+
+/** Context key under which a command's output contract is stored in `Command.annotations`. */
+export class CommandOutputAnnotation extends Context.Service<
+  CommandOutputAnnotation,
+  CommandOutputContract
+>()("supabase/CommandOutput") {}
+
+/** Declares the JSON result shape described by `schema`. */
+export const make = (schema: Schema.Decoder<unknown>): CommandOutputContract => ({ schema });
+
+/** Annotates a command with its output contract; the handler and its writes are unchanged. */
+export const withOutput = (contract: CommandOutputContract) =>
+  Command.annotate(CommandOutputAnnotation, contract);
+
+/**
+ * Returns why a serialized result doesn't match the contract, or `None` when it does. Keys the
+ * schema doesn't declare count as a mismatch.
+ */
+export const resultIssue = (
+  contract: CommandOutputContract,
+  json: string,
+): Option.Option<string> => {
+  const exit = Schema.decodeExit(Schema.fromJsonString(contract.schema), {
+    onExcessProperty: "error",
+  })(json);
+  return Exit.isSuccess(exit) ? Option.none() : Option.some(String(exit.cause));
+};