diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40a5222..b643b4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,8 @@ on: branches: [main] pull_request: branches: [main] + schedule: + - cron: "30 2 * * *" concurrency: group: ci-${{ github.ref }} @@ -45,12 +47,8 @@ jobs: - '.github/workflows/**' - '.github/scripts/**' docker_ci: - - 'go.mod' - - 'go.sum' - 'Dockerfile' - '.dockerignore' - - '.github/workflows/ci.yml' - - '.github/workflows/publish-image.yml' linux: name: quality (Linux) @@ -65,7 +63,34 @@ jobs: with: go-version-file: go.mod cache: true - cache-dependency-path: go.sum + cache-dependency-path: | + go.sum + examples/orders-api/go.sum + - name: Set up project tools + uses: jdx/mise-action@v4 + with: + cache: true + install: true + github_token: ${{ github.token }} + - name: Run check + run: MISE_JOBS=1 mise run check:core + + lint: + name: lint (Linux) + needs: changes + if: needs.changes.outputs.quality_ci == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: | + go.sum + examples/orders-api/go.sum - name: Set up project tools uses: jdx/mise-action@v4 with: @@ -79,21 +104,51 @@ jobs: key: golangci-lint-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.mise.toml', '.golangci.yml', 'go.mod', 'go.sum') }} restore-keys: | golangci-lint-${{ runner.os }}-${{ runner.arch }}- - - name: Run full check - run: mise run check + - name: Run lint + run: mise run lint - - name: Build macOS binaries - env: - CGO_ENABLED: "0" - GOOS: darwin - run: | - set -euo pipefail - for goarch in amd64 arm64; do - GOARCH="${goarch}" GOWORK=off go build \ - -trimpath \ - -o "${RUNNER_TEMP}/devctl-darwin-${goarch}" \ - ./cmd/devctl - done + e2e: + name: e2e (Linux) + needs: changes + if: needs.changes.outputs.quality_ci == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: | + go.sum + examples/orders-api/go.sum + - name: Set up project tools + uses: jdx/mise-action@v4 + with: + cache: true + install: true + github_token: ${{ github.token }} + - name: Run e2e + run: mise run e2e + + race: + name: race (Linux) + if: github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/main') + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: | + go.sum + examples/orders-api/go.sum + - name: Run race tests + run: GOWORK=off go test -race ./... docker: name: docker (linux/amd64) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..a053651 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,42 @@ +name: Main CI + +on: + push: + branches: [main] + +concurrency: + group: main-ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + macos-build: + name: ${{ format('build (macOS/{0})', matrix.goarch) }} + runs-on: ubuntu-latest + strategy: + matrix: + goarch: [amd64, arm64] + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: | + go.sum + examples/orders-api/go.sum + - name: Build macOS binary + env: + CGO_ENABLED: "0" + GOOS: darwin + GOARCH: ${{ matrix.goarch }} + run: | + set -euo pipefail + GOWORK=off go build \ + -trimpath \ + -o "${RUNNER_TEMP}/devctl-darwin-${GOARCH}" \ + ./cmd/devctl diff --git a/.mise.toml b/.mise.toml index 3c8fa1d..da2d168 100644 --- a/.mise.toml +++ b/.mise.toml @@ -28,8 +28,11 @@ run = "GOWORK=off go mod tidy -diff" [tasks.e2e] run = "GOWORK=off go test -tags=e2e ./e2e -v" +[tasks."check:core"] +depends = ["fmt:check", "test", "build", "mod:check", "docs:check", "example:orders:check"] + [tasks.check] -depends = ["fmt:check", "lint", "test:race", "build", "mod:check", "docs:check", "example:orders:check", "e2e"] +depends = ["check:core", "lint", "e2e"] [tasks."docs:generate"] run = "GOWORK=off go run ./cmd/devctl/internal/docgen" diff --git a/CONTEXT.md b/CONTEXT.md index c73643b..69d029a 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -130,5 +130,5 @@ An optional environment-backed gate for a Capability. It distinguishes construction of a runtime dependency from starting its background work. **Migration Target**: -The migration path and migration-only environment belonging to one SQLite, -PostgreSQL, or ClickHouse database Variant. +The migration configuration of one SQLite, PostgreSQL, or ClickHouse database +Variant, identified by its Connection and Variant. diff --git a/README.md b/README.md index 5c45a44..9969e6a 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ go install github.com/devctllabs/devctl/cmd/devctl@latest devctl --version ``` +For builds from the current source checkout, see the +[temporary sibling-module requirement](docs/development.md#working-with-sibling-modules). + ## Start a Project ```sh @@ -72,6 +75,7 @@ does not need `sync`, while a changed external Contract does. ## Documentation +- [Documentation index](docs/README.md) - [User guide](docs/user-guide/README.md) - [Command reference](docs/user-guide/reference/commands.md) - [Manifest reference](docs/user-guide/reference/manifest/README.md) diff --git a/cmd/devctl/internal/add/db.go b/cmd/devctl/internal/add/db.go index 5833830..a767fab 100644 --- a/cmd/devctl/internal/add/db.go +++ b/cmd/devctl/internal/add/db.go @@ -58,7 +58,7 @@ func newDBCmd(opts dbCmdOpts, build dbBuilder) *cli.Command { return &cli.Command{ Name: "db", Usage: "Add a database variant", - Description: "Add a SQLite, PostgreSQL, or ClickHouse Variant to a named database Connection. A migration target is declared by default; Devctl never writes SQL or applies migrations.", + Description: "Add a SQLite, PostgreSQL, or ClickHouse Variant to a named database Connection. A Migration Target is declared by default; use devctl migrate explicitly to create SQL files or run migrations.", UsageText: "devctl add db --kind ", UseShortOptionHandling: true, Arguments: []cli.Argument{&cli.StringArg{ diff --git a/cmd/devctl/internal/app/app.go b/cmd/devctl/internal/app/app.go index baf2ba5..7a2a769 100644 --- a/cmd/devctl/internal/app/app.go +++ b/cmd/devctl/internal/app/app.go @@ -10,6 +10,7 @@ import ( initcmd "github.com/devctllabs/devctl/cmd/devctl/internal/init" inspectcmd "github.com/devctllabs/devctl/cmd/devctl/internal/inspect" lintcmd "github.com/devctllabs/devctl/cmd/devctl/internal/lint" + migratecmd "github.com/devctllabs/devctl/cmd/devctl/internal/migrate" synccmd "github.com/devctllabs/devctl/cmd/devctl/internal/sync" validatecmd "github.com/devctllabs/devctl/cmd/devctl/internal/validate" "github.com/urfave/cli/v3" @@ -20,7 +21,7 @@ func New(releaseVersion, releaseCommit string) *cli.Command { return &cli.Command{ Name: "devctl", Usage: "Manage Devctl Go projects", - Description: "Devctl defines, validates, and materializes reproducible Go projects from a devctl.yaml manifest. Commands are non-interactive and keep manifest mutation, synchronization, linting, scaffolding, and generation explicit.", + Description: "Devctl defines, validates, and materializes reproducible Go projects from a devctl.yaml manifest. Commands are non-interactive and keep manifest mutation, synchronization, linting, scaffolding, generation, and migration execution explicit.", Version: buildVersion(releaseVersion, releaseCommit), Commands: []*cli.Command{ initcmd.NewCmd(), @@ -31,6 +32,7 @@ func New(releaseVersion, releaseCommit string) *cli.Command { synccmd.NewCmd(), gencmd.NewCmd(), lintcmd.NewCmd(), + migratecmd.NewCmd(), }, ExitErrHandler: func(context.Context, *cli.Command, error) {}, OnUsageError: func(_ context.Context, _ *cli.Command, err error, _ bool) error { diff --git a/cmd/devctl/internal/command/reporter.go b/cmd/devctl/internal/command/reporter.go index 9736458..cbc2871 100644 --- a/cmd/devctl/internal/command/reporter.go +++ b/cmd/devctl/internal/command/reporter.go @@ -6,6 +6,7 @@ import ( "github.com/devctllabs/devctl/internal/domain/contract" "github.com/devctllabs/devctl/internal/domain/failure" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" projectdomain "github.com/devctllabs/devctl/internal/domain/project" "github.com/urfave/cli/v3" "go.uber.org/zap" @@ -93,6 +94,22 @@ func classifyError(err error) errorReport { report.addDetail("reason", string(metadataErr.Reason)) report.addDetail("hint", metadataErr.Hint) } + var migrationErr *migrationdomain.Error + if errors.As(err, &migrationErr) { + report.addDetail("reason", migrationErr.Reason) + if migrationErr.Target != "" { + report.addDetail("target", migrationErr.Target) + } + if migrationErr.Path != "" { + report.addDetail("path", migrationErr.Path) + } + if migrationErr.EnvKey != "" { + report.addDetail("env", migrationErr.EnvKey) + } + if len(migrationErr.Candidates) > 0 { + report.addDetail("targets", migrationErr.Candidates) + } + } return report } diff --git a/cmd/devctl/internal/migrate/create.go b/cmd/devctl/internal/migrate/create.go new file mode 100644 index 0000000..6921f8d --- /dev/null +++ b/cmd/devctl/internal/migrate/create.go @@ -0,0 +1,9 @@ +package migrate + +import "github.com/urfave/cli/v3" + +func newCreateCmd(build migrateBuilder) *cli.Command { + command, cmd := newLeaf("create", "Create a pair of timestamped SQL files", build) + command.Arguments = []cli.Argument{&cli.StringArg{Name: "name", UsageText: "", Destination: &cmd.opts.name}} + return command +} diff --git a/cmd/devctl/internal/migrate/down.go b/cmd/devctl/internal/migrate/down.go new file mode 100644 index 0000000..f396639 --- /dev/null +++ b/cmd/devctl/internal/migrate/down.go @@ -0,0 +1,9 @@ +package migrate + +import "github.com/urfave/cli/v3" + +func newDownCmd(build migrateBuilder) *cli.Command { + command, cmd := newLeaf("down", "Roll back migrations (one step by default)", build) + command.Flags = append(command.Flags, &cli.IntFlag{Name: "steps", Value: 1, Usage: "roll back `N` migrations", Destination: &cmd.opts.steps}) + return command +} diff --git a/cmd/devctl/internal/migrate/leaf.go b/cmd/devctl/internal/migrate/leaf.go new file mode 100644 index 0000000..fec1290 --- /dev/null +++ b/cmd/devctl/internal/migrate/leaf.go @@ -0,0 +1,117 @@ +package migrate + +import ( + "context" + "errors" + "strings" + "time" + + commandruntime "github.com/devctllabs/devctl/cmd/devctl/internal/command" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + "github.com/devctllabs/go-libs/lifecycle" + "github.com/urfave/cli/v3" + "go.uber.org/zap" +) + +type migrateCmdOpts struct { + commandruntime.CommonCmdOpts + target string + envFile string + name string + steps int +} + +type migrateCmd struct { + opts migrateCmdOpts + operation migrationdomain.Operation + buildRuntime migrateBuilder +} + +func newLeaf(operation migrationdomain.Operation, usage string, build migrateBuilder) (*cli.Command, *migrateCmd) { + cmd := &migrateCmd{operation: operation, buildRuntime: build} + flags := append(cmd.opts.CommonFlags(), &cli.StringFlag{ + Name: "target", Usage: "select Migration Target `connection:variant`", Destination: &cmd.opts.target, + }) + if operation != migrationdomain.Create { + flags = append(flags, &cli.StringFlag{Name: "env-file", Usage: "read migration URL from this dotenv `path` after process environment", Destination: &cmd.opts.envFile}) + } + return &cli.Command{ + Name: string(operation), Usage: usage, Description: usage + ". Migration SQL is selected by migrations.path in the Manifest.", Flags: flags, + OnUsageError: func(_ context.Context, command *cli.Command, err error, _ bool) error { + return cmd.reporter(command).ReportError(cli.Exit(err, 2)) + }, + Action: cmd.Action, + }, cmd +} + +func (cmd *migrateCmd) reporter(command *cli.Command) *commandruntime.ErrorReporter { + return commandruntime.NewErrorReporter(cmd.opts.NewStderrLogger(command), cmd.opts.Verbose) +} + +func (cmd *migrateCmd) validate(command *cli.Command) error { + if command.Args().Len() != 0 { + return cli.Exit("unexpected positional arguments", 2) + } + if command.IsSet("target") { + connection, variant, found := strings.Cut(cmd.opts.target, ":") + if !found || connection == "" || variant == "" || strings.Contains(variant, ":") { + return cli.Exit("--target must be connection:variant", 2) + } + } + if command.IsSet("env-file") && cmd.opts.envFile == "" { + return cli.Exit("--env-file must not be empty", 2) + } + if cmd.operation == migrationdomain.Create && !migrationdomain.ValidName(cmd.opts.name) { + return cli.Exit("migration name must start with an ASCII letter or digit and contain only ASCII letters, digits, _ or -", 2) + } + if (cmd.operation == migrationdomain.Up && command.IsSet("steps") && cmd.opts.steps <= 0) || + (cmd.operation == migrationdomain.Down && cmd.opts.steps <= 0) { + return cli.Exit("--steps must be positive", 2) + } + return nil +} + +// Action executes one migration capability after parsing and validating command input. +func (cmd *migrateCmd) Action(ctx context.Context, command *cli.Command) error { + reporter := cmd.reporter(command) + if err := cmd.validate(command); err != nil { + return reporter.ReportError(err) + } + runtime, err := cmd.buildRuntime(ctx, cmd.opts.NewStderrLogger(command)) + if err != nil { + return reporter.ReportError(err) + } + result, operationErr := runtime.migrator.Migrate(ctx, migrationdomain.Command{ + ManifestPath: cmd.opts.ManifestPath, Target: cmd.opts.target, Operation: cmd.operation, + EnvFile: cmd.opts.envFile, Name: cmd.opts.name, Steps: cmd.opts.steps, + }) + shutdownErr := lifecycle.Shutdown(ctx, 5*time.Second, runtime.shutdown) + if err := errors.Join(operationErr, shutdownErr); err != nil { + var options []commandruntime.ErrorOption + if result.Changed || operationErr == nil { + options = append(options, commandruntime.WithPartialResult(migrationDTO(result))) + } + return reporter.ReportError(err, options...) + } + cmd.opts.NewStdoutLogger(command).Info("migration "+string(cmd.operation)+" completed", zap.Any("data", migrationDTO(result))) + return nil +} + +type stateDTO struct { + Target string `json:"target"` + Version *uint `json:"version"` + Dirty bool `json:"dirty"` + NoChange bool `json:"no_change,omitempty"` +} + +type createDTO struct { + Target string `json:"target"` + Files []string `json:"files,omitempty"` +} + +func migrationDTO(result migrationdomain.Result) any { + if result.State == nil { + return createDTO{Target: result.Target, Files: result.Files} + } + return stateDTO{Target: result.Target, Version: result.State.Version, Dirty: result.State.Dirty, NoChange: result.NoChange} +} diff --git a/cmd/devctl/internal/migrate/migrate.go b/cmd/devctl/internal/migrate/migrate.go new file mode 100644 index 0000000..338d57e --- /dev/null +++ b/cmd/devctl/internal/migrate/migrate.go @@ -0,0 +1,51 @@ +package migrate + +import ( + "context" + "errors" + "fmt" + + commandruntime "github.com/devctllabs/devctl/cmd/devctl/internal/command" + "github.com/devctllabs/devctl/internal/deps" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + "github.com/urfave/cli/v3" + "go.uber.org/zap" +) + +//go:generate go tool mockgen -destination mocks/migrate.go -package mocks -typed . migrator + +type migrator interface { + // Migrate executes one operation on a selected Migration Target. + Migrate(ctx context.Context, command migrationdomain.Command) (migrationdomain.Result, error) +} + +type migrateRuntime struct { + migrator migrator + shutdown func(context.Context) error +} + +type migrateBuilder func(ctx context.Context, logger *zap.Logger) (migrateRuntime, error) + +// NewCmd constructs the migration command namespace and its lazy executable leaves. +func NewCmd() *cli.Command { return newMigrateCmd(buildMigrate) } + +func newMigrateCmd(build migrateBuilder) *cli.Command { + return &cli.Command{ + Name: "migrate", + Usage: "Create and run database migrations", + Description: "Operate on one Manifest Migration Target. Select connection:variant explicitly, or omit --target when exactly one Migration Target exists.", + Commands: []*cli.Command{newCreateCmd(build), newUpCmd(build), newDownCmd(build), newStatusCmd(build)}, + } +} + +func buildMigrate(ctx context.Context, logger *zap.Logger) (migrateRuntime, error) { + container, err := deps.New(logger) + if err != nil { + return migrateRuntime{}, fmt.Errorf("deps.New: %w", err) + } + service, err := container.MigrationService() + if err != nil { + return migrateRuntime{}, errors.Join(fmt.Errorf("container.MigrationService: %w", err), commandruntime.Shutdown(ctx, container)) + } + return migrateRuntime{migrator: service, shutdown: container.Shutdown}, nil +} diff --git a/cmd/devctl/internal/migrate/migrate_test.go b/cmd/devctl/internal/migrate/migrate_test.go new file mode 100644 index 0000000..2436d7a --- /dev/null +++ b/cmd/devctl/internal/migrate/migrate_test.go @@ -0,0 +1,160 @@ +package migrate + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + + commandruntime "github.com/devctllabs/devctl/cmd/devctl/internal/command" + "github.com/devctllabs/devctl/cmd/devctl/internal/migrate/mocks" + "github.com/devctllabs/devctl/internal/domain/failure" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v3" + "go.uber.org/mock/gomock" + "go.uber.org/zap" +) + +func TestMigrationCommandsPassTypedInputAndEmitOneEvent(t *testing.T) { + t.Parallel() + for _, operation := range []migrationdomain.Operation{migrationdomain.Create, migrationdomain.Up, migrationdomain.Down, migrationdomain.Status} { + t.Run(string(operation), func(t *testing.T) { + t.Parallel() + input := migrationdomain.Command{Operation: operation, ManifestPath: "custom.yaml", Target: "primary:postgres"} + args := []string{"migrate", string(operation), "--file", "custom.yaml", "--target", "primary:postgres", "--json"} + result := migrationdomain.Result{Target: input.Target, State: &migrationdomain.State{Dirty: true}} + if operation == migrationdomain.Create { + input.Name = "create_orders" + args = append(args, input.Name) + result.State = nil + result.Files = []string{"migrations/1_create_orders.up.sql", "migrations/1_create_orders.down.sql"} + } else { + input.EnvFile = ".env.ci" + args = append(args, "--env-file", input.EnvFile) + } + if operation == migrationdomain.Up || operation == migrationdomain.Down { + input.Steps = 3 + args = append(args, "--steps", "3") + } + capability := mocks.NewMockmigrator(gomock.NewController(t)) + capability.EXPECT().Migrate(gomock.Any(), input).Return(result, nil) + closed := false + cmd := newMigrateCmd(func(context.Context, *zap.Logger) (migrateRuntime, error) { + return migrateRuntime{migrator: capability, shutdown: func(context.Context) error { closed = true; return nil }}, nil + }) + var stdout, stderr bytes.Buffer + cmd.Writer, cmd.ErrWriter = &stdout, &stderr + cmd.ExitErrHandler = func(context.Context, *cli.Command, error) {} + require.NoError(t, cmd.Run(context.Background(), args)) + require.True(t, closed) + require.Empty(t, stderr.String()) + var event struct { + Command string `json:"command"` + Data struct { + Target string `json:"target"` + Version *uint `json:"version"` + Dirty bool `json:"dirty"` + Files []string `json:"files"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal(stdout.Bytes(), &event)) + require.Equal(t, "migrate "+string(operation), event.Command) + require.Equal(t, input.Target, event.Data.Target) + if operation == migrationdomain.Create { + require.Equal(t, result.Files, event.Data.Files) + } else { + require.Nil(t, event.Data.Version) + require.True(t, event.Data.Dirty) + } + }) + } +} + +func TestMigrationHelpAndInvalidInputDoNotBuildRuntime(t *testing.T) { + t.Parallel() + for _, args := range [][]string{ + {"--help"}, {"up", "--help"}, {"create", "--help"}, + {"create"}, {"create", "../escape"}, {"create", "one", "two"}, + {"up", "unexpected"}, {"up", "--target", "primary"}, + {"up", "--steps", "0"}, {"up", "--steps", "-1"}, {"up", "--steps", "bad"}, + {"down", "--steps", "0"}, {"down", "--steps", "-1"}, {"down", "--steps", "bad"}, + {"status", "--env-file", ""}, + } { + t.Run("input_"+strings.Join(args, "_"), func(t *testing.T) { + t.Parallel() + cmd := newMigrateCmd(func(context.Context, *zap.Logger) (migrateRuntime, error) { + t.Fatal("runtime constructed for help or invalid input") + return migrateRuntime{}, nil + }) + var output bytes.Buffer + cmd.Writer, cmd.ErrWriter = &output, &output + cmd.ExitErrHandler = func(context.Context, *cli.Command, error) {} + err := cmd.Run(context.Background(), append([]string{"migrate"}, args...)) + if args[len(args)-1] == "--help" { + require.NoError(t, err) + } else { + require.Error(t, err) + require.Equal(t, 2, commandruntime.ExitCode(err)) + } + }) + } +} + +func TestMigrationErrorsJoinCleanupAndReportPartialWork(t *testing.T) { + t.Parallel() + capability := mocks.NewMockmigrator(gomock.NewController(t)) + cause, cleanup := errors.New("migration failed"), errors.New("cleanup failed") + version := uint(2) + capability.EXPECT().Migrate(gomock.Any(), migrationdomain.Command{Operation: migrationdomain.Down, Steps: 1}).Return( + migrationdomain.Result{Target: "primary:sqlite", State: &migrationdomain.State{Version: &version, Dirty: true}, Changed: true}, cause) + cmd := newMigrateCmd(func(context.Context, *zap.Logger) (migrateRuntime, error) { + return migrateRuntime{migrator: capability, shutdown: func(context.Context) error { return cleanup }}, nil + }) + var stdout, stderr bytes.Buffer + cmd.Writer, cmd.ErrWriter = &stdout, &stderr + cmd.ExitErrHandler = func(context.Context, *cli.Command, error) {} + err := cmd.Run(context.Background(), []string{"migrate", "down", "--json"}) + require.ErrorIs(t, err, cause) + require.ErrorIs(t, err, cleanup) + require.Empty(t, stdout.String()) + require.Equal(t, 1, bytes.Count(stderr.Bytes(), []byte(`"level":"error"`))) + require.Contains(t, stderr.String(), `"partial_result":{"target":"primary:sqlite"`) + require.Contains(t, stderr.String(), `"dirty":true`) +} + +func TestMigrationReportsActionableSafeContext(t *testing.T) { + t.Parallel() + capability := mocks.NewMockmigrator(gomock.NewController(t)) + cause := &migrationdomain.Error{Kind: failure.InvalidInput, Reason: "target_required", Candidates: []string{"primary:postgres", "primary:sqlite"}} + capability.EXPECT().Migrate(gomock.Any(), migrationdomain.Command{Operation: migrationdomain.Up}).Return(migrationdomain.Result{}, cause) + cmd := newMigrateCmd(func(context.Context, *zap.Logger) (migrateRuntime, error) { + return migrateRuntime{migrator: capability, shutdown: func(context.Context) error { return nil }}, nil + }) + var stdout, stderr bytes.Buffer + cmd.Writer, cmd.ErrWriter = &stdout, &stderr + cmd.ExitErrHandler = func(context.Context, *cli.Command, error) {} + require.ErrorIs(t, cmd.Run(context.Background(), []string{"migrate", "up", "--json"}), cause) + require.Empty(t, stdout.String()) + require.Contains(t, stderr.String(), `"reason":"target_required"`) + require.Contains(t, stderr.String(), `"targets":["primary:postgres","primary:sqlite"]`) +} + +func TestMigrationPartialResultDoesNotInventStateAfterReadFailure(t *testing.T) { + t.Parallel() + capability := mocks.NewMockmigrator(gomock.NewController(t)) + cause := errors.New("version read failed after SQL") + capability.EXPECT().Migrate(gomock.Any(), migrationdomain.Command{Operation: migrationdomain.Up}).Return( + migrationdomain.Result{Target: "primary:sqlite", Changed: true}, cause) + cmd := newMigrateCmd(func(context.Context, *zap.Logger) (migrateRuntime, error) { + return migrateRuntime{migrator: capability, shutdown: func(context.Context) error { return nil }}, nil + }) + var stdout, stderr bytes.Buffer + cmd.Writer, cmd.ErrWriter = &stdout, &stderr + cmd.ExitErrHandler = func(context.Context, *cli.Command, error) {} + require.ErrorIs(t, cmd.Run(context.Background(), []string{"migrate", "up", "--json"}), cause) + require.Empty(t, stdout.String()) + require.Contains(t, stderr.String(), `"partial_result":{"target":"primary:sqlite"}`) +} diff --git a/cmd/devctl/internal/migrate/mocks/migrate.go b/cmd/devctl/internal/migrate/mocks/migrate.go new file mode 100644 index 0000000..e0232e1 --- /dev/null +++ b/cmd/devctl/internal/migrate/mocks/migrate.go @@ -0,0 +1,81 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/devctllabs/devctl/cmd/devctl/internal/migrate (interfaces: migrator) +// +// Generated by this command: +// +// mockgen -destination mocks/migrate.go -package mocks -typed . migrator +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + migration "github.com/devctllabs/devctl/internal/domain/migration" + gomock "go.uber.org/mock/gomock" +) + +// Mockmigrator is a mock of migrator interface. +type Mockmigrator struct { + ctrl *gomock.Controller + recorder *MockmigratorMockRecorder + isgomock struct{} +} + +// MockmigratorMockRecorder is the mock recorder for Mockmigrator. +type MockmigratorMockRecorder struct { + mock *Mockmigrator +} + +// NewMockmigrator creates a new mock instance. +func NewMockmigrator(ctrl *gomock.Controller) *Mockmigrator { + mock := &Mockmigrator{ctrl: ctrl} + mock.recorder = &MockmigratorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *Mockmigrator) EXPECT() *MockmigratorMockRecorder { + return m.recorder +} + +// Migrate mocks base method. +func (m *Mockmigrator) Migrate(ctx context.Context, command migration.Command) (migration.Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Migrate", ctx, command) + ret0, _ := ret[0].(migration.Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Migrate indicates an expected call of Migrate. +func (mr *MockmigratorMockRecorder) Migrate(ctx, command any) *MockmigratorMigrateCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Migrate", reflect.TypeOf((*Mockmigrator)(nil).Migrate), ctx, command) + return &MockmigratorMigrateCall{Call: call} +} + +// MockmigratorMigrateCall wrap *gomock.Call +type MockmigratorMigrateCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockmigratorMigrateCall) Return(arg0 migration.Result, arg1 error) *MockmigratorMigrateCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockmigratorMigrateCall) Do(f func(context.Context, migration.Command) (migration.Result, error)) *MockmigratorMigrateCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockmigratorMigrateCall) DoAndReturn(f func(context.Context, migration.Command) (migration.Result, error)) *MockmigratorMigrateCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/cmd/devctl/internal/migrate/status.go b/cmd/devctl/internal/migrate/status.go new file mode 100644 index 0000000..7e5de4a --- /dev/null +++ b/cmd/devctl/internal/migrate/status.go @@ -0,0 +1,9 @@ +package migrate + +import "github.com/urfave/cli/v3" + +func newStatusCmd(build migrateBuilder) *cli.Command { + command, _ := newLeaf("status", "Show migration version and dirty state", build) + command.Description = "Read the current migration version. Opening a fresh target may initialize the metadata table and SQLite database file. Dirty state is a successful result." + return command +} diff --git a/cmd/devctl/internal/migrate/up.go b/cmd/devctl/internal/migrate/up.go new file mode 100644 index 0000000..9dbff89 --- /dev/null +++ b/cmd/devctl/internal/migrate/up.go @@ -0,0 +1,9 @@ +package migrate + +import "github.com/urfave/cli/v3" + +func newUpCmd(build migrateBuilder) *cli.Command { + command, cmd := newLeaf("up", "Apply all pending migrations, or a positive number with --steps", build) + command.Flags = append(command.Flags, &cli.IntFlag{Name: "steps", Usage: "apply positive `N` migrations; omit for all", Destination: &cmd.opts.steps}) + return command +} diff --git a/cmd/devctl/main_test.go b/cmd/devctl/main_test.go index 74694c2..b2cbeb8 100644 --- a/cmd/devctl/main_test.go +++ b/cmd/devctl/main_test.go @@ -51,7 +51,7 @@ func TestRootHelp(t *testing.T) { t.Parallel() binary := buildCLI(t) stdout := runCLI(t, binary, "--help") - for _, commandName := range []string{"init", "validate", "inspect", "enable", "add", "sync", "gen", "lint"} { + for _, commandName := range []string{"init", "validate", "inspect", "enable", "add", "sync", "gen", "lint", "migrate"} { require.Contains(t, stdout, commandName) } require.NotContains(t, stdout, "--file") @@ -494,7 +494,8 @@ languages: require.FileExists(t, filepath.Join(root, "migrations/primary/sqlite/.gitkeep")) var mise struct { Tasks map[string]struct { - Run string `toml:"run"` + Run string `toml:"run"` + Usage string `toml:"usage"` } `toml:"tasks"` } _, err := toml.DecodeFile(filepath.Join(root, ".mise.toml"), &mise) @@ -502,12 +503,28 @@ languages: require.Contains(t, mise.Tasks, "migrate:primary:sqlite:create") require.Contains(t, mise.Tasks, "migrate:primary:sqlite:up") require.Contains(t, mise.Tasks, "migrate:primary:sqlite:down") + require.Contains(t, mise.Tasks, "migrate:primary:sqlite:status") + require.Contains(t, mise.Tasks["migrate:primary:sqlite:up"].Usage, `flag "--steps "`) + // Exercise the generated wrapper, including a dotenv path containing spaces. + envFile := filepath.Join(root, "migration settings.env") + require.NoError(t, os.WriteFile(envFile, []byte("SAMPLE_DB_PRIMARY_SQLITE_MIGRATIONS_URL=sqlite://"+filepath.Join(root, "wrapper.db")+"\n"), 0o600)) + status := exec.CommandContext(context.Background(), "sh", "-c", mise.Tasks["migrate:primary:sqlite:status"].Run) + status.Dir = root + status.Env = append(os.Environ(), "PATH="+filepath.Dir(binary)+string(os.PathListSeparator)+os.Getenv("PATH"), "usage_env_file="+envFile) + statusOutput, err := status.CombinedOutput() + require.NoError(t, err, "%s", statusOutput) + require.FileExists(t, filepath.Join(root, "wrapper.db")) goMod, err := os.ReadFile(filepath.Join(root, "go.mod")) require.NoError(t, err) require.NotContains(t, string(goMod), "golang-migrate") migrationPath := filepath.Join(root, "migrations/primary/sqlite/20260830010000_create_users.up.sql") require.NoError(t, os.WriteFile(migrationPath, []byte("CREATE TABLE users (id INTEGER PRIMARY KEY);\n"), 0o644)) + up := exec.CommandContext(context.Background(), "sh", "-c", mise.Tasks["migrate:primary:sqlite:up"].Run) + up.Dir = root + up.Env = append(os.Environ(), "PATH="+filepath.Dir(binary)+string(os.PathListSeparator)+os.Getenv("PATH"), "usage_env_file="+envFile, "usage_steps=1") + upOutput, err := up.CombinedOutput() + require.NoError(t, err, "%s", upOutput) runCLI(t, binary, "init", "scaffold", "--file", manifestPath) content, err := os.ReadFile(migrationPath) require.NoError(t, err) diff --git a/cmd/devctl/migrate_test.go b/cmd/devctl/migrate_test.go new file mode 100644 index 0000000..04596d4 --- /dev/null +++ b/cmd/devctl/migrate_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMigrateCIWorkflow(t *testing.T) { + t.Parallel() + root := t.TempDir() + manifest := filepath.Join(root, "devctl.yaml") + require.NoError(t, os.WriteFile(manifest, []byte(`version: 1 +project: {name: orders-api, language: go} +env: {} +paths: {} +sources: {} +exports: {} +components: + db: + connections: + - name: primary + default: postgres + variants: + - name: postgres + kind: postgres + dsn_env: DB_DSN + - name: sqlite + kind: sqlite + dsn_env: CI_DSN + migrations: + path: db/migrations + database_env: MIGRATIONS_URL +languages: + go: {module: example.test/orders} +`), 0o644)) + // No go.mod or scaffold readiness: migration operations use only their inputs. + created := runCLI(t, buildCLI(t), "migrate", "create", "create_orders", "--file", manifest, "--json") + var event struct { + Data struct { + Target string `json:"target"` + Files []string `json:"files"` + Version *uint `json:"version"` + Dirty bool `json:"dirty"` + NoChange bool `json:"no_change"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal([]byte(created), &event)) + require.Equal(t, "primary:sqlite", event.Data.Target) + require.Len(t, event.Data.Files, 2) + upFile := filepath.Join(root, event.Data.Files[0]) + require.NoError(t, os.WriteFile(filepath.Join(root, event.Data.Files[0]), []byte("CREATE TABLE orders (id INTEGER PRIMARY KEY);"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, event.Data.Files[1]), []byte("DROP TABLE orders;"), 0o644)) + envFile := filepath.Join(root, "migration.env") + require.NoError(t, os.WriteFile(envFile, []byte("ORDERS_API_MIGRATIONS_URL=sqlite://"+filepath.Join(root, "orders.db")+"\n"), 0o600)) + for _, operation := range []string{"up", "up", "status", "down"} { + output := runCLI(t, buildCLI(t), "migrate", operation, "--target", "primary:sqlite", "--file", manifest, "--env-file", envFile, "--json") + event.Data.Version = nil + require.NoError(t, json.Unmarshal([]byte(output), &event)) + require.Equal(t, "primary:sqlite", event.Data.Target) + require.False(t, event.Data.Dirty) + if operation == "down" { + require.Nil(t, event.Data.Version) + } else { + require.NotNil(t, event.Data.Version) + } + } + require.FileExists(t, filepath.Join(root, "orders.db")) + + // SQL errors remain redacted even under --verbose and expose observed dirty state. + require.NoError(t, os.WriteFile(upFile, []byte("INVALID_SQL SUPER_SECRET_TOKEN"), 0o644)) + command := exec.CommandContext(context.Background(), buildCLI(t), "migrate", "up", "--file", manifest, "--env-file", envFile, "--json", "--verbose") + var stdout, stderr bytes.Buffer + command.Stdout, command.Stderr = &stdout, &stderr + var exitError *exec.ExitError + require.ErrorAs(t, command.Run(), &exitError) + require.Equal(t, 1, exitError.ExitCode()) + require.Empty(t, stdout.String()) + require.NotContains(t, stderr.String(), "SUPER_SECRET_TOKEN") + require.NotContains(t, stderr.String(), "sqlite://") + require.Contains(t, stderr.String(), `"partial_result"`) + require.Contains(t, stderr.String(), `"dirty":true`) +} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..a887465 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,42 @@ +# Devctl documentation + +These documents describe the current source tree. For a released binary, use +the documentation at its matching Git tag. + +## Product and architecture + +- [Product documentation](product/README.md): scope and capability index. +- [Project lifecycle](product/project-lifecycle.md): Manifest changes, validation, + inspection, and addressable operations. +- [Contracts and generation](product/contracts-and-generation.md): Sources, + Snapshots, synchronization, linting, and generated outputs. +- [Scaffold and ownership](product/scaffold-and-ownership.md): refresh rules, + user-owned inputs, and partial publication. +- [Runtime configuration](product/runtime-configuration.md): environment keys, + defaults, secrets, and Start Policy. +- [Generated application](product/generated-application.md): composition, + Scenarios, protocol boundaries, and Resources. +- [CLI automation](product/cli-automation.md): invocation, structured outcomes, + previews, and failure handling. +- [Migrations](product/migrations.md): target selection, execution, and recovery + rules. +- [Architecture overview](architecture/overview.md): components, boundaries, + workflow interactions, generated runtime, and operational constraints. +- [Domain glossary](../CONTEXT.md): canonical Project vocabulary. + +## Decisions + +- [Product Decision Records](product/README.md#product-decisions): accepted + workflow, ownership, reproducibility, runtime, automation, and migration choices. +- [Architecture Decision Records](adr/): technical decisions and their reasons. + +## Using and developing Devctl + +- [User guide](user-guide/README.md). +- [Command reference](user-guide/reference/commands.md). +- [Manifest reference](user-guide/reference/manifest/README.md). +- [Output and error contract](user-guide/reference/output-and-errors.md). +- [Development guide](development.md), including migration verification. + +Specs and implementation issues live in the local Markdown tracker under +`.scratch/`; see the [tracker conventions](agents/issue-tracker.md). diff --git a/docs/adr/0001-canonical-project-model-and-command-boundaries.md b/docs/adr/0001-canonical-project-model-and-command-boundaries.md index b7fa523..90933d6 100644 --- a/docs/adr/0001-canonical-project-model-and-command-boundaries.md +++ b/docs/adr/0001-canonical-project-model-and-command-boundaries.md @@ -2,6 +2,7 @@ - Status: accepted - Date: 2026-09-03 +- Command list amended by [ADR 0009](0009-explicit-migration-execution.md). ## Context @@ -13,7 +14,7 @@ and network effects difficult to predict. ## Decision -The v1 CLI has exactly eight top-level commands: `init`, `validate`, `inspect`, +The initial v1 CLI had eight top-level commands: `init`, `validate`, `inspect`, `enable`, `add`, `sync`, `lint`, and `gen`. Initialization is explicit through `init manifest` and `init scaffold`. Manifest mutations do not install tools, run generators, or change handwritten Go. `sync`, `lint`, and `gen` never diff --git a/docs/adr/0004-project-owned-generator-toolchain.md b/docs/adr/0004-project-owned-generator-toolchain.md index 7ef1281..5b1320c 100644 --- a/docs/adr/0004-project-owned-generator-toolchain.md +++ b/docs/adr/0004-project-owned-generator-toolchain.md @@ -2,6 +2,7 @@ - Status: accepted - Date: 2026-09-03 +- Migration tooling amended by [ADR 0009](0009-explicit-migration-execution.md). ## Context @@ -15,10 +16,11 @@ explicit override is selected. The consuming Project owns generator versions and native configuration. Go generators are pinned to exact published module versions with `go.mod` tool -directives. Mise owns exact executable runtime and task versions that are not -Go tools, including Node 24, quicktype 26.0.0, and golang-migrate 4.19.1 when -their capabilities are used. Generator objects in the Manifest do not expose a -competing `tool` selector. +directives. Mise owns executable runtime and task versions that are not Go +tools, including Node 24 and quicktype 26.0.0 when their capabilities are used. +Migration tasks moved from the external golang-migrate executable to +`devctl migrate` in [ADR 0009](0009-explicit-migration-execution.md). Generator +objects in the Manifest do not expose a competing `tool` selector. Devctl invokes tools through the Project environment, writes into temporary storage, validates expected output, and hands publication to the owning diff --git a/docs/adr/0005-canonical-runtime-config.md b/docs/adr/0005-canonical-runtime-config.md index 08d11f7..c270089 100644 --- a/docs/adr/0005-canonical-runtime-config.md +++ b/docs/adr/0005-canonical-runtime-config.md @@ -20,8 +20,9 @@ field paths are typed domain errors reported by `validate` as `runtime_config_conflict`. Every Go Project has an implicit `config` Target. Its default Managed Output is -`gen/config/config.gen.go`; `languages.go.generators.config.out` only overrides -that location. Newly initialized Manifests omit the redundant default block. +`gen/config/config.gen.go`; `languages.go.generators.config.out` selects the +output directory, defaulting to `gen/config`, while the filename remains +`config.gen.go`. Newly initialized Manifests omit the redundant default block. `devctl gen config` and `devctl init scaffold` use the same renderer for one grouped generated `Config` type and the root `.env.example`. Scaffold emits an adapter that imports the generated type rather than maintaining a second @@ -43,9 +44,12 @@ overridden through environment. `CommitRetry` inherits the main retry policy; through user-owned custom environment and explicit composition until they become Manifest features. -Migration database environment belongs only to example and inspect -projections. It is absent from application Runtime Config, and the migration -URL is distinct from the runtime database connection. Secret fields never +Migration database environment is absent from application Runtime Config, and +the migration URL is distinct from the runtime database connection. The +migration projection supplies its effective environment key to example and +inspect output and to the explicit migration workflow added in +[ADR 0009](0009-explicit-migration-execution.md). That workflow resolves its own +URL separately from application Runtime Config. Runtime secret fields never carry defaults: generated tags omit them, `.env.example` leaves them blank, and inspect reports only the secret marker. diff --git a/docs/adr/0006-machine-readable-cli-contract.md b/docs/adr/0006-machine-readable-cli-contract.md index ed87516..4e8fa7e 100644 --- a/docs/adr/0006-machine-readable-cli-contract.md +++ b/docs/adr/0006-machine-readable-cli-contract.md @@ -23,12 +23,15 @@ with stable `code` and `exit_code` fields. Safe scalar context belongs in `details.partial_result`; the reporter safely merges it with existing details, top-level error `data` is removed, and stdout remains empty. This pre-v1 wire change has no compatibility shim. Raw causes and external tool output appear -only under `--verbose`. +only under `--verbose`. Migration failures retain safe reasons and context even +under verbose logging, as added by +[ADR 0009](0009-explicit-migration-execution.md); SQL and credentials remain +unrendered. Invalid `validate` and `lint` findings are normal results: the structured event is written to stdout, stderr is empty, and the process exits `1`. Execution -errors also exit `1`, usage/help exits `2`, cancellation exits `130`, and -success exits `0`. +errors also exit `1`, invalid usage or help failure exits `2`, cancellation exits +`130`, and success exits `0`. Public failure codes remain `usage`, `invalid_input`, `not_found`, `conflict`, `unavailable`, `unsupported`, `cancelled`, and `internal`. Adapters add safe diff --git a/docs/adr/0009-explicit-migration-execution.md b/docs/adr/0009-explicit-migration-execution.md new file mode 100644 index 0000000..75562ee --- /dev/null +++ b/docs/adr/0009-explicit-migration-execution.md @@ -0,0 +1,47 @@ +# ADR 0009: Execute migrations through an explicit Devctl command + +- Status: accepted +- Date: 2026-10-19 + +Devctl adds `migrate create/up/down/status` as its ninth top-level command. +This amends the fixed command list in +[ADR 0001](0001-canonical-project-model-and-command-boundaries.md) and the +project-owned external migration executable in +[ADR 0004](0004-project-owned-generator-toolchain.md). Generated Mise tasks +retain their names but call Devctl, so local execution and CI share selection, +configuration, output, and failure rules. Scaffold and application startup never +apply migrations. + +A shared Project-domain projection derives ordered Migration Targets from +database Connections and Variants, including SQL paths and prefixed environment +keys. Inspect, scaffold, and execution consume it independently of the Target +Catalog and application Runtime Config, preserving the boundary in +[ADR 0005](0005-canonical-runtime-config.md). Selection and migration-only URL +resolution follow [PDR 0008](../product/decisions/0008-explicit-migration-target-selection.md); +the service uses semantic Project loading without unrelated generator readiness +checks. + +`go-libs/dbmigrate` owns SQL execution through caller-registered upstream drivers +instead of invoking a Project-installed migration executable. Devctl owns +configuration resolution, contained timestamped file creation, and CLI result +mapping. Status uses the same backend constructor as execution, accepting its +metadata and SQLite file initialization effects as recorded in +[PDR 0009](../product/decisions/0009-migration-status-and-recovery.md). + +The source filesystem stays open until the migrator closes; operation and close +errors are joined. State observed before and after an operation supports partial +results without attempting to undo completed SQL. Creation uses rooted exclusive +file operations and a version reservation because SQL is user-owned and must +never be replaced; ordinary failure cleans up owned files, but a pair is not +atomic across crashes. + +SQL cancellation and closing follow the library's waiting semantics; driver +timeouts remain operator-owned. PostgreSQL, SQLite, and the ClickHouse SQL +migration driver support builds with CGO disabled. ClickHouse remains separate +from the native application connection in +[ADR 0008](0008-native-clickhouse-runtime-and-migrations.md), non-transactional, +and receives unchanged SQL and URLs. Migration errors preserve causes +internally but do not render SQL or credentials, including under verbose logging. + +The current operation, path, no-change, dirty-state, and recovery rules are +maintained in the [migration product document](../product/migrations.md). diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md new file mode 100644 index 0000000..24deee7 --- /dev/null +++ b/docs/architecture/overview.md @@ -0,0 +1,168 @@ +# Architecture overview + +Devctl is a non-interactive CLI over a canonical Project Manifest. Explicit +workflows change the Manifest, publish Managed Outputs, or execute selected +operations. Generated applications have their own composition and runtime; +Devctl does not run them or apply their schema at startup. The +[domain glossary](../../CONTEXT.md) defines the vocabulary and the +[product index](../product/README.md) links observable behavior. + +## Components and boundaries + +| Component | Responsibility | +| --- | --- | +| CLI adapters | Parse and validate input, invoke one service root, map results and errors to the CLI contract | +| Services | Own workflow policy and coordinate consumer-owned ports | +| Domain | Define the Manifest, effective projections, operation data, pure invariants, and typed failures | +| Repositories | Own Manifest and workspace persistence, rooted file publication, migration files and database mechanics | +| Clients | Own HTTP/Git acquisition, generator routing, and external tool execution | +| Protocol analyzers | Validate OpenAPI and JSON Schema independently of workflow orchestration | +| Production composition | Register concrete providers and lazily resolve the graph needed by the selected command | + +Dependencies point `cmd -> service -> domain`. Business owners depend on their +ports; concrete adapters and integration resources are composed in +`internal/deps`. The Project service is the common gateway for discovery, +decoding, semantic validation, inspection, and Manifest mutation. +[ADR 0001](../adr/0001-canonical-project-model-and-command-boundaries.md) +records these boundaries. + +## Manifest and effective projections + +Discovery selects a Manifest and Project root. Structural decoding rejects +unknown/duplicate fields and wrong YAML types; domain validation establishes +semantic validity. `validate` additionally calls the Readiness checker for local +inputs, module/tool declarations, and native configs. Other workflows load a +semantically valid Project and check only their own execution prerequisites. + +The Target Catalog is the shared source of identities, references, Logical +Inputs, output locations, and supported operations for contract/config workflows. +Selection is deterministic; generation has an explicit dependency-sensitive +execution order. The Runtime Config catalog separately owns effective keys, +defaults, secret policy, and Go field paths. Its shared renderer serves scaffold +and config generation, while inspection consumes the same catalog. + +A separate ordered projection derives Migration Targets and their effective +environment keys. Inspect, scaffold, and migration execution consume that +projection. Migration-only settings appear in example/inspect output and stay +outside application Runtime Config; see +[ADR 0005](../adr/0005-canonical-runtime-config.md). + +## Contract acquisition and consumption + +`sync` selects Targets, materializes each external Source, then publishes a +normalized Snapshot. Source strategies isolate local containment, bounded HTTP +closures, temporary Git checkouts, and named Exports from upstream Devctl +Projects. The materialization result carries files, Entrypoint or Module Root, +and metadata needed to interpret the selected surface. + +Downstream workflows use a shared committed Snapshot loader and Target input +resolver. The loader validates metadata against the Target, and resolution maps +its Logical Input to local input files or a committed Resolved Input. Missing +required metadata fails lint/generation with sync guidance, while inspect omits +unavailable resolution and still describes the Project. + +Lint combines protocol analysis and Project-owned Buf checks. Generation routes +effective Targets to the shared config renderer, oapi-codegen, Buf, or quicktype. +Generator adapters own tool declarations, arguments, bounded diagnostics, and +expected output validation; shared process and temporary-output mechanics stay +behind those adapters. Tools render into temporary storage before publication. + +Only sync acquires configured Contract suppliers. Lint and generation consume +local or committed inputs, but external tools can require their own module or +plugin downloads. The architecture does not impose process-level network +isolation. See [ADR 0002](../adr/0002-contract-sources-exports-and-snapshots.md) +and [ADR 0004](../adr/0004-project-owned-generator-toolchain.md). + +## Publication and scaffold + +The workspace repository owns rooted containment and atomic file/tree +publication. Publication reports created, updated, unchanged, or removed as part +of the operation. Dry-run uses read-only inspection and performs no Source +acquisition, tool execution, publication, or pruning. + +Generation owns each selected output tree; sync owns the external-contract +namespace and can prune stale children during full/family execution. Atomicity +stops at a publication boundary: separate config/environment files, multiple +Targets, and scaffold artifacts are not one transaction. Later failures preserve +completed changes. See [ADR 0003](../adr/0003-managed-artifacts-and-publication.md). + +Scaffold compiles a private typed projection, renders its artifact plan, assigns +ownership, and preflights conflicts before publishing in stable order. Canonical +generator configs are managed; custom HTTP configs are excluded from publication +and checked for collisions with planned artifacts. Shared custom configs are +allowed. Scaffold preserves existing Seeds, uses a best-effort existence check +for their creation, and does not prune artifacts removed from the Manifest. + +## Generated application runtime + +Generated infrastructure and user-owned Provider Bindings form a second +composition boundary. Managed `provideX` functions register infrastructure; +`provideApplication` explicitly selects bindings and remains a Scaffold Seed. +Typed HTTP/gRPC registrars connect user behavior to runtime servers. Outbound +providers expose raw transport handles, leaving generated client/stub composition +to application code. + +An API Scenario resolves enabled server tasks. A consumer Scenario validates +its selected name and Start Policy before resolving one named Kafka runner. +Providers are registered eagerly and constructed lazily; shutdown owns resources +actually constructed by that graph. Consumer bindings select decoder, handler, +message types, and retry policy. The unfinished handler Seed fails closed. +These contracts are recorded in +[ADR 0007](../adr/0007-explicit-application-composition-and-runtime-scenarios.md). + +Database providers open only the selected Variant of a logical Connection. +SQLite/PostgreSQL expose transactional endpoints; ClickHouse retains its native +API and separate migration driver. Redis, S3, logging, and telemetry are owned +graph resources; health and pprof contribute operational tasks. Generated gRPC +clients currently use plaintext transport. Supported interfaces and constraints +are detailed in [Generated application](../product/generated-application.md). + +## Migration execution flow + +1. The CLI validates arguments before constructing dependencies. The migration + service loads the Project and selects one Migration Target from its Manifest. +2. `create` goes directly to the migration repository for contained file + creation. Database operations first resolve the migration URL through a + separate environment adapter, which reads an optional dotenv file without + modifying process state. +3. The repository opens a filesystem rooted at the Project and constructs + `go-libs/dbmigrate` with the selected SQL directory and resolved database URL. + Registered upstream drivers implement PostgreSQL, SQLite, and ClickHouse + migration behavior. The same constructor serves status and execution. +4. The repository reads state, performs the requested operation, and reads the + resulting state when applicable. Results preserve observed progress on + failure. The migrator closes before its source filesystem; close and operation + errors are joined rather than losing either failure. +5. The CLI emits one success result or a final error with an available partial + result. Migration errors retain their causes internally while rendering safe + reasons, target identities, and paths instead of SQL or credentials. + +[ADR 0009](../adr/0009-explicit-migration-execution.md) records the integration +choice. The observable selection, configuration, status, and recovery rules live +in the [migration product document](../product/migrations.md). + +## Operational constraints + +The CLI composition root owns resource cleanup after each command. Signals +cancel its context, and the final reporter preserves typed categories and +available partial progress. Findings remain normal results; execution errors go +to stderr. Migration errors retain causes without rendering SQL or credentials, +even under verbose logging. See +[ADR 0006](../adr/0006-machine-readable-cli-contract.md). + +Migration SQL is user-owned. File creation uses exclusive operations and a +per-version reservation; a pair is not atomic across crashes. Migration SQL and +URLs pass through unchanged. ClickHouse migrations remain non-transactional and +separate from the native application connection in +[ADR 0008](../adr/0008-native-clickhouse-runtime-and-migrations.md). + +The drivers support Devctl's builds with CGO disabled. Context checks precede +migration operations, but in-flight SQL and closing follow the library and +driver's waiting semantics; connection and query timeouts belong to the driver +URL. + +Release workflows package Devctl as a CLI archive or its own toolchain +container. Scaffold does not generate application deployment packaging. The +build/platform matrix, release and image workflows, temporary sibling-module +requirement, and automated versus live-database checks are described in the +[development guide](../development.md). diff --git a/docs/development.md b/docs/development.md index f24c20b..164b2fc 100644 --- a/docs/development.md +++ b/docs/development.md @@ -17,14 +17,18 @@ Run the complete local CI contract through Mise: mise run check ``` -`check` runs formatting, golangci-lint (including `govet`), dependency-graph -hygiene, race-enabled Go tests, build, generated documentation, the Orders API -example, and e2e workflows. The ordinary `mise run test` task remains -available for faster non-race iteration. - -The pull request CI runs this complete check on Linux and cross-builds the -Devctl binary for macOS amd64 and arm64 with CGO disabled. Native macOS jobs -are intentionally not part of the regular CI because the repository has no +`check` runs the core checks plus golangci-lint and e2e workflows. The +`check:core` task contains formatting, dependency-graph hygiene, Go tests, +build, generated documentation, and the Orders API example. The separate +`mise run test:race` task runs the race-enabled suite; CI runs it on pushes to +`main` and on the nightly schedule. + +Pull request CI runs the core, lint, and e2e checks in separate Linux jobs. +The separate Main CI workflow cross-builds the Devctl binary for macOS amd64 +and arm64 with CGO disabled on pushes to `main`. The Docker image check runs +in PR CI only when `Dockerfile` or `.dockerignore` changes. Race tests run +separately on pushes to `main` and on the nightly schedule. Native macOS jobs +are intentionally not part of regular CI because the repository has no platform-specific implementation. The release workflow still builds and publishes native macOS release archives for both architectures. @@ -34,6 +38,37 @@ Regenerate the CLI reference after changing visible command metadata: mise run docs:generate ``` +The command reference is generated from CLI metadata. `docs:check` checks that +reference; prose changes also need repository-local link and heading-anchor +checks. Product rules and architecture are indexed in [Documentation](README.md). + +## Release and image packaging + +| Artifact | Platforms | +| --- | --- | +| CLI release archive | Linux amd64, macOS amd64 and arm64 | +| Devctl container image | Linux amd64 and arm64 | + +The release helper derives stable and RC versions through `git-cliff` and +existing tags. Stable calculation ignores RC tags; RC versions carry a +sequential `-rc.N` suffix. Release creation requires new commits and an unused +remote tag, creates an annotated tag, and builds with `GOWORK=off` and CGO +disabled. Archives include the binary, README, and license. The GitHub Release +is created as a draft for manual publication. + +The image workflow accepts a published SemVer or RC release tag and checks its +commit/version metadata before building the image. Stable images receive both +the version tag and `latest`; RC images receive only their version tag. The +workflow publishes provenance attestations. The image packages Devctl, Go, +Git, Node, and quicktype; it is tooling for mounted Projects, not an application +deployment image generated by scaffold. + +The source of these rules is the +[version helper](../.github/scripts/compute_release_version.py), +[release workflow](../.github/workflows/release.yml), and +[image workflow](../.github/workflows/publish-image.yml). Packaging the current +checkout also requires resolving the local dependency constraint below. + ## Working with sibling modules For local work across sibling `devctl` and `go-libs` checkouts, create a Go @@ -43,10 +78,45 @@ workspace outside either repository, for example in their parent directory: go work init ./devctl ./go-libs/... ``` -Use that workspace only for interactive development. Do not commit `go.work` -or add `replace` directives to the released module. Before submitting, rerun -the checks above with `GOWORK=off` so they exercise the dependency graph used -by CI and release builds. +Use that workspace only for interactive development. Do not commit `go.work`. +Before submitting, rerun the checks above with `GOWORK=off` so an external +workspace does not hide dependency-graph problems. + +The current migration implementation temporarily requires +`github.com/devctllabs/go-libs/dbmigrate v0.0.0` with a `go.mod` replacement to +`../go-libs/dbmigrate`. Keep that module in a sibling `go-libs` checkout; the +replacement also applies with `GOWORK=off`. A standalone release, module +installation, or Docker build needs a published dependency version in place of +this local replacement. + +## Verifying migrations + +Run the focused CLI, service, and repository tests during migration changes: + +```sh +GOWORK=off go test ./cmd/devctl/internal/migrate ./internal/service/migration ./internal/repository/migration +GOWORK=off go test ./cmd/devctl -run 'TestMigrateCIWorkflow|TestInitScaffoldCreatesMigrationToolingWithoutOwningSQL' +``` + +Repository tests execute SQL against temporary SQLite databases, including +no-change application, dirty state, partial rollback, file containment, and +creation collisions. Environment tests cover URL precedence and safe failures. +The main-package workflow runs the built CLI against a temporary SQLite Project; +the scaffold test also exercises a generated Mise migration wrapper. + +Live PostgreSQL and ClickHouse verification is separate from these automated +tests. `mise run example:orders:check` runs the example's Go tests without a live +database; it does not apply the PostgreSQL SQL in `examples/orders-api`. + +To verify that example's actual migration, build the current Devctl binary and +put it on `PATH`, provide `ORDERS_API_DB_PRIMARY_POSTGRES_MIGRATIONS_URL` for an +isolated PostgreSQL database, and run `mise run migrate:primary:postgres:up` +from `examples/orders-api`. Check the resulting `orders` columns and constraints, +status, repeated up, down, and reapplication. The +[getting started guide](user-guide/getting-started.md) describes an API round +trip against the migrated schema. ClickHouse smoke verification likewise needs +an isolated live server and the selected target's migration URL; include +non-transactional failure and dirty-state behavior when changing its adapter. ## Ownership boundaries diff --git a/docs/product/README.md b/docs/product/README.md new file mode 100644 index 0000000..72e8701 --- /dev/null +++ b/docs/product/README.md @@ -0,0 +1,45 @@ +# Product documentation + +Devctl gives Project authors explicit workflows for defining, validating, and +materializing Go projects. Product documents describe implemented capabilities, +their observable behavior, and operational constraints. + +## Capabilities + +- [Project lifecycle](project-lifecycle.md): Manifest initialization and mutation, + discovery, validation, inspection, and effective Targets. +- [Contracts and generation](contracts-and-generation.md): bounded Sources, + committed Snapshots, linting, generators, and publication. +- [Scaffold and ownership](scaffold-and-ownership.md): Managed Outputs, Seeds, + custom configurations, and refresh behavior. +- [Runtime configuration](runtime-configuration.md): environment policy, + generated config, Start Policy, and migration separation. +- [Generated application](generated-application.md): explicit composition, + Scenarios, protocol integrations, and Resource contracts. +- [CLI automation](cli-automation.md): invocation, JSONL, findings, failures, + previews, and partial progress. +- [Migrations](migrations.md): create SQL pairs, apply or roll back migrations, + and inspect one database Variant's migration state. + +## Product decisions + +Accepted Product Decision Records preserve the reasons for significant behavior: + +- [PDR 0001: Manifest-first workflows](decisions/0001-manifest-first-workflows.md). +- [PDR 0002: Committed Contract Snapshots](decisions/0002-committed-contract-snapshots.md). +- [PDR 0003: Managed Outputs and Scaffold Seeds](decisions/0003-managed-outputs-and-scaffold-seeds.md). +- [PDR 0004: Project-owned toolchain](decisions/0004-project-owned-toolchain.md). +- [PDR 0005: Unified runtime configuration](decisions/0005-unified-runtime-configuration.md). +- [PDR 0006: Structured CLI contract](decisions/0006-structured-cli-contract.md). +- [PDR 0007: Explicit application composition](decisions/0007-explicit-application-composition.md). +- [PDR 0008: Explicit migration target selection](decisions/0008-explicit-migration-target-selection.md). +- [PDR 0009: Migration status and recovery](decisions/0009-migration-status-and-recovery.md). + +Capability documents state current behavior; PDRs preserve decision history. +Proposals, open questions, System Designs, and tickets belong in the +[local issue tracker](../agents/issue-tracker.md). They become current product +documentation after implementation. + +Use the [user guide](../user-guide/README.md) for command walkthroughs, the +[domain glossary](../../CONTEXT.md) for terms, and the +[architecture overview](../architecture/overview.md) for technical boundaries. diff --git a/docs/product/cli-automation.md b/docs/product/cli-automation.md new file mode 100644 index 0000000..01a6d5d --- /dev/null +++ b/docs/product/cli-automation.md @@ -0,0 +1,59 @@ +# CLI automation + +Devctl commands are non-interactive and expose structured outcomes for local +tools and CI. Users explicitly choose each workflow and inspect its result +before retrying a failed operation. The automation decision is recorded in +[PDR 0006](decisions/0006-structured-cli-contract.md). + +## Invocation and preview + +Executable leaves accept `--file`, `--json`, and `--verbose`. Flags and positional +arguments are validated before constructing the command's service graph. Help +does not require a Project or initialized integration resources. + +Text is the default output encoding. `--json` selects compact JSON Lines; +`--verbose` independently enables diagnostic detail. Common Manifest discovery +and target selection are described in [Project lifecycle](project-lifecycle.md). + +`sync` and `gen` offer `--dry-run`. Preview plans from local state without Source +acquisition, generator execution, publication, or pruning. It reports planned +actions, not a promise that later execution will succeed against changed remote +or workspace state. + +## Results, findings, and errors + +| Outcome | Output and exit behavior | +| --- | --- | +| Success | One final stdout event; command payload under `data`; exit `0` | +| Invalid validate/lint findings | Normal stdout result containing findings; exit `1` | +| Execution failure | Final stderr error with stable `code` and `exit_code`; stdout empty; exit `1` | +| Invalid CLI usage | Final stderr error; exit `2` | +| Cancellation | Final stderr error; exit `130` | + +JSON success is an event with `level`, `ts`, and `msg`, not a bare result object. +Errors expose safe context under `details`, and recovery information under +`details.partial_result`; they have no top-level `data`. Diagnostic events can +appear under verbose mode. Callers branch on stable failure categories rather +than human-readable messages. Exact fields and codes remain in the +[output reference](../user-guide/reference/output-and-errors.md). + +For validation and lint, exit `1` can mean completed analysis with findings. +Automation must inspect that command's stdout result instead of treating every +nonzero exit as an execution-error envelope. + +## Partial work and diagnostics + +Targets execute sequentially. Completed publication or SQL is not automatically +undone after a later operation or shutdown failure. A partial result describes +available completed progress, not a successful whole invocation. Planned work +alone does not justify a partial result. Review observed state before retrying. + +Ordinary diagnostics suppress raw causes and external tool output unless +verbose mode is selected. Migration SQL and credentials remain suppressed even +under verbose mode. Typed internal causes are retained for classification and +cancellation handling. + +SIGINT and SIGTERM cancel the Devctl command context. Cancellation does not +guarantee rollback or prompt interruption of every backend operation; migration +SQL and closing follow their documented +[driver waiting semantics](migrations.md#state-failures-and-recovery). diff --git a/docs/product/contracts-and-generation.md b/docs/product/contracts-and-generation.md new file mode 100644 index 0000000..e3b2653 --- /dev/null +++ b/docs/product/contracts-and-generation.md @@ -0,0 +1,103 @@ +# Contracts and generation + +Devctl manages OpenAPI 3.1, Proto, JSON Schema, and raw Kafka Contract surfaces. +A Contract Reference selects content from a bounded Source. External updates +become reviewable repository changes through explicit synchronization. + +## Sources and Exports + +| Source | Selection and acquisition | +| --- | --- | +| Local | File closures or Proto trees inside a Project-local containment root; sync succeeds without publication | +| URL | A URL-based OpenAPI or JSON Schema closure with a virtual committed Entrypoint | +| Git | A repository and revision, optionally narrowed to a contained path; supports file closures and Proto trees | +| Devctl | A repository and revision containing an upstream Project; selects a named Export, not a Source path | + +Ordinary Sources use a relative Contract path. Devctl Sources use an Export +whose surface must match the upstream effective HTTP server, gRPC server Module +Root, or Kafka producer. Only the selected upstream surface needs to be +materializable; unrelated upstream Readiness is not required. + +Raw Kafka Targets have no Source, Contract path, or Export reference and do not +materialize a Snapshot. Cross-project schema-backed Kafka consumption selects +JSON or Proto explicitly. + +URL acquisition requires HTTPS unless insecure HTTP was explicitly allowed. +Credentials are forbidden. Requests and redirects retain the original scheme, +host, and effective port; only relative references are followed, while absolute +references are ignored. Limits are 64 documents, 32 MiB per response, and +64 MiB per Snapshot. Query values distinguish fetch identities but are redacted +from diagnostics; fragments do not distinguish fetch identities. Distinct URLs +that collide at one committed path fail rather than overwrite one another. + +## Committed Contract state + +`sync` materializes external Contract Snapshots and publishes them under the +Project's managed external-contract namespace. Users review and commit that +state. A full or family sync may remove stale Target children; a targeted sync +never prunes siblings. `--dry-run` previews planned publication and removal +without acquiring Sources, invoking tools, or changing files. + +Snapshots retain their Entrypoint or Module Root and relevant supplier Buf +configuration and lock file. Devctl-sourced gRPC and schema-backed Kafka +snapshots carry `.devctl-contract.json` metadata describing their selected +surface: + +| Snapshot | Metadata fields | +| --- | --- | +| gRPC Proto | `kind: grpc`, `format: proto`, `module_root`, `buf_config` | +| Kafka JSON | `kind: kafka`, `format: json`, `topic`, `entrypoint` | +| Kafka Proto | `kind: kafka`, `format: proto`, `topic`, `entrypoint`, `module_root`, `buf_config` | + +Metadata rejects unknown fields and trailing JSON values. Paths are normalized, +relative, contained, and must resolve to the declared Snapshot content; a Proto +Kafka Entrypoint stays inside its Module Root. `buf.lock` is a Snapshot file, +not a metadata field. Local and HTTP/OpenAPI inputs have no sidecar. + +Required metadata that is missing, malformed, or inconsistent fails downstream +execution with `snapshot_metadata_invalid` and guidance to run `devctl sync`; +Devctl does not rediscover a schema heuristically. + +`lint`, `gen`, and inspection consume local or committed inputs without +reacquiring the configured supplier. This separates Contract updates from their +consumption, as recorded in +[PDR 0002](decisions/0002-committed-contract-snapshots.md). External tools may +still need network access for their own modules or plugins; operation without +network access also requires prepared tool dependencies and caches. + +## Lint and generation + +Lint checks Contract structure and conventions: OpenAPI operations and successful +responses, Proto naming and Buf lint rules, Kafka topic shape, and JSON Schema +validity and root `title`. Local Kafka schema filenames must be `.json` +or `.proto`; directories may vary, while the basename must match. This +filename rule does not apply to external Sources. Findings are normal output +with exit code `1` when invalid. Lint never synchronizes or generates. + +| Target input | Generator | +| --- | --- | +| Runtime Config | Shared Devctl config renderer | +| HTTP server/client OpenAPI | Project-declared oapi-codegen | +| gRPC and Kafka Proto | Project-declared Buf and its generator configuration | +| Kafka JSON Schema | Project-installed quicktype; root `title` owns the Go type name | +| Raw Kafka | No generated Contract code | + +The consuming Project owns tool versions and native generator configuration; +see [PDR 0004](decisions/0004-project-owned-toolchain.md). Canonical native +configs are Managed Outputs. Alternate configs are user-owned, must exist for +validation/generation, and are not created or replaced by scaffold. Multiple +Targets may consume one user-owned config. Supplier Buf configuration and the +consumer's `*.gen.yaml` have separate ownership. + +Generation executes config, HTTP server/client, gRPC server/client, then Kafka +consumer/producer Targets. Tools write into temporary storage before managed +publication. Each selected output tree replaces its previous tree and removes +stale files inside that boundary; unrelated Target trees remain untouched. +Config generation also publishes `.env.example` as a separate managed file. + +Publication is atomic at each file or tree boundary, not across the whole +workflow. A later failure preserves completed changes and returns their partial +result. Dry-run does not execute generators or publish outputs. See +[scaffold and ownership](scaffold-and-ownership.md), the +[workflow guide](../user-guide/contracts-and-generation.md), and the +[Source reference](../user-guide/reference/manifest/sources-and-exports.md). diff --git a/docs/product/decisions/0001-manifest-first-workflows.md b/docs/product/decisions/0001-manifest-first-workflows.md new file mode 100644 index 0000000..f571b7f --- /dev/null +++ b/docs/product/decisions/0001-manifest-first-workflows.md @@ -0,0 +1,17 @@ +# Keep Project changes and workflows explicit + +Status: Accepted +Date: 2026-09-03 + +One checked-in Manifest defines desired Project shape, and each Devctl workflow +has explicit filesystem, tool, and network effects. Manifest mutation, scaffold, +sync, lint, generation, and migration execution do not silently compose one +another, so users can review changes and run only the operations a change needs. +Stable effective Targets keep selection consistent across workflows instead of +giving the same declaration different meanings in different commands. + +The accepted rationale is recorded in +[ADR 0001](../../adr/0001-canonical-project-model-and-command-boundaries.md), +with migration execution added by +[ADR 0009](../../adr/0009-explicit-migration-execution.md). Current behavior is +described in [Project lifecycle](../project-lifecycle.md). diff --git a/docs/product/decisions/0002-committed-contract-snapshots.md b/docs/product/decisions/0002-committed-contract-snapshots.md new file mode 100644 index 0000000..ebe788a --- /dev/null +++ b/docs/product/decisions/0002-committed-contract-snapshots.md @@ -0,0 +1,21 @@ +# Consume external Contracts from committed Snapshots + +Status: Accepted +Date: 2026-09-03 + +External suppliers can change or become unavailable between builds, so `sync` +publishes bounded, reviewable Contract Snapshots into the consuming Project. +Lint and generation consume local or committed state and metadata without +reacquiring the supplier, trading automatic freshness for reproducible inputs +and explicit updates. Named Exports identify exact upstream surfaces rather +than relying on schema discovery heuristics. + +## Consequences + +Users commit Snapshots and refresh stale metadata through `sync`. Running +without network access also requires preparing tool and module dependencies; +committing supplier state does not sandbox external tools. + +The rationale is recorded in +[ADR 0002](../../adr/0002-contract-sources-exports-and-snapshots.md). Current +behavior is described in [Contracts and generation](../contracts-and-generation.md). diff --git a/docs/product/decisions/0003-managed-outputs-and-scaffold-seeds.md b/docs/product/decisions/0003-managed-outputs-and-scaffold-seeds.md new file mode 100644 index 0000000..46cc40f --- /dev/null +++ b/docs/product/decisions/0003-managed-outputs-and-scaffold-seeds.md @@ -0,0 +1,20 @@ +# Separate refreshable outputs from user-owned Seeds + +Status: Accepted +Date: 2026-09-03 + +Generated infrastructure needs deterministic refresh, while application code +must retain user intent. Devctl may replace complete Managed Outputs but creates +Scaffold Seeds only when absent, so ownership determines what refresh can +change. Publication is atomic at its file or tree boundary, and a later failure +preserves completed work instead of attempting workflow-wide rollback. + +## Consequences + +Users keep handwritten code outside managed trees and inspect partial results +before retrying. Seed creation accepts the documented external-writer race; +it is not an atomic cross-process create-if-absent guarantee. + +The rationale is recorded in +[ADR 0003](../../adr/0003-managed-artifacts-and-publication.md). Current +behavior is described in [Scaffold and ownership](../scaffold-and-ownership.md). diff --git a/docs/product/decisions/0004-project-owned-toolchain.md b/docs/product/decisions/0004-project-owned-toolchain.md new file mode 100644 index 0000000..0738258 --- /dev/null +++ b/docs/product/decisions/0004-project-owned-toolchain.md @@ -0,0 +1,16 @@ +# Keep generator versions and configuration in the Project + +Status: Accepted +Date: 2026-09-03 + +Generated code must be reproducible from versioned Project inputs, so generator +versions and native configuration belong to the consuming Project rather than +an independently changing global or Devctl-bundled policy. Go tool declarations +and Mise describe the selected toolchain, making upgrades reviewable dependency +changes. Canonical generator configs are managed, while alternate configs stay +user-owned and are consumed without scaffold replacement. + +The rationale is recorded in +[ADR 0004](../../adr/0004-project-owned-generator-toolchain.md). Current +behavior is described in [Contracts and generation](../contracts-and-generation.md) +and [Scaffold and ownership](../scaffold-and-ownership.md). diff --git a/docs/product/decisions/0005-unified-runtime-configuration.md b/docs/product/decisions/0005-unified-runtime-configuration.md new file mode 100644 index 0000000..18df5d3 --- /dev/null +++ b/docs/product/decisions/0005-unified-runtime-configuration.md @@ -0,0 +1,15 @@ +# Share one effective runtime environment contract + +Status: Accepted +Date: 2026-09-03 + +Independent defaults and environment-key rules previously let generated config, +scaffold consumers, and inspection disagree. All projections now share the +Manifest's effective keys, types, defaults, field groups, and secret handling so +the settings users inspect match the settings their application consumes. +Start Policy explicitly separates constructing a dependency from starting its +background work, while migration-only URLs remain outside application config. + +The rationale is recorded in +[ADR 0005](../../adr/0005-canonical-runtime-config.md). Current behavior is +described in [Runtime configuration](../runtime-configuration.md). diff --git a/docs/product/decisions/0006-structured-cli-contract.md b/docs/product/decisions/0006-structured-cli-contract.md new file mode 100644 index 0000000..c5c25f9 --- /dev/null +++ b/docs/product/decisions/0006-structured-cli-contract.md @@ -0,0 +1,15 @@ +# Expose structured command outcomes for automation + +Status: Accepted +Date: 2026-09-03 + +Automation needs stable outcomes without parsing human messages, so JSON mode +emits result and error events with defined streams, failure categories, and exit +codes. Validation and lint findings remain normal result data even when they +produce exit code `1`. Completed progress after a later failure belongs in an +error's partial result, preserving recovery information without presenting the +whole invocation as successful. + +The rationale is recorded in +[ADR 0006](../../adr/0006-machine-readable-cli-contract.md). Current behavior is +described in [CLI automation](../cli-automation.md). diff --git a/docs/product/decisions/0007-explicit-application-composition.md b/docs/product/decisions/0007-explicit-application-composition.md new file mode 100644 index 0000000..24fec53 --- /dev/null +++ b/docs/product/decisions/0007-explicit-application-composition.md @@ -0,0 +1,21 @@ +# Keep application composition explicit and user-owned + +Status: Accepted +Date: 2026-09-03 + +Automatic registry wiring would hide application intent and require runtime +discovery or mutation of user code. Generated infrastructure therefore exposes +providers and typed protocol boundaries, while a user-owned composition list +selects application behavior and lazy Scenarios construct the selected runtime +branch. Unimplemented Kafka handlers fail closed rather than acknowledge +messages without application work. + +## Consequences + +Adding a Component can create new binding Seeds but requires an explicit provider +call in existing composition. Refresh preserves those choices instead of +silently activating newly available behavior. + +The rationale is recorded in +[ADR 0007](../../adr/0007-explicit-application-composition-and-runtime-scenarios.md). +Current behavior is described in [Generated application](../generated-application.md). diff --git a/docs/product/decisions/0008-explicit-migration-target-selection.md b/docs/product/decisions/0008-explicit-migration-target-selection.md new file mode 100644 index 0000000..e65a902 --- /dev/null +++ b/docs/product/decisions/0008-explicit-migration-target-selection.md @@ -0,0 +1,22 @@ +# Select one migration target independently of runtime defaults + +Status: Accepted +Date: 2026-10-19 + +A Project can declare several database Connections and Variants, so using its +runtime default for migrations could change the destination when application +wiring changes. Migrations select one `connection:variant`, with automatic +selection only when exactly one migration-enabled Variant exists across the +Manifest, keeping database changes tied to an unambiguous declared target. +The migration URL comes from its dedicated environment key, an explicit +env-file, or its Manifest default, so deployment environments can supply their +own database location without changing target identity. + +## Consequences + +CI specifies the target explicitly. Runtime database configuration and implicit +dotenv loading are not migration fallbacks; a present empty URL fails rather +than selecting a lower-priority destination. Adding another migration-enabled +Variant makes previously implicit selection require `--target`. + +Current behavior is described in [Migrations](../migrations.md). diff --git a/docs/product/decisions/0009-migration-status-and-recovery.md b/docs/product/decisions/0009-migration-status-and-recovery.md new file mode 100644 index 0000000..cb0eb30 --- /dev/null +++ b/docs/product/decisions/0009-migration-status-and-recovery.md @@ -0,0 +1,19 @@ +# Report dirty migration state and keep recovery explicit + +Status: Accepted +Date: 2026-10-19 + +Operators need to inspect migration state after a failure, so `status` succeeds +when a database is dirty while `up` and `down` reject that state. Status uses +the normal migration backend and accepts metadata initialization, including +creation of a SQLite database file, to keep state interpretation consistent +with execution. Recovery remains an explicit operator action outside Devctl +because partial SQL effects cannot be safely inferred or repaired automatically. + +## Consequences + +Status exit success does not imply a clean database or an absence of writes; +automation must inspect `dirty`. Devctl provides no force or automatic repair +operation, and an operation error does not guarantee that no SQL took effect. + +Current behavior is described in [Migrations](../migrations.md). diff --git a/docs/product/generated-application.md b/docs/product/generated-application.md new file mode 100644 index 0000000..8fe81c9 --- /dev/null +++ b/docs/product/generated-application.md @@ -0,0 +1,70 @@ +# Generated application + +Scaffold provides Go infrastructure and executable foundations. Application +behavior stays in user-owned code, including provider composition, protocol +registrations, handlers, and repositories. Adding a Manifest Component does not +silently wire new application behavior into existing composition. + +## Composition and Scenarios + +`provideApplication` is the explicit user-owned provider list. Generated +infrastructure registers lazy dependencies; users connect those providers to +their application types. Devctl does not rewrite Go syntax or discover plugins +at runtime. See [PDR 0007](decisions/0007-explicit-application-composition.md). + +An API Scenario owns its dependency graph, selected HTTP/gRPC/health/pprof work, +and shutdown. A consumer Scenario owns one named Kafka runner. Construction is +lazy and shutdown closes resources constructed by that Scenario. Selecting an +unknown or disabled consumer fails before its Kafka client is constructed. +These are application errors, separate from Devctl's CLI error categories. + +The `cli` preset supplies a root command foundation; application-specific +commands and behavior remain user-owned. Scaffold creates API or consumer +command Seeds when those capabilities are present. + +## Protocol boundaries + +| Capability | Generated contract and user responsibility | +| --- | --- | +| HTTP server | Echo runtime with a typed `HTTPRegistrar`; users implement and register application routes | +| gRPC server | Runtime with a typed `GRPCRegistrar`; users register generated services | +| HTTP client | Raw `*http.Client` and base URL; users construct the generated OpenAPI client | +| gRPC client | Raw `*grpc.ClientConn`; users construct Proto stubs; the generated connection uses plaintext credentials | +| Kafka producer | Raw byte producer with separately available topic configuration; users compose application publishing behavior | +| Kafka consumer | Named config, decoder, batch handler, retry policy, and runner; users implement the handler and select message types | + +The generated gRPC client provider has no Manifest TLS/auth surface. Secure +transport composition requires user-owned wiring. Devctl does not add facades +around generated HTTP clients or Proto stubs. + +Kafka supports raw bytes, JSON Schema, and Proto, with binary or JSON Proto +encoding. Omitting `format` selects raw; schema-backed endpoints explicitly +choose `json` or `proto`. Schema-backed consumers initially bind `[]byte` too; +users opt into generated message types in their Provider Binding. Raw batch +handlers must not retain batch data. The initial handler returns a permanent +not-implemented error, so unfinished behavior cannot acknowledge messages +silently. + +## Resources and operations + +| Resource | Supported behavior | +| --- | --- | +| SQLite / PostgreSQL | One selected Variant per logical Connection, transactional endpoints, reader/writer access and transaction manager | +| ClickHouse | Native connection and eager ping; separate from transactional Variants, without transaction manager or reader/writer abstraction | +| Redis | Named client and owned close behavior; addresses exclude credential-bearing URLs; no generated health probe | +| S3 | Named AWS SDK connections with ambient/static credentials, endpoint, region and path-style options; named bucket wrappers | +| Logging / telemetry | Shared graph resources with environment-backed configuration and owned cleanup | +| Health / pprof | Configured operational server tasks; generated database probes are non-critical | + +Backend-specific ClickHouse semantics are recorded in +[ADR 0008](../adr/0008-native-clickhouse-runtime-and-migrations.md). Migration +execution is separate from runtime construction and application startup. + +Scaffold does not produce application Dockerfiles, Compose files, Helm charts, +or Kubernetes manifests. The repository's own container image packages the +Devctl CLI. Projects own their application deployment packaging. + +See [runtime configuration](runtime-configuration.md), +[scaffold ownership](scaffold-and-ownership.md), and the +[generated Project guide](../user-guide/generated-project.md) for the related +contracts and application workflow. diff --git a/docs/product/migrations.md b/docs/product/migrations.md new file mode 100644 index 0000000..d9be5e6 --- /dev/null +++ b/docs/product/migrations.md @@ -0,0 +1,106 @@ +# Migrations + +Devctl creates and executes user-owned SQL migrations for SQLite, PostgreSQL, +and ClickHouse database Variants. Execution is an explicit command; scaffold, +generation, and application startup do not apply migrations. + +## Operations + +| Operation | Behavior | +| --- | --- | +| `migrate create ` | Creates an empty timestamped `.up.sql` / `.down.sql` pair in the selected migration directory. Does not resolve a database URL or connect to a database. | +| `migrate up` | Applies all pending migrations by default, or a positive number selected by `--steps N`. Repeating an unbounded successful application succeeds with `no_change: true`. An empty SQL source fails. | +| `migrate down` | Rolls back one migration by default, or a positive number selected by `--steps N`. No confirmation is requested. A database with no applied version fails. | +| `migrate status` | Reports the current version and dirty state. No applied version is represented by `version: null`. Dirty state is a successful status result. | + +Each operation loads a semantically valid Manifest. Unrelated generator tools, +generated code, and application module readiness are not migration prerequisites. +Common command options include `--file`, `--target`, `--json`, and `--verbose`; +`--env-file` is available on `up`, `down`, and `status`. See the +[command reference](../user-guide/reference/commands.md) for the complete CLI. + +## Selecting a Migration Target + +A Migration Target belongs to one database Connection and Variant and is +addressed as `connection:variant`. Only Variants with a `migrations` block are +eligible. With no `--target`, Devctl selects automatically only when exactly +one eligible Variant exists in the whole Manifest. A runtime default +Connection does not influence migration selection. + +Multiple eligible Variants require an explicit target and produce an +`invalid_input` error listing candidates. No eligible target or an unknown +selector produces `not_found`; explicitly selecting an existing Variant with +migrations disabled produces `unsupported`. Each invocation operates on one +target. CI should always provide `--target`. + +The reason for this selection policy is recorded in +[PDR 0008](decisions/0008-explicit-migration-target-selection.md). + +## Database configuration and paths + +The selected Variant declares `migrations.path`, `migrations.database_env`, and +an optional `migrations.database_default`. The environment key includes the +Project's effective environment prefix. The URL is resolved in this order: + +1. A present process environment key. +2. That key in the explicitly supplied dotenv file. +3. The Manifest default. + +Presence controls precedence: an empty process or file value fails instead of +falling back. An explicitly supplied file must be readable and valid even when +the process environment already supplies the URL. Reading it does not change +the process environment. Devctl does not load `.env` implicitly or use an +application runtime DSN as a fallback. + +URLs must match the selected backend: `sqlite://`, `postgres://` or +`postgresql://`, or `clickhouse://`. Devctl passes the URL through without +rewriting its path or driver parameters. The same Manifest target can therefore +be used across environments by supplying a different migration URL. + +SQL paths are relative to the selected Manifest's directory and must stay +within that Project. Relative `--env-file` paths and relative SQLite database +URLs retain the invoking working directory's meaning, including when `--file` +selects a Manifest elsewhere. There is no command-line SQL directory override. + +## SQL ownership and creation + +SQL files remain user-owned, including after scaffold runs. `create` makes a +missing migration directory and names its pair +`YYYYMMDDhhmmss_.{up,down}.sql` using UTC with second precision. Names +start with an ASCII letter or digit and contain only ASCII letters, digits, +underscores, and hyphens. + +An existing version, concurrent creation in the same second, or an existing +destination produces a conflict without overwriting SQL. Ordinary creation +failure attempts to remove files created by that invocation; a process crash +can leave an incomplete pair or a `.devctl-create-` reservation for +operator inspection. + +Scaffold manages the Mise wrappers `migrate:::` followed +by `create`, `up`, `down`, or `status`. They invoke Devctl with an explicit +target; no separate migration executable is required. + +## State, failures, and recovery + +Opening a target for `status` may initialize its migration metadata table and, +for SQLite, create the database file. Status is therefore a successful state +inspection with possible initialization effects. Both `up` and `down` reject +dirty state as a conflict. Repair and force operations remain outside Devctl; +see [PDR 0009](decisions/0009-migration-status-and-recovery.md). + +A failed command can leave changes in the database. In particular, bounded +`up` or `down` requesting more steps than exist can apply available steps +before reporting the shortage. Devctl does not roll back completed migrations +after a later failure. When changes are observed, the error includes the +available partial result; SQL and credentials are omitted from diagnostics even +with `--verbose`. +Success reports the selected target and either created file paths or the +observed version and dirty state, following the +[output and error contract](../user-guide/reference/output-and-errors.md). + +Cancellation is checked before migration work; an in-flight SQL operation and +resource closing may wait for the driver. Operators configure timeouts through +the database driver URL. ClickHouse migrations are non-transactional and may +need manual recovery after partial application. Multi-statement files require +appropriate driver configuration, commonly `x-multi-statement=true`; Devctl +does not add it, and the driver's semicolon splitting has limitations. diff --git a/docs/product/project-lifecycle.md b/docs/product/project-lifecycle.md new file mode 100644 index 0000000..0730c70 --- /dev/null +++ b/docs/product/project-lifecycle.md @@ -0,0 +1,75 @@ +# Project lifecycle + +Devctl is a non-interactive control plane for Go Projects. A checked-in +`devctl.yaml` Manifest declares desired Project shape; application behavior and +protocol definitions remain explicit user-owned inputs. + +## Initialization and mutation + +Manifest v1 supports Go with `cli` and `http-service` initialization presets. +`init manifest` writes only the Manifest. `enable` adds runtime Capabilities; +`add` declares Resources, Sources, clients, and Kafka endpoints. These commands +do not install tools, scaffold an application, acquire Contracts, or run code +generation. + +An identical declaration succeeds unchanged. Replacing a different existing +declaration requires `--force`. `init manifest --force` rebuilds the complete +Manifest from its arguments and preset; it does not merge with existing content. +Other mutations operate on the selected semantically valid Manifest and publish +the updated document atomically. + +`init scaffold` is a separate operation that refreshes Managed Outputs and +creates missing Scaffold Seeds. It has no `--force` option. After a structural +change, users review the generated foundation and explicitly register new +Provider Bindings. See [scaffold and ownership](scaffold-and-ownership.md). + +## Discovery and paths + +Without `--file`, Devctl searches upward from the invoking directory for +`devctl.yaml`. An explicit file selects that Manifest and its directory as the +Project root; `--file` may be absolute. Manifest-declared paths are +Project-relative and must remain contained. Semantic validation checks path +declarations and managed-output conflicts; filesystem adapters enforce +containment when inputs and outputs are accessed. + +Migration env-file paths and SQLite database URLs have their own +[working-directory rules](migrations.md#database-configuration-and-paths). + +## Validation and inspection + +| Layer | What it establishes | +| --- | --- | +| Structural decoding | Valid YAML shape, known fields, correct scalar/container types, no duplicate keys | +| Semantic validity | Supported version/language, valid identity and paths, references, defaults, Resource rules, Runtime Config consistency | +| Readiness | Required local inputs, migration directories, module/tool declarations, and native generator configurations | + +`validate` checks all three layers. Findings are a normal result with exit code +`1`; loading or I/O failures use the execution-error contract. Other workflows +load a semantically valid Project and check their own prerequisites, without +requiring unrelated application or generator readiness. + +`inspect` reports the effective Project: paths, Targets, Resources, environment +keys and safe defaults. It is an inspection operation, not source acquisition. +Missing or invalid committed Snapshot Metadata simply omits `resolved_input`; +it does not prevent inspection of the Manifest. + +## Addressable operations + +The Target Catalog gives `sync`, `lint`, and `gen` stable selectors: +`config`, `http-server`, `http-client:`, `grpc-server`, +`grpc-client:`, `kafka-consumer:`, and `kafka-producer:`. +Every Go Project has an implicit config Target. Each Target declares which +operations it supports, and `inspect` lists the effective facts in stable order. + +An unknown family produces `invalid_input`; a known family with no applicable +Targets succeeds with an empty result. An unknown explicit Target produces +`not_found`; an existing Target without the requested operation produces +`unsupported`. Local Contract sync is a supported no-op. Raw Kafka Targets +have no sync operation. Migration Targets use their separate +`connection:variant` selection rules. + +Workflows remain explicit: `sync`, `lint`, and `gen` never invoke one another, +and applying migrations requires `migrate`. This boundary is recorded in +[PDR 0001](decisions/0001-manifest-first-workflows.md). See the +[Project workflow guide](../user-guide/project-workflow.md) for command sequences +and [CLI automation](cli-automation.md) for result and failure handling. diff --git a/docs/product/runtime-configuration.md b/docs/product/runtime-configuration.md new file mode 100644 index 0000000..e7321f6 --- /dev/null +++ b/docs/product/runtime-configuration.md @@ -0,0 +1,69 @@ +# Runtime configuration + +The Manifest defines one effective environment contract for generated Go config, +`.env.example`, inspection, and application consumers. Users can inspect its +keys, types, defaults, and secret markers before starting the application. +The decision to keep these projections consistent is recorded in +[PDR 0005](decisions/0005-unified-runtime-configuration.md). + +## Keys, values, and generated code + +The default environment prefix derives from the Project name: +`orders-api` becomes `ORDERS_API_`. `env.prefix` overrides it. Keys beginning +with `OTEL_` retain their ecosystem prefix. + +Built-in Component settings and custom environment declarations share key and +type rules. Custom values support string, bool, int, and duration types. +Effective key conflicts and generated Go field-path collisions are reported as +`runtime_config_conflict`, rather than producing inconsistent projections. + +Generated fields are grouped by meaning, such as `cfg.HTTP.Address` or +`cfg.DBPrimary.PostgresDSN`. Runtime values come from environment with declared +defaults. Secret fields have no rendered defaults: generated tags omit them, +`.env.example` leaves them blank, and inspection exposes a secret marker. + +Every Go Project has a config Target. Its default output is +`gen/config/config.gen.go`; `languages.go.generators.config.out` selects the +output directory while the filename remains `config.gen.go`. Both `gen config` +and scaffold render that config and the root `.env.example` from the same +effective settings. The example file is a managed template, not an implicit +request to load dotenv files into application processes. + +## Runtime Start Policy + +An absent `start` block means a Capability is always enabled. A present block +requires an environment key and creates a toggle; omitted `default` means +`false`. Runtime settings distinguish constructing a dependency from starting +its background work: a disabled Capability may still be constructed but does +not contribute a runtime task. + +Kafka consumer selection is stricter: an unknown or disabled selected consumer +fails before its client graph is constructed. API Scenarios run the enabled +HTTP, gRPC, health, and pprof tasks. Logging and telemetry are supporting +resources rather than additional API server roots. + +## Resource configuration + +Database Runtime Config selects one Variant of a logical Connection and its +runtime DSN. A single Variant can omit the Connection default; multiple +Variants require a valid default. ClickHouse remains separate from +transactional Variants in a Connection. + +Redis settings identify named connection addresses. S3 settings describe named +connections, ambient or static credentials, endpoint/region options, and bucket +selection. HTTP and gRPC clients have environment-backed base URLs or addresses. +Kafka settings cover brokers, topics, groups, batching, retry, rebalance, drain, +and shutdown behavior; Manifest topics are defaults and can be overridden +through environment. + +Kafka's default commit retry inherits the main retry policy, and rejection stops +the consumer. Additional library-specific choices remain in user-owned custom +settings and Provider Bindings until exposed by the Manifest. + +## Migration boundary + +Migration-only URLs appear in `.env.example` and inspection but are absent from +application Runtime Config. `migrate` resolves its selected environment key +separately, with its own explicit env-file and fallback rules. Runtime DSNs are +not migration fallbacks. See [migrations](migrations.md) and the +[environment reference](../user-guide/reference/manifest/project-and-language.md#env). diff --git a/docs/product/scaffold-and-ownership.md b/docs/product/scaffold-and-ownership.md new file mode 100644 index 0000000..dcc1766 --- /dev/null +++ b/docs/product/scaffold-and-ownership.md @@ -0,0 +1,64 @@ +# Scaffold and ownership + +`init scaffold` materializes the foundation described by a semantically valid +Manifest. It plans and preflights artifacts before publishing them in stable +order. Scaffold does not install dependencies, synchronize Contracts, run +external generators, or apply migrations. + +## File ownership + +| Owner | Examples | Refresh behavior | +| --- | --- | --- | +| Managed Output | `go.mod`, `.mise.toml`, `.golangci.yml`, `.env.example`, generated config, `internal/deps/*.gen.go`, canonical native generator configs | Devctl may replace contents | +| Scaffold Seed | Application commands, `internal/deps/application.go`, Provider Bindings, handlers, server OpenAPI seed, scaffolded README, migration placeholders | Created if absent; existing regular files remain user-owned | +| User-owned input | Migration SQL, alternate native generator configs, handwritten application behavior | Not refreshed by scaffold | + +Devctl-owned Go scaffold files use `*.gen.go` and a generated-file header. +Ownership also applies to non-Go files: manual dependency or task edits can be +replaced when `go.mod` or `.mise.toml` is refreshed. Review those diffs after +scaffold and dependency resolution. + +Managed Contract and generated-code trees belong entirely to their owning +workflow. Handwritten files inside those trees can be removed by replacement. +The distinction and partial-publication policy are recorded in +[PDR 0003](decisions/0003-managed-outputs-and-scaffold-seeds.md). + +## Native generator configs + +Canonical HTTP config paths are `tools/oapi/server.yaml` and +`tools/oapi/clients..yaml`. Canonical consumer Buf configs are +`tools/buf/grpc.gen.yaml` and `tools/buf/kafka.gen.yaml`. These remain managed +even when the Manifest explicitly spells out their canonical value. + +An alternate config path is user-owned. Scaffold does not create or overwrite +it; Readiness requires a regular, non-symlink file at a safe path, and `gen` +consumes it. Filesystem access enforces containment for alternate paths. +Several HTTP Targets may share a user-owned config. A custom HTTP config path +that equals a planned scaffold artifact is rejected before publication so one +file cannot acquire conflicting owners. + +## Refresh and composition + +Refresh creates missing Seeds and updates current Managed Outputs. It does not +delete stale scaffold files after a Component or Resource is removed. This +differs from cleanup inside managed `sync` and `gen` trees; users review and +remove obsolete scaffold files themselves. + +`internal/deps/application.go` contains the user-owned `provideApplication` +composition list. Scaffold can add infrastructure and new binding Seeds but +does not edit that list. Users explicitly call new providers and connect +application handlers, generated clients, and message types. See +[generated application behavior](generated-application.md) and +[PDR 0007](decisions/0007-explicit-application-composition.md). + +## Failures and concurrency + +Unsafe symlinks, non-regular destinations, and detectable plan conflicts fail +before publication. A later I/O failure can leave previously published files; +the result reports that progress and no workflow-wide rollback occurs. + +Seed creation preserves existing files but is not an atomic cross-process +create-if-absent operation. An external writer can race the existence check. +`init scaffold` has no force option; migration SQL creation has its own stronger +[version reservation and exclusive creation](migrations.md#sql-ownership-and-creation) +rules. diff --git a/docs/user-guide/concepts-and-ownership.md b/docs/user-guide/concepts-and-ownership.md index 0bdb8b1..53aae9c 100644 --- a/docs/user-guide/concepts-and-ownership.md +++ b/docs/user-guide/concepts-and-ownership.md @@ -26,7 +26,9 @@ when generated code is required. A **Source** is a bounded origin for Contracts. A Contract Reference selects an Entrypoint or named Export from that Source. External Contracts become committed **Contract Snapshots** after `sync`, so later lint and generation can -run offline and produce reviewable changes. +consume them without reacquiring the supplier. External tools may still require +network access or prepared dependency caches; see +[Contracts and generation](../product/contracts-and-generation.md). A **Target** is the effective unit addressed by `sync`, `lint`, or `gen`, for example `http-client:billing`. `inspect` shows the complete, deterministically diff --git a/docs/user-guide/contracts-and-generation.md b/docs/user-guide/contracts-and-generation.md index f0a794f..759e6cc 100644 --- a/docs/user-guide/contracts-and-generation.md +++ b/docs/user-guide/contracts-and-generation.md @@ -61,12 +61,21 @@ go mod tidy mise run check ``` -## Supplier and consumer Buf files +## Generator configuration ownership + +Canonical native generator configs are Managed Outputs. Alternate HTTP +`oapi_config` and Proto `buf_gen_config` files are user-owned: create them before +validation or generation; scaffold does not create or replace them. HTTP Targets +may share a custom config, but its path must not collide with a scaffold +artifact. Explicitly spelling a canonical config path keeps it managed. For Proto, the supplier's `buf.yaml` and adjacent `buf.lock` travel with the -Contract Snapshot. The consuming Project owns `*.gen.yaml`. Canonical consumer -configs are Managed Outputs; an explicitly selected `buf_gen_config` is -user-owned and must already exist before validation or generation. +Contract Snapshot. The consuming Project separately owns `*.gen.yaml` and its +generation policy. + +Lint and generation do not reacquire Contract suppliers. Their external tools +may still need module or plugin downloads, so prepare tool dependencies and +caches before relying on execution without network access. ## When to run each command diff --git a/docs/user-guide/generated-project.md b/docs/user-guide/generated-project.md index 3c7c7c2..d6f7f8a 100644 --- a/docs/user-guide/generated-project.md +++ b/docs/user-guide/generated-project.md @@ -10,6 +10,13 @@ Managed files may change on every refresh. Scaffold Seeds are yours after creation. Do not put handwritten files inside a Target directory owned by `sync` or `gen`, because publication may replace that complete tree. +Managed files also include `go.mod`, `.mise.toml`, `.golangci.yml`, +`.env.example`, and canonical generator configs. Review refreshed dependencies +and task definitions as well as Go code. Removing a Component or Resource from +the Manifest does not prune its old scaffold files; review and remove obsolete +files yourself. See [Scaffold and ownership](../product/scaffold-and-ownership.md) +for the complete contract. + ## Compose application behavior `internal/deps/application.go` is the user-owned composition list. Generated @@ -22,6 +29,10 @@ HTTP and gRPC behavior is registered through `HTTPRegistrar` and URLs or gRPC connections; application code constructs the generated client or stub it needs. +Generated gRPC client providers use plaintext transport. Use user-owned provider +composition when TLS or authentication is required; the Manifest does not expose +those options. + ## Run Scenarios The generated `Scenario` owns one lazy dependency graph and orderly shutdown: @@ -35,6 +46,9 @@ Unknown or disabled consumers fail before their Kafka client is constructed. The initial Kafka handler Seed returns a permanent not-implemented error so an unfinished consumer cannot acknowledge messages silently. +Application deployment packaging remains Project-owned. Scaffold does not +generate Dockerfiles, Compose files, Helm charts, or Kubernetes manifests. + ## Configure runtime behavior Generated Runtime Config is derived from the Manifest. `.env.example`, @@ -47,14 +61,99 @@ default is `false`. Secret entries never receive rendered defaults. ## Manage migrations For each database Variant with migrations, scaffold creates the directory and -pinned Mise tasks. Devctl does not write SQL or apply migrations. +Mise tasks that call `devctl migrate`. SQL files remain user-owned. ```sh mise run migrate:primary:postgres:create create_orders mise run migrate:primary:postgres:up +mise run migrate:primary:postgres:up --steps 1 mise run migrate:primary:postgres:down +mise run migrate:primary:postgres:status +``` + +The equivalent direct commands are: + +```sh +devctl migrate create create_orders --target primary:postgres +devctl migrate up --target primary:postgres +devctl migrate up --target primary:postgres --steps 1 +devctl migrate down --target primary:postgres --steps 1 +devctl migrate status --target primary:postgres +``` + +`--target` selects a Connection and Variant from the Manifest. It may be omitted +only when exactly one Variant has migrations enabled; the Connection's runtime +default does not affect selection. Use `inspect` to list migration resources. + +All four operations use that Variant's `migrations.path`, relative to the Project +root, including when `--file` selects a Manifest elsewhere. There is no directory +override on `migrate`. Set the path in the Manifest or with `add db +--migrations-path` when adding the Variant. + +`create` needs no database connection. It creates the directory if missing and +writes an empty UTC timestamped pair, such as +`20261019120000_create_orders.up.sql` and `.down.sql`. Names must start with an +ASCII letter or digit and contain only ASCII letters, digits, `_`, or `-`. +Existing versions and files are never overwritten. Ordinary creation errors +clean up files created by that invocation; a process crash can leave a partial +pair or `.devctl-create-` reservation to inspect and remove manually. + +`up` applies all pending migrations by default and accepts positive +`--steps N` when a bounded application is needed. Repeating an unbounded `up` +when all migrations are applied succeeds with `no_change: true`; an empty +migration directory is an error. `down` rolls back one migration by default, +accepts positive `--steps N`, and never prompts. A bounded `up` or rollback +larger than the available history can fail after partial execution; rollback +of a fresh database is an error. + +`status` returns `target`, `version` (null when nothing is applied), and `dirty`. +Dirty state is a successful status result, but prevents `up` and `down`. +Opening a target may initialize its version table; for SQLite it may also +create the database file. `status` therefore requires initialization privileges +on a fresh target. + +### Migration configuration and CI + +Runtime DSNs and migration URLs are independent. Devctl reads the selected +`migrations.database_env` with the Project environment prefix. Its precedence is +process environment, then an explicitly supplied `--env-file`, then Manifest +`migrations.database_default`. A present empty value is an error, and does not +fall through. Missing or malformed explicit env-files are errors even when the +process environment supplies the URL. No `.env` file is loaded automatically +and process environment is never modified. Dotenv parsing supports quoted +values, comments and `export`; variable expansion uses earlier entries in that +file, independently of process environment. + +```sh +devctl migrate up --target primary:postgres --env-file .env.local +mise run migrate:primary:postgres:up --env-file .env.local +``` + +Relative env-file paths and relative SQLite database URLs resolve against the +directory of invocation. SQL paths always resolve against the selected Project. +Migration commands load the semantically valid Manifest without requiring the +Project's generator tools, generated application code, or go.mod readiness. + +In a deployment job, provide the binary, Manifest, and SQL files. Set the migration +URL through the CI environment, for example +`ORDERS_API_DB_PRIMARY_POSTGRES_MIGRATIONS_URL` for the orders-api Project: + +```sh +devctl migrate up --file devctl.yaml --target primary:postgres --json ``` -Runtime DSNs and migration URLs are independent. ClickHouse migrations are -non-transactional; a failed multi-statement migration may require manual -inspection and recovery before forcing a version. +Staging and production can use the same Manifest and target with different URL +values. Explicit targets keep jobs stable when another Variant is added. +The command emits the standard single JSONL result event, exits 0 on success or +no-change, and exits nonzero on failure. Failed operations that observed progress +attach state to `details.partial_result`; they do not undo applied SQL. + +Cancellation before execution prevents migration work. Once SQL starts, +`dbmigrate` waits for its actual result, and closing resources may also wait for +the driver. Configure driver/statement timeouts in the migration URL; Devctl +does not provide an interrupting `--timeout`. + +ClickHouse migrations are non-transactional. Supply `x-multi-statement=true` +when needed; driver semicolon splitting still has limitations and can leave a +partial apply. Devctl leaves the URL and SQL unchanged. Inspect and repair dirty +databases before using external recovery tooling; this CLI has no `force`. diff --git a/docs/user-guide/getting-started.md b/docs/user-guide/getting-started.md index 70ee4ee..56bd90b 100644 --- a/docs/user-guide/getting-started.md +++ b/docs/user-guide/getting-started.md @@ -246,7 +246,7 @@ Wait until it is ready: docker exec devctl-orders-postgres pg_isready -U orders -d orders ``` -Configure both the runtime pool and the migration tool, then migrate: +Configure both the runtime pool and the separate migration URL, then migrate: ```sh export ORDERS_API_DB_PRIMARY_KIND=postgres diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index e0387c1..6e26689 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -15,6 +15,19 @@ devctl add db analytics --kind clickhouse Migration targets default to `migrations//`. Override the path with `--migrations-path`, or opt out with `--no-migrations`. +Create SQL and execute migrations explicitly: + +```sh +devctl migrate create create_orders --target primary:postgres +devctl migrate up --target primary:postgres --env-file .env.local +devctl migrate status --target primary:postgres +``` + +For CI, supply the migration URL through the process environment and use +`devctl migrate up --target primary:postgres --json`. See +[migration configuration and CI](generated-project.md#migration-configuration-and-ci) +for source precedence, path rules, and failure behavior. + A Connection with several Variants must name a default. ClickHouse cannot be mixed with SQLite or PostgreSQL Variants in the same logical Connection because its native runtime is not transactional. @@ -99,7 +112,7 @@ devctl add kafka-producer created \ --topic orders.created.v1 \ --format json \ --source contracts \ - --path orders-created.schema.json + --path orders.created.v1.json devctl add kafka-consumer billing \ --topic billing.events.v1 \ @@ -111,7 +124,8 @@ devctl add kafka-consumer billing \ ``` `raw` has no Contract files and supports inspect/lint but not sync or code -generation. JSON Schema roots need a non-empty `title`. Proto encoding is +generation. JSON Schema roots need a non-empty `title`. For local Sources, the +schema basename must be `.json` or `.proto`. Proto encoding is `binary` or `json`. ## Targeted previews diff --git a/docs/user-guide/reference/commands.md b/docs/user-guide/reference/commands.md index dccc192..cdf6605 100644 --- a/docs/user-guide/reference/commands.md +++ b/docs/user-guide/reference/commands.md @@ -4,7 +4,7 @@ ## CLI interface - devctl -Devctl defines, validates, and materializes reproducible Go projects from a devctl.yaml manifest. Commands are non-interactive and keep manifest mutation, synchronization, linting, scaffolding, and generation explicit. +Devctl defines, validates, and materializes reproducible Go projects from a devctl.yaml manifest. Commands are non-interactive and keep manifest mutation, synchronization, linting, scaffolding, generation, and migration execution explicit. Manage Devctl Go projects. @@ -293,7 +293,7 @@ Add a database variant. > devctl add db --kind -Add a SQLite, PostgreSQL, or ClickHouse Variant to a named database Connection. A migration target is declared by default; Devctl never writes SQL or applies migrations. +Add a SQLite, PostgreSQL, or ClickHouse Variant to a named database Connection. A Migration Target is declared by default; use devctl migrate explicitly to create SQL files or run migrations. Usage: @@ -841,3 +841,104 @@ The following flags are supported: | `--file="…"` | use `path` as the Manifest instead of discovering devctl.yaml | string | | *none* | | `--json` | emit compact JSONL events instead of text | bool | `false` | *none* | | `--verbose` | include debug diagnostics and raw causes on stderr | bool | `false` | *none* | + +### `migrate` command + +Create and run database migrations. + +Operate on one Manifest Migration Target. Select connection:variant explicitly, or omit --target when exactly one Migration Target exists. + +Usage: + +```bash +$ devctl [GLOBAL FLAGS] migrate [ARGUMENTS...] +``` + +### `migrate create` subcommand + +Create a pair of timestamped SQL files. + +Create a pair of timestamped SQL files. Migration SQL is selected by migrations.path in the Manifest. + +Usage: + +```bash +$ devctl [GLOBAL FLAGS] migrate create [COMMAND FLAGS] [ARGUMENTS...] +``` + +The following flags are supported: + +| Name | Description | Type | Default value | Environment variables | +|----------------|---------------------------------------------------------------|--------|:-------------:|:---------------------:| +| `--file="…"` | use `path` as the Manifest instead of discovering devctl.yaml | string | | *none* | +| `--json` | emit compact JSONL events instead of text | bool | `false` | *none* | +| `--verbose` | include debug diagnostics and raw causes on stderr | bool | `false` | *none* | +| `--target="…"` | select Migration Target `connection:variant` | string | | *none* | + +### `migrate up` subcommand + +Apply all pending migrations, or a positive number with --steps. + +Apply all pending migrations, or a positive number with --steps. Migration SQL is selected by migrations.path in the Manifest. + +Usage: + +```bash +$ devctl [GLOBAL FLAGS] migrate up [COMMAND FLAGS] [ARGUMENTS...] +``` + +The following flags are supported: + +| Name | Description | Type | Default value | Environment variables | +|------------------|----------------------------------------------------------------------|--------|:-------------:|:---------------------:| +| `--file="…"` | use `path` as the Manifest instead of discovering devctl.yaml | string | | *none* | +| `--json` | emit compact JSONL events instead of text | bool | `false` | *none* | +| `--verbose` | include debug diagnostics and raw causes on stderr | bool | `false` | *none* | +| `--target="…"` | select Migration Target `connection:variant` | string | | *none* | +| `--env-file="…"` | read migration URL from this dotenv `path` after process environment | string | | *none* | +| `--steps="…"` | apply positive `N` migrations; omit for all | int | `0` | *none* | + +### `migrate down` subcommand + +Roll back migrations (one step by default). + +Roll back migrations (one step by default). Migration SQL is selected by migrations.path in the Manifest. + +Usage: + +```bash +$ devctl [GLOBAL FLAGS] migrate down [COMMAND FLAGS] [ARGUMENTS...] +``` + +The following flags are supported: + +| Name | Description | Type | Default value | Environment variables | +|------------------|----------------------------------------------------------------------|--------|:-------------:|:---------------------:| +| `--file="…"` | use `path` as the Manifest instead of discovering devctl.yaml | string | | *none* | +| `--json` | emit compact JSONL events instead of text | bool | `false` | *none* | +| `--verbose` | include debug diagnostics and raw causes on stderr | bool | `false` | *none* | +| `--target="…"` | select Migration Target `connection:variant` | string | | *none* | +| `--env-file="…"` | read migration URL from this dotenv `path` after process environment | string | | *none* | +| `--steps="…"` | roll back `N` migrations | int | `1` | *none* | + +### `migrate status` subcommand + +Show migration version and dirty state. + +Read the current migration version. Opening a fresh target may initialize the metadata table and SQLite database file. Dirty state is a successful result. + +Usage: + +```bash +$ devctl [GLOBAL FLAGS] migrate status [COMMAND FLAGS] [ARGUMENTS...] +``` + +The following flags are supported: + +| Name | Description | Type | Default value | Environment variables | +|------------------|----------------------------------------------------------------------|--------|:-------------:|:---------------------:| +| `--file="…"` | use `path` as the Manifest instead of discovering devctl.yaml | string | | *none* | +| `--json` | emit compact JSONL events instead of text | bool | `false` | *none* | +| `--verbose` | include debug diagnostics and raw causes on stderr | bool | `false` | *none* | +| `--target="…"` | select Migration Target `connection:variant` | string | | *none* | +| `--env-file="…"` | read migration URL from this dotenv `path` after process environment | string | | *none* | diff --git a/docs/user-guide/reference/manifest/components.md b/docs/user-guide/reference/manifest/components.md index 7854118..964f091 100644 --- a/docs/user-guide/reference/manifest/components.md +++ b/docs/user-guide/reference/manifest/components.md @@ -116,7 +116,7 @@ components: topic_env: KAFKA_CREATED_TOPIC contract: source: contracts - path: orders-created.schema.json + path: orders.created.v1.json format: json env: {} ``` @@ -127,7 +127,7 @@ same `contract` shape: | Field | Required | Description | |---|:---:|---| -| `format` | yes | `raw`, `json`, or `proto` | +| `format` | no | `raw` (default), `json`, or `proto` | | `source` | schema-backed | Existing Source | | `path` | conditional | Entrypoint for non-Devctl Sources | | `export` | conditional | Kafka Export for a Devctl Source | @@ -135,10 +135,13 @@ same `contract` shape: | `message` | Proto only | Fully-qualified message name | | `encoding` | Proto only | `binary` or `json` | -Raw endpoints have no Source, path, Export, or Proto fields. JSON Schema roots -need a non-empty `title`, which owns the generated Go type name. Kafka Runtime -Config also derives brokers, batching, retry, rebalance, drain, and shutdown -settings from each effective endpoint. +Schema-backed endpoints must explicitly choose `json` or `proto`; omitting +`format` selects raw even if schema fields were provided, making those fields +invalid. Raw endpoints have no Source, path, Export, or Proto fields. For local +Sources, lint requires schema filenames `.json` or `.proto`. +JSON Schema roots need a non-empty `title`, which owns the generated Go type +name. Kafka Runtime Config also derives brokers, batching, retry, rebalance, +drain, and shutdown settings from each effective endpoint. ## Databases diff --git a/docs/user-guide/reference/manifest/project-and-language.md b/docs/user-guide/reference/manifest/project-and-language.md index 0145df2..802fd76 100644 --- a/docs/user-guide/reference/manifest/project-and-language.md +++ b/docs/user-guide/reference/manifest/project-and-language.md @@ -69,7 +69,7 @@ languages: module: example.com/orders-api generators: config: - out: gen/config/config.gen.go + out: gen/config http: oapi_config: tools/oapi/server.yaml server_out: gen/serverhttp @@ -96,7 +96,7 @@ languages: | Field | Required | Effective default | |---|:---:|---| | `languages.go.module` | yes | none | -| `generators.config.out` | no | `gen/config/config.gen.go` | +| `generators.config.out` | no | `gen/config` (output directory) | | `generators.http.oapi_config` | no | `tools/oapi/server.yaml` | | `generators.http.server_out` | no | `gen/serverhttp` | | `generators.http.client_out` | no | `gen/clienthttp` | @@ -110,6 +110,10 @@ omitted. Canonical native generator configs are Managed Outputs. An explicitly selected alternate config is user-owned, must already exist, and is never created or replaced by scaffold. +The config Target writes `config.gen.go` inside `generators.config.out`, so its +default resulting file is `gen/config/config.gen.go`. An explicitly declared +canonical native config path remains managed. + `languages.go.components.pprof` uses the same `start` and component `env` shapes described in [Components](components.md). The HTTP service preset uses `127.0.0.1:6060` and a disabled-by-default `PPROF_ENABLED` toggle. diff --git a/docs/user-guide/reference/output-and-errors.md b/docs/user-guide/reference/output-and-errors.md index cb415e8..9ea9dcb 100644 --- a/docs/user-guide/reference/output-and-errors.md +++ b/docs/user-guide/reference/output-and-errors.md @@ -2,6 +2,8 @@ Every executable leaf accepts `--json`; text output is the default. `--verbose` independently enables debug diagnostics and raw causes. +Migration failures keep database URLs, credentials, and SQL out of messages even +under `--verbose`; their safe reason and target context remain available. ## JSONL success @@ -69,6 +71,12 @@ The command-specific payload is under `data` on success and may appear under | `inspect` | `{project: {...}}` | | `lint` | `{valid, contracts, issues}` | | `sync`, `gen` | `{targets, changes, dry_run}` | +| `migrate create` | `{target, files}` | +| `migrate up`, `migrate down`, `migrate status` | `{target, version, dirty}`; up-to-date up adds `no_change: true` | + +Migration `version` is null before any migration and after a complete rollback. +Dirty `status` is a successful result with exit code 0. A failed migration may +include the observed version and dirty state in `details.partial_result`. Publication actions are `created`, `updated`, `unchanged`, and `removed`. Static previews use `planned_publish` and `planned_remove`. diff --git a/docs/user-guide/troubleshooting.md b/docs/user-guide/troubleshooting.md index fda638d..f3cc25c 100644 --- a/docs/user-guide/troubleshooting.md +++ b/docs/user-guide/troubleshooting.md @@ -78,3 +78,31 @@ statements, use a migration DSN with `x-multi-statement=true`. Driver statement splitting can still leave a partial apply. Inspect the database, complete or revert the intended changes manually, and only then force the migration version. Devctl does not alter the DSN or perform recovery. + +## Migration target or URL is missing + +`migrate` requires one Migration Target. If several Variants have migrations, +pass `--target connection:variant`; runtime defaults do not select migrations. +Use `inspect` to find the configured path and fully prefixed migration env key. + +Set that key in process environment or an explicit `--env-file`. The Manifest's +`database_default` is used only when the key is absent from both sources. An +explicit empty value fails rather than selecting another connection. Devctl does +not automatically load `.env` and never uses the runtime DSN as a fallback. + +When using `--file` from outside the Project, SQL files still resolve against +the Project root, while relative SQLite database URLs and env-file paths resolve +against your current working directory. + +## Migration database is dirty + +Run `devctl migrate status --target connection:variant --json` to inspect the +version and dirty state. Status exits 0 when it reads dirty state successfully; +`up` and `down` return `conflict`. Repair the database before changing its recorded +version with external recovery tooling. Devctl has no `force` command. + +`up --steps N` and `down --steps N` can partially apply or roll back before +reporting insufficient history. Review `details.partial_result` when present +and query status before retrying. +An empty SQL directory or rollback of a fresh database is an error; repeated +`up` on an up-to-date database succeeds. diff --git a/examples/orders-api/.mise.toml b/examples/orders-api/.mise.toml index 621d406..8e74d59 100644 --- a/examples/orders-api/.mise.toml +++ b/examples/orders-api/.mise.toml @@ -1,7 +1,6 @@ [tools] go = "1.26.0" golangci-lint = "2.12.2" -"go:github.com/golang-migrate/migrate/v4/cmd/migrate" = { version = "v4.19.1", tags = ["postgres"] } [tasks.fmt] run = "golangci-lint fmt" @@ -30,21 +29,39 @@ depends = ["fmt:check", "lint", "test"] description = "Create timestamped migration files in migrations/primary/postgres" usage = 'arg "" help="Migration name"' run = ''' -migrate create -ext sql -dir "migrations/primary/postgres" -format "20060102150405" "${usage_name?}" +devctl migrate create --target "primary:postgres" "${usage_name?}" ''' [tasks."migrate:primary:postgres:up"] description = "Apply migrations from migrations/primary/postgres" +usage = ''' +flag "--steps " help="Apply N migrations; omit to apply all" +flag "--env-file " help="Read migration URL from dotenv file" +''' run = ''' -database_url="${ORDERS_API_DB_PRIMARY_POSTGRES_MIGRATIONS_URL:?set ORDERS_API_DB_PRIMARY_POSTGRES_MIGRATIONS_URL}" -migrate -path "migrations/primary/postgres" -database "$database_url" up +set -- +if [ -n "${usage_steps:-}" ]; then set -- --steps "$usage_steps"; fi +if [ -n "${usage_env_file:-}" ]; then set -- "$@" --env-file "$usage_env_file"; fi +devctl migrate up --target "primary:postgres" "$@" ''' [tasks."migrate:primary:postgres:down"] description = "Roll back migrations from migrations/primary/postgres" -usage = 'arg "[steps]" default="1" help="Number of migrations"' -confirm = "Roll back migrate:primary:postgres migrations?" +usage = ''' +arg "[steps]" default="1" help="Number of migrations" +flag "--env-file " help="Read migration URL from dotenv file" +''' +run = ''' +set -- +if [ -n "${usage_env_file:-}" ]; then set -- --env-file "$usage_env_file"; fi +devctl migrate down --target "primary:postgres" --steps "${usage_steps?}" "$@" +''' + +[tasks."migrate:primary:postgres:status"] +description = "Show migration version and dirty state for primary:postgres" +usage = 'flag "--env-file " help="Read migration URL from dotenv file"' run = ''' -database_url="${ORDERS_API_DB_PRIMARY_POSTGRES_MIGRATIONS_URL:?set ORDERS_API_DB_PRIMARY_POSTGRES_MIGRATIONS_URL}" -migrate -path "migrations/primary/postgres" -database "$database_url" down "${usage_steps?}" +set -- +if [ -n "${usage_env_file:-}" ]; then set -- --env-file "$usage_env_file"; fi +devctl migrate status --target "primary:postgres" "$@" ''' diff --git a/go.mod b/go.mod index 2a3787b..350dd42 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,14 @@ go 1.26.0 require ( github.com/BurntSushi/toml v1.6.0 + github.com/ClickHouse/clickhouse-go v1.4.3 + github.com/devctllabs/go-libs/dbmigrate v0.1.0 github.com/devctllabs/go-libs/di v0.1.0 github.com/devctllabs/go-libs/filesystem v0.2.0 github.com/devctllabs/go-libs/lifecycle v0.2.0 github.com/devctllabs/go-libs/log v0.2.0 + github.com/golang-migrate/migrate/v4 v4.20.1 + github.com/joho/godotenv v1.5.1 github.com/pb33f/libopenapi v0.38.6 github.com/pb33f/libopenapi-validator v0.14.0 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 @@ -24,19 +28,26 @@ require ( github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/basgys/goxml2json v1.1.1-0.20231018121955-e66ee54ceaad // indirect github.com/buger/jsonparser v1.1.2 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/getkin/kin-openapi v0.142.0 // indirect github.com/go-openapi/jsonpointer v0.23.2 // indirect github.com/go-openapi/swag/jsonname v0.26.1 // indirect github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oapi-codegen/oapi-codegen/v2 v2.8.0 // indirect github.com/oasdiff/yaml v0.1.1 // indirect github.com/oasdiff/yaml3 v0.0.14 // indirect github.com/pb33f/jsonpath v0.8.2 // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/samber/do/v2 v2.1.0 // indirect github.com/samber/go-type-to-string v1.8.0 // indirect @@ -48,8 +59,13 @@ require ( go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.40.0 // indirect golang.org/x/tools v0.48.0 // indirect + modernc.org/libc v1.74.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.54.0 // indirect ) tool ( diff --git a/go.sum b/go.sum index 2ede17b..e793848 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,50 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= +github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/basgys/goxml2json v1.1.1-0.20231018121955-e66ee54ceaad h1:3swAvbzgfaI6nKuDDU7BiKfZRdF+h2ZwKgMHd8Ha4t8= github.com/basgys/goxml2json v1.1.1-0.20231018121955-e66ee54ceaad/go.mod h1:9+nBLYNWkvPcq9ep0owWUsPTLgL9ZXTsZWcCSVGGLJ0= github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow= github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/devctllabs/go-libs/dbmigrate v0.1.0 h1:QPEPuxDPfZyAm5sw5+pZt9Em0D3r/H+SjqTqf6vP7LA= +github.com/devctllabs/go-libs/dbmigrate v0.1.0/go.mod h1:MyLcZFXBWD6YaIMdl6dXyGJzsL5CMS3+iFvM3s8tfWA= github.com/devctllabs/go-libs/di v0.1.0 h1:pPhlKmdYyvKVYr/5eV51Ea9HcfqsqWCs3GZKG6jFGW8= github.com/devctllabs/go-libs/di v0.1.0/go.mod h1:kr9mSKElEmsDHgKsZbDGJhSCdmlcluT81PjbCvcOmbY= github.com/devctllabs/go-libs/filesystem v0.2.0 h1:PdPKAoFy90RoH0+8qOSvJnyr8ASj0L1RiFngBe/2bns= @@ -25,23 +53,48 @@ github.com/devctllabs/go-libs/lifecycle v0.2.0 h1:vafo21o5tjrU3Qf4tVa0a2A6a8NQPS github.com/devctllabs/go-libs/lifecycle v0.2.0/go.mod h1:/m8kzixQx7IhAi3jx7hFfICu96VxBKNuQxHd+tY6VIY= github.com/devctllabs/go-libs/log v0.2.0 h1:RICLkubslpX8CGSw/m+i2FxrRo64PzhE7hVFQXlwReM= github.com/devctllabs/go-libs/log v0.2.0/go.mod h1:KfCsyQUkit4D2kounIhY6AauWB/Lo5SMiHZZXaTtMJ4= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getkin/kin-openapi v0.142.0 h1:izj0vBdFprMhitfzaX8sTqztsEQyvwhssBoB6n8NO7w= github.com/getkin/kin-openapi v0.142.0/go.mod h1:3BH9M9XDe/y9M5DSvEocVYAYq1w0qrhJHjC/vZi0AaY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.23.2 h1:DK7R/3zAt4xTytxNkw7jARGPFI7rkaSsii58n8X45x0= github.com/go-openapi/jsonpointer v0.23.2/go.mod h1:noUOckXtq7b4bVkqw0sbHKieq9uEZRN7p6EF/dalc4w= github.com/go-openapi/swag/jsonname v0.26.1 h1:VReupaV6WxlAsCn0e4DUfgV6bPmINnPpyJDLqSfNPcE= github.com/go-openapi/swag/jsonname v0.26.1/go.mod h1:OvdW6BoWoj33pTfi7x9vFrgmT+fk7aw0BRwvCE0YOuc= github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/golang-migrate/migrate/v4 v4.20.1 h1:2N/ToVTKrKl58ynBpgeVJ4In7VcLCjWTZtm4eP1LxhU= +github.com/golang-migrate/migrate/v4 v4.20.1/go.mod h1:DDPgKVb4ovSWc4FwSPfV2Uz1160f4XBiTHTrAJtljmM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -58,8 +111,19 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -67,6 +131,38 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= +github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -88,6 +184,10 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pb33f/jsonpath v0.8.2 h1:Ou4C7zjYClBm97dfZjDCjdZGusJoynv/vrtiEKNfj2Y= github.com/pb33f/jsonpath v0.8.2/go.mod h1:zBV5LJW4OQOPatmQE2QdKpGQJvhDTlE5IEj6ASaRNTo= github.com/pb33f/libopenapi v0.38.6 h1:rnLshOSCSLx3JmN/MnftyYcdN1FfaeRkbaBE/ZHdIXg= @@ -98,9 +198,17 @@ github.com/pb33f/ordered-map/v2 v2.3.1 h1:5319HDO0aw4DA4gzi+zv4FXU9UlSs3xGZ40wcP github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7VvaHKbd1HnmQ= github.com/pb33f/testify v0.1.0 h1:g48/HDU/jn2COspS4nM0scptxiKTJ4DnbX/4ehK6IZ8= github.com/pb33f/testify v0.1.0/go.mod h1:nq283P/jJ8hXMmdhAqfj7BJIz0y+6IOHj9q0044rKt4= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -113,16 +221,27 @@ github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEV github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/speakeasy-api/jsonpath v0.6.3 h1:c+QPwzAOdrWvzycuc9HFsIZcxKIaWcNpC+xhOW9rJxU= github.com/speakeasy-api/jsonpath v0.6.3/go.mod h1:2cXloNuQ+RSXi5HTRaeBh7JEmjRXTiaKpFTdZiL7URI= github.com/speakeasy-api/openapi v1.24.0 h1:opoD27rupX7zBVPq1HkIGLeMOzNNA7JalhYP8q34i04= github.com/speakeasy-api/openapi v1.24.0/go.mod h1:g3+dIMe0AYgbbGvnlQZqesmjAVWSm9BmsjLevnefQrg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI= +github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/urfave/cli-docs/v3 v3.1.0 h1:Sa5xm19IpE5gpm6tZzXdfjdFxn67PnEsE4dpXF7vsKw= github.com/urfave/cli-docs/v3 v3.1.0/go.mod h1:59d+5Hz1h6GSGJ10cvcEkbIe3j233t4XDqI72UIx7to= github.com/urfave/cli/v3 v3.10.1 h1:7Kx9H50hrHbRbyxgO1KP6/BcbiGRz0uYh5YyQ30JEEY= @@ -131,6 +250,18 @@ github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9N github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= @@ -148,6 +279,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -191,6 +324,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= @@ -245,3 +379,31 @@ gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= +modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= +modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog= +modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/deps/container.go b/internal/deps/container.go index 64bd991..61cf9a0 100644 --- a/internal/deps/container.go +++ b/internal/deps/container.go @@ -6,6 +6,7 @@ import ( generateservice "github.com/devctllabs/devctl/internal/service/generate" lintservice "github.com/devctllabs/devctl/internal/service/lint" + migrationservice "github.com/devctllabs/devctl/internal/service/migration" "github.com/devctllabs/devctl/internal/service/project" "github.com/devctllabs/devctl/internal/service/scaffold" syncservice "github.com/devctllabs/devctl/internal/service/sync" @@ -30,6 +31,7 @@ func New(logger *zap.Logger) (*Container, error) { container.provideClients, container.provideServices, container.provideScaffoldServices, + container.provideMigration, } for _, provide := range providers { if err := provide(); err != nil { @@ -39,6 +41,15 @@ func New(logger *zap.Logger) (*Container, error) { return container, nil } +// MigrationService resolves the selected-target migration capability on demand. +func (c *Container) MigrationService() (*migrationservice.Service, error) { + service, err := resolve[*migrationservice.Service](c.di) + if err != nil { + return nil, fmt.Errorf("di.Resolve: %w", err) + } + return service, nil +} + // ProjectService resolves the project query and mutation root on demand. func (c *Container) ProjectService() (*project.Service, error) { service, err := resolve[*project.Service](c.di) diff --git a/internal/deps/migration.go b/internal/deps/migration.go new file mode 100644 index 0000000..73f01bd --- /dev/null +++ b/internal/deps/migration.go @@ -0,0 +1,41 @@ +package deps + +import ( + "fmt" + + migrationrepo "github.com/devctllabs/devctl/internal/repository/migration" + migrationservice "github.com/devctllabs/devctl/internal/service/migration" + projectservice "github.com/devctllabs/devctl/internal/service/project" + "github.com/devctllabs/go-libs/di" +) + +func (c *Container) provideMigration() error { + if err := di.Provide(c.di, func(di.Resolver) (*migrationrepo.Repository, error) { + return migrationrepo.NewRepository(), nil + }); err != nil { + return fmt.Errorf("di.Provide: %w", err) + } + if err := di.Provide(c.di, func(di.Resolver) (*migrationrepo.Environment, error) { + return migrationrepo.NewEnvironment(), nil + }); err != nil { + return fmt.Errorf("di.Provide: %w", err) + } + if err := di.Provide(c.di, func(resolver di.Resolver) (*migrationservice.Service, error) { + projects, err := resolve[*projectservice.Service](resolver) + if err != nil { + return nil, err + } + config, err := resolve[*migrationrepo.Environment](resolver) + if err != nil { + return nil, err + } + repository, err := resolve[*migrationrepo.Repository](resolver) + if err != nil { + return nil, err + } + return migrationservice.New(projects, config, repository), nil + }); err != nil { + return fmt.Errorf("di.Provide: %w", err) + } + return nil +} diff --git a/internal/domain/migration/command.go b/internal/domain/migration/command.go new file mode 100644 index 0000000..78d593e --- /dev/null +++ b/internal/domain/migration/command.go @@ -0,0 +1,71 @@ +// Package migration defines migration operations and their observable results. +package migration + +import ( + "regexp" + + projectdomain "github.com/devctllabs/devctl/internal/domain/project" +) + +type Operation string + +const ( + Create Operation = "create" + Up Operation = "up" + Down Operation = "down" + Status Operation = "status" +) + +// Command selects one Manifest migration target and operation. +type Command struct { + ManifestPath string + Target string + Operation Operation + EnvFile string + Name string + Steps int // Up uses zero for all pending migrations; Up and Down use positive limits. +} + +// State is the database's migration metadata; nil Version means no applied version. +type State struct { + Version *uint + Dirty bool +} + +// Result records observed state and files created by an operation. +type Result struct { + Target string + State *State + Files []string + NoChange bool + Changed bool +} + +var migrationName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) + +// ValidName reports whether name is a safe migration filename component. +func ValidName(name string) bool { return migrationName.MatchString(name) } + +// DatabaseConfig selects migration-only environment sources for one backend. +type DatabaseConfig struct { + Kind string + EnvKey string + Default string + EnvFile string +} + +// Execution contains the resolved inputs to a database migration operation. +type Execution struct { + Root string + Target projectdomain.MigrationTarget + Operation Operation + DatabaseURL string + Steps int // Up uses zero for all pending migrations; Up and Down use positive limits. +} + +// Creation selects the contained directory and filename of a new SQL pair. +type Creation struct { + Root string + Target projectdomain.MigrationTarget + Name string +} diff --git a/internal/domain/migration/errors.go b/internal/domain/migration/errors.go new file mode 100644 index 0000000..af6a26e --- /dev/null +++ b/internal/domain/migration/errors.go @@ -0,0 +1,20 @@ +package migration + +import "github.com/devctllabs/devctl/internal/domain/failure" + +// Error retains a cause without rendering credentials or SQL in its message. +type Error struct { + Kind failure.Category + Reason string + Target string + Path string + EnvKey string + Candidates []string + Cause error +} + +func (e *Error) Error() string { return "migration failed: " + e.Reason } + +func (e *Error) Unwrap() error { return e.Cause } + +func (e *Error) Category() failure.Category { return e.Kind } diff --git a/internal/domain/project/migration.go b/internal/domain/project/migration.go new file mode 100644 index 0000000..da8181e --- /dev/null +++ b/internal/domain/project/migration.go @@ -0,0 +1,39 @@ +package project + +import "sort" + +// MigrationTarget is the effective migration configuration of one database Variant. +type MigrationTarget struct { + Connection string + Variant string + Kind string + Path string + DatabaseEnv string + DatabaseDefault string +} + +// ID returns the CLI selector of this Migration Target. +func (t MigrationTarget) ID() string { return t.Connection + ":" + t.Variant } + +// MigrationTargets returns migration-enabled Variants in stable selector order. +func MigrationTargets(manifest Manifest) []MigrationTarget { + var targets []MigrationTarget + if manifest.Components.DB == nil { + return targets + } + prefix := runtimeConfigPrefix(manifest) + for _, connection := range manifest.Components.DB.Connections { + for _, variant := range connection.Variants { + if variant.Migrations == nil { + continue + } + migrations := variant.Migrations + targets = append(targets, MigrationTarget{ + Connection: connection.Name, Variant: variant.Name, Kind: variant.Kind, + Path: migrations.Path, DatabaseEnv: prefix + migrations.DatabaseEnv, DatabaseDefault: migrations.DatabaseDefault, + }) + } + } + sort.Slice(targets, func(i, j int) bool { return targets[i].ID() < targets[j].ID() }) + return targets +} diff --git a/internal/repository/migration/create.go b/internal/repository/migration/create.go new file mode 100644 index 0000000..ada72a1 --- /dev/null +++ b/internal/repository/migration/create.go @@ -0,0 +1,111 @@ +package migration + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "strconv" + "strings" + + "github.com/devctllabs/devctl/internal/domain/failure" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + "github.com/golang-migrate/migrate/v4/source" +) + +// Create reserves the timestamp and exclusively creates the two contained SQL files. +func (r *Repository) Create(ctx context.Context, creation migrationdomain.Creation) (result migrationdomain.Result, err error) { + if err := ctx.Err(); err != nil { + return result, fmt.Errorf("ctx.Err: %w", err) + } + if !fs.ValidPath(creation.Target.Path) || !migrationdomain.ValidName(creation.Name) { + return result, &migrationdomain.Error{Kind: failure.InvalidInput, Reason: "invalid_creation_path"} + } + root, err := os.OpenRoot(creation.Root) + if err != nil { + return result, migrationError(creation.Target.ID(), "source_open_failed", err, failure.Unavailable) + } + defer func() { err = errors.Join(err, closeError(creation.Target.ID(), root.Close())) }() + if err := root.MkdirAll(creation.Target.Path, 0o755); err != nil { + return result, migrationError(creation.Target.ID(), "directory_create_failed", err, failure.Unavailable) + } + version := r.now().UTC().Format("20060102150405") + reservation := path.Join(creation.Target.Path, ".devctl-create-"+version) + lock, err := root.OpenFile(reservation, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return result, migrationError(creation.Target.ID(), "version_reservation_failed", err, failure.Unavailable) + } + defer func() { + err = errors.Join(err, closeError(creation.Target.ID(), errors.Join(lock.Close(), root.Remove(reservation)))) + }() + if err := checkVersion(ctx, root.FS(), creation.Target.Path, version); err != nil { + return result, migrationError(creation.Target.ID(), "version_conflict", err, failure.Unavailable) + } + for _, direction := range []string{"up", "down"} { + name := path.Join(creation.Target.Path, version+"_"+creation.Name+"."+direction+".sql") + if err := createEmptyFile(ctx, root, name); err != nil { + cleanupErr := removeCreated(root, &result) + return result, migrationError(creation.Target.ID(), "file_create_failed", errors.Join(err, cleanupErr), failure.Unavailable) + } + result.Files = append(result.Files, name) + } + result.Changed = true + return result, nil +} + +func checkVersion(ctx context.Context, files fs.FS, directory, version string) error { + wanted, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return fmt.Errorf("strconv.ParseUint: %w", err) + } + err = fs.WalkDir(files, directory, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("ctx.Err: %w", err) + } + if entry.IsDir() || !strings.HasSuffix(name, ".sql") { + return nil + } + migration, parseErr := source.Parse(entry.Name()) + if parseErr == nil && uint64(migration.Version) == wanted { + return fmt.Errorf("migration version exists: %w", fs.ErrExist) + } + return nil + }) + if err != nil { + return fmt.Errorf("fs.WalkDir: %w", err) + } + return nil +} + +func createEmptyFile(ctx context.Context, root *os.Root, name string) error { + if err := ctx.Err(); err != nil { + return fmt.Errorf("ctx.Err: %w", err) + } + file, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return fmt.Errorf("root.OpenFile: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("file.Close: %w", errors.Join(err, root.Remove(name))) + } + return nil +} + +func removeCreated(root *os.Root, result *migrationdomain.Result) error { + var cleanupErr error + var remaining []string + for _, name := range result.Files { + if err := root.Remove(name); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("root.Remove: %w", err)) + remaining = append(remaining, name) + } + } + result.Files = remaining + result.Changed = len(remaining) > 0 + return cleanupErr +} diff --git a/internal/repository/migration/environment.go b/internal/repository/migration/environment.go new file mode 100644 index 0000000..a352cdc --- /dev/null +++ b/internal/repository/migration/environment.go @@ -0,0 +1,61 @@ +package migration + +import ( + "context" + "fmt" + "net/url" + "os" + + "github.com/devctllabs/devctl/internal/domain/failure" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + "github.com/joho/godotenv" +) + +// Environment resolves migration-only configuration without changing process state. +type Environment struct{} + +func NewEnvironment() *Environment { return &Environment{} } + +// ResolveDatabaseURL preserves key presence and validates a backend-specific URL. +func (e *Environment) ResolveDatabaseURL(ctx context.Context, config migrationdomain.DatabaseConfig) (string, error) { + if err := ctx.Err(); err != nil { + return "", fmt.Errorf("ctx.Err: %w", err) + } + var values map[string]string + if config.EnvFile != "" { + var err error + values, err = godotenv.Read(config.EnvFile) + if err != nil { + return "", &migrationdomain.Error{Kind: failure.InvalidInput, Reason: "env_file_invalid", Path: config.EnvFile, Cause: err} + } + } + value, exists := os.LookupEnv(config.EnvKey) + if !exists { + value, exists = values[config.EnvKey] + } + if !exists { + value = config.Default + } + if value == "" { + reason := "database_url_missing" + if exists { + reason = "database_url_empty" + } + return "", &migrationdomain.Error{Kind: failure.InvalidInput, Reason: reason, EnvKey: config.EnvKey} + } + parsed, err := url.Parse(value) + if err != nil || !matchesKind(parsed, config.Kind) { + return "", &migrationdomain.Error{Kind: failure.InvalidInput, Reason: "database_url_invalid", EnvKey: config.EnvKey, Cause: err} + } + return value, nil +} + +func matchesKind(value *url.URL, kind string) bool { + if value == nil { + return false + } + if kind == "postgres" { + return value.Scheme == "postgres" || value.Scheme == "postgresql" + } + return (kind == "sqlite" || kind == "clickhouse") && value.Scheme == kind +} diff --git a/internal/repository/migration/environment_test.go b/internal/repository/migration/environment_test.go new file mode 100644 index 0000000..98802f1 --- /dev/null +++ b/internal/repository/migration/environment_test.go @@ -0,0 +1,77 @@ +package migration + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/devctllabs/devctl/internal/domain/failure" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + "github.com/stretchr/testify/require" +) + +func TestMigrationURLPrecedenceAndPresence(t *testing.T) { + key := "DEVCTL_MIGRATION_TEST_URL" + fileOnlyKey := "DEVCTL_MIGRATION_FILE_ONLY_TEST" + before, beforeExists := os.LookupEnv(fileOnlyKey) + file := filepath.Join(t.TempDir(), "migration.env") + require.NoError(t, os.WriteFile(file, []byte(key+"='postgres://file/db'\n"+fileOnlyKey+"=from-file\n"), 0o600)) + config := migrationdomain.DatabaseConfig{Kind: "postgres", EnvKey: key, EnvFile: file, Default: "postgres://default/db"} + environment := NewEnvironment() + t.Setenv(key, "postgres://process/db") + value, err := environment.ResolveDatabaseURL(context.Background(), config) + require.NoError(t, err) + require.Equal(t, "postgres://process/db", value) + t.Setenv(key, "") + _, err = environment.ResolveDatabaseURL(context.Background(), config) + require.Equal(t, failure.InvalidInput, failure.CategoryOf(err)) + var configErr *migrationdomain.Error + require.ErrorAs(t, err, &configErr) + require.Equal(t, "database_url_empty", configErr.Reason) + require.NoError(t, os.Unsetenv(key)) + value, err = environment.ResolveDatabaseURL(context.Background(), config) + require.NoError(t, err) + require.Equal(t, "postgres://file/db", value) + after, afterExists := os.LookupEnv(fileOnlyKey) + require.Equal(t, beforeExists, afterExists) + require.Equal(t, before, after) + config.EnvFile = "" + value, err = environment.ResolveDatabaseURL(context.Background(), config) + require.NoError(t, err) + require.Equal(t, config.Default, value) + config.Default = "" + _, err = environment.ResolveDatabaseURL(context.Background(), config) + require.ErrorAs(t, err, &configErr) + require.Equal(t, "database_url_missing", configErr.Reason) +} + +func TestMigrationURLRejectsInvalidSourcesWithoutLeakingSecrets(t *testing.T) { + t.Setenv("DEVCTL_MIGRATION_TEST_URL", "postgres://process/db") + for _, content := range []string{"DEVCTL_MIGRATION_TEST_URL=", "DEVCTL_MIGRATION_TEST_URL=clickhouse://secret@host/db", "DEVCTL_MIGRATION_TEST_URL=postgres://user:SECRET_TOKEN@host/%zz", "invalid SECRET_TOKEN ?"} { + file := filepath.Join(t.TempDir(), "migration.env") + require.NoError(t, os.WriteFile(file, []byte(content), 0o600)) + require.NoError(t, os.Unsetenv("DEVCTL_MIGRATION_TEST_URL")) + _, err := NewEnvironment().ResolveDatabaseURL(context.Background(), migrationdomain.DatabaseConfig{Kind: "postgres", EnvKey: "DEVCTL_MIGRATION_TEST_URL", EnvFile: file, Default: "postgres://default/db"}) + require.Error(t, err) + require.Equal(t, failure.InvalidInput, failure.CategoryOf(err)) + require.NotContains(t, err.Error(), "SECRET_TOKEN") + require.NotContains(t, err.Error(), "secret@host") + } + t.Setenv("DEVCTL_MIGRATION_TEST_URL", "postgres://process/db") + _, err := NewEnvironment().ResolveDatabaseURL(context.Background(), migrationdomain.DatabaseConfig{Kind: "postgres", EnvFile: filepath.Join(t.TempDir(), "missing.env"), Default: "postgres://default/db"}) + require.Error(t, err) +} + +func TestMigrationURLPreservesSQLitePathsAndPostgresAlias(t *testing.T) { + t.Parallel() + for _, test := range []struct{ kind, value string }{ + {"sqlite", "sqlite://./data/app.db?_pragma=foreign_keys%281%29"}, + {"postgres", "postgresql://user:password@host/db"}, + {"clickhouse", "clickhouse://host/db?x-multi-statement=true"}, + } { + value, err := NewEnvironment().ResolveDatabaseURL(context.Background(), migrationdomain.DatabaseConfig{Kind: test.kind, EnvKey: "DEVCTL_UNUSED_MIGRATION_URL", Default: test.value}) + require.NoError(t, err) + require.Equal(t, test.value, value) + } +} diff --git a/internal/repository/migration/repository.go b/internal/repository/migration/repository.go new file mode 100644 index 0000000..7a9979e --- /dev/null +++ b/internal/repository/migration/repository.go @@ -0,0 +1,129 @@ +package migration + +import ( + "context" + "errors" + "fmt" + "io/fs" + "time" + + _ "github.com/ClickHouse/clickhouse-go" // Register the ClickHouse SQL driver used by its migration adapter. + "github.com/devctllabs/devctl/internal/domain/failure" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + "github.com/devctllabs/go-libs/dbmigrate" + devfs "github.com/devctllabs/go-libs/filesystem" + upstream "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/clickhouse" + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/database/sqlite" + "github.com/golang-migrate/migrate/v4/source" +) + +// Repository owns migration database and contained SQL-file mechanics. +type Repository struct{ now func() time.Time } + +func NewRepository() *Repository { return &Repository{now: time.Now} } + +// Execute keeps the contained source alive until the migration database is closed. +func (r *Repository) Execute(ctx context.Context, execution migrationdomain.Execution) (result migrationdomain.Result, err error) { + if err := ctx.Err(); err != nil { + return result, fmt.Errorf("ctx.Err: %w", err) + } + target := execution.Target.ID() + disk, err := devfs.Open(execution.Root) + if err != nil { + return result, migrationError(target, "source_open_failed", err, failure.Unavailable) + } + defer func() { err = errors.Join(err, closeError(target, disk.Close())) }() + migrator, err := dbmigrate.New(disk, dbmigrate.Config{DatabaseURL: execution.DatabaseURL, Directories: []string{execution.Target.Path}}) + if err != nil { + return result, migrationError(target, "migration_open_failed", err, failure.Unavailable) + } + defer func() { err = errors.Join(err, closeError(target, migrator.Close())) }() + return runMigration(ctx, migrator, execution) +} + +func runMigration(ctx context.Context, migrator *dbmigrate.Migrator, execution migrationdomain.Execution) (migrationdomain.Result, error) { + var result migrationdomain.Result + target := execution.Target.ID() + before, err := readState(migrator) + if err != nil { + return result, migrationError(target, "version_read_failed", err, failure.Unavailable) + } + var operationErr error + switch execution.Operation { + case migrationdomain.Status: + result.State = before + return result, nil + case migrationdomain.Up: + if execution.Steps > 0 { + operationErr = migrator.UpN(ctx, execution.Steps) + } else { + operationErr = migrator.Up(ctx) + } + case migrationdomain.Down: + operationErr = migrator.Down(ctx, execution.Steps) + case migrationdomain.Create: + operationErr = &migrationdomain.Error{Kind: failure.Unsupported, Reason: "unsupported_operation"} + default: + operationErr = &migrationdomain.Error{Kind: failure.Unsupported, Reason: "unsupported_operation"} + } + if execution.Operation == migrationdomain.Up && errors.Is(operationErr, upstream.ErrNoChange) { + operationErr = nil + result.NoChange = true + } + state, stateErr := readState(migrator) + result.State = state + completedChange := operationErr == nil && !result.NoChange + observedChange := state != nil && !sameState(before, state) + result.Changed = completedChange || observedChange + err = errors.Join(operationErr, stateErr) + if err != nil { + return result, migrationError(target, "migration_execution_failed", err, failure.Internal) + } + return result, nil +} + +func readState(migrator *dbmigrate.Migrator) (*migrationdomain.State, error) { + version, dirty, err := migrator.Version() + if errors.Is(err, upstream.ErrNilVersion) { + return &migrationdomain.State{}, nil + } + if err != nil { + return nil, fmt.Errorf("migrator.Version: %w", err) + } + return &migrationdomain.State{Version: &version, Dirty: dirty}, nil +} + +func sameState(left, right *migrationdomain.State) bool { + if left.Dirty != right.Dirty { + return false + } + if left.Version == nil || right.Version == nil { + return left.Version == nil && right.Version == nil + } + return *left.Version == *right.Version +} + +func closeError(target string, err error) error { + if err == nil { + return nil + } + return migrationError(target, "close_failed", err, failure.Unavailable) +} + +func migrationError(target, reason string, cause error, fallback failure.Category) error { + kind := fallback + var dirty upstream.ErrDirty + var duplicate source.ErrDuplicateMigration + var shortage upstream.ErrShortLimit + switch { + case errors.As(cause, &dirty), errors.As(cause, &duplicate), errors.Is(cause, fs.ErrExist), errors.Is(cause, upstream.ErrLocked), errors.Is(cause, upstream.ErrLockTimeout): + kind = failure.Conflict + case errors.Is(cause, fs.ErrNotExist): + kind = failure.NotFound + case errors.As(cause, &shortage), errors.Is(cause, dbmigrate.ErrInvalidSteps), errors.Is(cause, dbmigrate.ErrInvalidConfig), errors.Is(cause, fs.ErrInvalid): + kind = failure.InvalidInput + } + return &migrationdomain.Error{Kind: kind, Reason: reason, Target: target, Cause: cause} +} diff --git a/internal/repository/migration/repository_test.go b/internal/repository/migration/repository_test.go new file mode 100644 index 0000000..2341e68 --- /dev/null +++ b/internal/repository/migration/repository_test.go @@ -0,0 +1,248 @@ +package migration + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + "time" + + "github.com/devctllabs/devctl/internal/domain/failure" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + projectdomain "github.com/devctllabs/devctl/internal/domain/project" + upstream "github.com/golang-migrate/migrate/v4" + "github.com/stretchr/testify/require" +) + +func sqliteExecution(t *testing.T) migrationdomain.Execution { + t.Helper() + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "migrations"), 0o755)) + for name, sql := range map[string]string{ + "1_create_entries.up.sql": "CREATE TABLE entries (id INTEGER PRIMARY KEY);", + "1_create_entries.down.sql": "DROP TABLE entries;", + "2_create_orders.up.sql": "CREATE TABLE orders (id INTEGER PRIMARY KEY);", + "2_create_orders.down.sql": "DROP TABLE orders;", + } { + require.NoError(t, os.WriteFile(filepath.Join(root, "migrations", name), []byte(sql), 0o644)) + } + return migrationdomain.Execution{Root: root, Target: projectdomain.MigrationTarget{Connection: "primary", Variant: "sqlite", Kind: "sqlite", Path: "migrations"}, DatabaseURL: "sqlite://" + filepath.Join(root, "app.db"), Operation: migrationdomain.Status, Steps: 1} +} + +func TestSQLiteMigrationLifecycle(t *testing.T) { + t.Parallel() + execution := sqliteExecution(t) + repository := NewRepository() + result, err := repository.Execute(context.Background(), execution) + require.NoError(t, err) + require.Equal(t, &migrationdomain.State{}, result.State) + execution.Operation, execution.Steps = migrationdomain.Up, 0 + result, err = repository.Execute(context.Background(), execution) + require.NoError(t, err) + require.NotNil(t, result.State) + require.Equal(t, uint(2), *result.State.Version) + require.False(t, result.State.Dirty) + db, err := sql.Open("sqlite", filepath.Join(execution.Root, "app.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + var count int + require.NoError(t, db.QueryRowContext(context.Background(), "SELECT count(*) FROM orders").Scan(&count)) + result, err = repository.Execute(context.Background(), execution) + require.NoError(t, err) + require.True(t, result.NoChange) + require.False(t, result.Changed) + execution.Operation, execution.Steps = migrationdomain.Down, 1 + result, err = repository.Execute(context.Background(), execution) + require.NoError(t, err) + require.Equal(t, uint(1), *result.State.Version) + require.Error(t, db.QueryRowContext(context.Background(), "SELECT count(*) FROM orders").Scan(&count)) + result, err = repository.Execute(context.Background(), execution) + require.NoError(t, err) + require.Nil(t, result.State.Version) + _, err = repository.Execute(context.Background(), execution) + require.Error(t, err) +} + +func TestSQLiteMigrationUpSteps(t *testing.T) { + t.Parallel() + execution := sqliteExecution(t) + execution.Operation, execution.Steps = migrationdomain.Up, 1 + repository := NewRepository() + + result, err := repository.Execute(context.Background(), execution) + require.NoError(t, err) + require.Equal(t, uint(1), *result.State.Version) + require.True(t, result.Changed) + + result, err = repository.Execute(context.Background(), execution) + require.NoError(t, err) + require.Equal(t, uint(2), *result.State.Version) + require.True(t, result.Changed) + + execution.Steps = 0 + result, err = repository.Execute(context.Background(), execution) + require.NoError(t, err) + require.True(t, result.NoChange) + require.False(t, result.Changed) + + execution = sqliteExecution(t) + execution.Operation, execution.Steps = migrationdomain.Up, 5 + result, err = repository.Execute(context.Background(), execution) + require.Error(t, err) + var shortage upstream.ErrShortLimit + require.ErrorAs(t, err, &shortage) + require.Equal(t, uint(2), *result.State.Version) + require.True(t, result.Changed) +} + +func TestSQLiteDirtyAndPartialRollbackResults(t *testing.T) { + t.Parallel() + t.Run("failed SQL", func(t *testing.T) { + t.Parallel() + execution := sqliteExecution(t) + require.NoError(t, os.WriteFile(filepath.Join(execution.Root, "migrations/2_create_orders.up.sql"), []byte("INVALID_SQL SUPER_SECRET_TOKEN"), 0o644)) + execution.Operation, execution.Steps = migrationdomain.Up, 0 + result, err := NewRepository().Execute(context.Background(), execution) + require.Error(t, err) + require.NotContains(t, err.Error(), "SUPER_SECRET_TOKEN") + require.NotNil(t, result.State) + require.Equal(t, uint(2), *result.State.Version) + require.True(t, result.State.Dirty) + require.True(t, result.Changed) + result, err = NewRepository().Execute(context.Background(), execution) + require.Equal(t, failure.Conflict, failure.CategoryOf(err)) + require.False(t, result.Changed) + var dirty upstream.ErrDirty + require.ErrorAs(t, err, &dirty) + execution.Operation = migrationdomain.Status + result, err = NewRepository().Execute(context.Background(), execution) + require.NoError(t, err) + require.True(t, result.State.Dirty) + }) + t.Run("short rollback", func(t *testing.T) { + t.Parallel() + execution := sqliteExecution(t) + execution.Operation = migrationdomain.Up + _, err := NewRepository().Execute(context.Background(), execution) + require.NoError(t, err) + execution.Operation, execution.Steps = migrationdomain.Down, 5 + result, err := NewRepository().Execute(context.Background(), execution) + require.Error(t, err) + var shortage upstream.ErrShortLimit + require.ErrorAs(t, err, &shortage) + require.True(t, result.Changed) + require.Nil(t, result.State.Version) + }) +} + +func TestMigrationMissingEmptyAndCancelledSources(t *testing.T) { + t.Parallel() + for _, mode := range []string{"missing", "empty", "cancelled", "symlink"} { + t.Run(mode, func(t *testing.T) { + t.Parallel() + execution := sqliteExecution(t) + execution.Operation = migrationdomain.Up + ctx := context.Background() + switch mode { + case "missing": + execution.Target.Path = "missing" + case "empty": + execution.Target.Path = "empty" + require.NoError(t, os.Mkdir(filepath.Join(execution.Root, "empty"), 0o755)) + case "cancelled": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + case "symlink": + execution.Target.Path = "outside" + require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(execution.Root, "outside"))) + } + _, err := NewRepository().Execute(ctx, execution) + require.Error(t, err) + if mode == "cancelled" { + require.ErrorIs(t, err, context.Canceled) + } + if mode != "empty" { + _, statErr := os.Stat(filepath.Join(execution.Root, "app.db")) + require.ErrorIs(t, statErr, os.ErrNotExist) + } + }) + } +} + +func TestCreateMigrationPairAndRejectVersionCollision(t *testing.T) { + t.Parallel() + repository := NewRepository() + repository.now = func() time.Time { return time.Date(2026, 10, 19, 12, 0, 0, 0, time.FixedZone("offset", 3600)) } + creation := migrationdomain.Creation{Root: t.TempDir(), Target: projectdomain.MigrationTarget{Path: "db/migrations"}, Name: "create_orders"} + result, err := repository.Create(context.Background(), creation) + require.NoError(t, err) + require.Equal(t, []string{"db/migrations/20261019110000_create_orders.up.sql", "db/migrations/20261019110000_create_orders.down.sql"}, result.Files) + for _, file := range result.Files { + data, readErr := os.ReadFile(filepath.Join(creation.Root, file)) + require.NoError(t, readErr) + require.Empty(t, data) + } + creation.Name = "another_name" + _, err = repository.Create(context.Background(), creation) + require.Equal(t, failure.Conflict, failure.CategoryOf(err)) + entries, err := os.ReadDir(filepath.Join(creation.Root, creation.Target.Path)) + require.NoError(t, err) + require.Len(t, entries, 2) +} + +func TestCreateMigrationCleansUpFirstFileAndRejectsEscapes(t *testing.T) { + t.Parallel() + root := t.TempDir() + repository := NewRepository() + repository.now = func() time.Time { return time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) } + creation := migrationdomain.Creation{Root: root, Target: projectdomain.MigrationTarget{Path: "migrations"}, Name: "create_orders"} + require.NoError(t, os.MkdirAll(filepath.Join(root, "migrations/20261019120000_create_orders.down.sql"), 0o755)) + _, err := repository.Create(context.Background(), creation) + require.Error(t, err) + _, err = os.Stat(filepath.Join(root, "migrations/20261019120000_create_orders.up.sql")) + require.ErrorIs(t, err, os.ErrNotExist) + creation.Target.Path = "../outside" + _, err = repository.Create(context.Background(), creation) + require.Error(t, err) + creation.Target.Path = "linked" + require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(root, "linked"))) + _, err = repository.Create(context.Background(), creation) + require.Error(t, err) + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + _, err = repository.Create(cancelled, creation) + require.ErrorIs(t, err, context.Canceled) +} + +func TestConcurrentCreatesReserveOneMigrationVersion(t *testing.T) { + t.Parallel() + repository := NewRepository() + repository.now = func() time.Time { return time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) } + creation := migrationdomain.Creation{Root: t.TempDir(), Target: projectdomain.MigrationTarget{Path: "migrations"}} + start := make(chan struct{}) + results := make(chan error, 2) + for _, name := range []string{"one", "two"} { + go func() { + <-start + input := creation + input.Name = name + _, err := repository.Create(context.Background(), input) + results <- err + }() + } + close(start) + succeeded := 0 + for range 2 { + if err := <-results; err != nil { + require.Equal(t, failure.Conflict, failure.CategoryOf(err)) + } else { + succeeded++ + } + } + require.Equal(t, 1, succeeded) + entries, err := os.ReadDir(filepath.Join(creation.Root, "migrations")) + require.NoError(t, err) + require.Len(t, entries, 2) +} diff --git a/internal/service/migration/mocks/service.go b/internal/service/migration/mocks/service.go new file mode 100644 index 0000000..90ec3a1 --- /dev/null +++ b/internal/service/migration/mocks/service.go @@ -0,0 +1,247 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/devctllabs/devctl/internal/service/migration (interfaces: ProjectRepository,DatabaseConfigResolver,MigrationRepository) +// +// Generated by this command: +// +// mockgen -destination mocks/service.go -package mocks -typed . ProjectRepository,DatabaseConfigResolver,MigrationRepository +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + migration "github.com/devctllabs/devctl/internal/domain/migration" + project "github.com/devctllabs/devctl/internal/domain/project" + gomock "go.uber.org/mock/gomock" +) + +// MockProjectRepository is a mock of ProjectRepository interface. +type MockProjectRepository struct { + ctrl *gomock.Controller + recorder *MockProjectRepositoryMockRecorder + isgomock struct{} +} + +// MockProjectRepositoryMockRecorder is the mock recorder for MockProjectRepository. +type MockProjectRepositoryMockRecorder struct { + mock *MockProjectRepository +} + +// NewMockProjectRepository creates a new mock instance. +func NewMockProjectRepository(ctrl *gomock.Controller) *MockProjectRepository { + mock := &MockProjectRepository{ctrl: ctrl} + mock.recorder = &MockProjectRepositoryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProjectRepository) EXPECT() *MockProjectRepositoryMockRecorder { + return m.recorder +} + +// LoadProject mocks base method. +func (m *MockProjectRepository) LoadProject(ctx context.Context, manifestPath string) (project.Project, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadProject", ctx, manifestPath) + ret0, _ := ret[0].(project.Project) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadProject indicates an expected call of LoadProject. +func (mr *MockProjectRepositoryMockRecorder) LoadProject(ctx, manifestPath any) *MockProjectRepositoryLoadProjectCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadProject", reflect.TypeOf((*MockProjectRepository)(nil).LoadProject), ctx, manifestPath) + return &MockProjectRepositoryLoadProjectCall{Call: call} +} + +// MockProjectRepositoryLoadProjectCall wrap *gomock.Call +type MockProjectRepositoryLoadProjectCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockProjectRepositoryLoadProjectCall) Return(arg0 project.Project, arg1 error) *MockProjectRepositoryLoadProjectCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockProjectRepositoryLoadProjectCall) Do(f func(context.Context, string) (project.Project, error)) *MockProjectRepositoryLoadProjectCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockProjectRepositoryLoadProjectCall) DoAndReturn(f func(context.Context, string) (project.Project, error)) *MockProjectRepositoryLoadProjectCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// MockDatabaseConfigResolver is a mock of DatabaseConfigResolver interface. +type MockDatabaseConfigResolver struct { + ctrl *gomock.Controller + recorder *MockDatabaseConfigResolverMockRecorder + isgomock struct{} +} + +// MockDatabaseConfigResolverMockRecorder is the mock recorder for MockDatabaseConfigResolver. +type MockDatabaseConfigResolverMockRecorder struct { + mock *MockDatabaseConfigResolver +} + +// NewMockDatabaseConfigResolver creates a new mock instance. +func NewMockDatabaseConfigResolver(ctrl *gomock.Controller) *MockDatabaseConfigResolver { + mock := &MockDatabaseConfigResolver{ctrl: ctrl} + mock.recorder = &MockDatabaseConfigResolverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDatabaseConfigResolver) EXPECT() *MockDatabaseConfigResolverMockRecorder { + return m.recorder +} + +// ResolveDatabaseURL mocks base method. +func (m *MockDatabaseConfigResolver) ResolveDatabaseURL(ctx context.Context, config migration.DatabaseConfig) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveDatabaseURL", ctx, config) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveDatabaseURL indicates an expected call of ResolveDatabaseURL. +func (mr *MockDatabaseConfigResolverMockRecorder) ResolveDatabaseURL(ctx, config any) *MockDatabaseConfigResolverResolveDatabaseURLCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveDatabaseURL", reflect.TypeOf((*MockDatabaseConfigResolver)(nil).ResolveDatabaseURL), ctx, config) + return &MockDatabaseConfigResolverResolveDatabaseURLCall{Call: call} +} + +// MockDatabaseConfigResolverResolveDatabaseURLCall wrap *gomock.Call +type MockDatabaseConfigResolverResolveDatabaseURLCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockDatabaseConfigResolverResolveDatabaseURLCall) Return(arg0 string, arg1 error) *MockDatabaseConfigResolverResolveDatabaseURLCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockDatabaseConfigResolverResolveDatabaseURLCall) Do(f func(context.Context, migration.DatabaseConfig) (string, error)) *MockDatabaseConfigResolverResolveDatabaseURLCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockDatabaseConfigResolverResolveDatabaseURLCall) DoAndReturn(f func(context.Context, migration.DatabaseConfig) (string, error)) *MockDatabaseConfigResolverResolveDatabaseURLCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// MockMigrationRepository is a mock of MigrationRepository interface. +type MockMigrationRepository struct { + ctrl *gomock.Controller + recorder *MockMigrationRepositoryMockRecorder + isgomock struct{} +} + +// MockMigrationRepositoryMockRecorder is the mock recorder for MockMigrationRepository. +type MockMigrationRepositoryMockRecorder struct { + mock *MockMigrationRepository +} + +// NewMockMigrationRepository creates a new mock instance. +func NewMockMigrationRepository(ctrl *gomock.Controller) *MockMigrationRepository { + mock := &MockMigrationRepository{ctrl: ctrl} + mock.recorder = &MockMigrationRepositoryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMigrationRepository) EXPECT() *MockMigrationRepositoryMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockMigrationRepository) Create(ctx context.Context, creation migration.Creation) (migration.Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", ctx, creation) + ret0, _ := ret[0].(migration.Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create. +func (mr *MockMigrationRepositoryMockRecorder) Create(ctx, creation any) *MockMigrationRepositoryCreateCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockMigrationRepository)(nil).Create), ctx, creation) + return &MockMigrationRepositoryCreateCall{Call: call} +} + +// MockMigrationRepositoryCreateCall wrap *gomock.Call +type MockMigrationRepositoryCreateCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockMigrationRepositoryCreateCall) Return(arg0 migration.Result, arg1 error) *MockMigrationRepositoryCreateCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockMigrationRepositoryCreateCall) Do(f func(context.Context, migration.Creation) (migration.Result, error)) *MockMigrationRepositoryCreateCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockMigrationRepositoryCreateCall) DoAndReturn(f func(context.Context, migration.Creation) (migration.Result, error)) *MockMigrationRepositoryCreateCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Execute mocks base method. +func (m *MockMigrationRepository) Execute(ctx context.Context, execution migration.Execution) (migration.Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Execute", ctx, execution) + ret0, _ := ret[0].(migration.Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Execute indicates an expected call of Execute. +func (mr *MockMigrationRepositoryMockRecorder) Execute(ctx, execution any) *MockMigrationRepositoryExecuteCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Execute", reflect.TypeOf((*MockMigrationRepository)(nil).Execute), ctx, execution) + return &MockMigrationRepositoryExecuteCall{Call: call} +} + +// MockMigrationRepositoryExecuteCall wrap *gomock.Call +type MockMigrationRepositoryExecuteCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockMigrationRepositoryExecuteCall) Return(arg0 migration.Result, arg1 error) *MockMigrationRepositoryExecuteCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockMigrationRepositoryExecuteCall) Do(f func(context.Context, migration.Execution) (migration.Result, error)) *MockMigrationRepositoryExecuteCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockMigrationRepositoryExecuteCall) DoAndReturn(f func(context.Context, migration.Execution) (migration.Result, error)) *MockMigrationRepositoryExecuteCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/internal/service/migration/service.go b/internal/service/migration/service.go new file mode 100644 index 0000000..856711c --- /dev/null +++ b/internal/service/migration/service.go @@ -0,0 +1,142 @@ +package migration + +import ( + "context" + "fmt" + "strings" + + "github.com/devctllabs/devctl/internal/domain/failure" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + projectdomain "github.com/devctllabs/devctl/internal/domain/project" +) + +//go:generate go tool mockgen -destination mocks/service.go -package mocks -typed . ProjectRepository,DatabaseConfigResolver,MigrationRepository + +type ProjectRepository interface { + // LoadProject returns the semantically valid Project without global readiness checks. + LoadProject(ctx context.Context, manifestPath string) (projectdomain.Project, error) +} + +type DatabaseConfigResolver interface { + // ResolveDatabaseURL resolves migration-only sources without mutating process environment. + ResolveDatabaseURL(ctx context.Context, config migrationdomain.DatabaseConfig) (string, error) +} + +type MigrationRepository interface { + // Execute runs an operation and returns observed state, including partial progress on failure. + Execute(ctx context.Context, execution migrationdomain.Execution) (migrationdomain.Result, error) + // Create creates a new SQL pair without overwriting existing files or connecting to a database. + Create(ctx context.Context, creation migrationdomain.Creation) (migrationdomain.Result, error) +} + +type Service struct { + projects ProjectRepository + config DatabaseConfigResolver + migrations MigrationRepository +} + +func New(projects ProjectRepository, config DatabaseConfigResolver, migrations MigrationRepository) *Service { + return &Service{projects: projects, config: config, migrations: migrations} +} + +// Migrate operates on one selected Migration Target. +func (s *Service) Migrate(ctx context.Context, command migrationdomain.Command) (migrationdomain.Result, error) { + if err := ctx.Err(); err != nil { + return migrationdomain.Result{}, fmt.Errorf("ctx.Err: %w", err) + } + if err := validateCommand(command); err != nil { + return migrationdomain.Result{}, err + } + selected, err := s.projects.LoadProject(ctx, command.ManifestPath) + if err != nil { + return migrationdomain.Result{}, fmt.Errorf("projects.LoadProject: %w", err) + } + target, err := selectTarget(selected.Manifest, command.Target) + if err != nil { + return migrationdomain.Result{}, err + } + if command.Operation == migrationdomain.Create { + result, err := s.migrations.Create(ctx, migrationdomain.Creation{Root: selected.Root, Target: target, Name: command.Name}) + result.Target = target.ID() + if err != nil { + return result, fmt.Errorf("migrations.Create: %w", err) + } + return result, nil + } + databaseURL, err := s.config.ResolveDatabaseURL(ctx, migrationdomain.DatabaseConfig{ + Kind: target.Kind, EnvKey: target.DatabaseEnv, Default: target.DatabaseDefault, EnvFile: command.EnvFile, + }) + if err != nil { + return migrationdomain.Result{}, fmt.Errorf("config.ResolveDatabaseURL: %w", err) + } + result, err := s.migrations.Execute(ctx, migrationdomain.Execution{ + Root: selected.Root, Target: target, Operation: command.Operation, DatabaseURL: databaseURL, Steps: command.Steps, + }) + result.Target = target.ID() + if err != nil { + return result, fmt.Errorf("migrations.Execute: %w", err) + } + return result, nil +} + +func validateCommand(command migrationdomain.Command) error { + switch command.Operation { + case migrationdomain.Create: + if !migrationdomain.ValidName(command.Name) { + return &migrationdomain.Error{Kind: failure.InvalidInput, Reason: "invalid_name"} + } + case migrationdomain.Up: + if command.Steps < 0 { + return &migrationdomain.Error{Kind: failure.InvalidInput, Reason: "invalid_steps"} + } + case migrationdomain.Down: + if command.Steps <= 0 { + return &migrationdomain.Error{Kind: failure.InvalidInput, Reason: "invalid_steps"} + } + case migrationdomain.Status: + default: + return &migrationdomain.Error{Kind: failure.Unsupported, Reason: "unsupported_operation"} + } + return nil +} + +func selectTarget(manifest projectdomain.Manifest, id string) (projectdomain.MigrationTarget, error) { + targets := projectdomain.MigrationTargets(manifest) + if id == "" && len(targets) == 1 { + return targets[0], nil + } + for _, target := range targets { + if target.ID() == id { + return target, nil + } + } + if id == "" && len(targets) > 1 { + candidates := make([]string, len(targets)) + for index, target := range targets { + candidates[index] = target.ID() + } + return projectdomain.MigrationTarget{}, &migrationdomain.Error{Kind: failure.InvalidInput, Reason: "target_required", Candidates: candidates} + } + if disabledTarget(manifest, id) { + return projectdomain.MigrationTarget{}, &migrationdomain.Error{Kind: failure.Unsupported, Reason: "migrations_disabled", Target: id} + } + return projectdomain.MigrationTarget{}, &migrationdomain.Error{Kind: failure.NotFound, Reason: "target_not_found", Target: id} +} + +func disabledTarget(manifest projectdomain.Manifest, id string) bool { + if manifest.Components.DB == nil { + return false + } + connectionName, variantName, _ := strings.Cut(id, ":") + for _, connection := range manifest.Components.DB.Connections { + if connection.Name != connectionName { + continue + } + for _, variant := range connection.Variants { + if variant.Name == variantName && variant.Migrations == nil { + return true + } + } + } + return false +} diff --git a/internal/service/migration/service_test.go b/internal/service/migration/service_test.go new file mode 100644 index 0000000..6bea651 --- /dev/null +++ b/internal/service/migration/service_test.go @@ -0,0 +1,131 @@ +package migration + +import ( + "context" + "errors" + "testing" + + "github.com/devctllabs/devctl/internal/domain/failure" + migrationdomain "github.com/devctllabs/devctl/internal/domain/migration" + projectdomain "github.com/devctllabs/devctl/internal/domain/project" + "github.com/devctllabs/devctl/internal/service/migration/mocks" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func migrationProject() projectdomain.Project { + return projectdomain.Project{Root: "/project", Manifest: projectdomain.Manifest{ + Project: projectdomain.Identity{Name: "orders-api"}, + Components: projectdomain.Components{DB: &projectdomain.DB{Connections: []projectdomain.DBConnection{{ + Name: "primary", Default: "sqlite", Variants: []projectdomain.DBVariant{ + {Name: "sqlite", Kind: "sqlite"}, + {Name: "postgres", Kind: "postgres", Migrations: &projectdomain.DBMigrations{ + Path: "db/migrations", DatabaseEnv: "MIGRATIONS_URL", DatabaseDefault: "postgres://default/db", + }}, + }, + }}}}, + }} +} + +func TestMigrateSelectsOnlyMigrationVariantAndDispatchesOperations(t *testing.T) { + t.Parallel() + for _, operation := range []migrationdomain.Operation{migrationdomain.Create, migrationdomain.Up, migrationdomain.Down, migrationdomain.Status} { + t.Run(string(operation), func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + projects, config, repository := mocks.NewMockProjectRepository(ctrl), mocks.NewMockDatabaseConfigResolver(ctrl), mocks.NewMockMigrationRepository(ctrl) + selected := migrationProject() + target := projectdomain.MigrationTarget{Connection: "primary", Variant: "postgres", Kind: "postgres", Path: "db/migrations", DatabaseEnv: "ORDERS_API_MIGRATIONS_URL", DatabaseDefault: "postgres://default/db"} + projects.EXPECT().LoadProject(gomock.Any(), "custom.yaml").Return(selected, nil) + input := migrationdomain.Command{ManifestPath: "custom.yaml", Operation: operation, Name: "create_orders", EnvFile: ".env.ci", Steps: 2} + if operation == migrationdomain.Status { + input.Target = target.ID() + } + output := migrationdomain.Result{State: &migrationdomain.State{}} + if operation == migrationdomain.Create { + output = migrationdomain.Result{Files: []string{"db/migrations/1_create_orders.up.sql", "db/migrations/1_create_orders.down.sql"}, Changed: true} + repository.EXPECT().Create(gomock.Any(), migrationdomain.Creation{Root: selected.Root, Target: target, Name: input.Name}).Return(output, nil) + } else { + config.EXPECT().ResolveDatabaseURL(gomock.Any(), migrationdomain.DatabaseConfig{Kind: "postgres", EnvKey: target.DatabaseEnv, Default: target.DatabaseDefault, EnvFile: ".env.ci"}).Return("postgres://ci/db", nil) + repository.EXPECT().Execute(gomock.Any(), migrationdomain.Execution{Root: selected.Root, Target: target, Operation: operation, DatabaseURL: "postgres://ci/db", Steps: 2}).Return(output, nil) + } + result, err := New(projects, config, repository).Migrate(context.Background(), input) + require.NoError(t, err) + output.Target = target.ID() + require.Equal(t, output, result) + }) + } +} + +func TestMigrateSelectionErrorsDoNotResolveDatabase(t *testing.T) { + t.Parallel() + tests := []struct { + name string + target string + multiple bool + empty bool + kind failure.Category + reason string + }{ + {name: "ambiguous", multiple: true, kind: failure.InvalidInput, reason: "target_required"}, + {name: "unknown", target: "missing:postgres", kind: failure.NotFound, reason: "target_not_found"}, + {name: "disabled", target: "primary:sqlite", kind: failure.Unsupported, reason: "migrations_disabled"}, + {name: "no targets", empty: true, kind: failure.NotFound, reason: "target_not_found"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + projects, config, repository := mocks.NewMockProjectRepository(ctrl), mocks.NewMockDatabaseConfigResolver(ctrl), mocks.NewMockMigrationRepository(ctrl) + selected := migrationProject() + if test.multiple { + selected.Manifest.Components.DB.Connections[0].Variants[0].Migrations = &projectdomain.DBMigrations{Path: "migrations/sqlite", DatabaseEnv: "SQLITE_URL"} + } + if test.empty { + selected.Manifest.Components.DB = nil + } + projects.EXPECT().LoadProject(gomock.Any(), "").Return(selected, nil) + _, err := New(projects, config, repository).Migrate(context.Background(), migrationdomain.Command{Operation: migrationdomain.Up, Target: test.target}) + require.Error(t, err) + require.Equal(t, test.kind, failure.CategoryOf(err)) + var selectedErr *migrationdomain.Error + require.ErrorAs(t, err, &selectedErr) + require.Equal(t, test.reason, selectedErr.Reason) + if test.multiple { + require.Equal(t, []string{"primary:postgres", "primary:sqlite"}, selectedErr.Candidates) + } + }) + } +} + +func TestMigratePreservesFailuresAndPartialResults(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + projects, config, repository := mocks.NewMockProjectRepository(ctrl), mocks.NewMockDatabaseConfigResolver(ctrl), mocks.NewMockMigrationRepository(ctrl) + projects.EXPECT().LoadProject(gomock.Any(), "").Return(migrationProject(), nil) + config.EXPECT().ResolveDatabaseURL(gomock.Any(), gomock.Any()).Return("postgres://ci/db", nil) + cause := errors.New("sql failed") + version := uint(2) + partial := migrationdomain.Result{State: &migrationdomain.State{Version: &version, Dirty: true}, Changed: true} + repository.EXPECT().Execute(gomock.Any(), gomock.Any()).Return(partial, cause) + result, err := New(projects, config, repository).Migrate(context.Background(), migrationdomain.Command{Operation: migrationdomain.Up}) + require.ErrorIs(t, err, cause) + partial.Target = "primary:postgres" + require.Equal(t, partial, result) +} + +func TestMigrateRejectsNegativeUpSteps(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + projects, config, repository := mocks.NewMockProjectRepository(ctrl), mocks.NewMockDatabaseConfigResolver(ctrl), mocks.NewMockMigrationRepository(ctrl) + + _, err := New(projects, config, repository).Migrate(context.Background(), migrationdomain.Command{ + Operation: migrationdomain.Up, + Steps: -1, + }) + + require.Equal(t, failure.InvalidInput, failure.CategoryOf(err)) + var migrationErr *migrationdomain.Error + require.ErrorAs(t, err, &migrationErr) + require.Equal(t, "invalid_steps", migrationErr.Reason) +} diff --git a/internal/service/project/inspect.go b/internal/service/project/inspect.go index 91ff79c..6c150db 100644 --- a/internal/service/project/inspect.go +++ b/internal/service/project/inspect.go @@ -35,7 +35,7 @@ func inspectProject(selected projectdomain.Project, targets []projectdomain.Targ Name: manifest.Project.Name, Language: manifest.Project.Language, Module: manifest.Languages.Go.Module, EnvPrefix: catalog.Prefix(), Paths: effectivePaths(manifest), Targets: inspectionTargets(targets), - Env: effectiveEnv(catalog), Resources: effectiveResources(manifest, catalog.Prefix()), + Env: effectiveEnv(catalog), Resources: effectiveResources(manifest), }, nil } @@ -86,9 +86,19 @@ func (s *Service) inspectionTargets( return result } -func effectiveResources(manifest projectdomain.Manifest, envPrefix string) projectdomain.InspectionResources { +func effectiveResources(manifest projectdomain.Manifest) projectdomain.InspectionResources { resources := projectdomain.InspectionResources{} - appendDBResources(&resources, manifest.Components.DB, envPrefix) + if manifest.Components.DB != nil { + for _, connection := range manifest.Components.DB.Connections { + resources.DBConnections = append(resources.DBConnections, connection.Name) + } + } + for _, target := range projectdomain.MigrationTargets(manifest) { + resources.Migrations = append(resources.Migrations, projectdomain.InspectionMigration{ + Connection: target.Connection, Variant: target.Variant, Kind: target.Kind, + Path: target.Path, DatabaseEnv: target.DatabaseEnv, + }) + } if manifest.Components.Redis != nil { for _, value := range manifest.Components.Redis.Connections { resources.RedisConnections = append(resources.RedisConnections, value.Name) @@ -115,26 +125,3 @@ func effectiveResources(manifest projectdomain.Manifest, envPrefix string) proje }) return resources } - -func appendDBResources(resources *projectdomain.InspectionResources, database *projectdomain.DB, envPrefix string) { - if database == nil { - return - } - for _, connection := range database.Connections { - resources.DBConnections = append(resources.DBConnections, connection.Name) - appendMigrations(resources, connection, envPrefix) - } -} - -func appendMigrations(resources *projectdomain.InspectionResources, connection projectdomain.DBConnection, envPrefix string) { - for _, variant := range connection.Variants { - migrations := variant.Migrations - if migrations == nil { - continue - } - resources.Migrations = append(resources.Migrations, projectdomain.InspectionMigration{ - Connection: connection.Name, Variant: variant.Name, Kind: variant.Kind, - Path: migrations.Path, DatabaseEnv: envPrefix + migrations.DatabaseEnv, - }) - } -} diff --git a/internal/service/scaffold/integration_test.go b/internal/service/scaffold/integration_test.go index d4ae4ac..bacbb79 100644 --- a/internal/service/scaffold/integration_test.go +++ b/internal/service/scaffold/integration_test.go @@ -115,6 +115,153 @@ languages: require.NoError(t, err) } +func TestScaffoldPreservesCustomHTTPConfigs(t *testing.T) { + t.Parallel() + + for _, existing := range []bool{false, true} { + t.Run(fmt.Sprintf("existing=%t", existing), func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + manifestPath := filepath.Join(root, "devctl.yaml") + require.NoError(t, os.WriteFile(manifestPath, []byte(`version: 1 +project: {name: sample, language: go} +env: {} +paths: {external_contracts: api/external} +sources: + contracts: {type: local, path: api/contracts} +exports: {} +components: + http: + server: {openapi: contracts/service.yaml} + clients: + - {name: orders, source: contracts, path: orders.yaml, oapi_config: tools/oapi/orders.yaml} + - {name: catalog, source: contracts, path: catalog.yaml} +languages: + go: + module: example.test/sample + generators: + http: {oapi_config: tools/oapi/custom-server.yaml} +`), 0o644)) + customPaths := []string{"tools/oapi/custom-server.yaml", "tools/oapi/orders.yaml"} + const customContent = "# user-owned\npackage: custom\n" + if existing { + require.NoError(t, os.MkdirAll(filepath.Join(root, "tools/oapi"), 0o755)) + for _, path := range customPaths { + require.NoError(t, os.WriteFile(filepath.Join(root, path), []byte(customContent), 0o644)) + } + } + + result, err := scaffoldProject(context.Background(), manifestPath) + require.NoError(t, err) + require.FileExists(t, filepath.Join(root, "contracts/service.yaml")) + require.FileExists(t, filepath.Join(root, "tools/oapi/clients.catalog.yaml")) + for _, path := range customPaths { + if existing { + content, err := os.ReadFile(filepath.Join(root, path)) + require.NoError(t, err) + require.Equal(t, customContent, string(content)) + } else { + require.NoFileExists(t, filepath.Join(root, path)) + } + } + for _, change := range result.Files { + require.NotContains(t, customPaths, change.Path) + } + }) + } +} + +func TestScaffoldManagesCanonicalAndSharedHTTPConfigs(t *testing.T) { + t.Parallel() + + for _, explicit := range []bool{false, true} { + t.Run(fmt.Sprintf("explicit canonical=%t", explicit), func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + manifest := projectdomain.Manifest{ + Project: projectdomain.Identity{Name: "sample", Language: "go"}, + Sources: map[string]projectdomain.Source{"contracts": {Type: projectdomain.SourceLocal, Path: "api/contracts"}}, + Languages: projectdomain.Languages{Go: projectdomain.GoLanguage{Module: "example.test/sample"}}, + Components: projectdomain.Components{HTTP: &projectdomain.HTTP{ + Server: &projectdomain.HTTPServer{}, + Clients: []projectdomain.HTTPClient{ + {Name: "orders", Source: "contracts", Path: "orders.yaml", OAPIConfig: "tools/oapi/shared.yaml"}, + {Name: "catalog", Source: "contracts", Path: "catalog.yaml", OAPIConfig: "tools/oapi/shared.yaml"}, + {Name: "billing", Source: "contracts", Path: "billing.yaml"}, + }, + }}, + } + if explicit { + manifest.Languages.Go.Generators.HTTP = &projectdomain.HTTPGenerator{OAPIConfig: "tools/oapi/server.yaml"} + manifest.Components.HTTP.Clients[2].OAPIConfig = "tools/oapi/clients.billing.yaml" + } + service := scaffold.New(zap.NewNop(), fixedProjectRepository{project: projectdomain.Project{Root: root, Manifest: manifest}}, workspacerepo.NewFilesystemRepo()) + require.NoError(t, os.MkdirAll(filepath.Join(root, "tools/oapi"), 0o755)) + const existing = "# user content\npackage: custom\n" + for _, path := range []string{"tools/oapi/shared.yaml", "tools/oapi/server.yaml", "tools/oapi/clients.billing.yaml"} { + require.NoError(t, os.WriteFile(filepath.Join(root, path), []byte(existing), 0o644)) + } + + for attempt := range 2 { + result, err := service.Scaffold(context.Background(), scaffolddomain.Command{}) + require.NoError(t, err) + shared, err := os.ReadFile(filepath.Join(root, "tools/oapi/shared.yaml")) + require.NoError(t, err) + require.Equal(t, existing, string(shared)) + for _, path := range []string{"tools/oapi/server.yaml", "tools/oapi/clients.billing.yaml"} { + content, err := os.ReadFile(filepath.Join(root, path)) + require.NoError(t, err) + require.NotEqual(t, existing, string(content)) + action := scaffolddomain.FileUpdated + if attempt > 0 { + action = scaffolddomain.FileUnchanged + } + requireFileChange(t, result, path, action) + } + } + }) + } +} + +func TestScaffoldRejectsCustomHTTPConfigArtifactCollisions(t *testing.T) { + t.Parallel() + + for _, configPath := range []string{"go.mod", "api/openapi/swagger.yaml", "tools/oapi/clients.catalog.yaml"} { + t.Run(configPath, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + manifest := projectdomain.Manifest{ + Project: projectdomain.Identity{Name: "sample", Language: "go"}, + Sources: map[string]projectdomain.Source{"contracts": {Type: projectdomain.SourceLocal, Path: "api/contracts"}}, + Languages: projectdomain.Languages{Go: projectdomain.GoLanguage{ + Module: "example.test/sample", Generators: projectdomain.GoGenerators{HTTP: &projectdomain.HTTPGenerator{OAPIConfig: configPath}}, + }}, + Components: projectdomain.Components{HTTP: &projectdomain.HTTP{ + Server: &projectdomain.HTTPServer{}, + Clients: []projectdomain.HTTPClient{{Name: "catalog", Source: "contracts", Path: "catalog.yaml"}}, + }}, + } + service := scaffold.New(zap.NewNop(), fixedProjectRepository{project: projectdomain.Project{Root: root, Manifest: manifest}}, workspacerepo.NewFilesystemRepo()) + fullPath := filepath.Join(root, configPath) + require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0o755)) + const existing = "# preserve on conflict\n" + require.NoError(t, os.WriteFile(fullPath, []byte(existing), 0o644)) + + result, err := service.Scaffold(context.Background(), scaffolddomain.Command{}) + + require.Error(t, err) + require.Empty(t, result.Files) + content, err := os.ReadFile(fullPath) + require.NoError(t, err) + require.Equal(t, existing, string(content)) + require.NoDirExists(t, filepath.Join(root, "cmd")) + }) + } +} + func requireFileChange(t *testing.T, result scaffolddomain.Result, path string, action scaffolddomain.FileAction) { t.Helper() for _, change := range result.Files { diff --git a/internal/service/scaffold/planner.go b/internal/service/scaffold/planner.go index 8744683..22c40b4 100644 --- a/internal/service/scaffold/planner.go +++ b/internal/service/scaffold/planner.go @@ -98,6 +98,9 @@ func plan(m projectdomain.Manifest) ([]Artifact, error) { if err := ensureUniqueArtifactPaths(artifacts); err != nil { return nil, fmt.Errorf("ensureUniqueArtifactPaths: %w", err) } + if err := checkHTTPConfigOwnership(projection.http.targets, artifacts); err != nil { + return nil, fmt.Errorf("checkHTTPConfigOwnership: %w", err) + } sort.SliceStable(artifacts, func(i, j int) bool { return artifacts[i].Path < artifacts[j].Path }) if err := formatGoArtifacts(artifacts); err != nil { return nil, fmt.Errorf("formatGoArtifacts: %w", err) @@ -383,6 +386,9 @@ func httpArtifacts(projection httpProjection) ([]Artifact, error) { artifacts = append(artifacts, Artifact{target.Reference.Entrypoint, 0o644, seed, false}) templateName = "oapi-server.yaml" } + if target.Config != canonicalHTTPConfig(target) { + continue + } config, err := readTemplateAsset(templateName) if err != nil { return nil, fmt.Errorf("readTemplateAsset: %w", err) @@ -392,6 +398,29 @@ func httpArtifacts(projection httpProjection) ([]Artifact, error) { return artifacts, nil } +func canonicalHTTPConfig(target projectdomain.Target) string { + if target.Role == "server" { + return "tools/oapi/server.yaml" + } + return "tools/oapi/clients." + target.Name + ".yaml" +} + +func checkHTTPConfigOwnership(targets []projectdomain.Target, artifacts []Artifact) error { + planned := make(map[string]struct{}, len(artifacts)) + for _, artifact := range artifacts { + planned[canonicalArtifactPath(artifact.Path)] = struct{}{} + } + for _, target := range targets { + if target.Config == canonicalHTTPConfig(target) { + continue + } + if _, exists := planned[canonicalArtifactPath(target.Config)]; exists { + return fmt.Errorf("custom HTTP config path collides with scaffold artifact %q", target.Config) + } + } + return nil +} + func formatGoArtifacts(artifacts []Artifact) error { for index := range artifacts { artifact := &artifacts[index] diff --git a/internal/service/scaffold/planner_test.go b/internal/service/scaffold/planner_test.go index b3ce2c5..65b21a5 100644 --- a/internal/service/scaffold/planner_test.go +++ b/internal/service/scaffold/planner_test.go @@ -423,9 +423,9 @@ func TestPlanAppliesHTTPArtifactPolicies(t *testing.T) { artifacts := plannedArtifacts(t, manifest) require.True(t, artifacts["contracts/service.yaml"].CreateOnly) - require.False(t, artifacts["tools/oapi/custom-server.yaml"].CreateOnly) + require.NotContains(t, artifacts, "tools/oapi/custom-server.yaml") require.False(t, artifacts["tools/oapi/clients.catalog.yaml"].CreateOnly) - require.False(t, artifacts["tools/oapi/orders.yaml"].CreateOnly) + require.NotContains(t, artifacts, "tools/oapi/orders.yaml") require.NotContains(t, artifacts, "tools/oapi/server.yaml") require.NotContains(t, artifacts, "api/openapi/swagger.yaml") requireArtifactContains(t, artifacts, "go.mod", "tool github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen") @@ -595,21 +595,21 @@ func TestPlanScaffoldsMigrationTargetsAndMiseTasks(t *testing.T) { Confirm any `toml:"confirm"` } `toml:"tasks"` } - metadata, err := toml.Decode(string(artifacts[".mise.toml"].Content), &mise) + _, err := toml.Decode(string(artifacts[".mise.toml"].Content), &mise) require.NoError(t, err) - var migrateTool struct { - Version string `toml:"version"` - Tags []string `toml:"tags"` - } - require.NoError(t, metadata.PrimitiveDecode(mise.Tools["go:github.com/golang-migrate/migrate/v4/cmd/migrate"], &migrateTool)) - require.Equal(t, "v4.19.1", migrateTool.Version) - require.Equal(t, []string{"postgres", "sqlite"}, migrateTool.Tags) + require.NotContains(t, mise.Tools, "go:github.com/golang-migrate/migrate/v4/cmd/migrate") require.Equal(t, `arg "" help="Migration name"`, mise.Tasks["migrate:primary:sqlite:create"].Usage) - require.Contains(t, mise.Tasks["migrate:primary:sqlite:create"].Run, `-format "20060102150405" "${usage_name?}"`) - require.Contains(t, mise.Tasks["migrate:primary:sqlite:up"].Run, `${APP_DB_PRIMARY_SQLITE_MIGRATIONS_URL:-sqlite://./data/primary.db?_pragma=foreign_keys%281%29}`) - require.Equal(t, `arg "[steps]" default="1" help="Number of migrations"`, mise.Tasks["migrate:primary:sqlite:down"].Usage) - require.NotNil(t, mise.Tasks["migrate:primary:sqlite:down"].Confirm) - require.Contains(t, mise.Tasks["migrate:archive:postgres:up"].Run, `${APP_ARCHIVE_MIGRATIONS_URL:?set APP_ARCHIVE_MIGRATIONS_URL}`) + require.Contains(t, mise.Tasks["migrate:primary:sqlite:create"].Run, `devctl migrate create --target "primary:sqlite" "${usage_name?}"`) + require.Contains(t, mise.Tasks["migrate:primary:sqlite:up"].Usage, `flag "--steps "`) + require.Contains(t, mise.Tasks["migrate:primary:sqlite:up"].Run, `--steps "$usage_steps"`) + require.Contains(t, mise.Tasks["migrate:primary:sqlite:up"].Run, `devctl migrate up --target "primary:sqlite"`) + require.Contains(t, mise.Tasks["migrate:primary:sqlite:down"].Usage, `arg "[steps]" default="1" help="Number of migrations"`) + require.Nil(t, mise.Tasks["migrate:primary:sqlite:down"].Confirm) + require.Contains(t, mise.Tasks["migrate:primary:sqlite:down"].Run, `--steps "${usage_steps?}"`) + require.Contains(t, mise.Tasks["migrate:archive:postgres:up"].Run, `devctl migrate up --target "archive:postgres"`) + require.Contains(t, mise.Tasks["migrate:archive:postgres:status"].Run, `devctl migrate status --target "archive:postgres"`) + require.Contains(t, mise.Tasks["migrate:archive:postgres:status"].Usage, `flag "--env-file "`) + require.NotContains(t, string(artifacts[".mise.toml"].Content), "database_url=") require.NotContains(t, mise.Tasks["migrate:archive:postgres:down"].Run, "-all") } @@ -669,17 +669,11 @@ func TestPlanRejectsDuplicateArtifactPaths(t *testing.T) { path: "go.mod", }, { - name: "client generator configs collide", + name: "OpenAPI seed collides with canonical generator config", configure: func(manifest *projectdomain.Manifest) { - manifest.Sources = map[string]projectdomain.Source{ - "contracts": {Type: projectdomain.SourceLocal, Path: "api/contracts"}, - } - manifest.Components.HTTP = &projectdomain.HTTP{Clients: []projectdomain.HTTPClient{ - {Name: "catalog", Source: "contracts", Path: "catalog.yaml", OAPIConfig: "tools/oapi/shared.yaml"}, - {Name: "orders", Source: "contracts", Path: "orders.yaml", OAPIConfig: "tools/oapi/shared.yaml"}, - }} + manifest.Components.HTTP = &projectdomain.HTTP{Server: &projectdomain.HTTPServer{OpenAPI: "tools/oapi/server.yaml"}} }, - path: "tools/oapi/shared.yaml", + path: "tools/oapi/server.yaml", }, } diff --git a/internal/service/scaffold/projection.go b/internal/service/scaffold/projection.go index 2936501..8f869d1 100644 --- a/internal/service/scaffold/projection.go +++ b/internal/service/scaffold/projection.go @@ -363,68 +363,15 @@ func databaseVariantRequirements(kind string) []string { } func compileMiseTemplate(manifest projectdomain.Manifest, kafkaJSON bool) miseTemplateData { - prefix := manifest.Env.Prefix - if prefix == "" { - prefix = strings.ToUpper(strings.ReplaceAll(manifest.Project.Name, "-", "_")) + "_" - } - migrations, tags := compileMigrationTasks(manifest.Components.DB, prefix) - return miseTemplateData{JSON: kafkaJSON, Migrations: migrations, MigrationTags: tags} + return miseTemplateData{JSON: kafkaJSON, Migrations: compileMigrationTasks(manifest)} } -func compileMigrationTasks(database *projectdomain.DB, prefix string) ([]migrationTask, []string) { - if database == nil { - return nil, nil - } +func compileMigrationTasks(manifest projectdomain.Manifest) []migrationTask { var migrations []migrationTask - migrationKinds := make(map[string]struct{}) - for _, connection := range database.Connections { - connectionTasks := compileConnectionMigrationTasks(connection, prefix) - migrations = append(migrations, connectionTasks...) - for _, task := range connectionTasks { - migrationKinds[task.Kind] = struct{}{} - } - } - sort.Slice(migrations, func(i, j int) bool { return migrations[i].TaskPrefix < migrations[j].TaskPrefix }) - tags := sortedKeys(migrationKinds) - return migrations, tags -} - -func compileConnectionMigrationTasks(connection projectdomain.DBConnection, prefix string) []migrationTask { - var tasks []migrationTask - for _, variant := range connection.Variants { - if task, exists := compileMigrationTask(connection.Name, variant, prefix); exists { - tasks = append(tasks, task) - } - } - return tasks -} - -func compileMigrationTask( - connectionName string, - variant projectdomain.DBVariant, - prefix string, -) (migrationTask, bool) { - if variant.Migrations == nil { - return migrationTask{}, false - } - environment := prefix + variant.Migrations.DatabaseEnv - expression := "${" + environment + ":?set " + environment + "}" - if variant.Migrations.DatabaseDefault != "" { - expression = "${" + environment + ":-" + variant.Migrations.DatabaseDefault + "}" - } - return migrationTask{ - TaskPrefix: "migrate:" + connectionName + ":" + variant.Name, - Path: variant.Migrations.Path, DatabaseExpression: expression, Kind: variant.Kind, - }, true -} - -func sortedKeys[Value any](values map[string]Value) []string { - keys := make([]string, 0, len(values)) - for key := range values { - keys = append(keys, key) + for _, target := range projectdomain.MigrationTargets(manifest) { + migrations = append(migrations, migrationTask{TaskPrefix: "migrate:" + target.ID(), Target: target.ID(), Path: target.Path}) } - sort.Strings(keys) - return keys + return migrations } func manifestHasKafkaFormat(manifest projectdomain.Manifest, format string) bool { diff --git a/internal/service/scaffold/render.go b/internal/service/scaffold/render.go index e75fbac..dabe187 100644 --- a/internal/service/scaffold/render.go +++ b/internal/service/scaffold/render.go @@ -55,16 +55,14 @@ func renderGoMod(data goModTemplateData) (string, error) { } type migrationTask struct { - TaskPrefix string - Path string - DatabaseExpression string - Kind string + TaskPrefix string + Target string + Path string } type miseTemplateData struct { - JSON bool - Migrations []migrationTask - MigrationTags []string + JSON bool + Migrations []migrationTask } func renderMise(data miseTemplateData) (string, error) { diff --git a/internal/service/scaffold/templates/mise.toml b/internal/service/scaffold/templates/mise.toml index b8d73de..363f052 100644 --- a/internal/service/scaffold/templates/mise.toml +++ b/internal/service/scaffold/templates/mise.toml @@ -5,9 +5,6 @@ golangci-lint = "2.12.2" node = "24" "npm:quicktype" = "26.0.0" {{- end }} -{{- if .Migrations }} -"go:github.com/golang-migrate/migrate/v4/cmd/migrate" = { version = "v4.19.1", tags = [{{- range $index, $tag := .MigrationTags }}{{ if $index }}, {{ end }}{{ printf "%q" $tag }}{{- end }}] } -{{- end }} [tasks.fmt] run = "golangci-lint fmt" @@ -36,22 +33,40 @@ depends = ["fmt:check", "lint", "test"] description = "Create timestamped migration files in {{ .Path }}" usage = 'arg "" help="Migration name"' run = ''' -migrate create -ext sql -dir "{{ .Path }}" -format "20060102150405" "${usage_name?}" +devctl migrate create --target "{{ .Target }}" "${usage_name?}" ''' [tasks."{{ .TaskPrefix }}:up"] description = "Apply migrations from {{ .Path }}" +usage = ''' +flag "--steps " help="Apply N migrations; omit to apply all" +flag "--env-file " help="Read migration URL from dotenv file" +''' run = ''' -database_url="{{ .DatabaseExpression }}" -migrate -path "{{ .Path }}" -database "$database_url" up +set -- +if [ -n "${usage_steps:-}" ]; then set -- --steps "$usage_steps"; fi +if [ -n "${usage_env_file:-}" ]; then set -- "$@" --env-file "$usage_env_file"; fi +devctl migrate up --target "{{ .Target }}" "$@" ''' [tasks."{{ .TaskPrefix }}:down"] description = "Roll back migrations from {{ .Path }}" -usage = 'arg "[steps]" default="1" help="Number of migrations"' -confirm = "Roll back {{ .TaskPrefix }} migrations?" +usage = ''' +arg "[steps]" default="1" help="Number of migrations" +flag "--env-file " help="Read migration URL from dotenv file" +''' +run = ''' +set -- +if [ -n "${usage_env_file:-}" ]; then set -- --env-file "$usage_env_file"; fi +devctl migrate down --target "{{ .Target }}" --steps "${usage_steps?}" "$@" +''' + +[tasks."{{ .TaskPrefix }}:status"] +description = "Show migration version and dirty state for {{ .Target }}" +usage = 'flag "--env-file " help="Read migration URL from dotenv file"' run = ''' -database_url="{{ .DatabaseExpression }}" -migrate -path "{{ .Path }}" -database "$database_url" down "${usage_steps?}" +set -- +if [ -n "${usage_env_file:-}" ]; then set -- --env-file "$usage_env_file"; fi +devctl migrate status --target "{{ .Target }}" "$@" ''' {{ end }} diff --git a/internal/service/scaffold/testdata/full/.mise.toml b/internal/service/scaffold/testdata/full/.mise.toml index 30c9c17..7de7e7c 100644 --- a/internal/service/scaffold/testdata/full/.mise.toml +++ b/internal/service/scaffold/testdata/full/.mise.toml @@ -3,7 +3,6 @@ go = "1.26.0" golangci-lint = "2.12.2" node = "24" "npm:quicktype" = "26.0.0" -"go:github.com/golang-migrate/migrate/v4/cmd/migrate" = { version = "v4.19.1", tags = ["clickhouse"] } [tasks.fmt] run = "golangci-lint fmt" @@ -32,21 +31,39 @@ depends = ["fmt:check", "lint", "test"] description = "Create timestamped migration files in migrations/analytics/clickhouse" usage = 'arg "" help="Migration name"' run = ''' -migrate create -ext sql -dir "migrations/analytics/clickhouse" -format "20060102150405" "${usage_name?}" +devctl migrate create --target "analytics:clickhouse" "${usage_name?}" ''' [tasks."migrate:analytics:clickhouse:up"] description = "Apply migrations from migrations/analytics/clickhouse" +usage = ''' +flag "--steps " help="Apply N migrations; omit to apply all" +flag "--env-file " help="Read migration URL from dotenv file" +''' run = ''' -database_url="${SAMPLE_DB_ANALYTICS_CLICKHOUSE_MIGRATIONS_URL:?set SAMPLE_DB_ANALYTICS_CLICKHOUSE_MIGRATIONS_URL}" -migrate -path "migrations/analytics/clickhouse" -database "$database_url" up +set -- +if [ -n "${usage_steps:-}" ]; then set -- --steps "$usage_steps"; fi +if [ -n "${usage_env_file:-}" ]; then set -- "$@" --env-file "$usage_env_file"; fi +devctl migrate up --target "analytics:clickhouse" "$@" ''' [tasks."migrate:analytics:clickhouse:down"] description = "Roll back migrations from migrations/analytics/clickhouse" -usage = 'arg "[steps]" default="1" help="Number of migrations"' -confirm = "Roll back migrate:analytics:clickhouse migrations?" +usage = ''' +arg "[steps]" default="1" help="Number of migrations" +flag "--env-file " help="Read migration URL from dotenv file" +''' +run = ''' +set -- +if [ -n "${usage_env_file:-}" ]; then set -- --env-file "$usage_env_file"; fi +devctl migrate down --target "analytics:clickhouse" --steps "${usage_steps?}" "$@" +''' + +[tasks."migrate:analytics:clickhouse:status"] +description = "Show migration version and dirty state for analytics:clickhouse" +usage = 'flag "--env-file " help="Read migration URL from dotenv file"' run = ''' -database_url="${SAMPLE_DB_ANALYTICS_CLICKHOUSE_MIGRATIONS_URL:?set SAMPLE_DB_ANALYTICS_CLICKHOUSE_MIGRATIONS_URL}" -migrate -path "migrations/analytics/clickhouse" -database "$database_url" down "${usage_steps?}" +set -- +if [ -n "${usage_env_file:-}" ]; then set -- --env-file "$usage_env_file"; fi +devctl migrate status --target "analytics:clickhouse" "$@" '''