From 86329fc0f026b628d4d98e40d0bc73452571572c Mon Sep 17 00:00:00 2001 From: Mike Landau Date: Fri, 30 Oct 2026 10:07:49 -0700 Subject: [PATCH] test(pkgs): install and switch versions of every shipped package Add a testscript next to every package definition in pkgs/. Each one installs a version from devbox.json, switches to another with `devbox add`, and switches back by editing devbox.json, checking the version (and the definition's env, such as JAVA_HOME or the python virtual environment) at every step. TestPackageDefinitions runs them with $PKGS pointing at pkgs/ when DEVBOX_RUN_PKGDEF_TESTS=1, because they download real toolchains. TestShippedDefinitionsParse checks that every definition is valid and named after its directory; it always runs. TestShippedDefinitionsResolve resolves and locks the latest version of each against its vendor, behind the same flag. A pkgdef-tests workflow runs both on Linux and macOS when the definitions or their implementation change, and weekly to catch vendor changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/pkgdef-tests.yaml | 81 ++++++++++++++++++++++++ internal/pkgdef/pkgs_test.go | 97 +++++++++++++++++++++++++++++ pkgs/bun/bun.test.txt | 33 ++++++++++ pkgs/deno/deno.test.txt | 33 ++++++++++ pkgs/go/go.test.txt | 36 +++++++++++ pkgs/gradle/gradle.test.txt | 32 ++++++++++ pkgs/helm/helm.test.txt | 31 +++++++++ pkgs/java/java.test.txt | 37 +++++++++++ pkgs/kubectl/kubectl.test.txt | 31 +++++++++ pkgs/maven/maven.test.txt | 33 ++++++++++ pkgs/nodejs/nodejs.test.txt | 38 +++++++++++ pkgs/pnpm/pnpm.test.txt | 33 ++++++++++ pkgs/python/python.test.txt | 41 ++++++++++++ pkgs/rust/rust.test.txt | 40 ++++++++++++ pkgs/terraform/terraform.test.txt | 31 +++++++++ pkgs/uv/uv.test.txt | 33 ++++++++++ pkgs/yarn/yarn.test.txt | 33 ++++++++++ pkgs/zig/zig.test.txt | 32 ++++++++++ testscripts/testrunner/pkgdefs.go | 36 +++++++++++ testscripts/testscripts_test.go | 18 ++++++ 20 files changed, 779 insertions(+) create mode 100644 .github/workflows/pkgdef-tests.yaml create mode 100644 internal/pkgdef/pkgs_test.go create mode 100644 pkgs/bun/bun.test.txt create mode 100644 pkgs/deno/deno.test.txt create mode 100644 pkgs/go/go.test.txt create mode 100644 pkgs/gradle/gradle.test.txt create mode 100644 pkgs/helm/helm.test.txt create mode 100644 pkgs/java/java.test.txt create mode 100644 pkgs/kubectl/kubectl.test.txt create mode 100644 pkgs/maven/maven.test.txt create mode 100644 pkgs/nodejs/nodejs.test.txt create mode 100644 pkgs/pnpm/pnpm.test.txt create mode 100644 pkgs/python/python.test.txt create mode 100644 pkgs/rust/rust.test.txt create mode 100644 pkgs/terraform/terraform.test.txt create mode 100644 pkgs/uv/uv.test.txt create mode 100644 pkgs/yarn/yarn.test.txt create mode 100644 pkgs/zig/zig.test.txt create mode 100644 testscripts/testrunner/pkgdefs.go diff --git a/.github/workflows/pkgdef-tests.yaml b/.github/workflows/pkgdef-tests.yaml new file mode 100644 index 00000000000..102b0520b11 --- /dev/null +++ b/.github/workflows/pkgdef-tests.yaml @@ -0,0 +1,81 @@ +name: pkgdef-tests +# Installs every package definition in pkgs/ from its vendor and switches +# between two of its versions (see specs/package-definitions.md). Runs when +# the definitions or their implementation change, and weekly to catch vendors +# that change their downloads or metadata. + +concurrency: + group: pkgdef-tests-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + paths: + - pkgs/** + - internal/pkgdef/** + - internal/devbox/pkgdef.go + - internal/lock/pkgdef.go + - internal/plugin/pkgdef.go + - testscripts/testrunner/pkgdefs.go + - .github/workflows/pkgdef-tests.yaml + push: + branches: + - main + paths: + - pkgs/** + - internal/pkgdef/** + - internal/devbox/pkgdef.go + - internal/lock/pkgdef.go + - internal/plugin/pkgdef.go + - testscripts/testrunner/pkgdefs.go + - .github/workflows/pkgdef-tests.yaml + workflow_dispatch: + schedule: + - cron: '0 9 * * 1' # Mondays at 9:00 UTC + +permissions: + contents: read + +defaults: + run: + shell: bash + +env: + # Version lookups and asset digests go through the GitHub API. + DEVBOX_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NIX_CONFIG: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + DEVBOX_RUN_PKGDEF_TESTS: 1 + +jobs: + pkgdef-tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - name: clear directories to reduce disk usage + if: runner.os == 'Linux' + # Every package installs two toolchains (several GB in total). + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - uses: actions/checkout@v7 + - name: Mount golang cache + uses: actions/cache@v6 + with: + path: | + ~/.cache/go-build + ~/go/pkg + key: go-devbox-tests-${{ runner.os }}-${{ hashFiles('go.sum') }} + - name: Install devbox + uses: jetify-com/devbox-install-action@v0.15.0 + with: + enable-cache: true + - name: Resolve and lock the latest version of every package + run: devbox run go test -v -count=1 -run TestShippedDefinitionsResolve ./internal/pkgdef + - name: Install and switch versions of every package + run: devbox run go test -v -count=1 -timeout 50m -run TestPackageDefinitions ./testscripts diff --git a/internal/pkgdef/pkgs_test.go b/internal/pkgdef/pkgs_test.go new file mode 100644 index 00000000000..f4e60545cdb --- /dev/null +++ b/internal/pkgdef/pkgs_test.go @@ -0,0 +1,97 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package pkgdef + +import ( + "os" + "path/filepath" + "runtime" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// networkTestsEnv enables tests that resolve the shipped package +// definitions against the real vendors. +const networkTestsEnv = "DEVBOX_RUN_PKGDEF_TESTS" + +// shippedDefinitions returns the package definitions in pkgs/, keyed by +// package directory name. +func shippedDefinitions(t *testing.T) map[string]string { + t.Helper() + paths, err := filepath.Glob(filepath.Join("..", "..", "pkgs", "*", "*.pkg.json*")) + require.NoError(t, err) + require.NotEmpty(t, paths) + defs := map[string]string{} + for _, p := range paths { + defs[filepath.Base(filepath.Dir(p))] = p + } + return defs +} + +func loadShipped(t *testing.T, path string) *Definition { + t.Helper() + loaded, err := loadLocal(Ref{Source: path}, "") + require.NoError(t, err) + return loaded.Definition +} + +// TestShippedDefinitionsParse checks that every definition in pkgs/ is +// valid, is named after its directory and file, and supports every +// platform Devbox does. +func TestShippedDefinitionsParse(t *testing.T) { + for dir, path := range shippedDefinitions(t) { + t.Run(dir, func(t *testing.T) { + def := loadShipped(t, path) + assert.Equal(t, dir, def.Name, "definition name matches its directory") + assert.Contains(t, []string{dir + ".pkg.json", dir + ".pkg.jsonc"}, filepath.Base(path)) + assert.ElementsMatch(t, knownSystems, def.Platforms) + assert.NotEmpty(t, def.Binaries, "binaries lets install verify the result") + assert.NotEmpty(t, def.Homepage) + assert.NotEmpty(t, def.License) + }) + } +} + +// TestShippedDefinitionsResolve resolves the latest version of every +// definition in pkgs/ against the real vendor and locks it for every +// platform. It catches vendors that changed their URLs or metadata. +func TestShippedDefinitionsResolve(t *testing.T) { + if on, _ := strconv.ParseBool(os.Getenv(networkTestsEnv)); !on { + t.Skipf("Skipping network test. To enable, set %s=1.", networkTestsEnv) + } + client := NewClient() + client.CacheDir = t.TempDir() + for dir, path := range shippedDefinitions(t) { + t.Run(dir, func(t *testing.T) { + def := loadShipped(t, path) + resolved, err := client.Resolve(t.Context(), def, Latest) + require.NoError(t, err) + for system, artifact := range resolved.Artifacts { + assert.NotEmpty(t, artifact.URL, system) + assert.True(t, artifact.HasDigest(), "%s: the latest version should have a vendor checksum", system) + } + // Vendors drop platforms (Temurin 27 has no Intel macOS build), + // so only the platform running the test is required. + assert.Contains(t, resolved.Artifacts, hostSystem(t)) + var missing []string + for _, system := range knownSystems { + if _, ok := resolved.Artifacts[system]; !ok { + missing = append(missing, system) + } + } + t.Logf("%s@latest = %s (not published for: %v)", def.Name, resolved.Version, missing) + }) + } +} + +// hostSystem returns the nix system of the machine running the test. +func hostSystem(t *testing.T) string { + t.Helper() + arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[runtime.GOARCH] + require.NotEmpty(t, arch, "unsupported GOARCH %s", runtime.GOARCH) + return arch + "-" + runtime.GOOS +} diff --git a/pkgs/bun/bun.test.txt b/pkgs/bun/bun.test.txt new file mode 100644 index 00000000000..54c8a5acacc --- /dev/null +++ b/pkgs/bun/bun.test.txt @@ -0,0 +1,33 @@ +# Bun from its GitHub releases. +# +# Installs bun@1.1 from devbox.json, switches to 1.2 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/bun/bun.pkg.jsonc bun.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./bun.pkg.jsonc@1.1 +exec devbox run -- bun --version +stdout '^1\.1\.' +exec devbox run -- bunx --version +stdout '^1\.1\.' + +exec devbox add pkg:./bun.pkg.jsonc@1.2 +! devboxjson.packages.contains devbox.json pkg:./bun.pkg.jsonc@1.1 +! devboxlock.packages.contains devbox.lock pkg:./bun.pkg.jsonc@1.1 +exec devbox run -- bun --version +stdout '^1\.2\.' + +# 1.1 is still installed, so switching back needs no download. +cp devbox.1.1.json devbox.json +exec devbox run -- bun --version +stdout '^1\.1\.' + +-- devbox.json -- +{ + "packages": ["pkg:./bun.pkg.jsonc@1.1"] +} +-- devbox.1.1.json -- +{ + "packages": ["pkg:./bun.pkg.jsonc@1.1"] +} diff --git a/pkgs/deno/deno.test.txt b/pkgs/deno/deno.test.txt new file mode 100644 index 00000000000..4e3c994febf --- /dev/null +++ b/pkgs/deno/deno.test.txt @@ -0,0 +1,33 @@ +# Deno from its GitHub releases. Deno 1.x published no checksums, so the +# first install records one (trust on first use). +# +# Installs deno@1.46 from devbox.json, switches to 2 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/deno/deno.pkg.jsonc deno.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./deno.pkg.jsonc@1.46 +exec devbox run -- deno --version +stdout '^deno 1\.46\.' +grep '"sha256": "' devbox.lock + +exec devbox add pkg:./deno.pkg.jsonc@2 +! devboxjson.packages.contains devbox.json pkg:./deno.pkg.jsonc@1.46 +! devboxlock.packages.contains devbox.lock pkg:./deno.pkg.jsonc@1.46 +exec devbox run -- deno --version +stdout '^deno 2\.' + +# 1.46 is still installed, so switching back needs no download. +cp devbox.1.46.json devbox.json +exec devbox run -- deno --version +stdout '^deno 1\.46\.' + +-- devbox.json -- +{ + "packages": ["pkg:./deno.pkg.jsonc@1.46"] +} +-- devbox.1.46.json -- +{ + "packages": ["pkg:./deno.pkg.jsonc@1.46"] +} diff --git a/pkgs/go/go.test.txt b/pkgs/go/go.test.txt new file mode 100644 index 00000000000..8a2711d0f58 --- /dev/null +++ b/pkgs/go/go.test.txt @@ -0,0 +1,36 @@ +# Go from go.dev. +# +# Installs go@1.22 from devbox.json, switches to 1.27 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/go/go.pkg.jsonc go.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./go.pkg.jsonc@1.22 +exec devbox run -- go version +stdout 'go1\.22\.' +# The version in devbox.json runs even if go.mod asks for a newer one. +exec devbox run -- go env GOTOOLCHAIN +stdout '^local$' + +exec devbox add pkg:./go.pkg.jsonc@1.27 +! devboxjson.packages.contains devbox.json pkg:./go.pkg.jsonc@1.22 +! devboxlock.packages.contains devbox.lock pkg:./go.pkg.jsonc@1.22 +exec devbox run -- go version +stdout 'go1\.27\.' + +# 1.22 is still installed, so switching back needs no download. +cp devbox.1.22.json devbox.json +exec devbox run -- go version +stdout 'go1\.22\.' +exec devbox run -- go env GOTOOLCHAIN +stdout '^local$' + +-- devbox.json -- +{ + "packages": ["pkg:./go.pkg.jsonc@1.22"] +} +-- devbox.1.22.json -- +{ + "packages": ["pkg:./go.pkg.jsonc@1.22"] +} diff --git a/pkgs/gradle/gradle.test.txt b/pkgs/gradle/gradle.test.txt new file mode 100644 index 00000000000..48aeefc6b5c --- /dev/null +++ b/pkgs/gradle/gradle.test.txt @@ -0,0 +1,32 @@ +# Gradle from services.gradle.org, running on the Temurin JDK. +# +# Installs gradle@8 from devbox.json, switches to 9 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/java/java.pkg.jsonc java.pkg.jsonc +cp $PKGS/gradle/gradle.pkg.jsonc gradle.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./gradle.pkg.jsonc@8 +exec devbox run -- gradle --version +stdout '^Gradle 8\.' + +exec devbox add pkg:./gradle.pkg.jsonc@9 +! devboxjson.packages.contains devbox.json pkg:./gradle.pkg.jsonc@8 +! devboxlock.packages.contains devbox.lock pkg:./gradle.pkg.jsonc@8 +exec devbox run -- gradle --version +stdout '^Gradle 9\.' + +# 8 is still installed, so switching back needs no download. +cp devbox.8.json devbox.json +exec devbox run -- gradle --version +stdout '^Gradle 8\.' + +-- devbox.json -- +{ + "packages": ["pkg:./java.pkg.jsonc@21", "pkg:./gradle.pkg.jsonc@8"] +} +-- devbox.8.json -- +{ + "packages": ["pkg:./java.pkg.jsonc@21", "pkg:./gradle.pkg.jsonc@8"] +} diff --git a/pkgs/helm/helm.test.txt b/pkgs/helm/helm.test.txt new file mode 100644 index 00000000000..07b39b3d6f1 --- /dev/null +++ b/pkgs/helm/helm.test.txt @@ -0,0 +1,31 @@ +# Helm from get.helm.sh. +# +# Installs helm@3 from devbox.json, switches to 4 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/helm/helm.pkg.jsonc helm.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./helm.pkg.jsonc@3 +exec devbox run -- helm version --short +stdout '^v3\.' + +exec devbox add pkg:./helm.pkg.jsonc@4 +! devboxjson.packages.contains devbox.json pkg:./helm.pkg.jsonc@3 +! devboxlock.packages.contains devbox.lock pkg:./helm.pkg.jsonc@3 +exec devbox run -- helm version --short +stdout '^v4\.' + +# 3 is still installed, so switching back needs no download. +cp devbox.3.json devbox.json +exec devbox run -- helm version --short +stdout '^v3\.' + +-- devbox.json -- +{ + "packages": ["pkg:./helm.pkg.jsonc@3"] +} +-- devbox.3.json -- +{ + "packages": ["pkg:./helm.pkg.jsonc@3"] +} diff --git a/pkgs/java/java.test.txt b/pkgs/java/java.test.txt new file mode 100644 index 00000000000..9362a71b19d --- /dev/null +++ b/pkgs/java/java.test.txt @@ -0,0 +1,37 @@ +# Java: the Eclipse Temurin JDK from Adoptium. +# +# Installs java@17 from devbox.json, switches to 21 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/java/java.pkg.jsonc java.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./java.pkg.jsonc@17 +exec devbox run -- java -version +stderr 'openjdk version "17\.' +exec devbox run -- javac -version +stdout '^javac 17\.' +exec devbox run -- sh -c '"$JAVA_HOME/bin/java" -version' +stderr 'openjdk version "17\.' + +exec devbox add pkg:./java.pkg.jsonc@21 +! devboxjson.packages.contains devbox.json pkg:./java.pkg.jsonc@17 +! devboxlock.packages.contains devbox.lock pkg:./java.pkg.jsonc@17 +exec devbox run -- java -version +stderr 'openjdk version "21\.' + +# 17 is still installed, so switching back needs no download. +cp devbox.17.json devbox.json +exec devbox run -- java -version +stderr 'openjdk version "17\.' +exec devbox run -- sh -c 'echo $JAVA_HOME' +stdout '/java/17\.' + +-- devbox.json -- +{ + "packages": ["pkg:./java.pkg.jsonc@17"] +} +-- devbox.17.json -- +{ + "packages": ["pkg:./java.pkg.jsonc@17"] +} diff --git a/pkgs/kubectl/kubectl.test.txt b/pkgs/kubectl/kubectl.test.txt new file mode 100644 index 00000000000..2577c4ca2ad --- /dev/null +++ b/pkgs/kubectl/kubectl.test.txt @@ -0,0 +1,31 @@ +# kubectl from dl.k8s.io. 1.30 is several pages back in the GitHub releases. +# +# Installs kubectl@1.30 from devbox.json, switches to 1.33 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/kubectl/kubectl.pkg.jsonc kubectl.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./kubectl.pkg.jsonc@1.30 +exec devbox run -- kubectl version --client +stdout 'Client Version: v1\.30\.' + +exec devbox add pkg:./kubectl.pkg.jsonc@1.33 +! devboxjson.packages.contains devbox.json pkg:./kubectl.pkg.jsonc@1.30 +! devboxlock.packages.contains devbox.lock pkg:./kubectl.pkg.jsonc@1.30 +exec devbox run -- kubectl version --client +stdout 'Client Version: v1\.33\.' + +# 1.30 is still installed, so switching back needs no download. +cp devbox.1.30.json devbox.json +exec devbox run -- kubectl version --client +stdout 'Client Version: v1\.30\.' + +-- devbox.json -- +{ + "packages": ["pkg:./kubectl.pkg.jsonc@1.30"] +} +-- devbox.1.30.json -- +{ + "packages": ["pkg:./kubectl.pkg.jsonc@1.30"] +} diff --git a/pkgs/maven/maven.test.txt b/pkgs/maven/maven.test.txt new file mode 100644 index 00000000000..b8bd7aab307 --- /dev/null +++ b/pkgs/maven/maven.test.txt @@ -0,0 +1,33 @@ +# Maven from Maven Central (sha512 checksums), running on the Temurin JDK. +# +# Installs maven@3.8 from devbox.json, switches to 3.9 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/java/java.pkg.jsonc java.pkg.jsonc +cp $PKGS/maven/maven.pkg.jsonc maven.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./maven.pkg.jsonc@3.8 +exec devbox run -- mvn --version +stdout '^Apache Maven 3\.8\.' +grep '"sha512": "' devbox.lock + +exec devbox add pkg:./maven.pkg.jsonc@3.9 +! devboxjson.packages.contains devbox.json pkg:./maven.pkg.jsonc@3.8 +! devboxlock.packages.contains devbox.lock pkg:./maven.pkg.jsonc@3.8 +exec devbox run -- mvn --version +stdout '^Apache Maven 3\.9\.' + +# 3.8 is still installed, so switching back needs no download. +cp devbox.3.8.json devbox.json +exec devbox run -- mvn --version +stdout '^Apache Maven 3\.8\.' + +-- devbox.json -- +{ + "packages": ["pkg:./java.pkg.jsonc@21", "pkg:./maven.pkg.jsonc@3.8"] +} +-- devbox.3.8.json -- +{ + "packages": ["pkg:./java.pkg.jsonc@21", "pkg:./maven.pkg.jsonc@3.8"] +} diff --git a/pkgs/nodejs/nodejs.test.txt b/pkgs/nodejs/nodejs.test.txt new file mode 100644 index 00000000000..673227a220a --- /dev/null +++ b/pkgs/nodejs/nodejs.test.txt @@ -0,0 +1,38 @@ +# Node.js from nodejs.org, with npm. +# +# Installs nodejs@20 from devbox.json, switches to 22 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/nodejs/nodejs.pkg.jsonc nodejs.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./nodejs.pkg.jsonc@20 +exec devbox run -- node --version +stdout '^v20\.' +exec devbox run -- npm --version +stdout '^10\.' +# Global npm installs go to the project, not the shared install. +exec devbox run -- sh -c 'echo $NPM_CONFIG_PREFIX' +stdout '\.devbox/virtenv/pkgdef-nodejs/npm-global$' + +exec devbox add pkg:./nodejs.pkg.jsonc@22 +! devboxjson.packages.contains devbox.json pkg:./nodejs.pkg.jsonc@20 +! devboxlock.packages.contains devbox.lock pkg:./nodejs.pkg.jsonc@20 +exec devbox run -- node --version +stdout '^v22\.' + +# 20 is still installed, so switching back needs no download. +cp devbox.20.json devbox.json +exec devbox run -- node --version +stdout '^v20\.' +exec devbox run -- sh -c 'echo $NPM_CONFIG_PREFIX' +stdout 'npm-global$' + +-- devbox.json -- +{ + "packages": ["pkg:./nodejs.pkg.jsonc@20"] +} +-- devbox.20.json -- +{ + "packages": ["pkg:./nodejs.pkg.jsonc@20"] +} diff --git a/pkgs/pnpm/pnpm.test.txt b/pkgs/pnpm/pnpm.test.txt new file mode 100644 index 00000000000..440130e7f1b --- /dev/null +++ b/pkgs/pnpm/pnpm.test.txt @@ -0,0 +1,33 @@ +# pnpm: 9 comes from npm (sha512), 12 from GitHub release tarballs +# (GitHub asset digests). +# +# Installs pnpm@9 from devbox.json, switches to 12 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/pnpm/pnpm.pkg.jsonc pnpm.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./pnpm.pkg.jsonc@9 +exec devbox run -- pnpm --version +stdout '^9\.' +grep '"sha512": "' devbox.lock + +exec devbox add pkg:./pnpm.pkg.jsonc@12 +! devboxjson.packages.contains devbox.json pkg:./pnpm.pkg.jsonc@9 +! devboxlock.packages.contains devbox.lock pkg:./pnpm.pkg.jsonc@9 +exec devbox run -- pnpm --version +stdout '^12\.' + +# 9 is still installed, so switching back needs no download. +cp devbox.9.json devbox.json +exec devbox run -- pnpm --version +stdout '^9\.' + +-- devbox.json -- +{ + "packages": ["pkg:./pnpm.pkg.jsonc@9"] +} +-- devbox.9.json -- +{ + "packages": ["pkg:./pnpm.pkg.jsonc@9"] +} diff --git a/pkgs/python/python.test.txt b/pkgs/python/python.test.txt new file mode 100644 index 00000000000..17df026175e --- /dev/null +++ b/pkgs/python/python.test.txt @@ -0,0 +1,41 @@ +# CPython from python-build-standalone. +# +# Installs python@3.12 from devbox.json, switches to 3.13 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/python/python.pkg.jsonc python.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./python.pkg.jsonc@3.12 +exec devbox run -- python --version +stdout '^Python 3\.12\.' +# python and pip come from a virtual environment in the project. +exec devbox run -- python -c 'import sys; print(sys.prefix)' +stdout '/\.venv$' +exec devbox run -- pip --version +stdout 'python 3\.12' + +exec devbox add pkg:./python.pkg.jsonc@3.13 +! devboxjson.packages.contains devbox.json pkg:./python.pkg.jsonc@3.12 +! devboxlock.packages.contains devbox.lock pkg:./python.pkg.jsonc@3.12 +exec devbox run -- python --version +stdout '^Python 3\.13\.' +# Switching versions recreates the virtual environment. +exec devbox run -- pip --version +stdout 'python 3\.13' + +# 3.12 is still installed, so switching back needs no download. +cp devbox.3.12.json devbox.json +exec devbox run -- python --version +stdout '^Python 3\.12\.' +exec devbox run -- pip --version +stdout 'python 3\.12' + +-- devbox.json -- +{ + "packages": ["pkg:./python.pkg.jsonc@3.12"] +} +-- devbox.3.12.json -- +{ + "packages": ["pkg:./python.pkg.jsonc@3.12"] +} diff --git a/pkgs/rust/rust.test.txt b/pkgs/rust/rust.test.txt new file mode 100644 index 00000000000..1772871b685 --- /dev/null +++ b/pkgs/rust/rust.test.txt @@ -0,0 +1,40 @@ +# Rust: rustup-init installs rustup, which installs the toolchain into a +# shared RUSTUP_HOME; RUSTUP_TOOLCHAIN selects the version. +# +# Installs rust@1.85 from devbox.json, switches to 1.90 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/rust/rust.pkg.jsonc rust.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./rust.pkg.jsonc@1.85 +exec devbox run -- rustc --version +stdout '^rustc 1\.85\.' +exec devbox run -- cargo --version +stdout '^cargo 1\.85\.' +exec devbox run -- cargo clippy --version +stdout '^clippy 0\.1\.85' +exec devbox run -- sh -c 'echo $RUSTUP_TOOLCHAIN' +stdout '^1\.85\.' + +exec devbox add pkg:./rust.pkg.jsonc@1.90 +! devboxjson.packages.contains devbox.json pkg:./rust.pkg.jsonc@1.85 +! devboxlock.packages.contains devbox.lock pkg:./rust.pkg.jsonc@1.85 +exec devbox run -- rustc --version +stdout '^rustc 1\.90\.' +exec devbox run -- rustfmt --version +stdout '^rustfmt 1\.' + +# 1.85 is still installed, so switching back needs no download. +cp devbox.1.85.json devbox.json +exec devbox run -- rustc --version +stdout '^rustc 1\.85\.' + +-- devbox.json -- +{ + "packages": ["pkg:./rust.pkg.jsonc@1.85"] +} +-- devbox.1.85.json -- +{ + "packages": ["pkg:./rust.pkg.jsonc@1.85"] +} diff --git a/pkgs/terraform/terraform.test.txt b/pkgs/terraform/terraform.test.txt new file mode 100644 index 00000000000..3315dc26509 --- /dev/null +++ b/pkgs/terraform/terraform.test.txt @@ -0,0 +1,31 @@ +# Terraform from releases.hashicorp.com. +# +# Installs terraform@1.5 from devbox.json, switches to 1.9 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/terraform/terraform.pkg.jsonc terraform.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./terraform.pkg.jsonc@1.5 +exec devbox run -- terraform version +stdout '^Terraform v1\.5\.' + +exec devbox add pkg:./terraform.pkg.jsonc@1.9 +! devboxjson.packages.contains devbox.json pkg:./terraform.pkg.jsonc@1.5 +! devboxlock.packages.contains devbox.lock pkg:./terraform.pkg.jsonc@1.5 +exec devbox run -- terraform version +stdout '^Terraform v1\.9\.' + +# 1.5 is still installed, so switching back needs no download. +cp devbox.1.5.json devbox.json +exec devbox run -- terraform version +stdout '^Terraform v1\.5\.' + +-- devbox.json -- +{ + "packages": ["pkg:./terraform.pkg.jsonc@1.5"] +} +-- devbox.1.5.json -- +{ + "packages": ["pkg:./terraform.pkg.jsonc@1.5"] +} diff --git a/pkgs/uv/uv.test.txt b/pkgs/uv/uv.test.txt new file mode 100644 index 00000000000..2fce4368405 --- /dev/null +++ b/pkgs/uv/uv.test.txt @@ -0,0 +1,33 @@ +# uv from its GitHub releases. +# +# Installs uv@0.8 from devbox.json, switches to 0.9 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/uv/uv.pkg.jsonc uv.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./uv.pkg.jsonc@0.8 +exec devbox run -- uv --version +stdout '^uv 0\.8\.' +exec devbox run -- uvx --version +stdout '^uvx 0\.8\.' + +exec devbox add pkg:./uv.pkg.jsonc@0.9 +! devboxjson.packages.contains devbox.json pkg:./uv.pkg.jsonc@0.8 +! devboxlock.packages.contains devbox.lock pkg:./uv.pkg.jsonc@0.8 +exec devbox run -- uv --version +stdout '^uv 0\.9\.' + +# 0.8 is still installed, so switching back needs no download. +cp devbox.0.8.json devbox.json +exec devbox run -- uv --version +stdout '^uv 0\.8\.' + +-- devbox.json -- +{ + "packages": ["pkg:./uv.pkg.jsonc@0.8"] +} +-- devbox.0.8.json -- +{ + "packages": ["pkg:./uv.pkg.jsonc@0.8"] +} diff --git a/pkgs/yarn/yarn.test.txt b/pkgs/yarn/yarn.test.txt new file mode 100644 index 00000000000..8f9226cc301 --- /dev/null +++ b/pkgs/yarn/yarn.test.txt @@ -0,0 +1,33 @@ +# Yarn from npm: 1.x is the yarn package, 4.x is @yarnpkg/cli-dist. Yarn +# runs on Node.js, which is installed from its own definition. +# +# Installs yarn@1 from devbox.json, switches to 4 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/nodejs/nodejs.pkg.jsonc nodejs.pkg.jsonc +cp $PKGS/yarn/yarn.pkg.jsonc yarn.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./yarn.pkg.jsonc@1 +exec devbox run -- yarn --version +stdout '^1\.22\.' + +exec devbox add pkg:./yarn.pkg.jsonc@4 +! devboxjson.packages.contains devbox.json pkg:./yarn.pkg.jsonc@1 +! devboxlock.packages.contains devbox.lock pkg:./yarn.pkg.jsonc@1 +exec devbox run -- yarn --version +stdout '^4\.' + +# 1 is still installed, so switching back needs no download. +cp devbox.1.json devbox.json +exec devbox run -- yarn --version +stdout '^1\.22\.' + +-- devbox.json -- +{ + "packages": ["pkg:./nodejs.pkg.jsonc@22", "pkg:./yarn.pkg.jsonc@1"] +} +-- devbox.1.json -- +{ + "packages": ["pkg:./nodejs.pkg.jsonc@22", "pkg:./yarn.pkg.jsonc@1"] +} diff --git a/pkgs/zig/zig.test.txt b/pkgs/zig/zig.test.txt new file mode 100644 index 00000000000..cbf6cff52ac --- /dev/null +++ b/pkgs/zig/zig.test.txt @@ -0,0 +1,32 @@ +# Zig from ziglang.org. Tarball names changed in 0.14; the download index +# has the URL for each. +# +# Installs zig@0.13 from devbox.json, switches to 0.14 with `devbox add`, and +# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1. + +cp $PKGS/zig/zig.pkg.jsonc zig.pkg.jsonc + +exec devbox install +devboxlock.packages.contains devbox.lock pkg:./zig.pkg.jsonc@0.13 +exec devbox run -- zig version +stdout '^0\.13\.' + +exec devbox add pkg:./zig.pkg.jsonc@0.14 +! devboxjson.packages.contains devbox.json pkg:./zig.pkg.jsonc@0.13 +! devboxlock.packages.contains devbox.lock pkg:./zig.pkg.jsonc@0.13 +exec devbox run -- zig version +stdout '^0\.14\.' + +# 0.13 is still installed, so switching back needs no download. +cp devbox.0.13.json devbox.json +exec devbox run -- zig version +stdout '^0\.13\.' + +-- devbox.json -- +{ + "packages": ["pkg:./zig.pkg.jsonc@0.13"] +} +-- devbox.0.13.json -- +{ + "packages": ["pkg:./zig.pkg.jsonc@0.13"] +} diff --git a/testscripts/testrunner/pkgdefs.go b/testscripts/testrunner/pkgdefs.go new file mode 100644 index 00000000000..e7c6ccab559 --- /dev/null +++ b/testscripts/testrunner/pkgdefs.go @@ -0,0 +1,36 @@ +package testrunner + +import ( + "path/filepath" + "testing" + + "github.com/rogpeppe/go-internal/testscript" + "github.com/stretchr/testify/require" +) + +// RunPackageDefinitionTestscripts runs the testscripts that live next to +// the package definitions in pkgsDir (pkgs//.test.txt). Each +// script can reach the definitions through $PKGS, and runs with package +// definitions enabled. +func RunPackageDefinitionTestscripts(t *testing.T, pkgsDir string) { + pkgsDir, err := filepath.Abs(pkgsDir) + require.NoError(t, err) + scripts := globScripts(filepath.Join(pkgsDir, "*", "*.test.txt")) + require.NotEmpty(t, scripts, "no package definition test scripts found") + + params := getTestscriptParams(pkgsDir) + params.Dir = "" + params.Files = scripts + params.Setup = func(env *testscript.Env) error { + if err := setupTestEnv(env); err != nil { + return err + } + env.Setenv("PKGS", pkgsDir) + env.Setenv("DEVBOX_FEATURE_PKG_DEFS", "1") + // Version lookups use the GitHub API, whose anonymous rate limit is + // easy to hit when every package resolves a few versions. + propagateEnvVars(env, "GITHUB_TOKEN", "DEVBOX_GITHUB_API_TOKEN") + return nil + } + testscript.Run(t, params) +} diff --git a/testscripts/testscripts_test.go b/testscripts/testscripts_test.go index a9c97774e24..cab9651b30d 100644 --- a/testscripts/testscripts_test.go +++ b/testscripts/testscripts_test.go @@ -12,6 +12,10 @@ import ( // found in examples/.. and testscripts/.. const runProjectTests = "DEVBOX_RUN_PROJECT_TESTS" +// When true, runs the testscripts next to the package definitions in pkgs/, +// which install real toolchains from their vendors. +const runPkgDefTests = "DEVBOX_RUN_PKGDEF_TESTS" + func TestScripts(t *testing.T) { // To run a specific test, say, testscripts/foo/bar.test.text, then run // go test ./testscripts -run TestScripts/bar @@ -43,3 +47,17 @@ func TestScriptsWithProjects(t *testing.T) { testrunner.RunDevboxTestscripts(t, ".") } + +// TestPackageDefinitions installs every package definition in pkgs/ and +// switches between two of its versions. It downloads several gigabytes, so +// it only runs when DEVBOX_RUN_PKGDEF_TESTS is set. +// +// To run one package, say pkgs/nodejs, run +// go test ./testscripts -run TestPackageDefinitions/nodejs +func TestPackageDefinitions(t *testing.T) { + isOn, err := strconv.ParseBool(os.Getenv(runPkgDefTests)) + if err != nil || !isOn { + t.Skipf("Skipping TestPackageDefinitions. To enable, set %s=1.", runPkgDefTests) + } + testrunner.RunPackageDefinitionTestscripts(t, "../pkgs") +}