diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index 4c832d6..d63e01a 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -11,192 +11,17 @@ on: - "features/dotnet/config.json" workflow_dispatch: -permissions: - contents: read - packages: write - concurrency: group: "dotnet-${{ github.ref }}" cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - plan: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.plan.outputs.matrix }} - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Plan the publishes - id: plan - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm tsx scripts/tasks/features/dotnet/planPublish.ts --channel "$CHANNEL" - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} - - install-binaries: - needs: plan - if: needs.plan.outputs.matrix != '' - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - rid: linux-x64 - runner: ubuntu-latest - arm64: false - - rid: linux-arm64 - runner: ubuntu-24.04-arm - arm64: true - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Pack the Feature binary - uses: devcontainers/ci@v0.3 - with: - imageName: ${{ matrix.arm64 && format('ghcr.io/{0}/devcontainer', github.repository) || '' }} - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: ${{ matrix.arm64 && 'filter' || 'never' }} - refFilterForPush: ${{ matrix.arm64 && 'refs/heads/main' || '' }} - useNativeRunner: ${{ matrix.arm64 }} - platform: ${{ matrix.arm64 && 'linux/arm64' || '' }} - runCmd: pnpm tsx scripts/tasks/features/dotnet/packFeature.ts --rid "$RID" --out .ci/feature - env: | - RID=${{ matrix.rid }} - - - name: Upload the Feature archive - uses: actions/upload-artifact@v7 - with: - name: feature-${{ matrix.rid }} - path: .ci/feature/${{ matrix.rid }}.tar - - image: - needs: - - plan - - install-binaries - if: needs.plan.outputs.matrix != '' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Download the x64 Feature archive - uses: actions/download-artifact@v7 - with: - name: feature-linux-x64 - path: .ci/feature - - - name: Download the arm64 Feature archive - uses: actions/download-artifact@v7 - with: - name: feature-linux-arm64 - path: .ci/feature - - - name: Publish the image - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm tsx scripts/tasks/features/dotnet/publishImage.ts --component ${{ matrix.component }} --version ${{ matrix.key }} --channel "$CHANNEL" --install-dir .ci/feature - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} - - release-tags: - needs: - - plan - - image - - verify - if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.image.result == 'success' || needs.image.result == 'skipped') && needs.verify.result == 'success' }} + verify: + uses: ./.github/workflows/verify.yml permissions: contents: write packages: write - runs-on: ubuntu-latest - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Apply the release tags - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm tsx scripts/tasks/features/dotnet/releaseTags.ts --channel "$CHANNEL" ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && '--git-tags' || '' }} - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} - - verify: - needs: - - plan - - image - if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.image.result == 'success' || needs.image.result == 'skipped') }} - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - runner: - - ubuntu-latest - - ubuntu-24.04-arm - scenario: - - sdk - - runtime - - aspnet - - multi - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Verify the images - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm verify dotnet --scenario ${{ matrix.scenario }} --images pull --channel "$CHANNEL" - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} + with: + feature: dotnet + binaries: true + scenarios: '["sdk", "runtime", "aspnet", "multi"]' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bba9eaf..63c024f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,15 @@ permissions: jobs: build: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: ubuntu-latest + arm64: false + - label: ubuntu-24.04-arm + arm64: true + runs-on: ${{ matrix.label }} steps: - name: Checkout (GitHub) uses: actions/checkout@v7 @@ -29,11 +37,14 @@ jobs: uses: devcontainers/ci@v0.3 with: imageName: ghcr.io/${{ github.repository }}/devcontainer - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer + cacheFrom: ghcr.io/${{ github.repository }}/devcontainer${{ matrix.arm64 && ':latest-linux-arm64' || '' }} push: filter refFilterForPush: refs/heads/main + platform: ${{ matrix.arm64 && 'linux/arm64' || '' }} + useNativeRunner: ${{ matrix.arm64 }} - name: Lint + if: ${{ !matrix.arm64 }} uses: devcontainers/ci@v0.3 with: cacheFrom: ghcr.io/${{ github.repository }}/devcontainer @@ -41,6 +52,7 @@ jobs: runCmd: pnpm lint - name: Test + if: ${{ !matrix.arm64 }} uses: devcontainers/ci@v0.3 with: cacheFrom: ghcr.io/${{ github.repository }}/devcontainer diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml index 4674285..1258e25 100644 --- a/.github/workflows/renovate.yml +++ b/.github/workflows/renovate.yml @@ -34,6 +34,6 @@ jobs: with: cacheFrom: ghcr.io/${{ github.repository }}/devcontainer push: never - runCmd: pnpm tsx scripts/tasks/features/${{ matrix.feature }}/renovate.ts + runCmd: pnpm tsx scripts/tasks/features/pipeline.ts renovate --feature ${{ matrix.feature }} env: | GH_TOKEN=${{ secrets.RENOVATE_TOKEN }} diff --git a/.github/workflows/ripgrep.yml b/.github/workflows/ripgrep.yml index 42b7d13..06ccb6a 100644 --- a/.github/workflows/ripgrep.yml +++ b/.github/workflows/ripgrep.yml @@ -11,129 +11,17 @@ on: - "features/ripgrep/config.json" workflow_dispatch: -permissions: - contents: read - packages: write - concurrency: group: "ripgrep-${{ github.ref }}" cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - plan: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.plan.outputs.matrix }} - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Plan the publishes - id: plan - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm tsx scripts/tasks/features/ripgrep/planPublish.ts --channel "$CHANNEL" - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} - - image: - needs: plan - if: needs.plan.outputs.matrix != '' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Publish the image - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm tsx scripts/tasks/features/ripgrep/publishImage.ts --version ${{ matrix.key }} --channel "$CHANNEL" - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} - - release-tags: - needs: - - plan - - image - - verify - if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.image.result == 'success' || needs.image.result == 'skipped') && needs.verify.result == 'success' }} + verify: + uses: ./.github/workflows/verify.yml permissions: contents: write packages: write - runs-on: ubuntu-latest - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Apply the release tags - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm tsx scripts/tasks/features/ripgrep/releaseTags.ts --channel "$CHANNEL" ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && '--git-tags' || '' }} - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} - - verify: - needs: - - plan - - image - if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.image.result == 'success' || needs.image.result == 'skipped') }} - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - runner: - - ubuntu-latest - - ubuntu-24.04-arm - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Verify the image - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm verify ripgrep --images pull --channel "$CHANNEL" - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} + with: + feature: ripgrep + binaries: false + scenarios: '["default"]' diff --git a/.github/workflows/uv.yml b/.github/workflows/uv.yml index 5a6f4eb..d8de2d0 100644 --- a/.github/workflows/uv.yml +++ b/.github/workflows/uv.yml @@ -11,192 +11,17 @@ on: - "features/uv/config.json" workflow_dispatch: -permissions: - contents: read - packages: write - concurrency: group: "uv-${{ github.ref }}" cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - plan: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.plan.outputs.matrix }} - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Plan the publishes - id: plan - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm tsx scripts/tasks/features/uv/planPublish.ts --channel "$CHANNEL" - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} - - install-binaries: - needs: plan - if: needs.plan.outputs.matrix != '' - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - rid: linux-x64 - runner: ubuntu-latest - arm64: false - - rid: linux-arm64 - runner: ubuntu-24.04-arm - arm64: true - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Pack the Feature binary - uses: devcontainers/ci@v0.3 - with: - imageName: ${{ matrix.arm64 && format('ghcr.io/{0}/devcontainer', github.repository) || '' }} - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: ${{ matrix.arm64 && 'filter' || 'never' }} - refFilterForPush: ${{ matrix.arm64 && 'refs/heads/main' || '' }} - useNativeRunner: ${{ matrix.arm64 }} - platform: ${{ matrix.arm64 && 'linux/arm64' || '' }} - runCmd: pnpm tsx scripts/tasks/features/uv/packFeature.ts --rid "$RID" --out .ci/feature - env: | - RID=${{ matrix.rid }} - - - name: Upload the Feature archive - uses: actions/upload-artifact@v7 - with: - name: feature-${{ matrix.rid }} - path: .ci/feature/${{ matrix.rid }}.tar - - image: - needs: - - plan - - install-binaries - if: needs.plan.outputs.matrix != '' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Download the x64 Feature archive - uses: actions/download-artifact@v7 - with: - name: feature-linux-x64 - path: .ci/feature - - - name: Download the arm64 Feature archive - uses: actions/download-artifact@v7 - with: - name: feature-linux-arm64 - path: .ci/feature - - - name: Publish the image - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm tsx scripts/tasks/features/uv/publishImage.ts --component ${{ matrix.component }} --version ${{ matrix.key }} --channel "$CHANNEL" --install-dir .ci/feature - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} - - release-tags: - needs: - - plan - - image - - verify - if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.image.result == 'success' || needs.image.result == 'skipped') && needs.verify.result == 'success' }} + verify: + uses: ./.github/workflows/verify.yml permissions: contents: write packages: write - runs-on: ubuntu-latest - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Apply the release tags - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm tsx scripts/tasks/features/uv/releaseTags.ts --channel "$CHANNEL" ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && '--git-tags' || '' }} - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} - - verify: - needs: - - plan - - image - if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.image.result == 'success' || needs.image.result == 'skipped') }} - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - runner: - - ubuntu-latest - - ubuntu-24.04-arm - scenario: - - binaries - - python - - multi - - download - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v7 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - - name: Verify the images - uses: devcontainers/ci@v0.3 - with: - cacheFrom: ghcr.io/${{ github.repository }}/devcontainer - push: never - runCmd: pnpm verify uv --scenario ${{ matrix.scenario }} --images pull --channel "$CHANNEL" - env: | - CHANNEL=${{ github.head_ref || github.ref_name }} - GH_TOKEN=${{ github.token }} + with: + feature: uv + binaries: true + scenarios: '["binaries", "python", "multi", "download"]' diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..56643aa --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,236 @@ +name: verify + +on: + workflow_call: + inputs: + feature: + description: The image-based feature id + required: true + type: string + binaries: + description: Whether the feature publishes a Feature binary + required: false + type: boolean + default: false + scenarios: + description: The verify scenarios, as a JSON array + required: true + type: string + runners: + description: The runners the verify and smoke jobs fan out over, as a JSON array of {label, arm64} objects + required: false + type: string + default: '[{"label":"ubuntu-latest","arm64":false},{"label":"ubuntu-24.04-arm","arm64":true}]' + +permissions: + contents: read + packages: write + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + channel: ${{ steps.plan.outputs.channel }} + matrix: ${{ steps.plan.outputs.matrix }} + smoke: ${{ steps.plan.outputs.smoke }} + steps: + - name: Checkout (GitHub) + uses: actions/checkout@v7 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Plan the publishes + id: plan + uses: devcontainers/ci@v0.3 + with: + cacheFrom: ghcr.io/${{ github.repository }}/devcontainer + push: never + runCmd: pnpm tsx scripts/tasks/features/pipeline.ts plan-publish --feature ${{ inputs.feature }} --runners "$RUNNERS" --ref "$REF" --pull-request "$PR_NUMBER" --prefix ghcr.io/${{ github.repository }} + env: | + REF=${{ github.ref }} + PR_NUMBER=${{ github.event.pull_request.number }} + RUNNERS=${{ inputs.runners }} + GH_TOKEN=${{ github.token }} + + install-binaries: + needs: plan + if: inputs.binaries && needs.plan.outputs.matrix != '' + runs-on: ${{ matrix.label }} + strategy: + fail-fast: false + matrix: + include: + - rid: linux-x64 + label: ubuntu-latest + arm64: false + - rid: linux-arm64 + label: ubuntu-24.04-arm + arm64: true + steps: + - name: Checkout (GitHub) + uses: actions/checkout@v7 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Pack the Feature binary + uses: devcontainers/ci@v0.3 + with: + cacheFrom: ghcr.io/${{ github.repository }}/devcontainer${{ matrix.arm64 && ':latest-linux-arm64' || '' }} + push: never + runCmd: pnpm tsx scripts/tasks/features/pipeline.ts pack-feature --feature ${{ inputs.feature }} --rid "$RID" --out .ci/feature + env: | + RID=${{ matrix.rid }} + + - name: Upload the Feature archive + uses: actions/upload-artifact@v7 + with: + name: feature-${{ matrix.rid }} + path: .ci/feature/${{ matrix.rid }}.tar + + image: + needs: + - plan + - install-binaries + if: ${{ !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.matrix != '' && (needs.install-binaries.result == 'success' || needs.install-binaries.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - name: Checkout (GitHub) + uses: actions/checkout@v7 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Download the x64 Feature archive + if: inputs.binaries + uses: actions/download-artifact@v7 + with: + name: feature-linux-x64 + path: .ci/feature + + - name: Download the arm64 Feature archive + if: inputs.binaries + uses: actions/download-artifact@v7 + with: + name: feature-linux-arm64 + path: .ci/feature + + - name: Publish the image + uses: devcontainers/ci@v0.3 + with: + cacheFrom: ghcr.io/${{ github.repository }}/devcontainer + push: never + runCmd: pnpm tsx scripts/tasks/features/pipeline.ts publish-image --feature ${{ inputs.feature }} --component ${{ matrix.component }} --version ${{ matrix.key }} --channel "$CHANNEL" --prefix ghcr.io/${{ github.repository }} ${{ inputs.binaries && '--install-dir .ci/feature' || '' }} + env: | + CHANNEL=${{ needs.plan.outputs.channel }} + GH_TOKEN=${{ github.token }} + + verify: + needs: + - plan + - image + if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.image.result == 'success' || needs.image.result == 'skipped') }} + runs-on: ${{ matrix.runner.label }} + strategy: + fail-fast: false + matrix: + runner: ${{ fromJSON(inputs.runners) }} + scenario: ${{ fromJSON(inputs.scenarios) }} + steps: + - name: Checkout (GitHub) + uses: actions/checkout@v7 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Verify the images + uses: devcontainers/ci@v0.3 + with: + cacheFrom: ghcr.io/${{ github.repository }}/devcontainer${{ matrix.runner.arm64 && ':latest-linux-arm64' || '' }} + push: never + runCmd: pnpm verify ${{ inputs.feature }} --scenario ${{ matrix.scenario }} --images pull --channel "$CHANNEL" --prefix ghcr.io/${{ github.repository }} + env: | + CHANNEL=${{ needs.plan.outputs.channel }} + GH_TOKEN=${{ github.token }} + + smoke: + needs: + - plan + - image + if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.image.result == 'success' || needs.image.result == 'skipped') }} + runs-on: ${{ matrix.label }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.smoke) }} + steps: + - name: Checkout (GitHub) + uses: actions/checkout@v7 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Smoke the image + uses: devcontainers/ci@v0.3 + with: + cacheFrom: ghcr.io/${{ github.repository }}/devcontainer${{ matrix.arm64 && ':latest-linux-arm64' || '' }} + push: never + runCmd: pnpm verify ${{ inputs.feature }} --scenario smoke --component ${{ matrix.component }} --version ${{ matrix.key }} --images pull --channel "$CHANNEL" --prefix ghcr.io/${{ github.repository }} + env: | + CHANNEL=${{ needs.plan.outputs.channel }} + GH_TOKEN=${{ github.token }} + + release-tags: + needs: + - plan + - image + - verify + - smoke + if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.image.result == 'success' || needs.image.result == 'skipped') && needs.verify.result == 'success' && (needs.smoke.result == 'success' || needs.smoke.result == 'skipped') }} + permissions: + contents: write + packages: write + runs-on: ubuntu-latest + steps: + - name: Checkout (GitHub) + uses: actions/checkout@v7 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Apply the release tags + uses: devcontainers/ci@v0.3 + with: + cacheFrom: ghcr.io/${{ github.repository }}/devcontainer + push: never + runCmd: pnpm tsx scripts/tasks/features/pipeline.ts release-tags --feature ${{ inputs.feature }} --channel "$CHANNEL" --prefix ghcr.io/${{ github.repository }} + env: | + CHANNEL=${{ needs.plan.outputs.channel }} + GH_TOKEN=${{ github.token }} diff --git a/Directory.Build.props b/Directory.Build.props index 788e299..384741f 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -7,6 +7,12 @@ $(ConfigDirectory)Packages.props net10.0 + + $([MSBuild]::MakeRelative($(ProjectRoot), $(MSBuildProjectDirectory))) + + + $(MSBuildProjectName) + diff --git a/features/dotnet/config.json b/features/dotnet/config.json index dd84623..63155fb 100644 --- a/features/dotnet/config.json +++ b/features/dotnet/config.json @@ -1,5 +1,5 @@ { - "revision": 6, + "revision": 7, "components": { "sdk": { "versions": { diff --git a/features/ripgrep/config.json b/features/ripgrep/config.json index bc8f3eb..53ea9b2 100644 --- a/features/ripgrep/config.json +++ b/features/ripgrep/config.json @@ -1,5 +1,5 @@ { - "revision": 2, + "revision": 3, "versions": { "15.2.0": { "x86_64-unknown-linux-musl": { diff --git a/features/uv/config.json b/features/uv/config.json index 7fb1fc1..a2f90cb 100644 --- a/features/uv/config.json +++ b/features/uv/config.json @@ -1,5 +1,5 @@ { - "revision": 1, + "revision": 2, "components": { "uv": { "versions": { diff --git a/scripts/pushTags.ts b/scripts/pushTags.ts index 5aef7e2..6abf25c 100644 --- a/scripts/pushTags.ts +++ b/scripts/pushTags.ts @@ -4,9 +4,12 @@ import path from "node:path"; import git from "isomorphic-git"; import { readBlob } from "isomorphic-git"; import http from "isomorphic-git/http/node"; +import { z } from "zod"; import { projectRoot } from "@/scripts/project.js"; +const featureManifestSchema = z.object({ version: z.string().min(1) }); + const defaultBranch = "main"; const remote = (await git.listRemotes({ fs, dir: projectRoot })).at(0); if (remote === undefined) { @@ -42,7 +45,7 @@ for (const filepath of files) { oid: remoteHead, filepath, }); - const { version } = JSON.parse(decoder.decode(blob)) as { version: string }; + const { version } = featureManifestSchema.parse(JSON.parse(decoder.decode(blob))); const tag = `${path.basename(path.dirname(filepath))}_${version}`; if (tags.has(tag)) { console.log(`Tag ${tag} already exists, skipping...`); diff --git a/scripts/tasks/features/architecture.ts b/scripts/tasks/features/architecture.ts new file mode 100644 index 0000000..ba5f55b --- /dev/null +++ b/scripts/tasks/features/architecture.ts @@ -0,0 +1,33 @@ +import { stat } from "node:fs/promises"; + +export const dockerArchOptions = ["amd64", "arm64"] as const; +export type DockerArch = (typeof dockerArchOptions)[number]; + +export const rids = ["linux-x64", "linux-arm64"] as const; +export type Rid = (typeof rids)[number]; + +export const hostArch = (): DockerArch => { + switch (process.arch) { + case "x64": + return "amd64"; + case "arm64": + return "arm64"; + default: + throw new Error(`Unsupported host architecture: ${process.arch}`); + } +}; + +const ridByArch: Record = { amd64: "linux-x64", arm64: "linux-arm64" }; + +export const ridFor = (arch: DockerArch): Rid => ridByArch[arch]; + +export const isFile = async (candidate: string): Promise => { + try { + return (await stat(candidate)).isFile(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; + } +}; diff --git a/scripts/tasks/features/buildContext.ts b/scripts/tasks/features/buildContext.ts new file mode 100644 index 0000000..1cffde7 --- /dev/null +++ b/scripts/tasks/features/buildContext.ts @@ -0,0 +1,43 @@ +import { copyFile, mkdir, rm } from "node:fs/promises"; +import path from "node:path"; + +import { projectRoot } from "@/scripts/project.js"; +import { linkTree } from "@/scripts/tasks/features/linkTree.js"; + +import type { DockerArch } from "./architecture.js"; +import type { Feature } from "./feature.js"; +import { contextPath, payloadPath } from "./paths.js"; + +export const assembleBuildContext = async ( + feature: Feature, + component: string, + key: string, + arches: readonly DockerArch[], + featureBinaries: Partial>, +): Promise => { + const stage = feature.stage(component, key); + const context = contextPath(feature.id, component, key); + await rm(context, { recursive: true, force: true }); + await mkdir(path.join(context, "docker"), { recursive: true }); + await copyFile( + path.join(projectRoot, "features", feature.id, "Dockerfile"), + path.join(context, "docker", "Dockerfile"), + ); + for (const arch of arches) { + const stagedRoot = path.join(context, "linux", arch, "features", feature.id); + const staged = path.join(stagedRoot, ...stage.subdir); + await mkdir(staged, { recursive: true }); + await linkTree(payloadPath(feature.id, component, key, arch), staged); + for (const file of stage.files) { + await copyFile(path.join(projectRoot, "features", feature.id, file), path.join(stagedRoot, file)); + } + if (stage.binary) { + const featureBinary = featureBinaries[arch]; + if (featureBinary === undefined) { + throw new Error(`No ${feature.id} Feature binary for ${arch}`); + } + await copyFile(featureBinary, path.join(stagedRoot, "Feature")); + } + } + return context; +}; diff --git a/scripts/tasks/features/commands/arguments.ts b/scripts/tasks/features/commands/arguments.ts new file mode 100644 index 0000000..0060fed --- /dev/null +++ b/scripts/tasks/features/commands/arguments.ts @@ -0,0 +1,47 @@ +import { z } from "zod"; + +import { dockerArchOptions, hostArch } from "../architecture.js"; +import type { Feature } from "../feature.js"; +import { features } from "../features.js"; +import { defaultRefPrefix, mainChannel } from "../tags.js"; + +export const featureOption = { type: "string" } as const; + +export const channelOption = { + channel: { type: "string", default: mainChannel }, +} as const; + +export const prefixOption = { + prefix: { type: "string" }, +} as const; + +export const channelOptions = { ...channelOption, ...prefixOption } as const; + +export const targetOptions = { + component: { type: "string" }, + version: { type: "string" }, +} as const; + +export const nonEmptyStringSchema = z.string().min(1); + +export const featureSchema = z + .enum(Object.keys(features) as [keyof typeof features, ...(keyof typeof features)[]]) + .transform((id) => features[id]); + +export const archSchema = z.enum(dockerArchOptions).default(hostArch); + +export const componentSchemaFor = (feature: Feature) => { + const [only] = feature.components; + const schema = z.enum(feature.components); + return feature.components.length === 1 && only !== undefined ? schema.default(only) : schema; +}; + +export const targetSchemaFor = (feature: Feature) => + z + .object({ + component: componentSchemaFor(feature), + version: nonEmptyStringSchema, + }) + .transform(({ component, version }) => ({ component, key: version })); + +export const resolvePrefix = async (value: string | undefined): Promise => value ?? defaultRefPrefix(); diff --git a/scripts/tasks/features/commands/buildImage.ts b/scripts/tasks/features/commands/buildImage.ts new file mode 100644 index 0000000..d9cc540 --- /dev/null +++ b/scripts/tasks/features/commands/buildImage.ts @@ -0,0 +1,64 @@ +import path from "node:path"; +import { parseArgs } from "node:util"; + +import { $$ } from "@/scripts/shell.js"; + +import type { DockerArch } from "../architecture.js"; +import { hostArch, ridFor } from "../architecture.js"; +import { assembleBuildContext } from "../buildContext.js"; +import { requireBinary } from "../feature.js"; +import { imageEntry } from "../imageEntries.js"; +import { payloadPath } from "../paths.js"; +import { + archSchema, + channelOptions, + featureOption, + featureSchema, + resolvePrefix, + targetOptions, + targetSchemaFor, +} from "./arguments.js"; + +export const buildImage = async (args: string[]): Promise => { + const { values } = parseArgs({ + args, + options: { + feature: featureOption, + ...targetOptions, + arch: { type: "string" }, + tag: { type: "string", multiple: true }, + ...channelOptions, + force: { type: "boolean", default: false }, + }, + }); + const feature = featureSchema.parse(values.feature); + const { component, key } = targetSchemaFor(feature).parse(values); + const arch = archSchema.parse(values.arch); + const channel = values.channel; + const prefix = await resolvePrefix(values.prefix); + if (arch !== hostArch()) { + throw new Error( + `--arch ${arch} does not match the host architecture ${hostArch()}; local image builds build the current platform only`, + ); + } + if (!(await feature.payload.present(component, key, arch))) { + throw new Error( + `No payload at ${payloadPath(feature.id, component, key, arch)}; run the pipeline fetch-payload command for ${component} ${key} ${arch} first`, + ); + } + const featureBinaries: Partial> = {}; + if (feature.stage(component, key).binary) { + featureBinaries[arch] = await requireBinary(feature).ensure(ridFor(arch), values.force); + } + const context = await assembleBuildContext(feature, component, key, [arch], featureBinaries); + const entry = imageEntry(feature, await feature.document(), prefix, channel, { component, key }); + const refs = [...entry.refs, ...(values.tag ?? [])]; + await $$`docker buildx build ${[ + ...["--platform", `linux/${arch}`], + "--load", + ...["--file", path.join(context, "docker", "Dockerfile")], + ...refs.flatMap((ref) => ["--tag", ref]), + context, + ]}`; + console.log(refs.join("\n")); +}; diff --git a/scripts/tasks/features/commands/fetchPayload.ts b/scripts/tasks/features/commands/fetchPayload.ts new file mode 100644 index 0000000..fc5fa80 --- /dev/null +++ b/scripts/tasks/features/commands/fetchPayload.ts @@ -0,0 +1,18 @@ +import { parseArgs } from "node:util"; + +import { archSchema, featureOption, featureSchema, targetOptions, targetSchemaFor } from "./arguments.js"; + +export const fetchPayload = async (args: string[]): Promise => { + const { values } = parseArgs({ + args, + options: { + feature: featureOption, + ...targetOptions, + arch: { type: "string" }, + force: { type: "boolean", default: false }, + }, + }); + const feature = featureSchema.parse(values.feature); + const { component, key } = targetSchemaFor(feature).parse(values); + await feature.payload.ensure(component, key, archSchema.parse(values.arch), values.force); +}; diff --git a/scripts/tasks/features/commands/generateConfig.ts b/scripts/tasks/features/commands/generateConfig.ts new file mode 100644 index 0000000..54863e6 --- /dev/null +++ b/scripts/tasks/features/commands/generateConfig.ts @@ -0,0 +1,8 @@ +import { parseArgs } from "node:util"; + +import { featureOption, featureSchema } from "./arguments.js"; + +export const generateConfig = async (args: string[]): Promise => { + const { values } = parseArgs({ args, options: { feature: featureOption } }); + await featureSchema.parse(values.feature).generateConfig(); +}; diff --git a/scripts/tasks/features/commands/packFeature.ts b/scripts/tasks/features/commands/packFeature.ts new file mode 100644 index 0000000..18e63fd --- /dev/null +++ b/scripts/tasks/features/commands/packFeature.ts @@ -0,0 +1,38 @@ +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import { parseArgs } from "node:util"; + +import { z } from "zod"; + +import { hostArch, ridFor, rids } from "../architecture.js"; +import { requireBinary } from "../feature.js"; +import { createTar } from "../tar.js"; +import { featureOption, featureSchema, nonEmptyStringSchema } from "./arguments.js"; + +const ridSchema = z.enum(rids); + +const outSchema = nonEmptyStringSchema.transform((value) => path.resolve(value)); + +export const packFeature = async (args: string[]): Promise => { + const { values } = parseArgs({ + args, + options: { + feature: featureOption, + rid: { type: "string" }, + out: { type: "string" }, + force: { type: "boolean", default: false }, + }, + }); + const feature = featureSchema.parse(values.feature); + const binary = requireBinary(feature); + const rid = ridSchema.parse(values.rid); + if (rid !== ridFor(hostArch())) { + throw new Error(`--rid ${rid} does not match the host RID: a local pack builds ${ridFor(hostArch())}`); + } + const out = outSchema.parse(values.out); + const binaryPath = await binary.ensure(rid, values.force); + const archive = path.join(out, `${rid}.tar`); + await mkdir(out, { recursive: true }); + await createTar(archive, path.dirname(binaryPath), [path.basename(binaryPath)]); + console.log(`Feature binary archive written to ${archive}`); +}; diff --git a/scripts/tasks/features/commands/plan.ts b/scripts/tasks/features/commands/plan.ts new file mode 100644 index 0000000..942d53e --- /dev/null +++ b/scripts/tasks/features/commands/plan.ts @@ -0,0 +1,54 @@ +import { parseArgs } from "node:util"; + +import { z } from "zod"; + +import { imageEntries } from "../imageEntries.js"; +import { planPublish } from "../planPublish.js"; +import { deriveChannel } from "../tags.js"; +import { featureOption, featureSchema, nonEmptyStringSchema, prefixOption, resolvePrefix } from "./arguments.js"; + +const pullRequestSchema = z.preprocess( + (value: unknown) => (value === "" ? undefined : value), + z + .string() + .regex(/^[1-9][0-9]*$/u) + .transform(Number) + .pipe(z.int()) + .optional(), +); + +const runnerSchema = z.object({ + label: nonEmptyStringSchema, + arm64: z.boolean(), +}); + +const runnersSchema = z + .string() + .transform((value) => JSON.parse(value) as unknown) + .pipe(z.array(runnerSchema).min(1)); + +export const plan = async (args: string[]): Promise => { + const { values } = parseArgs({ + args, + options: { + feature: featureOption, + ...prefixOption, + runners: { type: "string" }, + ref: { type: "string" }, + "pull-request": { type: "string" }, + }, + }); + const feature = featureSchema.parse(values.feature); + const runners = runnersSchema.parse(values.runners); + const channel = deriveChannel( + nonEmptyStringSchema.parse(values.ref), + pullRequestSchema.parse(values["pull-request"]), + ); + const prefix = await resolvePrefix(values.prefix); + const document = await feature.document(); + await planPublish(imageEntries(feature, document, prefix, channel), { + channel, + prefix, + runners, + }); +}; diff --git a/scripts/tasks/features/commands/publishImage.ts b/scripts/tasks/features/commands/publishImage.ts new file mode 100644 index 0000000..01660cd --- /dev/null +++ b/scripts/tasks/features/commands/publishImage.ts @@ -0,0 +1,44 @@ +import { parseArgs } from "node:util"; + +import { dockerArchOptions } from "../architecture.js"; +import { assembleBuildContext } from "../buildContext.js"; +import { resolveFeatureBinaries } from "../featureBinary.js"; +import { pushImage } from "../image.js"; +import { imageEntry } from "../imageEntries.js"; +import { login } from "../registry.js"; +import { + channelOptions, + featureOption, + featureSchema, + resolvePrefix, + targetOptions, + targetSchemaFor, +} from "./arguments.js"; + +export const publishImage = async (args: string[]): Promise => { + const { values } = parseArgs({ + args, + options: { + feature: featureOption, + ...targetOptions, + ...channelOptions, + "install-dir": { type: "string" }, + }, + }); + const feature = featureSchema.parse(values.feature); + const { component, key } = targetSchemaFor(feature).parse(values); + const channel = values.channel; + const prefix = await resolvePrefix(values.prefix); + const entry = imageEntry(feature, await feature.document(), prefix, channel, { component, key }); + const featureBinaries = feature.stage(component, key).binary + ? await resolveFeatureBinaries(feature, values["install-dir"]) + : {}; + for (const arch of dockerArchOptions) { + await feature.payload.ensure(component, key, arch, false); + } + const context = await assembleBuildContext(feature, component, key, dockerArchOptions, featureBinaries); + await login(prefix); + const refs = [entry.canonical]; + await pushImage(context, refs); + console.log(refs.join("\n")); +}; diff --git a/scripts/tasks/features/commands/release.ts b/scripts/tasks/features/commands/release.ts new file mode 100644 index 0000000..86ee45f --- /dev/null +++ b/scripts/tasks/features/commands/release.ts @@ -0,0 +1,27 @@ +import { parseArgs } from "node:util"; + +import { imageEntries } from "../imageEntries.js"; +import { releaseTags } from "../release.js"; +import { tagPublished } from "../tagPublished.js"; +import { mainChannel } from "../tags.js"; +import { channelOptions, featureOption, featureSchema, resolvePrefix } from "./arguments.js"; + +export const release = async (args: string[]): Promise => { + const { values } = parseArgs({ + args, + options: { + feature: featureOption, + ...channelOptions, + }, + }); + const feature = featureSchema.parse(values.feature); + const channel = values.channel; + const prefix = await resolvePrefix(values.prefix); + const releases = imageEntries(feature, await feature.document(), prefix, channel); + await releaseTags(releases, prefix); + if (channel === mainChannel) { + for (const { gitTag } of releases) { + await tagPublished(gitTag); + } + } +}; diff --git a/scripts/tasks/features/commands/renovate.ts b/scripts/tasks/features/commands/renovate.ts new file mode 100644 index 0000000..3989f55 --- /dev/null +++ b/scripts/tasks/features/commands/renovate.ts @@ -0,0 +1,12 @@ +import { parseArgs } from "node:util"; + +import { refreshFeature } from "../renovate.js"; +import { featureOption, featureSchema } from "./arguments.js"; + +export const renovate = async (args: string[]): Promise => { + const { values } = parseArgs({ + args, + options: { feature: featureOption, "dry-run": { type: "boolean", default: false } }, + }); + await refreshFeature(featureSchema.parse(values.feature), values["dry-run"]); +}; diff --git a/scripts/tasks/features/config.ts b/scripts/tasks/features/config.ts index 8339ccb..b169d35 100644 --- a/scripts/tasks/features/config.ts +++ b/scripts/tasks/features/config.ts @@ -1,7 +1,35 @@ export const tagPattern = /^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$/u; +// The names a key derives before any channel suffix; tagNames takes the first as canonical. +export const mainTagNames = (key: string, revision: number, aliases: readonly string[]): readonly string[] => [ + `${key}-r${revision}`, + key, + ...aliases, +]; + export type AliasTable = Readonly>; +export interface ConfigNamespace { + name: string; + versions: Readonly>; + aliases: AliasTable; +} + +export interface ConfigDocument { + revision: number; + namespaces: readonly ConfigNamespace[]; +} + +export interface ConfigEntry { + component: string; + key: string; +} + +export const configEntries = (document: ConfigDocument): ConfigEntry[] => + document.namespaces.flatMap((namespace) => + Object.keys(namespace.versions).map((key) => ({ component: namespace.name, key })), + ); + export interface TagNamespace { source: string; revision: number; @@ -56,7 +84,7 @@ export const checkTagNamespace = (namespace: TagNamespace): void => { } const owners = new Map(); for (const key of Object.keys(versions)) { - for (const name of [`${key}-r${revision}`, key, ...aliases[key]]) { + for (const name of mainTagNames(key, revision, aliases[key])) { if (!tagPattern.test(name)) { throw new Error(`${source}: tag ${name} derived by ${key} is not a valid registry tag name`); } diff --git a/scripts/tasks/features/dotnet/buildContext.ts b/scripts/tasks/features/dotnet/buildContext.ts deleted file mode 100644 index 4ebfbdf..0000000 --- a/scripts/tasks/features/dotnet/buildContext.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { copyFile, mkdir, rm } from "node:fs/promises"; -import path from "node:path"; - -import { projectRoot } from "@/scripts/project.js"; -import { linkTree } from "@/scripts/tasks/features/linkTree.js"; - -import type { Component } from "./generateConfig.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; - -export const assembleBuildContext = async ( - component: Component, - key: string, - featureBinaries: Partial>, -): Promise => { - const context = paths.contextPath(component, key); - await rm(context, { recursive: true, force: true }); - await mkdir(path.join(context, "docker"), { recursive: true }); - await copyFile(path.join(projectRoot, "features/dotnet/Dockerfile"), path.join(context, "docker", "Dockerfile")); - for (const arch of paths.dockerArchOptions) { - const featureBinary = featureBinaries[arch]; - if (featureBinary === undefined) { - continue; - } - const stagedRoot = path.join(context, "linux", arch, "features", "dotnet"); - const staged = path.join(stagedRoot, component, key); - await mkdir(staged, { recursive: true }); - await linkTree(paths.payloadPath(component, key, arch), staged); - await copyFile(path.join(projectRoot, "features/dotnet/install"), path.join(stagedRoot, "install")); - await copyFile(featureBinary, path.join(stagedRoot, "Feature")); - } - return context; -}; diff --git a/scripts/tasks/features/dotnet/buildImage.ts b/scripts/tasks/features/dotnet/buildImage.ts deleted file mode 100644 index 5129e95..0000000 --- a/scripts/tasks/features/dotnet/buildImage.ts +++ /dev/null @@ -1,64 +0,0 @@ -import path from "node:path"; -import { parseArgs } from "node:util"; - -import { $$ } from "@/scripts/shell.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { assembleBuildContext } from "./buildContext.js"; -import { readConfig } from "./generateConfig.js"; -import { parseInvocation } from "./invocation.js"; -import { ensureFeatureBinary } from "./packBinary.js"; -import { paths } from "./paths.js"; -import { imageRefs, tagNames } from "./tags.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - component: { type: "string" }, - version: { type: "string" }, - arch: { type: "string" }, - tag: { type: "string", multiple: true }, - channel: { type: "string", default: mainChannel }, - prefix: { type: "string" }, - force: { type: "boolean", default: false }, - }, - }); - - const { component, key, arch } = parseInvocation(values); - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - if (arch !== paths.hostArch()) { - throw new Error( - `--arch ${arch} does not match the host architecture ${paths.hostArch()}; local image builds build the current platform only`, - ); - } - - const feature = await ensureFeatureBinary(paths.ridFor(arch), values.force); - const payload = paths.payloadPath(component, key, arch); - if (!(await paths.isFile(path.join(payload, "dotnet")))) { - throw new Error( - `No payload at ${payload}; run scripts/tasks/features/dotnet/fetchPayload.ts --component ${component} --version ${key} --arch ${arch} first`, - ); - } - - const context = await assembleBuildContext(component, key, { [arch]: feature }); - const config = await readConfig(); - const refs = [...imageRefs(prefix, component, tagNames(config, component, key, channel).all), ...(values.tag ?? [])]; - await $$`docker buildx build ${[ - ...["--platform", `linux/${arch}`], - "--load", - ...["--file", path.join(context, "docker", "Dockerfile")], - ...refs.flatMap((ref) => ["--tag", ref]), - context, - ]}`; - console.log(refs.join("\n")); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/dotnet/feature.ts b/scripts/tasks/features/dotnet/feature.ts new file mode 100644 index 0000000..5f2355c --- /dev/null +++ b/scripts/tasks/features/dotnet/feature.ts @@ -0,0 +1,36 @@ +import type { ConfigDocument } from "@/scripts/tasks/features/config.js"; +import type { Feature, FeatureStage } from "@/scripts/tasks/features/feature.js"; + +import type { FeatureConfig } from "./generateConfig.js"; +import { components, componentSchema, configPath, generateConfig, parseConfig, readConfig } from "./generateConfig.js"; +import { ensureFeatureBinary } from "./packBinary.js"; +import { ensurePayload, payloadPresent } from "./payload.js"; + +export const documentOf = (config: FeatureConfig): ConfigDocument => ({ + revision: config.revision, + namespaces: components.map((component) => ({ + name: component, + versions: config.components[component].versions, + aliases: config.components[component].aliases, + })), +}); + +export const dotnet: Feature = { + id: "dotnet", + configPath, + components, + document: async () => documentOf(await readConfig()), + parseDocument: (text, source) => documentOf(parseConfig(text, source)), + generateConfig, + repository: (component) => `dotnet/${componentSchema.parse(component)}`, + stage: (component, key): FeatureStage => { + componentSchema.parse(component); + return { subdir: [component, key], files: ["install"], binary: true }; + }, + payload: { + ensure: async (component, key, arch, force) => ensurePayload(componentSchema.parse(component), key, arch, force), + present: async (component, key, arch) => payloadPresent(componentSchema.parse(component), key, arch), + }, + binary: { ensure: ensureFeatureBinary }, + gitTag: (component, canonical) => `features-dotnet-${componentSchema.parse(component)}-${canonical}`, +}; diff --git a/scripts/tasks/features/dotnet/featureBinary.ts b/scripts/tasks/features/dotnet/featureBinary.ts deleted file mode 100644 index 7102706..0000000 --- a/scripts/tasks/features/dotnet/featureBinary.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { mkdir, rm } from "node:fs/promises"; -import path from "node:path"; - -import { extractTar } from "@/scripts/tasks/features/tar.js"; - -import { ensureFeatureBinary } from "./packBinary.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; - -const localBinary = async (arch: DockerArch): Promise => { - if (arch !== paths.hostArch()) { - throw new Error( - `--install-dir is required for the ${paths.ridFor(arch)} Feature binary: a local pack builds the host architecture only`, - ); - } - return ensureFeatureBinary(paths.ridFor(arch), false); -}; - -const archivedBinary = async (installDir: string, arch: DockerArch): Promise => { - const rid = paths.ridFor(arch); - const archive = path.join(installDir, `${rid}.tar`); - if (!(await paths.isFile(archive))) { - throw new Error(`No ${rid} Feature archive at ${archive}`); - } - const directory = path.join(installDir, rid); - await rm(directory, { recursive: true, force: true }); - await mkdir(directory, { recursive: true }); - await extractTar(archive, directory); - const binary = path.join(directory, "Feature"); - if (!(await paths.isFile(binary))) { - throw new Error(`The ${rid} Feature archive has no Feature entry`); - } - return binary; -}; - -export const resolveFeatureBinaries = async (installDir?: string): Promise> => { - const directory = installDir === undefined ? undefined : path.resolve(installDir); - const binaries = {} as Record; - for (const arch of paths.dockerArchOptions) { - binaries[arch] = directory === undefined ? await localBinary(arch) : await archivedBinary(directory, arch); - } - return binaries; -}; diff --git a/scripts/tasks/features/dotnet/fetchPayload.ts b/scripts/tasks/features/dotnet/fetchPayload.ts deleted file mode 100644 index 07cc2d9..0000000 --- a/scripts/tasks/features/dotnet/fetchPayload.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { parseArgs } from "node:util"; - -import { parseInvocation } from "./invocation.js"; -import { ensurePayload } from "./payload.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - component: { type: "string" }, - version: { type: "string" }, - arch: { type: "string" }, - force: { type: "boolean", default: false }, - }, - }); - - const { component, key, arch } = parseInvocation(values); - await ensurePayload(component, key, arch, values.force); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/dotnet/generateConfig.ts b/scripts/tasks/features/dotnet/generateConfig.ts index 0aca696..c9f99cd 100644 --- a/scripts/tasks/features/dotnet/generateConfig.ts +++ b/scripts/tasks/features/dotnet/generateConfig.ts @@ -14,6 +14,8 @@ export const configPath = path.resolve(projectRoot, "features/dotnet/config.json export const components = ["sdk", "runtime", "aspnet"] as const; export type Component = (typeof components)[number]; +export const componentSchema = z.enum(components); + export const rids = ["linux-x64", "linux-arm64"] as const; export type Rid = (typeof rids)[number]; @@ -83,7 +85,7 @@ const componentConfigSchema = z.strictObject({ }); export const featureConfigSchema = z.strictObject({ - revision: z.number(), + revision: z.int().min(1), components: z.strictObject({ sdk: componentConfigSchema, runtime: componentConfigSchema, @@ -248,12 +250,3 @@ export const generateConfig = async (): Promise => { await prettier.format(JSON.stringify(config), { ...prettierOptions, filepath: configPath }), ); }; - -if (import.meta.main) { - try { - await generateConfig(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/dotnet/invocation.ts b/scripts/tasks/features/dotnet/invocation.ts deleted file mode 100644 index 4766be0..0000000 --- a/scripts/tasks/features/dotnet/invocation.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { Component } from "./generateConfig.js"; -import { components } from "./generateConfig.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; - -export interface Invocation { - component: Component; - key: string; - arch: DockerArch; -} - -export const parseComponent = (value: string | undefined): Component => { - const component = components.find((candidate) => candidate === value); - if (component === undefined) { - throw new Error(`--component must be one of: ${components.join(", ")}`); - } - return component; -}; - -export const parseKey = (value: string | undefined): string => { - if (value === undefined || value === "") { - throw new Error("--version is required"); - } - return value; -}; - -export const parseInvocation = (values: { component?: string; version?: string; arch?: string }): Invocation => { - const component = parseComponent(values.component); - const key = parseKey(values.version); - const arch = - values.arch === undefined - ? paths.hostArch() - : paths.dockerArchOptions.find((candidate) => candidate === values.arch); - if (arch === undefined) { - throw new Error(`--arch must be one of: ${paths.dockerArchOptions.join(", ")}`); - } - return { component, key, arch }; -}; diff --git a/scripts/tasks/features/dotnet/packBinary.ts b/scripts/tasks/features/dotnet/packBinary.ts index bf1703a..82980ee 100644 --- a/scripts/tasks/features/dotnet/packBinary.ts +++ b/scripts/tasks/features/dotnet/packBinary.ts @@ -2,22 +2,23 @@ import path from "node:path"; import { project$$ } from "@/scripts/shell.js"; -import type { Rid } from "./generateConfig.js"; +import type { Rid } from "../architecture.js"; +import { isFile } from "../architecture.js"; import type { Assembly } from "./paths.js"; -import { paths } from "./paths.js"; +import { packPath } from "./paths.js"; const packBinary = async (assembly: Assembly, rid: Rid, force: boolean): Promise => { - const binaryPath = path.join(paths.packPath(assembly, rid), assembly); - if (!force && (await paths.isFile(binaryPath))) { + const binaryPath = path.join(packPath(assembly, rid), assembly); + if (!force && (await isFile(binaryPath))) { return binaryPath; } await project$$`dotnet publish ${[ ...["--configuration", "Release"], ...["--runtime", rid], `features/dotnet/${assembly}/${assembly}.csproj`, - `-p:PublishDir=${paths.packPath(assembly, rid)}/`, + `-p:PublishDir=${packPath(assembly, rid)}/`, ]}`; - if (!(await paths.isFile(binaryPath))) { + if (!(await isFile(binaryPath))) { throw new Error(`dotnet publish did not produce ${binaryPath}`); } return binaryPath; diff --git a/scripts/tasks/features/dotnet/packFeature.ts b/scripts/tasks/features/dotnet/packFeature.ts deleted file mode 100644 index a453b53..0000000 --- a/scripts/tasks/features/dotnet/packFeature.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { mkdir } from "node:fs/promises"; -import path from "node:path"; -import { parseArgs } from "node:util"; - -import { createTar } from "@/scripts/tasks/features/tar.js"; - -import { rids } from "./generateConfig.js"; -import { ensureFeatureBinary } from "./packBinary.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - rid: { type: "string" }, - out: { type: "string" }, - force: { type: "boolean", default: false }, - }, - }); - - const rid = rids.find((candidate) => candidate === values.rid); - if (rid === undefined) { - throw new Error(`--rid must be one of: ${rids.join(", ")}`); - } - if (values.out === undefined || values.out === "") { - throw new Error("--out is required"); - } - const binary = await ensureFeatureBinary(rid, values.force); - const out = path.resolve(values.out); - const archive = path.join(out, `${rid}.tar`); - await mkdir(out, { recursive: true }); - await createTar(archive, path.dirname(binary), [path.basename(binary)]); - console.log(`Feature binary archive written to ${archive}`); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/dotnet/paths.ts b/scripts/tasks/features/dotnet/paths.ts index 800474f..c066f26 100644 --- a/scripts/tasks/features/dotnet/paths.ts +++ b/scripts/tasks/features/dotnet/paths.ts @@ -1,57 +1,9 @@ -import { stat } from "node:fs/promises"; import path from "node:path"; import { workspaces } from "@/scripts/project.js"; -import type { Component, Rid } from "./generateConfig.js"; - -const dockerArchOptions = ["amd64", "arm64"] as const; -export type DockerArch = (typeof dockerArchOptions)[number]; +import type { Rid } from "../architecture.js"; export type Assembly = "Build" | "Feature"; -const hostArch = (): DockerArch => { - switch (process.arch) { - case "x64": - return "amd64"; - case "arm64": - return "arm64"; - default: - throw new Error(`Unsupported host architecture: ${process.arch}`); - } -}; - -const rids: Record = { amd64: "linux-x64", arm64: "linux-arm64" }; - -const ridFor = (arch: DockerArch): Rid => rids[arch]; - -const payloadRoot = path.resolve(workspaces, "payload/dotnet"); - -const payloadPath = (component: Component, key: string, arch: DockerArch): string => - path.join(payloadRoot, component, key, arch); - -const contextPath = (component: Component, key: string): string => path.join(payloadRoot, "ctx", component, key); - -const packPath = (assembly: Assembly, rid: Rid): string => path.join(workspaces, "bin/dotnet", assembly, rid); - -const isFile = async (candidate: string): Promise => { - try { - return (await stat(candidate)).isFile(); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return false; - } - throw error; - } -}; - -export const paths = { - dockerArchOptions, - hostArch, - ridFor, - payloadRoot, - payloadPath, - contextPath, - packPath, - isFile, -} as const; +export const packPath = (assembly: Assembly, rid: Rid): string => path.join(workspaces, "bin/dotnet", assembly, rid); diff --git a/scripts/tasks/features/dotnet/payload.ts b/scripts/tasks/features/dotnet/payload.ts index 8b16b89..d8dfa03 100644 --- a/scripts/tasks/features/dotnet/payload.ts +++ b/scripts/tasks/features/dotnet/payload.ts @@ -3,11 +3,15 @@ import path from "node:path"; import { $$ } from "@/scripts/shell.js"; +import type { DockerArch } from "../architecture.js"; +import { hostArch, isFile, ridFor } from "../architecture.js"; +import { payloadPath } from "../paths.js"; import type { Component } from "./generateConfig.js"; import { configPath } from "./generateConfig.js"; import { ensureBuildBinary } from "./packBinary.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; + +export const payloadPresent = (component: Component, key: string, arch: DockerArch): Promise => + isFile(path.join(payloadPath("dotnet", component, key, arch), "dotnet")); export const ensurePayload = async ( component: Component, @@ -15,12 +19,12 @@ export const ensurePayload = async ( arch: DockerArch, force: boolean, ): Promise => { - const outPath = paths.payloadPath(component, key, arch); - if (!force && (await paths.isFile(path.join(outPath, "dotnet")))) { + const outPath = payloadPath("dotnet", component, key, arch); + if (!force && (await payloadPresent(component, key, arch))) { console.log(`Payload already present at ${outPath}; skipping fetch`); return; } - const buildBinary = await ensureBuildBinary(paths.ridFor(paths.hostArch()), force); + const buildBinary = await ensureBuildBinary(ridFor(hostArch()), force); const tempPath = `${outPath}.tmp`; await rm(tempPath, { recursive: true, force: true }); await $$`${[ diff --git a/scripts/tasks/features/dotnet/planPublish.ts b/scripts/tasks/features/dotnet/planPublish.ts deleted file mode 100644 index c0d292f..0000000 --- a/scripts/tasks/features/dotnet/planPublish.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { parseArgs } from "node:util"; - -import { planPublish } from "@/scripts/tasks/features/planPublish.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { components, readConfig } from "./generateConfig.js"; -import { imageRef, tagNames } from "./tags.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - channel: { type: "string", default: mainChannel }, - out: { type: "string", default: ".ci/plan.json" }, - prefix: { type: "string" }, - }, - }); - - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const config = await readConfig(); - const pairs = components.flatMap((component) => - Object.keys(config.components[component].versions).map((key) => ({ component, key })), - ); - await planPublish( - pairs.map(({ component, key }) => { - const tags = tagNames(config, component, key, channel); - return { - label: `${component} ${key}`, - canonical: imageRef(prefix, component, tags.canonical), - tags: tags.all, - entry: { component, key }, - }; - }), - { channel, prefix, out: values.out }, - ); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/dotnet/publishImage.ts b/scripts/tasks/features/dotnet/publishImage.ts deleted file mode 100644 index d50ead1..0000000 --- a/scripts/tasks/features/dotnet/publishImage.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { parseArgs } from "node:util"; - -import { pushImage } from "@/scripts/tasks/features/image.js"; -import { login } from "@/scripts/tasks/features/registry.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { assembleBuildContext } from "./buildContext.js"; -import { resolveFeatureBinaries } from "./featureBinary.js"; -import { readConfig } from "./generateConfig.js"; -import { parseComponent, parseKey } from "./invocation.js"; -import { paths } from "./paths.js"; -import { ensurePayload } from "./payload.js"; -import { imageRef, tagNames } from "./tags.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - component: { type: "string" }, - version: { type: "string" }, - channel: { type: "string", default: mainChannel }, - "install-dir": { type: "string" }, - prefix: { type: "string" }, - }, - }); - - const component = parseComponent(values.component); - const key = parseKey(values.version); - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const config = await readConfig(); - const canonical = tagNames(config, component, key, channel).canonical; - const featureBinaries = await resolveFeatureBinaries(values["install-dir"]); - for (const arch of paths.dockerArchOptions) { - await ensurePayload(component, key, arch, false); - } - const context = await assembleBuildContext(component, key, featureBinaries); - await login(prefix); - const refs = [imageRef(prefix, component, canonical)]; - await pushImage(context, refs); - console.log(refs.join("\n")); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/dotnet/releaseTags.ts b/scripts/tasks/features/dotnet/releaseTags.ts deleted file mode 100644 index 4781331..0000000 --- a/scripts/tasks/features/dotnet/releaseTags.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { parseArgs } from "node:util"; - -import type { ReleaseEntry } from "@/scripts/tasks/features/release.js"; -import { releaseTags } from "@/scripts/tasks/features/release.js"; -import { tagPublished } from "@/scripts/tasks/features/tagPublished.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { components, readConfig } from "./generateConfig.js"; -import { imageRef, imageRefs, tagNames } from "./tags.js"; - -interface FeatureReleaseEntry extends ReleaseEntry { - gitTag: string; -} - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - channel: { type: "string", default: mainChannel }, - prefix: { type: "string" }, - "git-tags": { type: "boolean", default: false }, - }, - }); - - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const config = await readConfig(); - const entries: FeatureReleaseEntry[] = components.flatMap((component) => - Object.keys(config.components[component].versions).map((key) => { - const tags = tagNames(config, component, key, channel); - return { - label: `${component} ${key}`, - canonical: imageRef(prefix, component, tags.canonical), - tags: imageRefs(prefix, component, tags.claims), - gitTag: `features-dotnet-${component}-${tagNames(config, component, key, mainChannel).canonical}`, - }; - }), - ); - await releaseTags(entries, prefix); - if (values["git-tags"]) { - for (const { gitTag } of entries) { - await tagPublished(gitTag); - } - } -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/dotnet/renovate.ts b/scripts/tasks/features/dotnet/renovate.ts deleted file mode 100644 index 0125371..0000000 --- a/scripts/tasks/features/dotnet/renovate.ts +++ /dev/null @@ -1,48 +0,0 @@ -import path from "node:path"; -import { parseArgs } from "node:util"; - -import { projectRoot } from "@/scripts/project.js"; -import type { ConfigDocument } from "@/scripts/tasks/features/renovate.js"; -import { refreshFeatureConfig } from "@/scripts/tasks/features/renovate.js"; - -import type { FeatureConfig } from "./generateConfig.js"; -import { components, configPath, generateConfig, parseConfig, readConfig } from "./generateConfig.js"; - -const document = (config: FeatureConfig): ConfigDocument => ({ - revision: config.revision, - namespaces: components.map((component) => ({ - name: component, - versions: config.components[component].versions, - aliases: config.components[component].aliases, - })), -}); - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - "dry-run": { type: "boolean", default: false }, - }, - }); - - await refreshFeatureConfig( - { - branch: "renovate-dotnet", - title: "Update dotnet version pins", - filepath: path.relative(projectRoot, configPath), - generate: generateConfig, - read: readConfig, - parse: parseConfig, - document, - }, - values["dry-run"], - ); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/dotnet/tags.ts b/scripts/tasks/features/dotnet/tags.ts deleted file mode 100644 index 7e496a6..0000000 --- a/scripts/tasks/features/dotnet/tags.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { TagSet, TagSource } from "@/scripts/tasks/features/tags.js"; -import { tagNames as baseTagNames } from "@/scripts/tasks/features/tags.js"; - -import type { Component, FeatureConfig } from "./generateConfig.js"; - -const sourceOf = (config: FeatureConfig, component: Component, key: string): TagSource => { - const entry = config.components[component].versions[key]; - if (entry === undefined) { - const available = Object.keys(config.components[component].versions).join(", "); - throw new Error(`Component ${component} has no version ${key} (available: ${available})`); - } - return { key, revision: config.revision, aliases: config.components[component].aliases[key] }; -}; - -export const tagNames = (config: FeatureConfig, component: Component, key: string, channel: string): TagSet => - baseTagNames(sourceOf(config, component, key), channel); - -export const imageRef = (prefix: string, component: Component, tag: string): string => - `${prefix}/dotnet/${component}:${tag}`; - -export const imageRefs = (prefix: string, component: Component, tags: readonly string[]): string[] => - tags.map((tag) => imageRef(prefix, component, tag)); diff --git a/scripts/tasks/features/feature.ts b/scripts/tasks/features/feature.ts new file mode 100644 index 0000000..a2c51d5 --- /dev/null +++ b/scripts/tasks/features/feature.ts @@ -0,0 +1,35 @@ +import type { DockerArch, Rid } from "./architecture.js"; +import type { ConfigDocument } from "./config.js"; + +export interface FeatureStage { + subdir: readonly string[]; + files: readonly string[]; + binary: boolean; +} + +export interface Feature { + id: string; + configPath: string; + components: readonly string[]; + document(): Promise; + parseDocument(text: string, source: string): ConfigDocument; + generateConfig(): Promise; + repository(component: string): string; + stage(component: string, key: string): FeatureStage; + payload: { + ensure(component: string, key: string, arch: DockerArch, force: boolean): Promise; + present(component: string, key: string, arch: DockerArch): Promise; + }; + binary?: { + ensure(rid: Rid, force: boolean): Promise; + }; + gitTag(component: string, canonical: string): string; +} + +export const requireBinary = (feature: Feature): NonNullable => { + const binary = feature.binary; + if (binary === undefined) { + throw new Error(`Feature ${feature.id} has no Feature binary`); + } + return binary; +}; diff --git a/scripts/tasks/features/uv/featureBinary.ts b/scripts/tasks/features/featureBinary.ts similarity index 54% rename from scripts/tasks/features/uv/featureBinary.ts rename to scripts/tasks/features/featureBinary.ts index 7102706..7ffa6f9 100644 --- a/scripts/tasks/features/uv/featureBinary.ts +++ b/scripts/tasks/features/featureBinary.ts @@ -3,23 +3,24 @@ import path from "node:path"; import { extractTar } from "@/scripts/tasks/features/tar.js"; -import { ensureFeatureBinary } from "./packBinary.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; +import type { DockerArch } from "./architecture.js"; +import { dockerArchOptions, hostArch, isFile, ridFor } from "./architecture.js"; +import type { Feature } from "./feature.js"; +import { requireBinary } from "./feature.js"; -const localBinary = async (arch: DockerArch): Promise => { - if (arch !== paths.hostArch()) { +const localBinary = async (binary: NonNullable, arch: DockerArch): Promise => { + if (arch !== hostArch()) { throw new Error( - `--install-dir is required for the ${paths.ridFor(arch)} Feature binary: a local pack builds the host architecture only`, + `--install-dir is required for the ${ridFor(arch)} Feature binary: a local pack builds the host architecture only`, ); } - return ensureFeatureBinary(paths.ridFor(arch), false); + return binary.ensure(ridFor(arch), false); }; const archivedBinary = async (installDir: string, arch: DockerArch): Promise => { - const rid = paths.ridFor(arch); + const rid = ridFor(arch); const archive = path.join(installDir, `${rid}.tar`); - if (!(await paths.isFile(archive))) { + if (!(await isFile(archive))) { throw new Error(`No ${rid} Feature archive at ${archive}`); } const directory = path.join(installDir, rid); @@ -27,17 +28,21 @@ const archivedBinary = async (installDir: string, arch: DockerArch): Promise> => { +export const resolveFeatureBinaries = async ( + feature: Feature, + installDir?: string, +): Promise> => { + const binary = requireBinary(feature); const directory = installDir === undefined ? undefined : path.resolve(installDir); const binaries = {} as Record; - for (const arch of paths.dockerArchOptions) { - binaries[arch] = directory === undefined ? await localBinary(arch) : await archivedBinary(directory, arch); + for (const arch of dockerArchOptions) { + binaries[arch] = directory === undefined ? await localBinary(binary, arch) : await archivedBinary(directory, arch); } return binaries; }; diff --git a/scripts/tasks/features/features.ts b/scripts/tasks/features/features.ts new file mode 100644 index 0000000..ce6b38f --- /dev/null +++ b/scripts/tasks/features/features.ts @@ -0,0 +1,6 @@ +import { dotnet } from "./dotnet/feature.js"; +import type { Feature } from "./feature.js"; +import { ripgrep } from "./ripgrep/feature.js"; +import { uv } from "./uv/feature.js"; + +export const features: Readonly> = { dotnet, ripgrep, uv }; diff --git a/scripts/tasks/features/imageEntries.ts b/scripts/tasks/features/imageEntries.ts new file mode 100644 index 0000000..1cdd0ec --- /dev/null +++ b/scripts/tasks/features/imageEntries.ts @@ -0,0 +1,56 @@ +import type { ConfigDocument, ConfigEntry } from "./config.js"; +import { configEntries } from "./config.js"; +import type { Feature } from "./feature.js"; +import type { TagSource } from "./tags.js"; +import { mainChannel, tagNames } from "./tags.js"; + +export interface ImageEntry { + entry: ConfigEntry; + label: string; + canonical: string; + refs: readonly string[]; + claims: readonly string[]; + gitTag: string; +} + +const imageRef = (feature: Feature, prefix: string, component: string, tag: string): string => + `${prefix}/${feature.repository(component)}:${tag}`; + +const tagSource = (feature: Feature, document: ConfigDocument, { component, key }: ConfigEntry): TagSource => { + const namespace = document.namespaces.find((candidate) => candidate.name === component); + if (namespace === undefined) { + throw new Error(`${feature.configPath}: the config has no component ${component}`); + } + if (!Object.hasOwn(namespace.versions, key)) { + throw new Error(`${feature.configPath}: component ${component} has no version ${key}`); + } + return { key, revision: document.revision, aliases: namespace.aliases[key] }; +}; + +export const imageEntry = ( + feature: Feature, + document: ConfigDocument, + prefix: string, + channel: string, + entry: ConfigEntry, +): ImageEntry => { + const source = tagSource(feature, document, entry); + const tags = tagNames(source, channel); + const { component, key } = entry; + const refsOf = (names: readonly string[]): string[] => names.map((tag) => imageRef(feature, prefix, component, tag)); + return { + entry, + label: `${component} ${key}`, + canonical: imageRef(feature, prefix, component, tags.canonical), + refs: refsOf(tags.all), + claims: refsOf(tags.claims), + gitTag: feature.gitTag(component, tagNames(source, mainChannel).canonical), + }; +}; + +export const imageEntries = ( + feature: Feature, + document: ConfigDocument, + prefix: string, + channel: string, +): ImageEntry[] => configEntries(document).map((entry) => imageEntry(feature, document, prefix, channel, entry)); diff --git a/scripts/tasks/features/paths.ts b/scripts/tasks/features/paths.ts new file mode 100644 index 0000000..8d92ed7 --- /dev/null +++ b/scripts/tasks/features/paths.ts @@ -0,0 +1,11 @@ +import path from "node:path"; + +import { workspaces } from "@/scripts/project.js"; + +import type { DockerArch } from "./architecture.js"; + +export const payloadPath = (feature: string, component: string, key: string, arch: DockerArch): string => + path.join(workspaces, "payload", feature, component, key, arch); + +export const contextPath = (feature: string, component: string, key: string): string => + path.join(workspaces, "payload", feature, "ctx", component, key); diff --git a/scripts/tasks/features/pipeline.ts b/scripts/tasks/features/pipeline.ts new file mode 100644 index 0000000..128772d --- /dev/null +++ b/scripts/tasks/features/pipeline.ts @@ -0,0 +1,37 @@ +import { z } from "zod"; + +import { buildImage } from "./commands/buildImage.js"; +import { fetchPayload } from "./commands/fetchPayload.js"; +import { generateConfig } from "./commands/generateConfig.js"; +import { packFeature } from "./commands/packFeature.js"; +import { plan } from "./commands/plan.js"; +import { publishImage } from "./commands/publishImage.js"; +import { release } from "./commands/release.js"; +import { renovate } from "./commands/renovate.js"; + +const commands: Readonly Promise>> = { + "generate-config": generateConfig, + renovate, + "plan-publish": plan, + "fetch-payload": fetchPayload, + "pack-feature": packFeature, + "build-image": buildImage, + "publish-image": publishImage, + "release-tags": release, +}; + +const commandSchema = z.enum(Object.keys(commands) as [string, ...string[]]); + +const main = async (): Promise => { + const [name, ...args] = process.argv.slice(2); + await commands[commandSchema.parse(name)](args); +}; + +if (import.meta.main) { + try { + await main(); + } catch (error) { + console.error(error); + process.exitCode = 1; + } +} diff --git a/scripts/tasks/features/planPublish.ts b/scripts/tasks/features/planPublish.ts index 38db2a5..c80d327 100644 --- a/scripts/tasks/features/planPublish.ts +++ b/scripts/tasks/features/planPublish.ts @@ -1,53 +1,54 @@ -import { appendFile, mkdir, writeFile } from "node:fs/promises"; -import path from "node:path"; +import { appendFile } from "node:fs/promises"; import { $ } from "execa"; +import type { ConfigEntry } from "./config.js"; +import type { ImageEntry } from "./imageEntries.js"; import { login } from "./registry.js"; import { mainChannel } from "./tags.js"; const $$inspect = $({ reject: false, stdio: "ignore" }); -export interface PlanCandidate { +export interface PlanRunner { label: string; - canonical: string; - tags: readonly string[]; - entry: Entry; + arm64: boolean; } export interface PlanOptions { channel: string; prefix: string; - out: string; + // The runners the smoke matrix fans out over, with the architecture each one builds for. + runners: readonly PlanRunner[]; } -export const planPublish = async ( - candidates: readonly PlanCandidate[], - options: PlanOptions, -): Promise => { +export const planPublish = async (entries: readonly ImageEntry[], options: PlanOptions): Promise => { if (options.channel === mainChannel) { await login(options.prefix); } - const listed: Entry[] = []; - for (const candidate of candidates) { + const listed: ConfigEntry[] = []; + for (const entry of entries) { if (options.channel === mainChannel) { - const inspect = await $$inspect`docker buildx imagetools inspect ${candidate.canonical}`; + const inspect = await $$inspect`docker buildx imagetools inspect ${entry.canonical}`; if (inspect.exitCode === 0) { - console.log(`${candidate.label}: already published (${candidate.canonical})`); + console.log(`${entry.label}: already published (${entry.canonical})`); continue; } } - console.log(`${candidate.label}: to publish (${candidate.tags.join(", ")})`); - listed.push(candidate.entry); + console.log(`${entry.label}: to publish (${entry.refs.join(", ")})`); + listed.push(entry.entry); } - const plan = { include: listed }; - const outPath = path.resolve(options.out); - await mkdir(path.dirname(outPath), { recursive: true }); - await writeFile(outPath, `${JSON.stringify(plan, null, 2)}\n`); + const matrix = { include: listed }; + const smoke = { + include: entries.flatMap(({ entry }) => + options.runners.map(({ label, arm64 }) => ({ label, arm64, component: entry.component, key: entry.key })), + ), + }; const githubOutput = process.env.GITHUB_OUTPUT; if (githubOutput !== undefined && githubOutput !== "") { - await appendFile(githubOutput, `matrix=${listed.length === 0 ? "" : JSON.stringify(plan)}\n`); + await appendFile(githubOutput, `matrix=${listed.length === 0 ? "" : JSON.stringify(matrix)}\n`); + await appendFile(githubOutput, `smoke=${JSON.stringify(smoke)}\n`); + await appendFile(githubOutput, `channel=${options.channel}\n`); } - console.log(`${listed.length} of ${candidates.length} entries to publish (${outPath})`); + console.log(`${listed.length} of ${entries.length} entries to publish on channel ${options.channel}`); }; diff --git a/scripts/tasks/features/release.ts b/scripts/tasks/features/release.ts index 3c57646..d307825 100644 --- a/scripts/tasks/features/release.ts +++ b/scripts/tasks/features/release.ts @@ -1,16 +1,11 @@ import { $ } from "execa"; +import type { ImageEntry } from "./imageEntries.js"; import { login } from "./registry.js"; const $$inspect = $({ reject: false, stdio: "ignore" }); -export interface ReleaseEntry { - label: string; - canonical: string; - tags: readonly string[]; -} - -export const releaseTags = async (entries: readonly ReleaseEntry[], prefix: string): Promise => { +export const releaseTags = async (entries: readonly ImageEntry[], prefix: string): Promise => { await login(prefix); for (const entry of entries) { const inspect = await $$inspect`docker buildx imagetools inspect ${entry.canonical}`; @@ -19,7 +14,7 @@ export const releaseTags = async (entries: readonly ReleaseEntry[], prefix: stri } } for (const entry of entries) { - await $`docker buildx imagetools create ${entry.tags.flatMap((tag) => ["--tag", tag])} ${entry.canonical}`; - console.log(`${entry.label}: applied ${entry.tags.join(", ")} to ${entry.canonical}`); + await $`docker buildx imagetools create ${entry.claims.flatMap((tag) => ["--tag", tag])} ${entry.canonical}`; + console.log(`${entry.label}: applied ${entry.claims.join(", ")} to ${entry.canonical}`); } }; diff --git a/scripts/tasks/features/renovate.ts b/scripts/tasks/features/renovate.ts index 1e3a603..1387973 100644 --- a/scripts/tasks/features/renovate.ts +++ b/scripts/tasks/features/renovate.ts @@ -1,4 +1,5 @@ import fs from "node:fs/promises"; +import path from "node:path"; import { Octokit } from "@octokit/rest"; import git from "isomorphic-git"; @@ -7,6 +8,9 @@ import http from "isomorphic-git/http/node"; import { getRemoteInfo } from "@/scripts/git.js"; import { projectRoot } from "@/scripts/project.js"; +import type { ConfigDocument } from "./config.js"; +import type { Feature } from "./feature.js"; + const textDecoder = new TextDecoder(); const isNotFound = (error: unknown): boolean => @@ -25,17 +29,6 @@ export const headFile = async (oid: string, filepath: string): Promise>; - aliases: Readonly>; -} - -export interface ConfigDocument { - revision: number; - namespaces: readonly ConfigNamespace[]; -} - const isObject = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); @@ -128,37 +121,20 @@ const emptyDocument = (document: ConfigDocument): ConfigDocument => ({ namespaces: document.namespaces.map(({ name }) => ({ name, versions: {}, aliases: {} })), }); -export interface FeatureConfigRefresh { - branch: string; - title: string; - filepath: string; - generate: () => Promise; - read: () => Promise; - parse: (text: string, source: string) => Config; - document: (config: Config) => ConfigDocument; -} - -// A refresh regenerates the config, then compares it against HEAD and opens or updates the feature's branch. -export const refreshFeatureConfig = async ( - options: FeatureConfigRefresh, - dryRun: boolean, -): Promise => { - await options.generate(); +export const refreshFeature = async (feature: Feature, dryRun: boolean): Promise => { + await feature.generateConfig(); + const filepath = path.relative(projectRoot, feature.configPath); await refresh( { - branch: options.branch, - title: options.title, - filepath: options.filepath, + branch: `renovate-${feature.id}`, + title: `Update ${feature.id} version pins`, + filepath, body: async (headOid) => { - const text = await headFile(headOid, options.filepath); - const previous = text === undefined ? undefined : options.parse(text, `${options.filepath} at HEAD`); - const next = options.document(await options.read()); - const changes = configChanges( - previous === undefined ? emptyDocument(next) : options.document(previous), - next, - options.filepath, - ); - return changeBody(changes, options.filepath); + const text = await headFile(headOid, filepath); + const previous = text === undefined ? undefined : feature.parseDocument(text, `${filepath} at HEAD`); + const next = await feature.document(); + const changes = configChanges(previous ?? emptyDocument(next), next, filepath); + return changeBody(changes, filepath); }, }, dryRun, diff --git a/scripts/tasks/features/ripgrep/buildContext.ts b/scripts/tasks/features/ripgrep/buildContext.ts deleted file mode 100644 index d96a983..0000000 --- a/scripts/tasks/features/ripgrep/buildContext.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { copyFile, mkdir, rm } from "node:fs/promises"; -import path from "node:path"; - -import { projectRoot } from "@/scripts/project.js"; -import { linkTree } from "@/scripts/tasks/features/linkTree.js"; - -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; - -export const assembleBuildContext = async (key: string, arches: readonly DockerArch[]): Promise => { - const context = paths.contextPath(key); - await rm(context, { recursive: true, force: true }); - await mkdir(path.join(context, "docker"), { recursive: true }); - await copyFile(path.join(projectRoot, "features/ripgrep/Dockerfile"), path.join(context, "docker", "Dockerfile")); - for (const arch of arches) { - const staged = path.join(context, "linux", arch, "features", "ripgrep"); - await mkdir(staged, { recursive: true }); - await linkTree(paths.payloadPath(key, arch), staged); - await copyFile(path.join(projectRoot, "features/ripgrep/install"), path.join(staged, "install")); - } - return context; -}; diff --git a/scripts/tasks/features/ripgrep/buildImage.ts b/scripts/tasks/features/ripgrep/buildImage.ts deleted file mode 100644 index 751feab..0000000 --- a/scripts/tasks/features/ripgrep/buildImage.ts +++ /dev/null @@ -1,62 +0,0 @@ -import path from "node:path"; -import { parseArgs } from "node:util"; - -import { $$ } from "@/scripts/shell.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { assembleBuildContext } from "./buildContext.js"; -import { readConfig } from "./generateConfig.js"; -import { parseInvocation } from "./invocation.js"; -import { paths } from "./paths.js"; -import { derive, loadSection, payloadPresent } from "./payload.js"; -import { imageRefs, tagNames } from "./tags.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - version: { type: "string" }, - arch: { type: "string" }, - channel: { type: "string", default: mainChannel }, - tag: { type: "string", multiple: true }, - prefix: { type: "string" }, - }, - }); - - const { key, arch } = parseInvocation(values); - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - if (arch !== paths.hostArch()) { - throw new Error( - `--arch ${arch} does not match the host architecture ${paths.hostArch()}; local image builds build the current platform only`, - ); - } - - const section = await loadSection(key); - const payload = derive(section, arch); - if (!(await payloadPresent(payload))) { - throw new Error( - `No payload at ${payload.cache}; run scripts/tasks/features/ripgrep/fetchPayload.ts --version ${key} --arch ${arch} first`, - ); - } - - const context = await assembleBuildContext(key, [arch]); - const config = await readConfig(); - const refs = [...imageRefs(prefix, tagNames(config, key, channel).all), ...(values.tag ?? [])]; - await $$`docker buildx build ${[ - ...["--platform", `linux/${arch}`], - "--load", - ...["--file", path.join(context, "docker", "Dockerfile")], - ...refs.flatMap((ref) => ["--tag", ref]), - context, - ]}`; - console.log(refs.join("\n")); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/ripgrep/feature.ts b/scripts/tasks/features/ripgrep/feature.ts new file mode 100644 index 0000000..f1735e5 --- /dev/null +++ b/scripts/tasks/features/ripgrep/feature.ts @@ -0,0 +1,31 @@ +import type { ConfigDocument } from "@/scripts/tasks/features/config.js"; +import type { Feature, FeatureStage } from "@/scripts/tasks/features/feature.js"; + +import type { FeatureConfig } from "./generateConfig.js"; +import { configPath, generateConfig, parseConfig, readConfig } from "./generateConfig.js"; +import { derive, ensurePayload, loadSection, payloadPresent } from "./payload.js"; + +const component = "ripgrep"; + +export const documentOf = (config: FeatureConfig): ConfigDocument => ({ + revision: config.revision, + namespaces: [{ name: component, versions: config.versions, aliases: config.aliases }], +}); + +const stage: FeatureStage = { subdir: [], files: ["install"], binary: false }; + +export const ripgrep: Feature = { + id: "ripgrep", + configPath, + components: [component], + document: async () => documentOf(await readConfig()), + parseDocument: (text, source) => documentOf(parseConfig(text, source)), + generateConfig, + repository: () => "ripgrep", + stage: () => stage, + payload: { + ensure: async (_component, key, arch, force) => ensurePayload(await loadSection(key), arch, force), + present: async (_component, key, arch) => payloadPresent(derive(await loadSection(key), arch)), + }, + gitTag: (_component, canonical) => `features-ripgrep-${canonical}`, +}; diff --git a/scripts/tasks/features/ripgrep/fetchPayload.ts b/scripts/tasks/features/ripgrep/fetchPayload.ts deleted file mode 100644 index fa907e9..0000000 --- a/scripts/tasks/features/ripgrep/fetchPayload.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { parseArgs } from "node:util"; - -import { parseInvocation } from "./invocation.js"; -import { ensurePayload, loadSection } from "./payload.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - version: { type: "string" }, - arch: { type: "string" }, - force: { type: "boolean", default: false }, - }, - }); - - const { key, arch } = parseInvocation(values); - await ensurePayload(await loadSection(key), arch, values.force); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/ripgrep/generateConfig.ts b/scripts/tasks/features/ripgrep/generateConfig.ts index b4edaa8..7354e80 100644 --- a/scripts/tasks/features/ripgrep/generateConfig.ts +++ b/scripts/tasks/features/ripgrep/generateConfig.ts @@ -43,7 +43,7 @@ const versionEntrySchema = z.strictObject({ const aliasTableSchema = z.record(z.string(), z.array(z.string())); export const featureConfigSchema = z.strictObject({ - revision: z.number(), + revision: z.int().min(1), versions: z.record(z.string(), versionEntrySchema), aliases: aliasTableSchema, }); @@ -183,12 +183,3 @@ export const generateConfig = async (): Promise => { await prettier.format(JSON.stringify(config), { ...prettierOptions, filepath: configPath }), ); }; - -if (import.meta.main) { - try { - await generateConfig(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/ripgrep/invocation.ts b/scripts/tasks/features/ripgrep/invocation.ts deleted file mode 100644 index b632dc1..0000000 --- a/scripts/tasks/features/ripgrep/invocation.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; - -export interface Invocation { - key: string; - arch: DockerArch; -} - -export const parseKey = (value: string | undefined): string => { - if (value === undefined || value === "") { - throw new Error("--version is required"); - } - return value; -}; - -export const parseInvocation = (values: { version?: string; arch?: string }): Invocation => { - const key = parseKey(values.version); - const arch = - values.arch === undefined - ? paths.hostArch() - : paths.dockerArchOptions.find((candidate) => candidate === values.arch); - if (arch === undefined) { - throw new Error(`--arch must be one of: ${paths.dockerArchOptions.join(", ")}`); - } - return { key, arch }; -}; diff --git a/scripts/tasks/features/ripgrep/paths.ts b/scripts/tasks/features/ripgrep/paths.ts deleted file mode 100644 index 0fae7cd..0000000 --- a/scripts/tasks/features/ripgrep/paths.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { stat } from "node:fs/promises"; -import path from "node:path"; - -import { workspaces } from "@/scripts/project.js"; - -import type { Target } from "./generateConfig.js"; - -const dockerArchOptions = ["amd64", "arm64"] as const; -export type DockerArch = (typeof dockerArchOptions)[number]; - -const hostArch = (): DockerArch => { - switch (process.arch) { - case "x64": - return "amd64"; - case "arm64": - return "arm64"; - default: - throw new Error(`Unsupported host architecture: ${process.arch}`); - } -}; - -const targets: Record = { - amd64: "x86_64-unknown-linux-musl", - arm64: "aarch64-unknown-linux-musl", -}; - -const payloadRoot = path.resolve(workspaces, "payload/ripgrep"); - -const payloadPath = (key: string, arch: DockerArch): string => path.join(payloadRoot, key, arch); - -const contextPath = (key: string): string => path.join(payloadRoot, "ctx", key); - -const isFile = async (candidate: string): Promise => { - try { - return (await stat(candidate)).isFile(); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return false; - } - throw error; - } -}; - -export const paths = { - dockerArchOptions, - hostArch, - targets, - payloadRoot, - payloadPath, - contextPath, - isFile, -} as const; diff --git a/scripts/tasks/features/ripgrep/payload.ts b/scripts/tasks/features/ripgrep/payload.ts index f4ea908..6940f56 100644 --- a/scripts/tasks/features/ripgrep/payload.ts +++ b/scripts/tasks/features/ripgrep/payload.ts @@ -4,10 +4,16 @@ import path from "node:path"; import { $$ } from "@/scripts/shell.js"; import { download } from "@/scripts/tasks/features/download.js"; -import type { Artifact, FeatureConfig, VersionEntry } from "./generateConfig.js"; +import type { DockerArch } from "../architecture.js"; +import { isFile } from "../architecture.js"; +import { payloadPath } from "../paths.js"; +import type { Artifact, FeatureConfig, Target, VersionEntry } from "./generateConfig.js"; import { configPath, readConfig } from "./generateConfig.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; + +const targets: Record = { + amd64: "x86_64-unknown-linux-musl", + arm64: "aarch64-unknown-linux-musl", +}; const entries = [ "rg", @@ -45,18 +51,19 @@ export const sectionOf = (config: FeatureConfig, key: string): PayloadSection => export const loadSection = async (key: string): Promise => sectionOf(await readConfig(), key); export const derive = (section: PayloadSection, arch: DockerArch): Payload => { - const target = paths.targets[arch]; + const target = targets[arch]; + const cache = payloadPath("ripgrep", "ripgrep", section.key, arch); return { key: section.key, target, - cache: paths.payloadPath(section.key, arch), - present: path.join(paths.payloadPath(section.key, arch), "rg"), + cache, + present: path.join(cache, "rg"), top: `ripgrep-${section.key}-${target}`, archive: section.entry[target], }; }; -export const payloadPresent = (payload: Payload): Promise => paths.isFile(payload.present); +export const payloadPresent = (payload: Payload): Promise => isFile(payload.present); export const ensurePayload = async (section: PayloadSection, arch: DockerArch, force: boolean): Promise => { const payload = derive(section, arch); @@ -82,7 +89,7 @@ export const ensurePayload = async (section: PayloadSection, arch: DockerArch, f ...entries.map((entry) => `${payload.top}/${entry}`), ]}`; for (const entry of entries) { - if (!(await paths.isFile(path.join(extractedPath, entry)))) { + if (!(await isFile(path.join(extractedPath, entry)))) { throw new Error(`${payload.archive.url}: the archive has no ${entry}`); } } diff --git a/scripts/tasks/features/ripgrep/planPublish.ts b/scripts/tasks/features/ripgrep/planPublish.ts deleted file mode 100644 index 619e3a1..0000000 --- a/scripts/tasks/features/ripgrep/planPublish.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { parseArgs } from "node:util"; - -import { planPublish } from "@/scripts/tasks/features/planPublish.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { readConfig } from "./generateConfig.js"; -import { imageRef, tagNames } from "./tags.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - channel: { type: "string", default: mainChannel }, - out: { type: "string", default: ".ci/plan.json" }, - prefix: { type: "string" }, - }, - }); - - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const config = await readConfig(); - await planPublish( - Object.keys(config.versions).map((key) => { - const tags = tagNames(config, key, channel); - return { - label: key, - canonical: imageRef(prefix, tags.canonical), - tags: tags.all, - entry: { key }, - }; - }), - { channel, prefix, out: values.out }, - ); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/ripgrep/publishImage.ts b/scripts/tasks/features/ripgrep/publishImage.ts deleted file mode 100644 index e953e9d..0000000 --- a/scripts/tasks/features/ripgrep/publishImage.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { parseArgs } from "node:util"; - -import { pushImage } from "@/scripts/tasks/features/image.js"; -import { login } from "@/scripts/tasks/features/registry.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { assembleBuildContext } from "./buildContext.js"; -import { readConfig } from "./generateConfig.js"; -import { parseKey } from "./invocation.js"; -import { paths } from "./paths.js"; -import { ensurePayload, loadSection } from "./payload.js"; -import { imageRef, tagNames } from "./tags.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - version: { type: "string" }, - channel: { type: "string", default: mainChannel }, - prefix: { type: "string" }, - }, - }); - - const key = parseKey(values.version); - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const config = await readConfig(); - const canonical = tagNames(config, key, channel).canonical; - const section = await loadSection(key); - for (const arch of paths.dockerArchOptions) { - await ensurePayload(section, arch, false); - } - const context = await assembleBuildContext(key, paths.dockerArchOptions); - await login(prefix); - const refs = [imageRef(prefix, canonical)]; - await pushImage(context, refs); - console.log(refs.join("\n")); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/ripgrep/releaseTags.ts b/scripts/tasks/features/ripgrep/releaseTags.ts deleted file mode 100644 index 1ce5a9a..0000000 --- a/scripts/tasks/features/ripgrep/releaseTags.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { parseArgs } from "node:util"; - -import type { ReleaseEntry } from "@/scripts/tasks/features/release.js"; -import { releaseTags } from "@/scripts/tasks/features/release.js"; -import { tagPublished } from "@/scripts/tasks/features/tagPublished.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { readConfig } from "./generateConfig.js"; -import { imageRef, imageRefs, tagNames } from "./tags.js"; - -interface FeatureReleaseEntry extends ReleaseEntry { - gitTag: string; -} - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - channel: { type: "string", default: mainChannel }, - prefix: { type: "string" }, - "git-tags": { type: "boolean", default: false }, - }, - }); - - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const config = await readConfig(); - const entries: FeatureReleaseEntry[] = Object.keys(config.versions).map((key) => { - const tags = tagNames(config, key, channel); - return { - label: key, - canonical: imageRef(prefix, tags.canonical), - tags: imageRefs(prefix, tags.claims), - gitTag: `features-ripgrep-${tagNames(config, key, mainChannel).canonical}`, - }; - }); - await releaseTags(entries, prefix); - if (values["git-tags"]) { - for (const { gitTag } of entries) { - await tagPublished(gitTag); - } - } -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/ripgrep/renovate.ts b/scripts/tasks/features/ripgrep/renovate.ts deleted file mode 100644 index d803557..0000000 --- a/scripts/tasks/features/ripgrep/renovate.ts +++ /dev/null @@ -1,44 +0,0 @@ -import path from "node:path"; -import { parseArgs } from "node:util"; - -import { projectRoot } from "@/scripts/project.js"; -import type { ConfigDocument } from "@/scripts/tasks/features/renovate.js"; -import { refreshFeatureConfig } from "@/scripts/tasks/features/renovate.js"; - -import type { FeatureConfig } from "./generateConfig.js"; -import { configPath, generateConfig, parseConfig, readConfig } from "./generateConfig.js"; - -const document = (config: FeatureConfig): ConfigDocument => ({ - revision: config.revision, - namespaces: [{ name: "ripgrep", versions: config.versions, aliases: config.aliases }], -}); - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - "dry-run": { type: "boolean", default: false }, - }, - }); - - await refreshFeatureConfig( - { - branch: "renovate-ripgrep", - title: "Update ripgrep version pins", - filepath: path.relative(projectRoot, configPath), - generate: generateConfig, - read: readConfig, - parse: parseConfig, - document, - }, - values["dry-run"], - ); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/ripgrep/tags.ts b/scripts/tasks/features/ripgrep/tags.ts deleted file mode 100644 index d6de045..0000000 --- a/scripts/tasks/features/ripgrep/tags.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { TagSet, TagSource } from "@/scripts/tasks/features/tags.js"; -import { tagNames as baseTagNames } from "@/scripts/tasks/features/tags.js"; - -import type { FeatureConfig } from "./generateConfig.js"; - -const sourceOf = (config: FeatureConfig, key: string): TagSource => { - const entry = config.versions[key]; - if (entry === undefined) { - const available = Object.keys(config.versions).join(", "); - throw new Error(`Version ${key} is not tracked (available: ${available})`); - } - return { key, revision: config.revision, aliases: config.aliases[key] }; -}; - -export const tagNames = (config: FeatureConfig, key: string, channel: string): TagSet => - baseTagNames(sourceOf(config, key), channel); - -export const imageRef = (prefix: string, tag: string): string => `${prefix}/ripgrep:${tag}`; - -export const imageRefs = (prefix: string, tags: readonly string[]): string[] => - tags.map((tag) => imageRef(prefix, tag)); diff --git a/scripts/tasks/features/tags.ts b/scripts/tasks/features/tags.ts index 29a55e4..55aaaa7 100644 --- a/scripts/tasks/features/tags.ts +++ b/scripts/tasks/features/tags.ts @@ -1,5 +1,5 @@ import { getRemoteInfo } from "@/scripts/git.js"; -import { tagPattern } from "@/scripts/tasks/features/config.js"; +import { mainTagNames, tagPattern } from "@/scripts/tasks/features/config.js"; export const mainChannel = "main"; @@ -23,11 +23,25 @@ const channelSuffix = (channel: string): string => { return suffix; }; -const mainTagNames = (source: TagSource): readonly string[] => [ - `${source.key}-r${source.revision}`, - source.key, - ...source.aliases, -]; +export const deriveChannel = (ref: string, pullRequest: number | undefined): string => { + if (pullRequest !== undefined) { + return `pr-${pullRequest}`; + } + if (ref === "refs/heads/main") { + return mainChannel; + } + const branch = "refs/heads/"; + if (ref.startsWith(branch)) { + return `branch-${channelSuffix(ref.slice(branch.length))}`; + } + const tag = "refs/tags/"; + if (ref.startsWith(tag)) { + return `tag-${channelSuffix(ref.slice(tag.length))}`; + } + throw new Error( + `Cannot derive a channel from ref ${ref}: expected refs/heads/, refs/tags/ or a pull request number`, + ); +}; const channelTagNames = (names: readonly string[], channel: string): readonly string[] => { if (channel === mainChannel) { @@ -45,7 +59,7 @@ const channelTagNames = (names: readonly string[], channel: string): readonly st }; export const tagNames = (source: TagSource, channel: string): TagSet => { - const names = channelTagNames(mainTagNames(source), channel).map((tag) => { + const names = channelTagNames(mainTagNames(source.key, source.revision, source.aliases), channel).map((tag) => { if (!tagPattern.test(tag)) { throw new Error(`Channel ${channel}: derived tag ${tag} is not a valid registry tag`); } @@ -55,8 +69,6 @@ export const tagNames = (source: TagSource, channel: string): TagSet => { return { canonical, claims, all: names }; }; -export const canonicalTag = (source: TagSource, channel: string): string => tagNames(source, channel).canonical; - export const defaultRefPrefix = async (): Promise => { const { owner, repo } = await getRemoteInfo(); return `ghcr.io/${owner}/${repo}`; diff --git a/scripts/tasks/features/uv/buildContext.ts b/scripts/tasks/features/uv/buildContext.ts deleted file mode 100644 index 4b2efeb..0000000 --- a/scripts/tasks/features/uv/buildContext.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { copyFile, mkdir, rm } from "node:fs/promises"; -import path from "node:path"; - -import { projectRoot } from "@/scripts/project.js"; -import { linkTree } from "@/scripts/tasks/features/linkTree.js"; - -import type { Component } from "./generateConfig.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; - -export const assembleBuildContext = async ( - component: Component, - key: string, - arches: readonly DockerArch[], - featureBinaries: Partial>, -): Promise => { - const context = paths.contextPath(component, key); - await rm(context, { recursive: true, force: true }); - await mkdir(path.join(context, "docker"), { recursive: true }); - await copyFile(path.join(projectRoot, "features/uv/Dockerfile"), path.join(context, "docker", "Dockerfile")); - for (const arch of arches) { - const stagedRoot = path.join(context, "linux", arch, "features", "uv"); - if (component === "uv") { - const featureBinary = featureBinaries[arch]; - if (featureBinary === undefined) { - throw new Error(`No Feature binary for ${arch}`); - } - await mkdir(path.join(stagedRoot, "uv"), { recursive: true }); - await linkTree(paths.payloadPath(component, key, arch), path.join(stagedRoot, "uv")); - await copyFile(path.join(projectRoot, "features/uv/install"), path.join(stagedRoot, "install")); - await copyFile(path.join(projectRoot, "features/uv/onCreate"), path.join(stagedRoot, "onCreate")); - await copyFile(featureBinary, path.join(stagedRoot, "Feature")); - } else { - await mkdir(path.join(stagedRoot, "python"), { recursive: true }); - await linkTree(paths.payloadPath(component, key, arch), path.join(stagedRoot, "python")); - } - } - return context; -}; diff --git a/scripts/tasks/features/uv/buildImage.ts b/scripts/tasks/features/uv/buildImage.ts deleted file mode 100644 index ad9ccd7..0000000 --- a/scripts/tasks/features/uv/buildImage.ts +++ /dev/null @@ -1,70 +0,0 @@ -import path from "node:path"; -import { parseArgs } from "node:util"; - -import { $$ } from "@/scripts/shell.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { assembleBuildContext } from "./buildContext.js"; -import { readConfig } from "./generateConfig.js"; -import { parseInvocation } from "./invocation.js"; -import { ensureFeatureBinary } from "./packBinary.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; -import { derive, payloadPresent, sectionOf } from "./payload.js"; -import { imageRefs, tagNames } from "./tags.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - component: { type: "string" }, - version: { type: "string" }, - arch: { type: "string" }, - tag: { type: "string", multiple: true }, - channel: { type: "string", default: mainChannel }, - prefix: { type: "string" }, - force: { type: "boolean", default: false }, - }, - }); - - const { component, key, arch } = parseInvocation(values); - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - if (arch !== paths.hostArch()) { - throw new Error( - `--arch ${arch} does not match the host architecture ${paths.hostArch()}; local image builds build the current platform only`, - ); - } - - const featureBinaries: Partial> = {}; - if (component === "uv") { - featureBinaries[arch] = await ensureFeatureBinary(paths.ridFor(arch), values.force); - } - const config = await readConfig(); - const payload = derive(sectionOf(config, component, key), arch); - if (!(await payloadPresent(payload))) { - throw new Error( - `No payload at ${payload.cache}; run scripts/tasks/features/uv/fetchPayload.ts --component ${component} --version ${key} --arch ${arch} first`, - ); - } - - const context = await assembleBuildContext(component, key, [arch], featureBinaries); - const tags = tagNames(config, component, key, channel); - const refs = [...imageRefs(prefix, component, tags.all), ...(values.tag ?? [])]; - await $$`docker buildx build ${[ - ...["--platform", `linux/${arch}`], - "--load", - ...["--file", path.join(context, "docker", "Dockerfile")], - ...refs.flatMap((ref) => ["--tag", ref]), - context, - ]}`; - console.log(refs.join("\n")); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/uv/feature.ts b/scripts/tasks/features/uv/feature.ts new file mode 100644 index 0000000..b1f63ab --- /dev/null +++ b/scripts/tasks/features/uv/feature.ts @@ -0,0 +1,42 @@ +import type { ConfigDocument } from "@/scripts/tasks/features/config.js"; +import type { Feature, FeatureStage } from "@/scripts/tasks/features/feature.js"; + +import type { Component, FeatureConfig } from "./generateConfig.js"; +import { components, componentSchema, configPath, generateConfig, parseConfig, readConfig } from "./generateConfig.js"; +import { ensureFeatureBinary } from "./packBinary.js"; +import { derive, ensurePayload, payloadPresent, sectionOf } from "./payload.js"; + +const repositories: Record = { uv: "uv", python: "uv/python" }; + +const stages: Record = { + uv: { subdir: ["uv"], files: ["install", "onCreate"], binary: true }, + python: { subdir: ["python"], files: [], binary: false }, +}; + +export const documentOf = (config: FeatureConfig): ConfigDocument => ({ + revision: config.revision, + namespaces: components.map((component) => ({ + name: component, + versions: config.components[component].versions, + aliases: config.components[component].aliases, + })), +}); + +export const uv: Feature = { + id: "uv", + configPath, + components, + document: async () => documentOf(await readConfig()), + parseDocument: (text, source) => documentOf(parseConfig(text, source)), + generateConfig, + repository: (component) => repositories[componentSchema.parse(component)], + stage: (component) => stages[componentSchema.parse(component)], + payload: { + ensure: async (component, key, arch, force) => + ensurePayload(sectionOf(await readConfig(), componentSchema.parse(component), key), arch, force), + present: async (component, key, arch) => + payloadPresent(derive(sectionOf(await readConfig(), componentSchema.parse(component), key), arch)), + }, + binary: { ensure: ensureFeatureBinary }, + gitTag: (component, canonical) => `features-uv-${componentSchema.parse(component)}-${canonical}`, +}; diff --git a/scripts/tasks/features/uv/fetchPayload.ts b/scripts/tasks/features/uv/fetchPayload.ts deleted file mode 100644 index cfe297c..0000000 --- a/scripts/tasks/features/uv/fetchPayload.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { parseArgs } from "node:util"; - -import { parseInvocation } from "./invocation.js"; -import { ensurePayload, loadSection } from "./payload.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - component: { type: "string" }, - version: { type: "string" }, - arch: { type: "string" }, - force: { type: "boolean", default: false }, - }, - }); - - const { component, key, arch } = parseInvocation(values); - await ensurePayload(await loadSection(component, key), arch, values.force); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/uv/generateConfig.ts b/scripts/tasks/features/uv/generateConfig.ts index 9b93d1b..d66e6ac 100644 --- a/scripts/tasks/features/uv/generateConfig.ts +++ b/scripts/tasks/features/uv/generateConfig.ts @@ -8,9 +8,10 @@ import prettierOptions from "@/.config/prettier/.prettierrc.json" with { type: " import { projectRoot } from "@/scripts/project.js"; import { checkTagNamespace } from "@/scripts/tasks/features/config.js"; +import type { DockerArch, Rid } from "../architecture.js"; +import { dockerArchOptions, ridFor, rids } from "../architecture.js"; import { verifyAttestation } from "./attestations.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; +import { gnuTriples, muslTriples } from "./paths.js"; const uvIndexUrl = "https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson"; const pythonIndexUrl = "https://raw.githubusercontent.com/astral-sh/versions/main/v1/python-build-standalone.ndjson"; @@ -24,8 +25,7 @@ export const configPath = path.resolve(projectRoot, "features/uv/config.json"); export const components = ["uv", "python"] as const; export type Component = (typeof components)[number]; -export const rids = ["linux-x64", "linux-arm64"] as const; -export type Rid = (typeof rids)[number]; +export const componentSchema = z.enum(components); const indexArtifactSchema = z.object({ platform: z.string(), @@ -67,7 +67,7 @@ const pythonComponentSchema = z.strictObject({ }); export const featureConfigSchema = z.strictObject({ - revision: z.number(), + revision: z.int().min(1), components: z.strictObject({ uv: uvComponentSchema, python: pythonComponentSchema, @@ -110,7 +110,7 @@ const resolveArtifacts = async ( variant: string, ): Promise> => { const resolved = Object.fromEntries( - paths.dockerArchOptions.map((arch) => { + dockerArchOptions.map((arch) => { const platform = platforms[arch]; const matches = artifacts.filter( (artifact) => @@ -120,7 +120,7 @@ const resolveArtifacts = async ( throw new Error(`Release ${version}: expected one ${platform} ${variant} archive, found ${matches.length}`); } const [artifact] = matches; - return [paths.ridFor(arch), { url: artifact.url, sha256: artifact.sha256 }] as const; + return [ridFor(arch), { url: artifact.url, sha256: artifact.sha256 }] as const; }), ) as Record; for (const rid of rids) { @@ -129,12 +129,28 @@ const resolveArtifacts = async ( return resolved; }; +const compareVersions = (left: string, right: string): number => { + const leftParts = left.split(".").map(Number); + const rightParts = right.split(".").map(Number); + for (const [index, part] of leftParts.entries()) { + if (part !== rightParts[index]) { + return part - rightParts[index]; + } + } + return 0; +}; + const trackedUvRelease = (releases: readonly IndexRelease[]): IndexRelease => { - const release = releases.find((candidate) => stablePattern.test(candidate.version)); - if (release === undefined) { + const stable = releases.filter((release) => stablePattern.test(release.version)); + const tracked = stable.reduce( + (greatest, release) => + greatest === undefined || compareVersions(release.version, greatest.version) > 0 ? release : greatest, + undefined, + ); + if (tracked === undefined) { throw new Error("The uv version index carries no stable release"); } - return release; + return tracked; }; const pythonReleaseParts = (release: IndexRelease): { version: string; build: string } => { @@ -163,17 +179,6 @@ const trackedPythonReleases = (releases: readonly IndexRelease[]): { build: stri return { build, entries }; }; -const compareVersions = (left: string, right: string): number => { - const leftParts = left.split(".").map(Number); - const rightParts = right.split(".").map(Number); - for (const [index, part] of leftParts.entries()) { - if (part !== rightParts[index]) { - return part - rightParts[index]; - } - } - return 0; -}; - const pythonVersions = async (releases: readonly IndexRelease[]): Promise => { const { build, entries } = trackedPythonReleases(releases); const resolved = await Promise.all( @@ -184,7 +189,7 @@ const pythonVersions = async (releases: readonly IndexRelease[]): Promise => { uvSlug, uvRelease.version, uvRelease.artifacts, - paths.muslTriples, + muslTriples, "default", ), }, @@ -274,12 +279,3 @@ export const generateConfig = async (): Promise => { await prettier.format(JSON.stringify(config), { ...prettierOptions, filepath: configPath }), ); }; - -if (import.meta.main) { - try { - await generateConfig(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/uv/invocation.ts b/scripts/tasks/features/uv/invocation.ts deleted file mode 100644 index 4766be0..0000000 --- a/scripts/tasks/features/uv/invocation.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { Component } from "./generateConfig.js"; -import { components } from "./generateConfig.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; - -export interface Invocation { - component: Component; - key: string; - arch: DockerArch; -} - -export const parseComponent = (value: string | undefined): Component => { - const component = components.find((candidate) => candidate === value); - if (component === undefined) { - throw new Error(`--component must be one of: ${components.join(", ")}`); - } - return component; -}; - -export const parseKey = (value: string | undefined): string => { - if (value === undefined || value === "") { - throw new Error("--version is required"); - } - return value; -}; - -export const parseInvocation = (values: { component?: string; version?: string; arch?: string }): Invocation => { - const component = parseComponent(values.component); - const key = parseKey(values.version); - const arch = - values.arch === undefined - ? paths.hostArch() - : paths.dockerArchOptions.find((candidate) => candidate === values.arch); - if (arch === undefined) { - throw new Error(`--arch must be one of: ${paths.dockerArchOptions.join(", ")}`); - } - return { component, key, arch }; -}; diff --git a/scripts/tasks/features/uv/packBinary.ts b/scripts/tasks/features/uv/packBinary.ts index 90ee515..d243ff2 100644 --- a/scripts/tasks/features/uv/packBinary.ts +++ b/scripts/tasks/features/uv/packBinary.ts @@ -2,21 +2,22 @@ import path from "node:path"; import { project$$ } from "@/scripts/shell.js"; -import type { Rid } from "./generateConfig.js"; -import { paths } from "./paths.js"; +import type { Rid } from "../architecture.js"; +import { isFile } from "../architecture.js"; +import { packPath } from "./paths.js"; export const ensureFeatureBinary = async (rid: Rid, force: boolean): Promise => { - const binaryPath = path.join(paths.packPath(rid), "Feature"); - if (!force && (await paths.isFile(binaryPath))) { + const binaryPath = path.join(packPath(rid), "Feature"); + if (!force && (await isFile(binaryPath))) { return binaryPath; } await project$$`dotnet publish ${[ ...["--configuration", "Release"], ...["--runtime", rid], "features/uv/Feature/Feature.csproj", - `-p:PublishDir=${paths.packPath(rid)}/`, + `-p:PublishDir=${packPath(rid)}/`, ]}`; - if (!(await paths.isFile(binaryPath))) { + if (!(await isFile(binaryPath))) { throw new Error(`dotnet publish did not produce ${binaryPath}`); } return binaryPath; diff --git a/scripts/tasks/features/uv/packFeature.ts b/scripts/tasks/features/uv/packFeature.ts deleted file mode 100644 index a453b53..0000000 --- a/scripts/tasks/features/uv/packFeature.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { mkdir } from "node:fs/promises"; -import path from "node:path"; -import { parseArgs } from "node:util"; - -import { createTar } from "@/scripts/tasks/features/tar.js"; - -import { rids } from "./generateConfig.js"; -import { ensureFeatureBinary } from "./packBinary.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - rid: { type: "string" }, - out: { type: "string" }, - force: { type: "boolean", default: false }, - }, - }); - - const rid = rids.find((candidate) => candidate === values.rid); - if (rid === undefined) { - throw new Error(`--rid must be one of: ${rids.join(", ")}`); - } - if (values.out === undefined || values.out === "") { - throw new Error("--out is required"); - } - const binary = await ensureFeatureBinary(rid, values.force); - const out = path.resolve(values.out); - const archive = path.join(out, `${rid}.tar`); - await mkdir(out, { recursive: true }); - await createTar(archive, path.dirname(binary), [path.basename(binary)]); - console.log(`Feature binary archive written to ${archive}`); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/uv/paths.ts b/scripts/tasks/features/uv/paths.ts index 069f824..b83c33a 100644 --- a/scripts/tasks/features/uv/paths.ts +++ b/scripts/tasks/features/uv/paths.ts @@ -1,73 +1,21 @@ -import { stat } from "node:fs/promises"; import path from "node:path"; import { workspaces } from "@/scripts/project.js"; -import type { Component, Rid } from "./generateConfig.js"; +import type { DockerArch, Rid } from "../architecture.js"; -const dockerArchOptions = ["amd64", "arm64"] as const; -export type DockerArch = (typeof dockerArchOptions)[number]; - -const hostArch = (): DockerArch => { - switch (process.arch) { - case "x64": - return "amd64"; - case "arm64": - return "arm64"; - default: - throw new Error(`Unsupported host architecture: ${process.arch}`); - } -}; - -const rids: Record = { amd64: "linux-x64", arm64: "linux-arm64" }; - -const muslTriples: Record = { +export const muslTriples: Record = { amd64: "x86_64-unknown-linux-musl", arm64: "aarch64-unknown-linux-musl", }; -const gnuTriples: Record = { +export const gnuTriples: Record = { amd64: "x86_64-unknown-linux-gnu", arm64: "aarch64-unknown-linux-gnu", }; const machines: Record = { amd64: "x86_64", arm64: "aarch64" }; -const ridFor = (arch: DockerArch): Rid => rids[arch]; - -const machineOf = (arch: DockerArch): string => machines[arch]; - -const payloadRoot = path.resolve(workspaces, "payload/uv"); - -const payloadPath = (component: Component, version: string, arch: DockerArch): string => - path.join(payloadRoot, component, version, arch); - -const contextPath = (component: Component, version: string): string => - path.join(payloadRoot, "ctx", component, version); - -const packPath = (rid: Rid): string => path.join(workspaces, "bin/uv/Feature", rid); - -const isFile = async (candidate: string): Promise => { - try { - return (await stat(candidate)).isFile(); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return false; - } - throw error; - } -}; +export const machineOf = (arch: DockerArch): string => machines[arch]; -export const paths = { - dockerArchOptions, - hostArch, - ridFor, - machineOf, - muslTriples, - gnuTriples, - payloadRoot, - payloadPath, - contextPath, - packPath, - isFile, -} as const; +export const packPath = (rid: Rid): string => path.join(workspaces, "bin/uv/Feature", rid); diff --git a/scripts/tasks/features/uv/payload.ts b/scripts/tasks/features/uv/payload.ts index 06b9c9d..ae28b2e 100644 --- a/scripts/tasks/features/uv/payload.ts +++ b/scripts/tasks/features/uv/payload.ts @@ -4,10 +4,12 @@ import path from "node:path"; import { $$ } from "@/scripts/shell.js"; import { download } from "@/scripts/tasks/features/download.js"; +import type { DockerArch } from "../architecture.js"; +import { isFile, ridFor } from "../architecture.js"; +import { payloadPath } from "../paths.js"; import type { Artifact, Component, FeatureConfig, PythonVersion, UvVersion } from "./generateConfig.js"; import { configPath, readConfig } from "./generateConfig.js"; -import type { DockerArch } from "./paths.js"; -import { paths } from "./paths.js"; +import { machineOf, muslTriples } from "./paths.js"; const pythonUrlPrefix = "https://github.com/astral-sh/python-build-standalone/releases/download/"; const uvBinaries = ["uv", "uvx"] as const; @@ -106,7 +108,7 @@ export const loadSection = async ( ): Promise => sectionOf(await readConfig(), component, key); const pythonArchive = (section: PythonSection, arch: DockerArch): PythonArchive => { - const { url, sha256 } = section.entry[paths.ridFor(arch)]; + const { url, sha256 } = section.entry[ridFor(arch)]; if (!url.startsWith(pythonUrlPrefix)) { throw new Error(`${url}: a python archive url must start with ${pythonUrlPrefix}`); } @@ -130,7 +132,7 @@ const pythonEntry = ( throw new Error(`Version ${section.key} is not ..`); } const [, major, minor, patch] = match; - const machine = paths.machineOf(arch); + const machine = machineOf(arch); const key = `cpython-${section.entry.pythonVersion}-linux-${machine}-gnu`; const entry: PythonDownloadEntry = { name: "cpython", @@ -149,16 +151,19 @@ const pythonEntry = ( return { key, entry }; }; -export const derive = (section: PayloadSection, arch: DockerArch): Payload => { - const cache = paths.payloadPath(section.component, section.key, arch); +export function derive(section: UvSection, arch: DockerArch): UvPayload; +export function derive(section: PythonSection, arch: DockerArch): PythonPayload; +export function derive(section: PayloadSection, arch: DockerArch): Payload; +export function derive(section: PayloadSection, arch: DockerArch): Payload { + const cache = payloadPath("uv", section.component, section.key, arch); if (section.component === "uv") { return { component: "uv", key: section.key, cache, present: path.join(cache, "uv"), - triple: paths.muslTriples[arch], - archive: section.entry[paths.ridFor(arch)], + triple: muslTriples[arch], + archive: section.entry[ridFor(arch)], }; } const archive = pythonArchive(section, arch); @@ -177,29 +182,9 @@ export const derive = (section: PayloadSection, arch: DockerArch): Payload => { entryPath: path.join(cache, "entries", `${entryKey}.json`), entry, }; -}; - -export const payloadPresent = (payload: Payload): Promise => paths.isFile(payload.present); - -export interface PythonPayloadPaths { - archive: string; - entry: string; - build: string; - name: string; } -export const pythonPayloadPaths = (section: PythonSection, arch: DockerArch): PythonPayloadPaths => { - const payload = derive(section, arch); - if (payload.component !== "python") { - throw new Error(`Version ${section.key} is not a python payload`); - } - return { - archive: payload.archivePath, - entry: payload.entryPath, - build: payload.build, - name: payload.archive.name, - }; -}; +export const payloadPresent = (payload: Payload): Promise => isFile(payload.present); const fetchUvPayload = async (payload: UvPayload): Promise => { const tempPath = `${payload.cache}.tmp`; @@ -221,7 +206,7 @@ const fetchUvPayload = async (payload: UvPayload): Promise => { ...uvBinaries.map((binary) => `${top}/${binary}`), ]}`; for (const binary of uvBinaries) { - if (!(await paths.isFile(path.join(extractedPath, binary)))) { + if (!(await isFile(path.join(extractedPath, binary)))) { throw new Error(`${payload.archive.url}: the archive has no ${binary}`); } } diff --git a/scripts/tasks/features/uv/planPublish.ts b/scripts/tasks/features/uv/planPublish.ts deleted file mode 100644 index c0d292f..0000000 --- a/scripts/tasks/features/uv/planPublish.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { parseArgs } from "node:util"; - -import { planPublish } from "@/scripts/tasks/features/planPublish.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { components, readConfig } from "./generateConfig.js"; -import { imageRef, tagNames } from "./tags.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - channel: { type: "string", default: mainChannel }, - out: { type: "string", default: ".ci/plan.json" }, - prefix: { type: "string" }, - }, - }); - - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const config = await readConfig(); - const pairs = components.flatMap((component) => - Object.keys(config.components[component].versions).map((key) => ({ component, key })), - ); - await planPublish( - pairs.map(({ component, key }) => { - const tags = tagNames(config, component, key, channel); - return { - label: `${component} ${key}`, - canonical: imageRef(prefix, component, tags.canonical), - tags: tags.all, - entry: { component, key }, - }; - }), - { channel, prefix, out: values.out }, - ); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/uv/publishImage.ts b/scripts/tasks/features/uv/publishImage.ts deleted file mode 100644 index 86da99b..0000000 --- a/scripts/tasks/features/uv/publishImage.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { parseArgs } from "node:util"; - -import { pushImage } from "@/scripts/tasks/features/image.js"; -import { login } from "@/scripts/tasks/features/registry.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { assembleBuildContext } from "./buildContext.js"; -import { resolveFeatureBinaries } from "./featureBinary.js"; -import { readConfig } from "./generateConfig.js"; -import { parseComponent, parseKey } from "./invocation.js"; -import { paths } from "./paths.js"; -import { ensurePayload, sectionOf } from "./payload.js"; -import { imageRef, tagNames } from "./tags.js"; - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - component: { type: "string" }, - version: { type: "string" }, - channel: { type: "string", default: mainChannel }, - "install-dir": { type: "string" }, - prefix: { type: "string" }, - }, - }); - - const component = parseComponent(values.component); - const key = parseKey(values.version); - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const config = await readConfig(); - const section = sectionOf(config, component, key); - const canonical = tagNames(config, component, key, channel).canonical; - const featureBinaries = component === "uv" ? await resolveFeatureBinaries(values["install-dir"]) : {}; - for (const arch of paths.dockerArchOptions) { - await ensurePayload(section, arch, false); - } - const context = await assembleBuildContext(component, key, paths.dockerArchOptions, featureBinaries); - await login(prefix); - const refs = [imageRef(prefix, component, canonical)]; - await pushImage(context, refs); - console.log(refs.join("\n")); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/uv/releaseTags.ts b/scripts/tasks/features/uv/releaseTags.ts deleted file mode 100644 index 6cc080a..0000000 --- a/scripts/tasks/features/uv/releaseTags.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { parseArgs } from "node:util"; - -import type { ReleaseEntry } from "@/scripts/tasks/features/release.js"; -import { releaseTags } from "@/scripts/tasks/features/release.js"; -import { tagPublished } from "@/scripts/tasks/features/tagPublished.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; - -import { components, readConfig } from "./generateConfig.js"; -import { imageRef, imageRefs, tagNames } from "./tags.js"; - -interface FeatureReleaseEntry extends ReleaseEntry { - gitTag: string; -} - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - channel: { type: "string", default: mainChannel }, - prefix: { type: "string" }, - "git-tags": { type: "boolean", default: false }, - }, - }); - - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const config = await readConfig(); - const entries: FeatureReleaseEntry[] = components.flatMap((component) => - Object.keys(config.components[component].versions).map((key) => { - const tags = tagNames(config, component, key, channel); - return { - label: `${component} ${key}`, - canonical: imageRef(prefix, component, tags.canonical), - tags: imageRefs(prefix, component, tags.claims), - gitTag: `features-uv-${component}-${tagNames(config, component, key, mainChannel).canonical}`, - }; - }), - ); - await releaseTags(entries, prefix); - if (values["git-tags"]) { - for (const { gitTag } of entries) { - await tagPublished(gitTag); - } - } -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/uv/renovate.ts b/scripts/tasks/features/uv/renovate.ts deleted file mode 100644 index cf4cbca..0000000 --- a/scripts/tasks/features/uv/renovate.ts +++ /dev/null @@ -1,48 +0,0 @@ -import path from "node:path"; -import { parseArgs } from "node:util"; - -import { projectRoot } from "@/scripts/project.js"; -import type { ConfigDocument } from "@/scripts/tasks/features/renovate.js"; -import { refreshFeatureConfig } from "@/scripts/tasks/features/renovate.js"; - -import type { FeatureConfig } from "./generateConfig.js"; -import { components, configPath, generateConfig, parseConfig, readConfig } from "./generateConfig.js"; - -const document = (config: FeatureConfig): ConfigDocument => ({ - revision: config.revision, - namespaces: components.map((component) => ({ - name: component, - versions: config.components[component].versions, - aliases: config.components[component].aliases, - })), -}); - -const main = async (): Promise => { - const { values } = parseArgs({ - options: { - "dry-run": { type: "boolean", default: false }, - }, - }); - - await refreshFeatureConfig( - { - branch: "renovate-uv", - title: "Update uv version pins", - filepath: path.relative(projectRoot, configPath), - generate: generateConfig, - read: readConfig, - parse: parseConfig, - document, - }, - values["dry-run"], - ); -}; - -if (import.meta.main) { - try { - await main(); - } catch (error) { - console.error(error); - process.exitCode = 1; - } -} diff --git a/scripts/tasks/features/uv/tags.ts b/scripts/tasks/features/uv/tags.ts deleted file mode 100644 index a70306f..0000000 --- a/scripts/tasks/features/uv/tags.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { TagSet, TagSource } from "@/scripts/tasks/features/tags.js"; -import { tagNames as baseTagNames } from "@/scripts/tasks/features/tags.js"; - -import type { Component, FeatureConfig } from "./generateConfig.js"; - -const repositories: Record = { uv: "uv", python: "uv/python" }; - -const sourceOf = (config: FeatureConfig, component: Component, key: string): TagSource => { - const entry = config.components[component].versions[key]; - if (entry === undefined) { - const available = Object.keys(config.components[component].versions).join(", "); - throw new Error(`Component ${component} has no version ${key} (available: ${available})`); - } - return { key, revision: config.revision, aliases: config.components[component].aliases[key] }; -}; - -export const tagNames = (config: FeatureConfig, component: Component, key: string, channel: string): TagSet => - baseTagNames(sourceOf(config, component, key), channel); - -export const imageRef = (prefix: string, component: Component, tag: string): string => - `${prefix}/${repositories[component]}:${tag}`; - -export const imageRefs = (prefix: string, component: Component, tags: readonly string[]): string[] => - tags.map((tag) => imageRef(prefix, component, tag)); diff --git a/scripts/verify.ts b/scripts/verify.ts index 8496354..d751cff 100644 --- a/scripts/verify.ts +++ b/scripts/verify.ts @@ -1,6 +1,8 @@ import { readdir } from "node:fs/promises"; import path from "node:path"; +import { z } from "zod"; + import { projectRoot } from "@/scripts/project.js"; const main = async (): Promise => { @@ -10,15 +12,14 @@ const main = async (): Promise => { .filter((entry) => entry.isDirectory()) .map((entry) => entry.name) .sort(); - if (name === undefined || !names.includes(name)) { - throw new Error(`First argument must be a feature name (one of: ${names.join(", ")})`); - } + const featureSchema = z.enum(names as [string, ...string[]]); + const feature = featureSchema.parse(name); - const { verify } = (await import(`@/scripts/verify/${name}/verify.js`)) as { + const { verify } = (await import(`@/scripts/verify/${feature}/verify.js`)) as { verify?: (argv: string[]) => Promise; }; if (verify === undefined) { - throw new Error(`scripts/verify/${name}/verify.ts does not export verify(argv: string[])`); + throw new Error(`scripts/verify/${feature}/verify.ts does not export verify(argv: string[])`); } await verify(rest); }; diff --git a/scripts/verify/arguments.ts b/scripts/verify/arguments.ts new file mode 100644 index 0000000..f998355 --- /dev/null +++ b/scripts/verify/arguments.ts @@ -0,0 +1,88 @@ +import { parseArgs } from "node:util"; + +import { z } from "zod"; + +import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; + +export const imagesModes = ["build", "pull"] as const; +export type ImagesMode = (typeof imagesModes)[number]; + +export const imagesSchema = z.enum(imagesModes); + +export const verifyOptions = { + force: { type: "boolean", default: false }, + rmi: { type: "boolean", default: false }, + scenario: { type: "string", multiple: true }, + component: { type: "string" }, + version: { type: "string" }, + images: { type: "string", default: "build" }, + channel: { type: "string", default: mainChannel }, + prefix: { type: "string" }, +} as const; + +export interface VerifyOptions { + images: ImagesMode; + channel: string; + prefix: string; + force: boolean; + rmi: boolean; +} + +export interface VerifyArguments { + options: VerifyOptions; + scenarios: readonly string[]; + component: string | undefined; + version: string | undefined; +} + +export const parseVerifyArguments = async (argv: string[]): Promise => { + const { values } = parseArgs({ args: argv, options: verifyOptions }); + return { + options: { + images: imagesSchema.parse(values.images), + channel: values.channel, + prefix: values.prefix ?? (await defaultRefPrefix()), + force: values.force, + rmi: values.rmi, + }, + scenarios: values.scenario ?? [], + component: values.component, + version: values.version, + }; +}; + +export const selectionSchema = ( + nameSchema: z.ZodType, + names: readonly Name[], + defaultSelection: readonly Name[], +) => + z + .array(nameSchema) + .default([]) + .transform((selected) => + selected.length === 0 ? defaultSelection : names.filter((name) => selected.includes(name)), + ); + +export const targetSchema = (componentSchema: z.ZodType) => + z + .object({ + component: componentSchema.optional(), + version: z.string().min(1).optional(), + }) + .refine((value) => (value.component === undefined) === (value.version === undefined), { + message: "--component and --version must be provided together", + }) + .transform((value) => + value.component === undefined || value.version === undefined + ? undefined + : { component: value.component, key: value.version }, + ); + +export const requireSmokeTarget = ( + scenarios: readonly string[], + target: { component: Component; key: string } | undefined, +): void => { + if (target === undefined && scenarios.includes("smoke")) { + throw new Error("--scenario smoke requires --component and --version"); + } +}; diff --git a/scripts/verify/uv/assertions.ts b/scripts/verify/assertions.ts similarity index 71% rename from scripts/verify/uv/assertions.ts rename to scripts/verify/assertions.ts index 3456148..22a8a82 100644 --- a/scripts/verify/uv/assertions.ts +++ b/scripts/verify/assertions.ts @@ -1,8 +1,9 @@ import type { CommandResult } from "./lifecycle.js"; export interface Expectation { - stdout: readonly string[]; + stdout?: readonly string[]; sorted?: boolean; + exitCode?: number; } const linesOf = (text: string): string[] => (text.trim() === "" ? [] : text.trimEnd().split("\n")); @@ -10,16 +11,17 @@ const linesOf = (text: string): string[] => (text.trim() === "" ? [] : text.trim const sameLines = (actual: readonly string[], expected: readonly string[]): boolean => actual.length === expected.length && actual.every((line, index) => line === expected[index]); -const assertExec = (result: CommandResult, expectation: Expectation): void => { +export const assertExec = (result: CommandResult, expectation: Expectation): void => { const stdout = linesOf(result.stdout); const sorted = expectation.sorted === true; const order = (lines: readonly string[]): string[] => (sorted ? [...lines].sort() : [...lines]); + const exitCode = expectation.exitCode ?? 0; const problems: string[] = []; - if (result.exitCode !== 0) { - problems.push(`exit code ${result.exitCode ?? "none"}`); + if (result.exitCode !== exitCode) { + problems.push(`exit code ${result.exitCode ?? "none"}, expected ${exitCode}`); } - if (!sameLines(order(stdout), order(expectation.stdout))) { + if (expectation.stdout !== undefined && !sameLines(order(stdout), order(expectation.stdout))) { problems.push(`stdout ${JSON.stringify(stdout)}, expected ${JSON.stringify(expectation.stdout)}`); } if (result.stderr.trim() !== "") { diff --git a/scripts/verify/dotnet/assertions.ts b/scripts/verify/dotnet/assertions.ts deleted file mode 100644 index 3456148..0000000 --- a/scripts/verify/dotnet/assertions.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { CommandResult } from "./lifecycle.js"; - -export interface Expectation { - stdout: readonly string[]; - sorted?: boolean; -} - -const linesOf = (text: string): string[] => (text.trim() === "" ? [] : text.trimEnd().split("\n")); - -const sameLines = (actual: readonly string[], expected: readonly string[]): boolean => - actual.length === expected.length && actual.every((line, index) => line === expected[index]); - -const assertExec = (result: CommandResult, expectation: Expectation): void => { - const stdout = linesOf(result.stdout); - const sorted = expectation.sorted === true; - const order = (lines: readonly string[]): string[] => (sorted ? [...lines].sort() : [...lines]); - - const problems: string[] = []; - if (result.exitCode !== 0) { - problems.push(`exit code ${result.exitCode ?? "none"}`); - } - if (!sameLines(order(stdout), order(expectation.stdout))) { - problems.push(`stdout ${JSON.stringify(stdout)}, expected ${JSON.stringify(expectation.stdout)}`); - } - if (result.stderr.trim() !== "") { - problems.push(`stderr ${JSON.stringify(result.stderr)}`); - } - if (problems.length === 0) { - return; - } - throw new Error(`devcontainer exec ${result.command.join(" ")}:\n${problems.join("\n")}`); -}; - -export const assertions = { exec: assertExec } as const; diff --git a/scripts/verify/dotnet/scenarios.ts b/scripts/verify/dotnet/scenarios.ts index 7a13508..d265b08 100644 --- a/scripts/verify/dotnet/scenarios.ts +++ b/scripts/verify/dotnet/scenarios.ts @@ -1,31 +1,31 @@ // spell-checker:ignore compinit import path from "node:path"; +import { z } from "zod"; + import type { Component } from "@/scripts/tasks/features/dotnet/generateConfig.js"; -import type { Expectation } from "./assertions.js"; -import type { TestImage, Workspace } from "./lifecycle.js"; -import { lifecycle } from "./lifecycle.js"; -import type { Selection } from "./selection.js"; -import type { Serve } from "./serve.js"; +import { selectionSchema } from "../arguments.js"; +import type { ComponentImage, TestImage, Workspace } from "../lifecycle.js"; +import { lifecycle } from "../lifecycle.js"; +import type { Exec, Scenario, ScenarioVariant } from "../run.js"; +import type { Serve } from "../serve.js"; +import type { Selection, SelectionTarget } from "./selection.js"; +import { staging } from "./staging.js"; -const names = ["sdk", "runtime", "aspnet", "multi"] as const; -export type ScenarioName = (typeof names)[number]; +const names = ["sdk", "runtime", "aspnet", "multi", "smoke"] as const; +type ScenarioName = (typeof names)[number]; +type ImageScenarioName = Exclude; -const appNames = ["console", "web"] as const; -export type AppName = (typeof appNames)[number]; +export const scenarioSelectionSchema = selectionSchema( + z.enum(names), + names, + names.filter((name) => name !== "smoke"), +); -export interface Exec { - command: readonly string[]; - env?: Readonly>; - expect: Expectation; -} +type AppName = "console" | "web"; -export interface ScenarioContext { - selection: Selection; - root: string; - workspace: Workspace; -} +const testImagePrefix = "features-verify-dotnet"; interface ScenarioDefinition { components: readonly Component[]; @@ -41,26 +41,12 @@ type ScenarioVariantDefinition = { skipWorkloadIntegrityCheck?: boolean; } & ({ execs: readonly Exec[] } | { buildFailure: string }); -export type ScenarioVariant = { - name: string; - workspace: Workspace; - testImage: TestImage; - skipWorkloadIntegrityCheck?: boolean; -} & ({ execs: readonly Exec[] } | { buildFailure: string }); - -export interface Scenario extends ScenarioDefinition { - name: ScenarioName; +interface ScenarioContext { + selection: Selection; + target: SelectionTarget | undefined; + root: string; workspace: Workspace; - testImage: TestImage; - execs: readonly Exec[]; - remoteEnv?: Readonly>; - serve?: Serve; - variants: readonly ScenarioVariant[]; -} - -export interface BuildStage { - testImage: TestImage; - publishes: readonly Exec[]; + build: Workspace | undefined; } interface ScenarioBehavior { @@ -70,15 +56,47 @@ interface ScenarioBehavior { variants?: readonly ScenarioVariantDefinition[]; } -const testImagePrefix = "features-verify-dotnet"; - -const definitions: Record = { +const definitions: Record = { sdk: { components: ["sdk"], assets: ["hello.cs", "tool"], directories: ["pushed"] }, runtime: { components: ["runtime"], assets: [], directories: [], app: "console" }, aspnet: { components: ["aspnet"], assets: [], directories: [], app: "web" }, multi: { components: ["sdk", "runtime", "aspnet"], assets: [], directories: [] }, }; +const remoteEnvOf = ( + skipWorkloadIntegrityCheck = true, + extra: Readonly> = {}, +): Readonly> => ({ + DOTNET_NOLOGO: "1", + ...(skipWorkloadIntegrityCheck ? { DOTNET_SKIP_WORKLOAD_INTEGRITY_CHECK: "1" } : {}), + ...extra, +}); + +const componentsOf = (name: ImageScenarioName, selection: Selection): ComponentImage[] => + definitions[name].components.map((component) => selection.components[component]); + +type ComponentArg = readonly [string, ComponentImage]; + +const testImageOf = (name: string, components: readonly ComponentImage[], variant?: string): TestImage => { + const [label] = components; + if (label === undefined) { + throw new Error(`The ${name} test image declares no components`); + } + const componentArgs: readonly ComponentArg[] = + components.length === 1 + ? [["DOTNET_IMAGE_REF", label]] + : components.map((component) => [`DOTNET_${component.component.toUpperCase()}_IMAGE_REF`, component]); + return { + tag: `${testImagePrefix}:${variant === undefined ? name : `${name}-${variant}`}-${label.key}`, + components, + dockerfile: path.join( + import.meta.dirname, + componentArgs.length === 1 ? "test-image.Dockerfile" : "test-image-multi.Dockerfile", + ), + args: componentArgs.map(([arg, component]) => `${arg}=${component.ref}`), + }; +}; + const completionFiles = [ "/usr/local/share/bash-completion/completions/dotnet", "/usr/local/share/zsh/site-functions/_dotnet", @@ -213,7 +231,7 @@ const runtimeBehavior = (): ScenarioBehavior => ({ const aspnetBehavior = ({ workspace }: ScenarioContext): ScenarioBehavior => ({ execs: [], - remoteEnv: { ASPNETCORE_URLS: `http://unix:${workspace.containerPath}/app.sock` }, + remoteEnv: remoteEnvOf(true, { ASPNETCORE_URLS: `http://unix:${workspace.containerPath}/app.sock` }), serve: { command: ["dotnet", "app/web.dll"], socketPath: path.join(workspace.path, "app.sock"), @@ -240,7 +258,7 @@ const multiBehavior = ({ selection }: ScenarioContext): ScenarioBehavior => ({ ], }); -const behaviors: Record ScenarioBehavior> = { +const behaviors: Record ScenarioBehavior> = { sdk: sdkBehavior, runtime: runtimeBehavior, aspnet: aspnetBehavior, @@ -249,50 +267,132 @@ const behaviors: Record ScenarioBeha const appOutput = (app: AppName): string => `out/${app}`; -const testImageOf = (name: ScenarioName, selection: Selection, variant?: string): TestImage => { - const components = definitions[name].components.map((component) => selection.components[component]); - const [label] = components; - if (label === undefined) { - throw new Error(`Scenario ${name} declares no components`); +const appOf = (selected: readonly ScenarioName[]): AppName[] => [ + ...new Set( + selected.flatMap((name) => { + if (name === "smoke") { + return []; + } + const { app } = definitions[name]; + return app === undefined ? [] : [app]; + }), + ), +]; + +const variantImage = ( + name: ImageScenarioName, + definition: ScenarioVariantDefinition, + context: ScenarioContext, +): TestImage => { + const image = testImageOf(name, componentsOf(name, context.selection), definition.name); + if (definition.dockerfile === undefined) { + return image; + } + if (definitions[name].components.length !== 1) { + throw new Error(`The ${name} scenario declares ${definition.name} with a dockerfile and multiple components`); } - const image = variant === undefined ? name : `${name}-${variant}`; - return { tag: `${testImagePrefix}:${image}-${label.key}`, components }; + return { + ...image, + dockerfile: path.join(import.meta.dirname, definition.dockerfile), + files: (definition.files ?? []).map((file) => ({ name: file, source: path.join(import.meta.dirname, file) })), + }; }; const materializeVariant = ( - name: ScenarioName, + name: ImageScenarioName, definition: ScenarioVariantDefinition, context: ScenarioContext, ): ScenarioVariant => { - const image = testImageOf(name, context.selection, definition.name); const variant = { name: definition.name, workspace: lifecycle.workspaceAt(context.root, `${name}-${definition.name}`), - testImage: { ...image, dockerfile: definition.dockerfile, files: definition.files }, - skipWorkloadIntegrityCheck: definition.skipWorkloadIntegrityCheck, + testImage: variantImage(name, definition, context), + remoteEnv: remoteEnvOf(definition.skipWorkloadIntegrityCheck), }; return "buildFailure" in definition ? { ...variant, buildFailure: definition.buildFailure } : { ...variant, execs: definition.execs }; }; +const smokeScenario = (context: ScenarioContext): Scenario => { + const { selection, target, workspace } = context; + if (target === undefined) { + throw new Error("The smoke scenario requires a target component and version"); + } + const image = selection.components[target.component]; + const execs: Exec[] = []; + switch (image.component) { + case "sdk": + execs.push({ command: ["dotnet", "--version"], expect: { stdout: [image.key] } }); + break; + case "runtime": + execs.push({ + command: ["dotnet", "--list-runtimes"], + expect: { stdout: [`Microsoft.NETCore.App ${image.key} [/opt/dotnet/shared/Microsoft.NETCore.App]`] }, + }); + break; + case "aspnet": + execs.push({ + command: ["dotnet", "--list-runtimes"], + expect: { + stdout: [ + `Microsoft.AspNetCore.App ${image.key} [/opt/dotnet/shared/Microsoft.AspNetCore.App]`, + `Microsoft.NETCore.App ${image.key} [/opt/dotnet/shared/Microsoft.NETCore.App]`, + ], + sorted: true, + }, + }); + break; + } + // The runtime host prints its usage and exits 129 for --help when no SDK is installed; + // --info is the host command every component answers with exit 0. + execs.push({ command: ["dotnet", "--info"], expect: { exitCode: 0 } }); + return { + name: "smoke", + workspace, + testImage: testImageOf(`smoke-${image.component}`, [image]), + execs, + remoteEnv: remoteEnvOf(), + }; +}; + const createScenario = (name: ScenarioName, context: ScenarioContext): Scenario => { + if (name === "smoke") { + return smokeScenario(context); + } + const definition = definitions[name]; const behavior = behaviors[name](context); return { - ...definitions[name], name, workspace: context.workspace, - testImage: testImageOf(name, context.selection), + testImage: testImageOf(name, componentsOf(name, context.selection)), execs: behavior.execs, - remoteEnv: behavior.remoteEnv, + remoteEnv: behavior.remoteEnv ?? remoteEnvOf(), serve: behavior.serve, + stage: async (workspace) => { + await staging.assets(workspace, definition.assets, definition.directories); + if (definition.app === undefined) { + return; + } + if (context.build === undefined) { + throw new Error(`The ${name} scenario needs the ${definition.app} app, but no build stage was created`); + } + await staging.app(workspace, path.join(context.build.path, appOutput(definition.app))); + }, variants: (behavior.variants ?? []).map((variant) => materializeVariant(name, variant, context)), }; }; -const createBuildStage = (apps: readonly AppName[], selection: Selection): BuildStage => ({ - testImage: testImageOf("sdk", selection), - publishes: apps.map((app) => ({ +interface BuildStageContext { + selection: Selection; + workspace: Workspace; +} + +const createBuildStage = (apps: readonly AppName[], { selection, workspace }: BuildStageContext): Scenario => ({ + name: "build", + workspace, + testImage: testImageOf("sdk", componentsOf("sdk", selection)), + execs: apps.map((app) => ({ command: [ ...["dotnet", "publish", `${app}/${app}.csproj`], ...["--configuration", "Release"], @@ -302,26 +402,12 @@ const createBuildStage = (apps: readonly AppName[], selection: Selection): Build env: { DefaultTargetFramework: selection.tfm }, expect: { stdout: [] }, })), + remoteEnv: remoteEnvOf(), + stage: (staged) => staging.assets(staged, apps, []), }); -const select = (values: readonly string[]): ScenarioName[] => { - if (values.length === 0) { - return [...names]; - } - const selected = values.map((value) => { - const name = names.find((candidate) => candidate === value); - if (name === undefined) { - throw new Error(`--scenario must be one of: ${names.join(", ")}`); - } - return name; - }); - return names.filter((name) => selected.includes(name)); -}; - export const scenarios = { - apps: appNames, - appOutput, + appOf, create: createScenario, createBuildStage, - select, } as const; diff --git a/scripts/verify/dotnet/selection.ts b/scripts/verify/dotnet/selection.ts index a2ff622..188e108 100644 --- a/scripts/verify/dotnet/selection.ts +++ b/scripts/verify/dotnet/selection.ts @@ -5,11 +5,12 @@ import { z } from "zod"; import { projectRoot } from "@/scripts/project.js"; import { claimedKey } from "@/scripts/tasks/features/config.js"; +import { documentOf, dotnet as dotnetFeature } from "@/scripts/tasks/features/dotnet/feature.js"; import type { Component, FeatureConfig } from "@/scripts/tasks/features/dotnet/generateConfig.js"; import { configPath, readConfig } from "@/scripts/tasks/features/dotnet/generateConfig.js"; -import { imageRef, tagNames } from "@/scripts/tasks/features/dotnet/tags.js"; +import { imageEntry } from "@/scripts/tasks/features/imageEntries.js"; -import { remoteUser } from "./lifecycle.js"; +import { remoteUser } from "../lifecycle.js"; const userInitFeaturePath = path.resolve(projectRoot, ".devcontainer/features/src/user-init/devcontainer-feature.json"); const dataRootSource = "XDG_DATA_HOME-${devcontainerId}"; @@ -24,6 +25,11 @@ export interface ComponentImage { ref: string; } +export interface SelectionTarget { + component: Component; + key: string; +} + export interface Selection { components: Record; tfm: string; @@ -33,11 +39,14 @@ export interface Selection { const componentImage = ( config: FeatureConfig, component: Component, + key: string | undefined, channel: string, prefix: string, ): ComponentImage => { - const key = claimedKey(config.components[component].aliases, "lts", `${configPath} component ${component}`); - return { component, key, ref: imageRef(prefix, component, tagNames(config, component, key, channel).canonical) }; + const selected = + key ?? claimedKey(config.components[component].aliases, "lts", `${configPath} component ${component}`); + const { canonical } = imageEntry(dotnetFeature, documentOf(config), prefix, channel, { component, key: selected }); + return { component, key: selected, ref: canonical }; }; const tfmOf = (sdkVersion: string): string => { @@ -61,14 +70,16 @@ const readGlobalPackagesPath = async (): Promise => { return posix.join(dataRoot, remoteUser, "NuGet/global-packages"); }; -const derive = async (channel: string, prefix: string): Promise => { +const derive = async (channel: string, prefix: string, target?: SelectionTarget): Promise => { const config = await readConfig(); - const sdk = componentImage(config, "sdk", channel, prefix); + const keyOf = (component: Component): string | undefined => + target?.component === component ? target.key : undefined; + const sdk = componentImage(config, "sdk", keyOf("sdk"), channel, prefix); return { components: { sdk, - runtime: componentImage(config, "runtime", channel, prefix), - aspnet: componentImage(config, "aspnet", channel, prefix), + runtime: componentImage(config, "runtime", keyOf("runtime"), channel, prefix), + aspnet: componentImage(config, "aspnet", keyOf("aspnet"), channel, prefix), }, tfm: tfmOf(sdk.key), globalPackagesPath: await readGlobalPackagesPath(), diff --git a/scripts/verify/dotnet/staging.ts b/scripts/verify/dotnet/staging.ts index 3914b4f..fd60e9e 100644 --- a/scripts/verify/dotnet/staging.ts +++ b/scripts/verify/dotnet/staging.ts @@ -1,18 +1,9 @@ import { cp, mkdir } from "node:fs/promises"; import path from "node:path"; -import { projectRoot } from "@/scripts/project.js"; - -import type { Workspace } from "./lifecycle.js"; +import type { Workspace } from "../lifecycle.js"; const assetsPath = path.join(import.meta.dirname, "assets"); -const featureSourcePath = path.join(projectRoot, ".devcontainer/features/src"); - -const stageFeatures = async (workspace: Workspace): Promise => { - const devcontainerPath = path.join(workspace.path, ".devcontainer"); - await cp(path.join(featureSourcePath, "features"), path.join(devcontainerPath, "features"), { recursive: true }); - await cp(path.join(featureSourcePath, "user-init"), path.join(devcontainerPath, "user-init"), { recursive: true }); -}; const stageAssets = async ( workspace: Workspace, @@ -31,8 +22,4 @@ const stageApp = async (workspace: Workspace, sourcePath: string): Promise await cp(sourcePath, path.join(workspace.path, "app"), { recursive: true }); }; -export const staging = { - features: stageFeatures, - assets: stageAssets, - app: stageApp, -} as const; +export const staging = { assets: stageAssets, app: stageApp } as const; diff --git a/scripts/verify/dotnet/verify.ts b/scripts/verify/dotnet/verify.ts index d9e56d4..114ff03 100644 --- a/scripts/verify/dotnet/verify.ts +++ b/scripts/verify/dotnet/verify.ts @@ -1,175 +1,41 @@ -import { mkdtempDisposable } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { parseArgs } from "node:util"; +import { z } from "zod"; -import type { Component } from "@/scripts/tasks/features/dotnet/generateConfig.js"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; +import { components } from "@/scripts/tasks/features/dotnet/generateConfig.js"; -import { assertions } from "./assertions.js"; -import type { TestImage, Workspace } from "./lifecycle.js"; -import { imagesModes, lifecycle } from "./lifecycle.js"; -import { scenarios } from "./scenarios.js"; -import type { ComponentImage } from "./selection.js"; +import { parseVerifyArguments, requireSmokeTarget, targetSchema } from "../arguments.js"; +import { lifecycle } from "../lifecycle.js"; +import { runSuite } from "../run.js"; +import { scenarios, scenarioSelectionSchema } from "./scenarios.js"; import { selection } from "./selection.js"; -import { serve } from "./serve.js"; -import { staging } from "./staging.js"; - -const componentImages = (images: readonly TestImage[]): ComponentImage[] => { - const byComponent = new Map(); - for (const image of images) { - for (const component of image.components) { - byComponent.set(component.component, component); - } - } - return [...byComponent.values()]; -}; export const verify = async (argv: string[]): Promise => { - const { values } = parseArgs({ - args: argv, - options: { - force: { type: "boolean", default: false }, - rmi: { type: "boolean", default: false }, - scenario: { type: "string", multiple: true }, - images: { type: "string", default: "build" }, - channel: { type: "string", default: mainChannel }, - prefix: { type: "string" }, - }, - }); - - const images = imagesModes.find((candidate) => candidate === values.images); - if (images === undefined) { - throw new Error(`--images must be one of: ${imagesModes.join(", ")}`); - } - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const selected = scenarios.select(values.scenario ?? []); - const derived = await selection.derive(channel, prefix); + const { options, scenarios: requested, component, version } = await parseVerifyArguments(argv); + const selected = scenarioSelectionSchema.parse(requested); + const target = targetSchema(z.enum(components)).parse({ component, version }); + requireSmokeTarget(selected, target); + const derived = await selection.derive(options.channel, options.prefix, target); console.log( `>>> verify dotnet: selection sdk=${derived.components.sdk.key} runtime=${derived.components.runtime.key} aspnet=${derived.components.aspnet.key} tfm=${derived.tfm}`, ); - const tempPath = await mkdtempDisposable(path.join(tmpdir(), "devcontainer-verify-dotnet-")); - const ctxPath = path.join(tempPath.path, "ctx"); - const runs = selected.map((name) => - scenarios.create(name, { - selection: derived, - root: tempPath.path, - workspace: lifecycle.workspaceAt(tempPath.path, name), - }), - ); - const variants = runs.flatMap((run) => run.variants); - const apps = [...new Set(runs.flatMap((run) => (run.app === undefined ? [] : [run.app])))]; - const build = - apps.length === 0 - ? undefined - : { - workspace: lifecycle.workspaceAt(tempPath.path, "build"), - ...scenarios.createBuildStage(apps, derived), - }; - const testImages = new Map(); - for (const run of runs) { - testImages.set(run.testImage.tag, run.testImage); - } - for (const variant of variants) { - testImages.set(variant.testImage.tag, variant.testImage); - } - if (build !== undefined) { - testImages.set(build.testImage.tag, build.testImage); - } - const workspaces: Workspace[] = [ - ...runs.map((run) => run.workspace), - ...variants.map((variant) => variant.workspace), - ...(build === undefined ? [] : [build.workspace]), - ]; - - try { - console.log(">>> verify dotnet: provision"); - await lifecycle.provision(componentImages([...testImages.values()]), { - images, - channel, - prefix, - force: values.force, - }); - - console.log(">>> verify dotnet: test images"); - for (const image of testImages.values()) { - await lifecycle.buildTestImage(ctxPath, image); - } - - console.log(">>> verify dotnet: workspaces"); - for (const run of runs) { - await lifecycle.createWorkspace(run.workspace, { image: run.testImage.tag, remoteEnv: run.remoteEnv }); - await staging.features(run.workspace); - await staging.assets(run.workspace, run.assets, run.directories); - } - for (const variant of variants) { - await lifecycle.createWorkspace(variant.workspace, { - image: variant.testImage.tag, - skipWorkloadIntegrityCheck: variant.skipWorkloadIntegrityCheck, - }); - await staging.features(variant.workspace); - } - if (build !== undefined) { - await lifecycle.createWorkspace(build.workspace, { image: build.testImage.tag }); - await staging.features(build.workspace); - await staging.assets(build.workspace, scenarios.apps, []); - } - - if (build !== undefined) { - console.log(">>> verify dotnet: build stage"); - await lifecycle.buildContainer(build.workspace); - await lifecycle.startContainer(build.workspace); - for (const publish of build.publishes) { - assertions.exec(await lifecycle.execInContainer(build.workspace, publish), publish.expect); - } - for (const run of runs) { - if (run.app !== undefined) { - await staging.app(run.workspace, path.join(build.workspace.path, scenarios.appOutput(run.app))); - } - } - } - - for (const run of runs) { - console.log(`>>> verify dotnet: scenario ${run.name}`); - await lifecycle.buildContainer(run.workspace); - await lifecycle.startContainer(run.workspace); - for (const exec of run.execs) { - assertions.exec(await lifecycle.execInContainer(run.workspace, exec), exec.expect); - } - if (run.serve !== undefined) { - const server = lifecycle.startInContainer(run.workspace, run.serve.command); - try { - await serve.assertResponse(server, run.serve); - } finally { - await server.kill(); - } - } - for (const variant of run.variants) { - console.log(`>>> verify dotnet: scenario ${run.name} variant ${variant.name}`); - if ("buildFailure" in variant) { - await lifecycle.expectBuildFailure(variant.workspace, variant.buildFailure); - continue; - } - await lifecycle.buildContainer(variant.workspace); - await lifecycle.startContainer(variant.workspace); - for (const exec of variant.execs) { - assertions.exec(await lifecycle.execInContainer(variant.workspace, exec), exec.expect); - } - } - } - } finally { - console.log(">>> verify dotnet: cleanup"); - for (const workspace of workspaces) { - await lifecycle.removeContainer(workspace); - await lifecycle.removeWorkspaceImages(workspace); - } - if (values.rmi) { - for (const image of testImages.values()) { - await lifecycle.removeImage(image.tag); - } - } - await tempPath.remove(); - } + await runSuite({ + ...options, + name: "dotnet", + create: (root) => { + const apps = scenarios.appOf(selected); + const build = apps.length === 0 ? undefined : lifecycle.workspaceAt(root, "build"); + return [ + ...(build === undefined ? [] : [scenarios.createBuildStage(apps, { selection: derived, workspace: build })]), + ...selected.map((name) => + scenarios.create(name, { + selection: derived, + target, + root, + workspace: lifecycle.workspaceAt(root, name), + build, + }), + ), + ]; + }, + }); }; diff --git a/scripts/verify/dotnet/lifecycle.ts b/scripts/verify/lifecycle.ts similarity index 75% rename from scripts/verify/dotnet/lifecycle.ts rename to scripts/verify/lifecycle.ts index 29cec88..cbcc27c 100644 --- a/scripts/verify/dotnet/lifecycle.ts +++ b/scripts/verify/lifecycle.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { cp, mkdir, writeFile } from "node:fs/promises"; +import { cp, mkdir, mkdtemp, writeFile } from "node:fs/promises"; import path, { posix } from "node:path"; import { $ } from "execa"; @@ -8,19 +8,41 @@ import { projectRoot } from "@/scripts/project.js"; import { project$$ } from "@/scripts/shell.js"; import { login } from "@/scripts/tasks/features/registry.js"; -import type { ComponentImage } from "./selection.js"; +import type { ImagesMode } from "./arguments.js"; const baseImage = "mcr.microsoft.com/devcontainers/base:debian"; -export const remoteUser = "verify"; +const cliFeatures = ["features", "user-init"]; +const featureSourcePath = path.join(projectRoot, ".devcontainer/features/src"); const $$docker = $({ reject: false, stdin: "ignore", stderr: "ignore" }); const $$capture = $({ reject: false, stdio: ["ignore", "pipe", "pipe"], verbose: "full", cwd: projectRoot }); +export const remoteUser = "verify"; + export interface Workspace { path: string; containerPath: string; } +export interface ComponentImage { + component: string; + key: string; + ref: string; +} + +export interface TestImageFile { + name: string; + source: string; +} + +export interface TestImage { + tag: string; + components: readonly ComponentImage[]; + dockerfile: string; + args: readonly string[]; + files?: readonly TestImageFile[]; +} + export interface Command { command: readonly string[]; env?: Readonly>; @@ -40,17 +62,16 @@ export interface StartedCommand { kill(): Promise; } -export interface TestImage { - tag: string; - components: readonly ComponentImage[]; - dockerfile?: string; - files?: readonly string[]; -} - export interface WorkspaceConfig { - image: string; remoteEnv?: Readonly>; - skipWorkloadIntegrityCheck?: boolean; + runArgs?: readonly string[]; +} + +export interface ProvisionOptions { + images: ImagesMode; + channel: string; + prefix: string; + force: boolean; } const workspaceAt = (root: string, name: string): Workspace => { @@ -69,17 +90,8 @@ const workspaceImages = async (workspace: Workspace): Promise => { return stdout.split("\n").filter((tag) => tag.startsWith(name)); }; -export const imagesModes = ["build", "pull"] as const; -export type ImagesMode = (typeof imagesModes)[number]; - -export interface ProvisionOptions { - images: ImagesMode; - channel: string; - prefix: string; - force: boolean; -} - const provisionComponent = async ( + feature: string, { component, key, ref }: ComponentImage, options: ProvisionOptions, ): Promise => { @@ -87,21 +99,34 @@ const provisionComponent = async ( await project$$`docker pull ${ref}`; return; } - const args = [...["--component", component], ...["--version", key], ...(options.force ? ["--force"] : [])]; - await project$$`tsx scripts/tasks/features/dotnet/fetchPayload.ts ${args}`; - await project$$`tsx scripts/tasks/features/dotnet/buildImage.ts ${[ + const args = [ + ...["--feature", feature], + ...["--component", component], + ...["--version", key], + ...(options.force ? ["--force"] : []), + ]; + await project$$`tsx scripts/tasks/features/pipeline.ts fetch-payload ${args}`; + await project$$`tsx scripts/tasks/features/pipeline.ts build-image ${[ ...args, ...["--channel", options.channel], ...["--prefix", options.prefix], ]}`; }; -const provision = async (components: readonly ComponentImage[], options: ProvisionOptions): Promise => { +const provision = async ( + feature: string, + images: readonly ComponentImage[], + options: ProvisionOptions, +): Promise => { if (options.images === "pull") { await login(options.prefix); } - for (const component of components) { - await provisionComponent(component, options); + const unique = new Map(); + for (const image of images) { + unique.set(`${image.component}:${image.key}`, image); + } + for (const image of unique.values()) { + await provisionComponent(feature, image, options); } const inspect = await $$docker`docker image inspect ${baseImage}`; if (inspect.exitCode !== 0) { @@ -109,50 +134,34 @@ const provision = async (components: readonly ComponentImage[], options: Provisi } }; -const testImageBuild = (image: TestImage): { dockerfile: string; args: string[] } => { - const [only] = image.components; - if (image.components.length === 1) { - return { dockerfile: image.dockerfile ?? "test-image.Dockerfile", args: [`DOTNET_IMAGE_REF=${only.ref}`] }; - } - if (image.dockerfile !== undefined) { - throw new Error(`${image.tag}: a variant dockerfile is only supported for single-component test images`); - } - return { - dockerfile: "test-image-multi.Dockerfile", - args: image.components.map(({ component, ref }) => `DOTNET_${component.toUpperCase()}_IMAGE_REF=${ref}`), - }; -}; - -const buildTestImage = async (ctxPath: string, image: TestImage): Promise => { - const { dockerfile, args } = testImageBuild(image); - await mkdir(ctxPath, { recursive: true }); +const buildTestImage = async (root: string, image: TestImage): Promise => { + const ctxPath = await mkdtemp(path.join(root, "ctx-")); for (const file of image.files ?? []) { - await cp(path.join(import.meta.dirname, file), path.join(ctxPath, file)); + await cp(file.source, path.join(ctxPath, file.name)); } await project$$`docker buildx build ${[ "--network=none", "--load", ...["--tag", image.tag], - ...["--file", path.join(import.meta.dirname, dockerfile)], - ...args.flatMap((arg) => ["--build-arg", arg]), + ...["--file", image.dockerfile], + ...image.args.flatMap((arg) => ["--build-arg", arg]), ctxPath, ]}`; }; -const createWorkspace = async (workspace: Workspace, config: WorkspaceConfig): Promise => { +const createWorkspace = async (workspace: Workspace, image: string, config: WorkspaceConfig = {}): Promise => { const devcontainerPath = path.join(workspace.path, ".devcontainer"); await mkdir(devcontainerPath, { recursive: true }); + for (const feature of cliFeatures) { + await cp(path.join(featureSourcePath, feature), path.join(devcontainerPath, feature), { recursive: true }); + } const data = { - image: config.image, + image, features: { "./features": {}, "./user-init": {} }, remoteUser, containerUser: remoteUser, - runArgs: ["--network=none"], - remoteEnv: { - DOTNET_NOLOGO: "1", - ...(config.skipWorkloadIntegrityCheck === false ? {} : { DOTNET_SKIP_WORKLOAD_INTEGRITY_CHECK: "1" }), - ...config.remoteEnv, - }, + runArgs: config.runArgs ?? ["--network=none"], + remoteEnv: config.remoteEnv === undefined ? undefined : { ...config.remoteEnv }, }; await writeFile(path.join(devcontainerPath, "devcontainer.json"), `${JSON.stringify(data, null, 2)}\n`); }; diff --git a/scripts/verify/ripgrep/assertions.ts b/scripts/verify/ripgrep/assertions.ts deleted file mode 100644 index 3456148..0000000 --- a/scripts/verify/ripgrep/assertions.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { CommandResult } from "./lifecycle.js"; - -export interface Expectation { - stdout: readonly string[]; - sorted?: boolean; -} - -const linesOf = (text: string): string[] => (text.trim() === "" ? [] : text.trimEnd().split("\n")); - -const sameLines = (actual: readonly string[], expected: readonly string[]): boolean => - actual.length === expected.length && actual.every((line, index) => line === expected[index]); - -const assertExec = (result: CommandResult, expectation: Expectation): void => { - const stdout = linesOf(result.stdout); - const sorted = expectation.sorted === true; - const order = (lines: readonly string[]): string[] => (sorted ? [...lines].sort() : [...lines]); - - const problems: string[] = []; - if (result.exitCode !== 0) { - problems.push(`exit code ${result.exitCode ?? "none"}`); - } - if (!sameLines(order(stdout), order(expectation.stdout))) { - problems.push(`stdout ${JSON.stringify(stdout)}, expected ${JSON.stringify(expectation.stdout)}`); - } - if (result.stderr.trim() !== "") { - problems.push(`stderr ${JSON.stringify(result.stderr)}`); - } - if (problems.length === 0) { - return; - } - throw new Error(`devcontainer exec ${result.command.join(" ")}:\n${problems.join("\n")}`); -}; - -export const assertions = { exec: assertExec } as const; diff --git a/scripts/verify/ripgrep/lifecycle.ts b/scripts/verify/ripgrep/lifecycle.ts deleted file mode 100644 index 41cf20e..0000000 --- a/scripts/verify/ripgrep/lifecycle.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { createHash } from "node:crypto"; -import { mkdir, writeFile } from "node:fs/promises"; -import path, { posix } from "node:path"; - -import { $ } from "execa"; - -import { projectRoot } from "@/scripts/project.js"; -import { project$$ } from "@/scripts/shell.js"; -import { login } from "@/scripts/tasks/features/registry.js"; - -const baseImage = "mcr.microsoft.com/devcontainers/base:debian"; -const remoteUser = "verify"; -export const testImagePrefix = "features-verify-ripgrep"; - -const $$docker = $({ reject: false, stdin: "ignore", stderr: "ignore" }); -const $$capture = $({ reject: false, stdio: ["ignore", "pipe", "pipe"], verbose: "full", cwd: projectRoot }); - -export interface Workspace { - path: string; - containerPath: string; -} - -export interface Command { - command: readonly string[]; -} - -export interface CommandResult { - command: readonly string[]; - exitCode: number | undefined; - stdout: string; - stderr: string; -} - -export interface WorkspaceConfig { - image: string; -} - -const workspaceAt = (root: string, name: string): Workspace => { - const directory = `workspace-${name}`; - return { path: path.join(root, directory), containerPath: posix.join("/workspaces", directory) }; -}; - -const workspaceImageName = (workspace: Workspace): string => { - const folderHash = createHash("sha256").update(workspace.path).digest("hex"); - return `vsc-${path.basename(workspace.path)}-${folderHash}`; -}; - -const workspaceImages = async (workspace: Workspace): Promise => { - const name = workspaceImageName(workspace); - const { stdout } = await $$docker`docker images ${["--format", "{{.Repository}}:{{.Tag}}"]}`; - return stdout.split("\n").filter((tag) => tag.startsWith(name)); -}; - -export const imagesModes = ["build", "pull"] as const; -export type ImagesMode = (typeof imagesModes)[number]; - -export interface ProvisionOptions { - images: ImagesMode; - key: string; - ref: string; - channel: string; - prefix: string; - force: boolean; -} - -const provision = async (options: ProvisionOptions): Promise => { - if (options.images === "pull") { - await login(options.prefix); - await project$$`docker pull ${options.ref}`; - } else { - await project$$`tsx scripts/tasks/features/ripgrep/fetchPayload.ts ${[ - ...["--version", options.key], - ...(options.force ? ["--force"] : []), - ]}`; - await project$$`tsx scripts/tasks/features/ripgrep/buildImage.ts ${[ - ...["--version", options.key], - ...["--channel", options.channel], - ...["--prefix", options.prefix], - ]}`; - } - const inspect = await $$docker`docker image inspect ${baseImage}`; - if (inspect.exitCode !== 0) { - await project$$`docker pull ${baseImage}`; - } -}; - -const buildTestImage = async (ctxPath: string, key: string, ref: string): Promise => { - await mkdir(ctxPath, { recursive: true }); - await project$$`docker buildx build ${[ - "--network=none", - "--load", - ...["--tag", `${testImagePrefix}:${key}`], - ...["--file", path.join(import.meta.dirname, "test-image.Dockerfile")], - ...["--build-arg", `RIPGREP_IMAGE_REF=${ref}`], - ctxPath, - ]}`; -}; - -const createWorkspace = async (workspace: Workspace, config: WorkspaceConfig): Promise => { - const devcontainerPath = path.join(workspace.path, ".devcontainer"); - await mkdir(devcontainerPath, { recursive: true }); - const data = { - image: config.image, - features: { "./features": {}, "./user-init": {} }, - remoteUser, - containerUser: remoteUser, - runArgs: ["--network=none"], - }; - await writeFile(path.join(devcontainerPath, "devcontainer.json"), `${JSON.stringify(data, null, 2)}\n`); -}; - -const buildContainer = async (workspace: Workspace): Promise => { - await project$$`devcontainer build ${["--workspace-folder", workspace.path]}`; -}; - -const startContainer = async (workspace: Workspace): Promise => { - await project$$`devcontainer up ${["--remove-existing-container", ...["--workspace-folder", workspace.path]]}`; - if ((await workspaceImages(workspace)).length === 0) { - throw new Error( - `devcontainer up ${workspace.path}: no images matching ${workspaceImageName(workspace)}*; the devcontainer CLI folder image naming may have changed`, - ); - } -}; - -const execInContainer = async (workspace: Workspace, command: Command): Promise => { - const result = await $$capture`devcontainer exec ${["--workspace-folder", workspace.path]} ${command.command}`; - return { command: command.command, exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr }; -}; - -const removeContainer = async (workspace: Workspace): Promise => { - const filter = `label=devcontainer.config_file=${path.join(workspace.path, ".devcontainer/devcontainer.json")}`; - const { stdout } = await $$docker`docker ps -a ${[...["--filter", filter], ...["--format", "{{.ID}}"]]}`; - const ids = [...new Set(stdout.split("\n").filter(Boolean))]; - if (ids.length > 0) { - await $$docker`docker rm --force ${ids}`; - } -}; - -const removeImage = async (tag: string): Promise => { - const inspect = await $$docker`docker image inspect ${tag}`; - if (inspect.exitCode !== 0) { - return; - } - await project$$`docker image rm ${tag}`; -}; - -const removeWorkspaceImages = async (workspace: Workspace): Promise => { - const tags = await workspaceImages(workspace); - if (tags.length > 0) { - await project$$`docker image rm ${tags}`; - } -}; - -export const lifecycle = { - provision, - buildTestImage, - createWorkspace, - workspaceAt, - buildContainer, - startContainer, - execInContainer, - removeContainer, - removeImage, - removeWorkspaceImages, -} as const; diff --git a/scripts/verify/ripgrep/selection.ts b/scripts/verify/ripgrep/selection.ts index 714f3af..6f8e9b5 100644 --- a/scripts/verify/ripgrep/selection.ts +++ b/scripts/verify/ripgrep/selection.ts @@ -1,19 +1,24 @@ import { claimedKey } from "@/scripts/tasks/features/config.js"; +import { imageEntry } from "@/scripts/tasks/features/imageEntries.js"; +import { documentOf, ripgrep as ripgrepFeature } from "@/scripts/tasks/features/ripgrep/feature.js"; import type { FeatureConfig } from "@/scripts/tasks/features/ripgrep/generateConfig.js"; import { configPath, readConfig } from "@/scripts/tasks/features/ripgrep/generateConfig.js"; -import { imageRef, tagNames } from "@/scripts/tasks/features/ripgrep/tags.js"; -export interface Selection { - key: string; - ref: string; -} +import type { ComponentImage } from "../lifecycle.js"; -const latestKey = (config: FeatureConfig): string => claimedKey(config.aliases, "latest", configPath); +export type Selection = ComponentImage; -const derive = async (channel: string, prefix: string): Promise => { +const selectedKey = (config: FeatureConfig, key: string | undefined): string => + key ?? claimedKey(config.aliases, "latest", configPath); + +const derive = async (channel: string, prefix: string, key?: string): Promise => { const config = await readConfig(); - const key = latestKey(config); - return { key, ref: imageRef(prefix, tagNames(config, key, channel).canonical) }; + const selected = selectedKey(config, key); + const { canonical } = imageEntry(ripgrepFeature, documentOf(config), prefix, channel, { + component: "ripgrep", + key: selected, + }); + return { component: "ripgrep", key: selected, ref: canonical }; }; export const selection = { derive } as const; diff --git a/scripts/verify/ripgrep/staging.ts b/scripts/verify/ripgrep/staging.ts deleted file mode 100644 index 84657d9..0000000 --- a/scripts/verify/ripgrep/staging.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { cp } from "node:fs/promises"; -import path from "node:path"; - -import { projectRoot } from "@/scripts/project.js"; - -import type { Workspace } from "./lifecycle.js"; - -const featureSourcePath = path.join(projectRoot, ".devcontainer/features/src"); - -const stageFeatures = async (workspace: Workspace): Promise => { - const devcontainerPath = path.join(workspace.path, ".devcontainer"); - await cp(path.join(featureSourcePath, "features"), path.join(devcontainerPath, "features"), { recursive: true }); - await cp(path.join(featureSourcePath, "user-init"), path.join(devcontainerPath, "user-init"), { recursive: true }); -}; - -export const staging = { features: stageFeatures } as const; diff --git a/scripts/verify/ripgrep/verify.ts b/scripts/verify/ripgrep/verify.ts index 0744f6c..04397bd 100644 --- a/scripts/verify/ripgrep/verify.ts +++ b/scripts/verify/ripgrep/verify.ts @@ -1,20 +1,25 @@ -import { mkdtempDisposable } from "node:fs/promises"; -import { tmpdir } from "node:os"; import path from "node:path"; -import { parseArgs } from "node:util"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; +import { z } from "zod"; -import type { Expectation } from "./assertions.js"; -import { assertions } from "./assertions.js"; -import { imagesModes, lifecycle, testImagePrefix } from "./lifecycle.js"; +import { parseVerifyArguments, selectionSchema } from "../arguments.js"; +import { lifecycle } from "../lifecycle.js"; +import type { Exec } from "../run.js"; +import { runSuite } from "../run.js"; import { selection } from "./selection.js"; -import { staging } from "./staging.js"; -interface Exec { - command: readonly string[]; - expect: Expectation; -} +const names = ["default", "smoke"] as const; +type ScenarioName = (typeof names)[number]; + +const scenarioSelectionSchema = selectionSchema(z.enum(names), names, ["default"]); + +const targetSchema = z.object({ + component: z.literal("ripgrep").optional(), + version: z.string().min(1).optional(), +}); + +const testImagePrefix = "features-verify-ripgrep"; +const testImageDockerfile = path.join(import.meta.dirname, "test-image.Dockerfile"); const installedFiles = [ "/usr/local/share/man/man1/rg.1", @@ -53,62 +58,44 @@ const defaultExecs = (version: string): Exec[] => [ }, ]; -export const verify = async (argv: string[]): Promise => { - const { values } = parseArgs({ - args: argv, - options: { - force: { type: "boolean", default: false }, - rmi: { type: "boolean", default: false }, - images: { type: "string", default: "build" }, - channel: { type: "string", default: mainChannel }, - prefix: { type: "string" }, - }, - }); - - const images = imagesModes.find((candidate) => candidate === values.images); - if (images === undefined) { - throw new Error(`--images must be one of: ${imagesModes.join(", ")}`); - } - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const derived = await selection.derive(channel, prefix); - console.log(`>>> verify ripgrep: selection version=${derived.key} ref=${derived.ref}`); - - const tempPath = await mkdtempDisposable(path.join(tmpdir(), "devcontainer-verify-ripgrep-")); - const workspace = lifecycle.workspaceAt(tempPath.path, "default"); - const testImage = `${testImagePrefix}:${derived.key}`; - - try { - console.log(">>> verify ripgrep: provision"); - await lifecycle.provision({ - images, - key: derived.key, - ref: derived.ref, - channel, - prefix, - force: values.force, - }); +const smokeExecs = (version: string): Exec[] => [ + { + command: ["sh", "-c", "rg --version | awk 'NR == 1 { print $1, $2 }'"], + expect: { stdout: [`ripgrep ${version}`] }, + }, + { + command: ["sh", "-c", "rg --version | grep -Fx 'features:+pcre2'"], + expect: { stdout: ["features:+pcre2"] }, + }, + { command: ["rg", "--help"], expect: { exitCode: 0 } }, +]; - console.log(">>> verify ripgrep: test image"); - await lifecycle.buildTestImage(path.join(tempPath.path, "ctx"), derived.key, derived.ref); +const execsOf: Record readonly Exec[]> = { + default: defaultExecs, + smoke: smokeExecs, +}; - console.log(">>> verify ripgrep: workspace"); - await lifecycle.createWorkspace(workspace, { image: testImage }); - await staging.features(workspace); +export const verify = async (argv: string[]): Promise => { + const { options, scenarios: requested, component, version } = await parseVerifyArguments(argv); + const selected = scenarioSelectionSchema.parse(requested); + const { version: key } = targetSchema.parse({ component, version }); + const derived = await selection.derive(options.channel, options.prefix, key); + console.log(`>>> verify ripgrep: selection version=${derived.key} ref=${derived.ref}`); - console.log(">>> verify ripgrep: scenario default"); - await lifecycle.buildContainer(workspace); - await lifecycle.startContainer(workspace); - for (const exec of defaultExecs(derived.key)) { - assertions.exec(await lifecycle.execInContainer(workspace, exec), exec.expect); - } - } finally { - console.log(">>> verify ripgrep: cleanup"); - await lifecycle.removeContainer(workspace); - await lifecycle.removeWorkspaceImages(workspace); - if (values.rmi) { - await lifecycle.removeImage(testImage); - } - await tempPath.remove(); - } + await runSuite({ + ...options, + name: "ripgrep", + create: (root) => + selected.map((name) => ({ + name, + workspace: lifecycle.workspaceAt(root, name), + testImage: { + tag: `${testImagePrefix}:${derived.key}`, + components: [derived], + dockerfile: testImageDockerfile, + args: [`RIPGREP_IMAGE_REF=${derived.ref}`], + }, + execs: execsOf[name](derived.key), + })), + }); }; diff --git a/scripts/verify/run.ts b/scripts/verify/run.ts new file mode 100644 index 0000000..063a41a --- /dev/null +++ b/scripts/verify/run.ts @@ -0,0 +1,128 @@ +import { mkdtempDisposable } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import type { VerifyOptions } from "./arguments.js"; +import type { Expectation } from "./assertions.js"; +import { assertions } from "./assertions.js"; +import type { TestImage, Workspace } from "./lifecycle.js"; +import { lifecycle } from "./lifecycle.js"; +import type { Serve } from "./serve.js"; +import { serve } from "./serve.js"; + +export interface Exec { + command: readonly string[]; + env?: Readonly>; + expect: Expectation; +} + +export interface ScenarioWorkspace { + workspace: Workspace; + testImage: TestImage; + remoteEnv?: Readonly>; + stage?: (workspace: Workspace) => Promise; +} + +export interface Scenario extends ScenarioWorkspace { + name: string; + execs: readonly Exec[]; + restart?: readonly Exec[]; + serve?: Serve; + variants?: readonly ScenarioVariant[]; +} + +export type ScenarioVariant = ScenarioWorkspace & { name: string } & ( + { execs: readonly Exec[] } | { buildFailure: string } + ); + +export interface Suite extends VerifyOptions { + name: string; + create(root: string): readonly Scenario[] | Promise; +} + +const execAll = async (workspace: Workspace, execs: readonly Exec[]): Promise => { + for (const exec of execs) { + assertions.exec(await lifecycle.execInContainer(workspace, exec), exec.expect); + } +}; + +const prepareWorkspace = async ({ workspace, testImage, remoteEnv, stage }: ScenarioWorkspace): Promise => { + await lifecycle.createWorkspace(workspace, testImage.tag, { remoteEnv }); + await stage?.(workspace); +}; + +const runVariant = async (feature: string, scenario: string, variant: ScenarioVariant): Promise => { + console.log(`>>> verify ${feature}: scenario ${scenario} variant ${variant.name}`); + await prepareWorkspace(variant); + if ("buildFailure" in variant) { + await lifecycle.expectBuildFailure(variant.workspace, variant.buildFailure); + return; + } + await lifecycle.buildContainer(variant.workspace); + await lifecycle.startContainer(variant.workspace); + await execAll(variant.workspace, variant.execs); +}; + +const runScenario = async (feature: string, scenario: Scenario): Promise => { + console.log(`>>> verify ${feature}: scenario ${scenario.name}`); + await prepareWorkspace(scenario); + await lifecycle.buildContainer(scenario.workspace); + await lifecycle.startContainer(scenario.workspace); + await execAll(scenario.workspace, scenario.execs); + if (scenario.serve !== undefined) { + const server = lifecycle.startInContainer(scenario.workspace, scenario.serve.command); + try { + await serve.assertResponse(server, scenario.serve); + } finally { + await server.kill(); + } + } + if (scenario.restart !== undefined) { + console.log(`>>> verify ${feature}: scenario ${scenario.name} restart`); + await lifecycle.startContainer(scenario.workspace); + await execAll(scenario.workspace, scenario.restart); + } + for (const variant of scenario.variants ?? []) { + await runVariant(feature, scenario.name, variant); + } +}; + +export const runSuite = async (suite: Suite): Promise => { + const tempPath = await mkdtempDisposable(path.join(tmpdir(), `devcontainer-verify-${suite.name}-`)); + const workspaces: Workspace[] = []; + const testImages = new Map(); + try { + const scenarios = await suite.create(tempPath.path); + for (const scenario of scenarios) { + for (const { workspace, testImage } of [scenario, ...(scenario.variants ?? [])]) { + workspaces.push(workspace); + testImages.set(testImage.tag, testImage); + } + } + console.log(`>>> verify ${suite.name}: provision`); + await lifecycle.provision( + suite.name, + [...testImages.values()].flatMap((image) => image.components), + suite, + ); + console.log(`>>> verify ${suite.name}: test images`); + for (const image of testImages.values()) { + await lifecycle.buildTestImage(tempPath.path, image); + } + for (const scenario of scenarios) { + await runScenario(suite.name, scenario); + } + } finally { + console.log(`>>> verify ${suite.name}: cleanup`); + for (const workspace of workspaces) { + await lifecycle.removeContainer(workspace); + await lifecycle.removeWorkspaceImages(workspace); + } + if (suite.rmi) { + for (const tag of testImages.keys()) { + await lifecycle.removeImage(tag); + } + } + await tempPath.remove(); + } +}; diff --git a/scripts/verify/dotnet/serve.ts b/scripts/verify/serve.ts similarity index 100% rename from scripts/verify/dotnet/serve.ts rename to scripts/verify/serve.ts diff --git a/scripts/verify/uv/lifecycle.ts b/scripts/verify/uv/lifecycle.ts deleted file mode 100644 index fac0a05..0000000 --- a/scripts/verify/uv/lifecycle.ts +++ /dev/null @@ -1,214 +0,0 @@ -import { createHash } from "node:crypto"; -import { cp, mkdir, writeFile } from "node:fs/promises"; -import path, { posix } from "node:path"; - -import { $ } from "execa"; - -import { projectRoot } from "@/scripts/project.js"; -import { project$$ } from "@/scripts/shell.js"; -import { login } from "@/scripts/tasks/features/registry.js"; - -import type { ComponentImage } from "./selection.js"; - -const baseImage = "mcr.microsoft.com/devcontainers/base:debian"; -export const remoteUser = "verify"; - -const $$docker = $({ reject: false, stdin: "ignore", stderr: "ignore" }); -const $$capture = $({ reject: false, stdio: ["ignore", "pipe", "pipe"], verbose: "full", cwd: projectRoot }); - -export interface Workspace { - path: string; - containerPath: string; -} - -export interface Command { - command: readonly string[]; - env?: Readonly>; -} - -export interface CommandResult { - command: readonly string[]; - exitCode: number | undefined; - stdout: string; - stderr: string; -} - -export interface TestImageFile { - name: string; - source: string; -} - -export interface TestImage { - tag: string; - components: readonly ComponentImage[]; - argNames: readonly string[]; - dockerfile: string; - files?: readonly TestImageFile[]; -} - -export interface WorkspaceConfig { - image: string; - remoteEnv?: Readonly>; -} - -const workspaceAt = (root: string, name: string): Workspace => { - const directory = `workspace-${name}`; - return { path: path.join(root, directory), containerPath: posix.join("/workspaces", directory) }; -}; - -const workspaceImageName = (workspace: Workspace): string => { - const folderHash = createHash("sha256").update(workspace.path).digest("hex"); - return `vsc-${path.basename(workspace.path)}-${folderHash}`; -}; - -const workspaceImages = async (workspace: Workspace): Promise => { - const name = workspaceImageName(workspace); - const { stdout } = await $$docker`docker images ${["--format", "{{.Repository}}:{{.Tag}}"]}`; - return stdout.split("\n").filter((tag) => tag.startsWith(name)); -}; - -export const imagesModes = ["build", "pull"] as const; -export type ImagesMode = (typeof imagesModes)[number]; - -export interface ProvisionOptions { - images: ImagesMode; - channel: string; - prefix: string; - force: boolean; -} - -const provisionComponent = async ( - { component, key, ref }: ComponentImage, - options: ProvisionOptions, -): Promise => { - if (options.images === "pull") { - await project$$`docker pull ${ref}`; - return; - } - const args = [...["--component", component], ...["--version", key], ...(options.force ? ["--force"] : [])]; - await project$$`tsx scripts/tasks/features/uv/fetchPayload.ts ${args}`; - await project$$`tsx scripts/tasks/features/uv/buildImage.ts ${[ - ...args, - ...["--channel", options.channel], - ...["--prefix", options.prefix], - ]}`; -}; - -const provision = async (components: readonly ComponentImage[], options: ProvisionOptions): Promise => { - if (options.images === "pull") { - await login(options.prefix); - } - const unique = new Map(); - for (const component of components) { - unique.set(`${component.component}:${component.key}`, component); - } - for (const component of unique.values()) { - await provisionComponent(component, options); - } - const inspect = await $$docker`docker image inspect ${baseImage}`; - if (inspect.exitCode !== 0) { - await project$$`docker pull ${baseImage}`; - } -}; - -const buildTestImage = async (ctxPath: string, image: TestImage): Promise => { - if (image.argNames.length !== image.components.length) { - throw new Error(`${image.tag}: ${image.argNames.length} build arg names for ${image.components.length} components`); - } - await mkdir(ctxPath, { recursive: true }); - for (const file of image.files ?? []) { - await cp(file.source, path.join(ctxPath, file.name)); - } - await project$$`docker buildx build ${[ - "--network=none", - "--load", - ...["--tag", image.tag], - ...["--file", path.join(import.meta.dirname, image.dockerfile)], - ...image.components.flatMap((component, index) => ["--build-arg", `${image.argNames[index]}=${component.ref}`]), - ctxPath, - ]}`; -}; - -const createWorkspace = async (workspace: Workspace, config: WorkspaceConfig): Promise => { - const devcontainerPath = path.join(workspace.path, ".devcontainer"); - await mkdir(devcontainerPath, { recursive: true }); - const data = { - image: config.image, - features: { "./features": {}, "./user-init": {} }, - remoteUser, - containerUser: remoteUser, - runArgs: ["--network=none"], - remoteEnv: { ...config.remoteEnv }, - }; - await writeFile(path.join(devcontainerPath, "devcontainer.json"), `${JSON.stringify(data, null, 2)}\n`); -}; - -const buildContainer = async (workspace: Workspace): Promise => { - await project$$`devcontainer build ${["--workspace-folder", workspace.path]}`; -}; - -const startContainer = async (workspace: Workspace): Promise => { - await project$$`devcontainer up ${["--remove-existing-container", ...["--workspace-folder", workspace.path]]}`; - if ((await workspaceImages(workspace)).length === 0) { - throw new Error( - `devcontainer up ${workspace.path}: no images matching ${workspaceImageName(workspace)}*; the devcontainer CLI folder image naming may have changed`, - ); - } -}; - -const execInContainer = async (workspace: Workspace, command: Command): Promise => { - const env = Object.entries(command.env ?? {}).flatMap(([name, value]) => ["--remote-env", `${name}=${value}`]); - const result = await $$capture`devcontainer exec ${["--workspace-folder", workspace.path]} ${env} ${command.command}`; - return { command: command.command, exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr }; -}; - -const expectBuildFailure = async (workspace: Workspace, message: string): Promise => { - const result = await $$capture`devcontainer build ${["--workspace-folder", workspace.path]}`; - if (result.exitCode === 0) { - throw new Error(`devcontainer build ${workspace.path}: expected a failing build`); - } - const output = `${result.stdout}\n${result.stderr}`; - if (!output.includes(message)) { - throw new Error( - `devcontainer build ${workspace.path}: expected ${JSON.stringify(message)} in the output:\n${output}`, - ); - } -}; - -const removeContainer = async (workspace: Workspace): Promise => { - const filter = `label=devcontainer.config_file=${path.join(workspace.path, ".devcontainer/devcontainer.json")}`; - const { stdout } = await $$docker`docker ps -a ${[...["--filter", filter], ...["--format", "{{.ID}}"]]}`; - const ids = [...new Set(stdout.split("\n").filter(Boolean))]; - if (ids.length > 0) { - await $$docker`docker rm --force ${ids}`; - } -}; - -const removeImage = async (tag: string): Promise => { - const inspect = await $$docker`docker image inspect ${tag}`; - if (inspect.exitCode !== 0) { - return; - } - await project$$`docker image rm ${tag}`; -}; - -const removeWorkspaceImages = async (workspace: Workspace): Promise => { - const tags = await workspaceImages(workspace); - if (tags.length > 0) { - await project$$`docker image rm ${tags}`; - } -}; - -export const lifecycle = { - provision, - buildTestImage, - createWorkspace, - workspaceAt, - buildContainer, - startContainer, - execInContainer, - expectBuildFailure, - removeContainer, - removeImage, - removeWorkspaceImages, -} as const; diff --git a/scripts/verify/uv/scenarios.ts b/scripts/verify/uv/scenarios.ts index 03bd74f..6132a62 100644 --- a/scripts/verify/uv/scenarios.ts +++ b/scripts/verify/uv/scenarios.ts @@ -1,86 +1,72 @@ import { writeFile } from "node:fs/promises"; import path, { posix } from "node:path"; -import type { Expectation } from "./assertions.js"; -import type { TestImage, Workspace } from "./lifecycle.js"; -import { lifecycle } from "./lifecycle.js"; -import type { ComponentImage, Selection } from "./selection.js"; +import { z } from "zod"; -const names = ["binaries", "python", "multi", "download"] as const; -export type ScenarioName = (typeof names)[number]; +import type { Component } from "@/scripts/tasks/features/uv/generateConfig.js"; + +import { selectionSchema } from "../arguments.js"; +import type { ComponentImage, TestImage, Workspace } from "../lifecycle.js"; +import { lifecycle } from "../lifecycle.js"; +import type { Exec, Scenario, ScenarioVariant } from "../run.js"; +import type { Selection, SelectionTarget } from "./selection.js"; +import { previousOf } from "./selection.js"; +import { staging } from "./staging.js"; + +const names = ["binaries", "python", "multi", "download", "smoke"] as const; +type ScenarioName = (typeof names)[number]; + +export const scenarioSelectionSchema = selectionSchema( + z.enum(names), + names, + names.filter((name) => name !== "smoke"), +); const testImagePrefix = "features-verify-uv"; const featureDir = "/opt/devcontainer-config/features/uv"; const malformedOptionsMessage = "install: /opt/devcontainer-config/features/uv/options.json: The JSON value could not be converted to DevcontainerConfig.Uv.Options. Path: $.defaultPythonVersion | LineNumber: 0 | BytePositionInLine: 27."; -export interface Exec { - command: readonly string[]; - env?: Readonly>; - expect: Expectation; -} - -export interface ScenarioContext { +interface ScenarioContext { selection: Selection; + target: SelectionTarget | undefined; root: string; workspace: Workspace; + force: boolean; } -export interface ScenarioVariant { - name: string; - workspace: Workspace; - testImage: TestImage; - buildFailure: string; -} - -export interface Scenario { - name: ScenarioName; - workspace: Workspace; - testImage: TestImage; - execs: readonly Exec[]; - restart?: readonly Exec[]; - remoteEnv?: Readonly>; - downloadMirror?: boolean; - variants: readonly ScenarioVariant[]; -} - -interface ScenarioDefinition { - dockerfile: string; - argNames: readonly string[]; -} +type ComponentArg = readonly [string, ComponentImage]; -const definitions: Record = { - binaries: { dockerfile: "test-image.Dockerfile", argNames: ["UV_IMAGE_REF"] }, - python: { dockerfile: "test-image-python.Dockerfile", argNames: ["UV_IMAGE_REF", "UV_PYTHON_IMAGE_REF"] }, - multi: { - dockerfile: "test-image-multi.Dockerfile", - argNames: ["UV_IMAGE_REF", "UV_PYTHON_LATEST_IMAGE_REF", "UV_PYTHON_PREVIOUS_IMAGE_REF"], - }, - download: { dockerfile: "test-image.Dockerfile", argNames: ["UV_IMAGE_REF"] }, -}; - -const componentImages: Record ComponentImage[]> = { - binaries: (selection) => [selection.uv], - python: (selection) => [selection.uv, selection.python], - multi: (selection) => [selection.uv, selection.python, selection.previous], - download: (selection) => [selection.uv], -}; - -const testImageOf = (name: ScenarioName, selection: Selection, variant?: string): TestImage => { - const components = componentImages[name](selection); - const [label] = components; - if (label === undefined) { - throw new Error(`Scenario ${name} declares no components`); +const testImage = ( + name: string, + dockerfile: string, + componentArgs: readonly ComponentArg[], + variant?: string, +): TestImage => { + const [first] = componentArgs; + if (first === undefined) { + throw new Error(`The ${name} test image declares no components`); } - const image = variant === undefined ? name : `${name}-${variant}`; + const [, label] = first; return { - tag: `${testImagePrefix}:${image}-${label.key}`, - components, - argNames: definitions[name].argNames, - dockerfile: definitions[name].dockerfile, + tag: `${testImagePrefix}:${variant === undefined ? name : `${name}-${variant}`}-${label.key}`, + components: componentArgs.map(([, component]) => component), + dockerfile: path.join(import.meta.dirname, dockerfile), + args: componentArgs.map(([arg, component]) => `${arg}=${component.ref}`), }; }; +const optionsImage = async ( + name: string, + root: string, + image: TestImage, + options: Readonly>, +): Promise => { + const source = path.join(root, `${name}-options.json`); + await writeFile(source, `${JSON.stringify(options)}\n`); + return { ...image, files: [{ name: "options.json", source }] }; +}; + const minorOf = (pythonVersion: string): string => { const [major, minor] = pythonVersion.split("."); return `${major}.${minor}`; @@ -172,7 +158,7 @@ const pythonRestartExecs = (selection: Selection): Exec[] => { const multiExecs = (selection: Selection): Exec[] => { const latest = selection.python.pythonVersion; - const previous = selection.previous.pythonVersion; + const previous = previousOf(selection).pythonVersion; return [ { command: ["uv", "python", "list", "--only-installed"], @@ -188,7 +174,7 @@ const multiExecs = (selection: Selection): Exec[] => { }; const downloadExecs = (selection: Selection): Exec[] => { - const pythonVersion = selection.previous.pythonVersion; + const pythonVersion = previousOf(selection).pythonVersion; return [ { command: ["uv", "python", "list", "--only-installed"], @@ -204,12 +190,34 @@ const downloadExecs = (selection: Selection): Exec[] => { ]; }; +const smokeExecs = (selection: Selection, component: Component): Exec[] => { + const execs: Exec[] = [ + { command: ["uv", "--version"], expect: { stdout: [`uv ${selection.uv.key} (${selection.muslTriple})`] } }, + { command: ["uv", "--help"], expect: { exitCode: 0 } }, + ]; + if (component === "python") { + const pythonVersion = selection.python.pythonVersion; + execs.push( + { command: ["sh", "-c", '"$XDG_BIN_HOME/python" -V'], expect: { stdout: [`Python ${pythonVersion}`] } }, + { + command: ["uv", "python", "list", "--only-installed"], + expect: { stdout: [pythonListLine(selection, pythonVersion)] }, + }, + ); + } + return execs; +}; + const malformedVariant = (selection: Selection, root: string): ScenarioVariant => ({ name: "options-malformed", workspace: lifecycle.workspaceAt(root, "binaries-options-malformed"), testImage: { - ...testImageOf("binaries", selection, "options-malformed"), - dockerfile: "test-image-options-malformed.Dockerfile", + ...testImage( + "binaries", + "test-image-options-malformed.Dockerfile", + [["UV_IMAGE_REF", selection.uv]], + "options-malformed", + ), files: [ { name: "options-malformed.json", source: path.join(import.meta.dirname, "assets", "options-malformed.json") }, ], @@ -217,91 +225,85 @@ const malformedVariant = (selection: Selection, root: string): ScenarioVariant = buildFailure: malformedOptionsMessage, }); -const optionsTestImage = async ( - name: ScenarioName, - selection: Selection, - root: string, - options: Readonly>, -): Promise => { - const source = path.join(root, `${name}-options.json`); - await writeFile(source, `${JSON.stringify(options)}\n`); - return { ...testImageOf(name, selection), files: [{ name: "options.json", source }] }; -}; - const createScenario = async (name: ScenarioName, context: ScenarioContext): Promise => { - const { selection, root, workspace } = context; + const { selection, target, root, workspace, force } = context; const remoteEnv = { XDG_BIN_HOME: posix.join(workspace.containerPath, ".bin") }; switch (name) { case "binaries": { - const testImage = await optionsTestImage(name, selection, root, {}); + const image = testImage(name, "test-image.Dockerfile", [["UV_IMAGE_REF", selection.uv]]); return { name, workspace, - testImage, + testImage: await optionsImage(name, root, image, {}), execs: binariesExecs(selection), remoteEnv, variants: [malformedVariant(selection, root)], }; } case "python": { - const testImage = await optionsTestImage(name, selection, root, { - defaultPythonVersion: selection.python.pythonVersion, - }); + const image = testImage(name, "test-image-python.Dockerfile", [ + ["UV_IMAGE_REF", selection.uv], + ["UV_PYTHON_IMAGE_REF", selection.python], + ]); return { name, workspace, - testImage, + testImage: await optionsImage(name, root, image, { defaultPythonVersion: selection.python.pythonVersion }), execs: pythonExecs(selection), restart: pythonRestartExecs(selection), remoteEnv, - variants: [], }; } case "multi": return { name, workspace, - testImage: testImageOf(name, selection), + testImage: testImage(name, "test-image-multi.Dockerfile", [ + ["UV_IMAGE_REF", selection.uv], + ["UV_PYTHON_LATEST_IMAGE_REF", selection.python], + ["UV_PYTHON_PREVIOUS_IMAGE_REF", previousOf(selection)], + ]), execs: multiExecs(selection), remoteEnv, - variants: [], }; + case "smoke": { + if (target === undefined) { + throw new Error("The smoke scenario requires a target component and version"); + } + const image = testImage(`smoke-${target.component}`, "test-image-python.Dockerfile", [ + ["UV_IMAGE_REF", selection.uv], + ["UV_PYTHON_IMAGE_REF", target.component === "python" ? selection.python : selection.uv], + ]); + const options: Readonly> = + target.component === "python" ? { defaultPythonVersion: selection.python.pythonVersion } : {}; + return { + name, + workspace, + testImage: await optionsImage(`smoke-${target.component}`, root, image, options), + execs: smokeExecs(selection, target.component), + remoteEnv, + }; + } case "download": { - const testImage = await optionsTestImage(name, selection, root, { - defaultPythonVersion: minorOf(selection.previous.pythonVersion), - }); + const image = testImage(name, "test-image.Dockerfile", [["UV_IMAGE_REF", selection.uv]]); return { name, workspace, - testImage, + testImage: await optionsImage(name, root, image, { + defaultPythonVersion: minorOf(previousOf(selection).pythonVersion), + }), execs: downloadExecs(selection), remoteEnv: { ...remoteEnv, UV_PYTHON_INSTALL_MIRROR: `file://${workspace.containerPath}/mirror`, UV_PYTHON_DOWNLOADS_JSON_URL: `${workspace.containerPath}/downloads.json`, }, - downloadMirror: true, - variants: [], + stage: (staged) => staging.downloadMirror(staged, selection, force), }; } } }; -const select = (values: readonly string[]): ScenarioName[] => { - if (values.length === 0) { - return [...names]; - } - const selected = values.map((value) => { - const name = names.find((candidate) => candidate === value); - if (name === undefined) { - throw new Error(`--scenario must be one of: ${names.join(", ")}`); - } - return name; - }); - return names.filter((name) => selected.includes(name)); -}; - export const scenarios = { create: createScenario, - select, } as const; diff --git a/scripts/verify/uv/selection.ts b/scripts/verify/uv/selection.ts index 4145710..052c7f7 100644 --- a/scripts/verify/uv/selection.ts +++ b/scripts/verify/uv/selection.ts @@ -4,14 +4,15 @@ import path, { posix } from "node:path"; import { z } from "zod"; import { projectRoot } from "@/scripts/project.js"; +import { hostArch } from "@/scripts/tasks/features/architecture.js"; import { claimedKey } from "@/scripts/tasks/features/config.js"; +import { imageEntry } from "@/scripts/tasks/features/imageEntries.js"; +import { documentOf, uv as uvFeature } from "@/scripts/tasks/features/uv/feature.js"; import type { Component, FeatureConfig, PythonVersion } from "@/scripts/tasks/features/uv/generateConfig.js"; import { configPath, readConfig } from "@/scripts/tasks/features/uv/generateConfig.js"; -import type { DockerArch } from "@/scripts/tasks/features/uv/paths.js"; -import { paths } from "@/scripts/tasks/features/uv/paths.js"; -import { imageRef, tagNames } from "@/scripts/tasks/features/uv/tags.js"; +import { machineOf, muslTriples } from "@/scripts/tasks/features/uv/paths.js"; -import { remoteUser } from "./lifecycle.js"; +import { remoteUser } from "../lifecycle.js"; const userInitFeaturePath = path.resolve(projectRoot, ".devcontainer/features/src/user-init/devcontainer-feature.json"); const dataRootSource = "XDG_DATA_HOME-${devcontainerId}"; @@ -31,10 +32,15 @@ export interface PythonImage extends ComponentImage { pythonVersion: string; } +export interface SelectionTarget { + component: Component; + key: string; +} + export interface Selection { uv: ComponentImage; python: PythonImage; - previous: PythonImage; + previous: PythonImage | undefined; xdgDataPath: string; machine: string; muslTriple: string; @@ -54,22 +60,8 @@ const compareVersions = (left: string, right: string): number => { const comparePython = (left: PythonVersion, right: PythonVersion): number => compareVersions(left.pythonVersion, right.pythonVersion) || Number(left.build) - Number(right.build); -const previousPython = (config: FeatureConfig, latest: string): { key: string; entry: PythonVersion } => { - const { versions } = config.components.python; - const latestEntry = versions[latest]; - if (latestEntry === undefined) { - throw new Error(`${configPath}: component python has no version ${latest}`); - } - const below = Object.entries(versions) - .filter(([key, entry]) => key !== latest && comparePython(entry, latestEntry) < 0) - .sort((left, right) => comparePython(left[1], right[1])); - const previous = below.at(-1); - if (previous === undefined) { - throw new Error(`${configPath}: component python carries no version below ${latest}`); - } - const [key, entry] = previous; - return { key, entry }; -}; +const refOf = (config: FeatureConfig, component: Component, key: string, channel: string, prefix: string): string => + imageEntry(uvFeature, documentOf(config), prefix, channel, { component, key }).canonical; const componentImage = ( config: FeatureConfig, @@ -77,11 +69,7 @@ const componentImage = ( key: string, channel: string, prefix: string, -): ComponentImage => ({ - component, - key, - ref: imageRef(prefix, component, tagNames(config, component, key, channel).canonical), -}); +): ComponentImage => ({ component, key, ref: refOf(config, component, key, channel, prefix) }); const pythonImage = (config: FeatureConfig, key: string, channel: string, prefix: string): PythonImage => { const entry = config.components.python.versions[key]; @@ -91,11 +79,28 @@ const pythonImage = (config: FeatureConfig, key: string, channel: string, prefix return { component: "python", key, - ref: imageRef(prefix, "python", tagNames(config, "python", key, channel).canonical), + ref: refOf(config, "python", key, channel, prefix), pythonVersion: entry.pythonVersion, }; }; +const previousPython = ( + config: FeatureConfig, + latest: string, + channel: string, + prefix: string, +): PythonImage | undefined => { + const latestEntry = config.components.python.versions[latest]; + if (latestEntry === undefined) { + throw new Error(`${configPath}: component python has no version ${latest}`); + } + const below = Object.entries(config.components.python.versions) + .filter(([key, entry]) => key !== latest && comparePython(entry, latestEntry) < 0) + .sort((left, right) => comparePython(left[1], right[1])); + const previous = below.at(-1); + return previous === undefined ? undefined : pythonImage(config, previous[0], channel, prefix); +}; + const readXdgDataPath = async (): Promise => { const data: unknown = JSON.parse(await readFile(userInitFeaturePath, "utf-8")); const result = userInitFeatureSchema.safeParse(data); @@ -109,20 +114,32 @@ const readXdgDataPath = async (): Promise => { return posix.join(dataRoot, remoteUser); }; -const derive = async (channel: string, prefix: string): Promise => { +const derive = async (channel: string, prefix: string, target?: SelectionTarget): Promise => { const config = await readConfig(); - const uvKey = claimedKey(config.components.uv.aliases, "latest", `${configPath} component uv`); - const pythonKey = claimedKey(config.components.python.aliases, "latest", `${configPath} component python`); - const previous = previousPython(config, pythonKey); - const arch: DockerArch = paths.hostArch(); + const uvKey = + target?.component === "uv" + ? target.key + : claimedKey(config.components.uv.aliases, "latest", `${configPath} component uv`); + const pythonKey = + target?.component === "python" + ? target.key + : claimedKey(config.components.python.aliases, "latest", `${configPath} component python`); + const arch = hostArch(); return { uv: componentImage(config, "uv", uvKey, channel, prefix), python: pythonImage(config, pythonKey, channel, prefix), - previous: pythonImage(config, previous.key, channel, prefix), + previous: previousPython(config, pythonKey, channel, prefix), xdgDataPath: await readXdgDataPath(), - machine: paths.machineOf(arch), - muslTriple: paths.muslTriples[arch], + machine: machineOf(arch), + muslTriple: muslTriples[arch], }; }; -export const selection = { derive } as const; +export const previousOf = (selection: Selection): PythonImage => { + if (selection.previous === undefined) { + throw new Error(`${configPath}: component python carries no version below ${selection.python.key}`); + } + return selection.previous; +}; + +export const selection = { derive, previousOf } as const; diff --git a/scripts/verify/uv/staging.ts b/scripts/verify/uv/staging.ts index bc6e265..a9b3383 100644 --- a/scripts/verify/uv/staging.ts +++ b/scripts/verify/uv/staging.ts @@ -1,34 +1,23 @@ import { cp, mkdir } from "node:fs/promises"; import path from "node:path"; -import { projectRoot } from "@/scripts/project.js"; import { project$$ } from "@/scripts/shell.js"; -import { paths } from "@/scripts/tasks/features/uv/paths.js"; -import { loadSection, pythonPayloadPaths } from "@/scripts/tasks/features/uv/payload.js"; +import { hostArch } from "@/scripts/tasks/features/architecture.js"; +import { derive, loadSection } from "@/scripts/tasks/features/uv/payload.js"; -import type { Workspace } from "./lifecycle.js"; +import type { Workspace } from "../lifecycle.js"; import type { Selection } from "./selection.js"; - -const featureSourcePath = path.join(projectRoot, ".devcontainer/features/src"); - -const stageFeatures = async (workspace: Workspace): Promise => { - const devcontainerPath = path.join(workspace.path, ".devcontainer"); - await cp(path.join(featureSourcePath, "features"), path.join(devcontainerPath, "features"), { recursive: true }); - await cp(path.join(featureSourcePath, "user-init"), path.join(devcontainerPath, "user-init"), { recursive: true }); -}; +import { previousOf } from "./selection.js"; const stageDownloadMirror = async (workspace: Workspace, selection: Selection, force: boolean): Promise => { - const args = [...["--component", "python"], ...["--version", selection.previous.key], ...(force ? ["--force"] : [])]; - await project$$`tsx scripts/tasks/features/uv/fetchPayload.ts ${args}`; - const section = await loadSection("python", selection.previous.key); - const payload = pythonPayloadPaths(section, paths.hostArch()); + const key = previousOf(selection).key; + const args = [...["--feature", "uv"], ...["--component", "python"], ...["--version", key]]; + await project$$`tsx scripts/tasks/features/pipeline.ts fetch-payload ${args} ${force ? ["--force"] : []}`; + const payload = derive(await loadSection("python", key), hostArch()); const mirrorPath = path.join(workspace.path, "mirror", payload.build); await mkdir(mirrorPath, { recursive: true }); - await cp(payload.archive, path.join(mirrorPath, payload.name)); - await cp(payload.entry, path.join(workspace.path, "downloads.json")); + await cp(payload.archivePath, path.join(mirrorPath, payload.archive.name)); + await cp(payload.entryPath, path.join(workspace.path, "downloads.json")); }; -export const staging = { - features: stageFeatures, - downloadMirror: stageDownloadMirror, -} as const; +export const staging = { downloadMirror: stageDownloadMirror } as const; diff --git a/scripts/verify/uv/verify.ts b/scripts/verify/uv/verify.ts index 3be1a0c..796266f 100644 --- a/scripts/verify/uv/verify.ts +++ b/scripts/verify/uv/verify.ts @@ -1,126 +1,37 @@ -import { mkdtempDisposable } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { parseArgs } from "node:util"; +import { z } from "zod"; -import { defaultRefPrefix, mainChannel } from "@/scripts/tasks/features/tags.js"; +import { components } from "@/scripts/tasks/features/uv/generateConfig.js"; -import { assertions } from "./assertions.js"; -import type { TestImage, Workspace } from "./lifecycle.js"; -import { imagesModes, lifecycle } from "./lifecycle.js"; -import { scenarios } from "./scenarios.js"; -import type { ComponentImage } from "./selection.js"; +import { parseVerifyArguments, requireSmokeTarget, targetSchema } from "../arguments.js"; +import { lifecycle } from "../lifecycle.js"; +import { runSuite } from "../run.js"; +import { scenarios, scenarioSelectionSchema } from "./scenarios.js"; import { selection } from "./selection.js"; -import { staging } from "./staging.js"; - -const componentImages = (images: readonly TestImage[]): ComponentImage[] => images.flatMap((image) => image.components); export const verify = async (argv: string[]): Promise => { - const { values } = parseArgs({ - args: argv, - options: { - force: { type: "boolean", default: false }, - rmi: { type: "boolean", default: false }, - scenario: { type: "string", multiple: true }, - images: { type: "string", default: "build" }, - channel: { type: "string", default: mainChannel }, - prefix: { type: "string" }, - }, - }); - - const images = imagesModes.find((candidate) => candidate === values.images); - if (images === undefined) { - throw new Error(`--images must be one of: ${imagesModes.join(", ")}`); - } - const channel = values.channel; - const prefix = values.prefix ?? (await defaultRefPrefix()); - const selected = scenarios.select(values.scenario ?? []); - const derived = await selection.derive(channel, prefix); + const { options, scenarios: requested, component, version } = await parseVerifyArguments(argv); + const selected = scenarioSelectionSchema.parse(requested); + const target = targetSchema(z.enum(components)).parse({ component, version }); + requireSmokeTarget(selected, target); + const derived = await selection.derive(options.channel, options.prefix, target); console.log( - `>>> verify uv: selection uv=${derived.uv.key} python=${derived.python.key} previous=${derived.previous.key}`, - ); - - const tempPath = await mkdtempDisposable(path.join(tmpdir(), "devcontainer-verify-uv-")); - const ctxPath = path.join(tempPath.path, "ctx"); - const runs = await Promise.all( - selected.map((name) => - scenarios.create(name, { - selection: derived, - root: tempPath.path, - workspace: lifecycle.workspaceAt(tempPath.path, name), - }), - ), + `>>> verify uv: selection uv=${derived.uv.key} python=${derived.python.key} previous=${derived.previous?.key ?? "none"}`, ); - const variants = runs.flatMap((run) => run.variants); - const testImages = new Map(); - for (const run of runs) { - testImages.set(run.testImage.tag, run.testImage); - } - for (const variant of variants) { - testImages.set(variant.testImage.tag, variant.testImage); - } - const workspaces: Workspace[] = [ - ...runs.map((run) => run.workspace), - ...variants.map((variant) => variant.workspace), - ]; - try { - console.log(">>> verify uv: provision"); - await lifecycle.provision(componentImages([...testImages.values()]), { - images, - channel, - prefix, - force: values.force, - }); - - console.log(">>> verify uv: test images"); - for (const image of testImages.values()) { - await lifecycle.buildTestImage(ctxPath, image); - } - - console.log(">>> verify uv: workspaces"); - for (const run of runs) { - await lifecycle.createWorkspace(run.workspace, { image: run.testImage.tag, remoteEnv: run.remoteEnv }); - await staging.features(run.workspace); - if (run.downloadMirror === true) { - await staging.downloadMirror(run.workspace, derived, values.force); - } - } - for (const variant of variants) { - await lifecycle.createWorkspace(variant.workspace, { image: variant.testImage.tag }); - await staging.features(variant.workspace); - } - - for (const run of runs) { - console.log(`>>> verify uv: scenario ${run.name}`); - await lifecycle.buildContainer(run.workspace); - await lifecycle.startContainer(run.workspace); - for (const exec of run.execs) { - assertions.exec(await lifecycle.execInContainer(run.workspace, exec), exec.expect); - } - if (run.restart !== undefined) { - console.log(`>>> verify uv: scenario ${run.name} restart`); - await lifecycle.startContainer(run.workspace); - for (const exec of run.restart) { - assertions.exec(await lifecycle.execInContainer(run.workspace, exec), exec.expect); - } - } - for (const variant of run.variants) { - console.log(`>>> verify uv: scenario ${run.name} variant ${variant.name}`); - await lifecycle.expectBuildFailure(variant.workspace, variant.buildFailure); - } - } - } finally { - console.log(">>> verify uv: cleanup"); - for (const workspace of workspaces) { - await lifecycle.removeContainer(workspace); - await lifecycle.removeWorkspaceImages(workspace); - } - if (values.rmi) { - for (const image of testImages.values()) { - await lifecycle.removeImage(image.tag); - } - } - await tempPath.remove(); - } + await runSuite({ + ...options, + name: "uv", + create: async (root) => + Promise.all( + selected.map((name) => + scenarios.create(name, { + selection: derived, + target, + root, + workspace: lifecycle.workspaceAt(root, name), + force: options.force, + }), + ), + ), + }); };