plugins.yml

name: Plugins

# Official bytes live in COS. Every app download goes through the statistics API.
on:
  push:
    branches: [dev]
    # Only plugin-relevant inputs launch this pipeline: app releases, lockfile
    # churn and build-tool edits must not republish unchanged online plugins.
    paths:
      - 'market/**'
      - 'packages/plugin-ui-sdk/**'
      - 'packages/plugin-build/**'
      - 'crates/tf-plugin-api/**'
      - 'crates/tf-plugin-guest/**'
      - 'wit/**'
      - 'scripts/publish-plugins.sh'
      - 'scripts/cos-setup.sh'
      - 'scripts/migrate-plugin-assets.sh'
      - '.github/workflows/plugins.yml'
  workflow_dispatch:
    inputs:
      bootstrap:
        description: 'Allow first COS catalog initialization after checking existing assets'
        type: boolean
        default: false

permissions:
  contents: read

concurrency:
  group: plugins-production
  cancel-in-progress: false

env:
  CARGO_TERM_COLOR: always
  RELEASE_TAG: plugins
  PLUGIN_ASSET_BASE_URL: https://plugins.toolforge.devlive.org
  PLUGIN_STATS_BASE_URL: https://stats.toolforge.devlive.org

jobs:
  build:
    if: github.ref == 'refs/heads/dev'
    runs-on: ubuntu-24.04
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          fetch-depth: 0
          persist-credentials: false
      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: 22
      - uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.1.0
      - name: Install WASM target and build trusted tools without signing secrets
        run: |
          # Pin a specific toolchain: a channel name (stable) makes rustup sync
          # channel updates on every shim call, and a mid-build sync replaced
          # rustc while cargo was compiling (run 39738297419).
          rustup toolchain install 1.99.1 --profile minimal --target wasm32-wasip2
          rustup default 1.99.1
          cargo build --locked --release -p xtask
          pnpm install --frozen-lockfile
          shellcheck scripts/cos-setup.sh scripts/publish-plugins.sh scripts/migrate-plugin-assets.sh
          cargo test --locked -p xtask
      - name: Read complete GitHub history without changing historical bytes
        env:
          GH_TOKEN: ${{ github.token }}
        run: scripts/migrate-plugin-assets.sh dist/plugins
      - name: Install COSCLI and write its temporary credentials configuration
        env:
          COS_SECRET_ID: ${{ secrets.COS_SECRET_ID }}
          COS_SECRET_KEY: ${{ secrets.COS_SECRET_KEY }}
          COS_SESSION_TOKEN: ${{ secrets.COS_SESSION_TOKEN }}
          COS_REGION: ${{ vars.COS_REGION || 'ap-singapore' }}
          COS_BUCKET: toolforge-1258585472
        run: scripts/cos-setup.sh
      - name: Read existing COS catalog and find unpublished versions
        env:
          BOOTSTRAP: ${{ inputs.bootstrap || false }}
          COS_BUCKET: toolforge-1258585472
          COSCLI_BIN: ${{ runner.temp }}/coscli
          COS_CONFIG_PATH: ${{ runner.temp }}/cos.yaml
        run: |
          set -euo pipefail
          # The bucket is private: read through authenticated COSCLI, never anonymous HTTP.
          if "$COSCLI_BIN" cp "cos://$COS_BUCKET/release-catalog.json" "$RUNNER_TEMP/prior-catalog.json" -c "$COS_CONFIG_PATH" --disable-log --fail-output=false --process-log=false; then
            jq -e '.schemaVersion == 1 and (.sourceCommit | test("^[a-f0-9]{40}$"))' "$RUNNER_TEMP/prior-catalog.json" >/dev/null
          elif [[ $BOOTSTRAP == true ]]; then
            rm -f "$RUNNER_TEMP/prior-catalog.json"
          else
            echo '::error::Cannot read COS catalog through COSCLI; first publication requires a bootstrap run'; exit 1
          fi
          python3 - <<'PY'
          import hashlib, json, os, pathlib, re, subprocess, tomllib
          root = pathlib.Path('dist/plugins')
          assets = {a['name'] for a in json.loads((root / 'previous-release.json').read_text())['assets']}
          prior_path = pathlib.Path(os.environ['RUNNER_TEMP']) / 'prior-catalog.json'
          prior = json.loads(prior_path.read_text()) if prior_path.exists() else None
          # A previous run may have published COS but only partially mirrored GitHub.
          # Recover its exact signed bytes rather than rebuilding an existing version.
          if prior:
              for item in prior['plugins']:
                  name = f"{item['id']}-{item['version']}.tfp"
                  if name in assets:
                      continue
                  key = f"packages/{item['id']}/{item['version']}/{item['sha256']}.tfp"
                  if item['objectKey'] != key or not re.fullmatch(r'[a-z0-9][a-z0-9.+-]*\.tfp', name) or not re.fullmatch(r'[a-f0-9]{64}', item['sha256']):
                      raise SystemExit('Unsafe existing COS catalog metadata')
                  target = root / 'previous' / name
                  target.parent.mkdir(parents=True, exist_ok=True)
                  target.unlink(missing_ok=True)
                  subprocess.run([os.environ['COSCLI_BIN'], 'cp', f"cos://{os.environ['COS_BUCKET']}/{key}", str(target), '-c', os.environ['COS_CONFIG_PATH'], '--disable-log', '--fail-output=false', '--process-log=false'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
                  if target.stat().st_size != item['size'] or hashlib.sha256(target.read_bytes()).hexdigest() != item['sha256']:
                      raise SystemExit('Cannot recover verified historical COS bytes')
                  assets.add(name)
          # Compare only packages reachable from each WASM backend. Native App-only lock
          # updates must not force every online plugin to publish a new version.
          cargo_changed = set()
          manifests = {}
          if prior:
              changed = subprocess.check_output(['git', 'diff', '--name-only', prior['sourceCommit'], 'HEAD', '--', 'Cargo.lock'], text=True)
              if changed.strip():
                  old_lock = tomllib.loads(subprocess.check_output(['git', 'show', prior['sourceCommit'] + ':Cargo.lock'], text=True))
                  new_lock = tomllib.loads(pathlib.Path('Cargo.lock').read_text())
                  key = lambda p: (p['name'], p.get('source'))
                  old_packages = {key(p): p for p in old_lock['package']}
                  new_packages = {key(p): p for p in new_lock['package']}
                  metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--locked', '--format-version', '1', '--filter-platform', 'wasm32-wasip2'], text=True))
                  packages = {p['id']: p for p in metadata['packages']}
                  nodes = {n['id']: n for n in metadata['resolve']['nodes']}
                  manifests = {str(pathlib.Path(p['manifest_path']).resolve()): p['id'] for p in metadata['packages']}
                  for ident, package in packages.items():
                      entry = key(package)
                      old, new = old_packages.get(entry), new_packages.get(entry)
                      if old is None or new is None:
                          cargo_changed.add(ident)
                          continue
                      # Local path crates carry release metadata versions: a
                      # version-only bump does not change compiled plugin bytes.
                      # A crates.io release is always a real input change.
                      if package.get('source') is None:
                          drop = {'version'}
                          old = {field: value for field, value in old.items() if field not in drop}
                          new = {field: value for field, value in new.items() if field not in drop}
                      if old != new:
                          cargo_changed.add(ident)
          # SDK changes listed in .github/plugin-sdk-host-only.txt only affect the host;
          # when every commit touching the SDK since the last publication is listed,
          # those paths do not count as inputs of already published plugins.
          sdk_paths = ['packages/plugin-ui-sdk', 'packages/plugin-build', 'crates/tf-plugin-api', 'crates/tf-plugin-guest', 'wit']
          host_only = set()
          ack = pathlib.Path('.github/plugin-sdk-host-only.txt')
          for line in ack.read_text().splitlines() if ack.exists() else []:
              line = line.strip()
              if not line or line.startswith('#'):
                  continue
              sha = line.split()[0]
              if not re.fullmatch(r'[a-f0-9]{40}', sha):
                  raise SystemExit(f'Invalid commit in {ack}: {sha}')
              host_only.add(sha)
          sdk_host_only = False
          if prior:
              sdk_commits = subprocess.check_output(['git', 'log', '--format=%H', f"{prior['sourceCommit']}..HEAD", '--', *sdk_paths], text=True).split()
              sdk_host_only = bool(sdk_commits) and set(sdk_commits) <= host_only
              if sdk_host_only:
                  print(f'SDK changes since the last publication are host-only: {len(sdk_commits)} commit(s)')
          pending = []
          for path in sorted(pathlib.Path('market').glob('*/manifest.json')):
              manifest = json.loads(path.read_text())
              slug = path.parent.name
              if not slug.replace('-', '').isalnum():
                  raise SystemExit('Unsafe plugin slug')
              name = f"{manifest['id']}-{manifest['version']}.tfp"
              if name not in assets:
                  pending.append(slug)
              elif prior:
                  watched = [str(path.parent)] + ([] if sdk_host_only else sdk_paths)
                  changed = subprocess.check_output(['git', 'diff', '--name-only', prior['sourceCommit'], 'HEAD', '--', *watched], text=True)
                  # pnpm-lock.yaml is deliberately absent: shared UI dependencies
                  # load from the host bundle (window.__TF_SHARED__) and never
                  # enter plugin bytes, so one plugin's dependency additions must
                  # not force every published plugin to re-version.
                  inputs = [line for line in changed.splitlines() if line not in ('crates/tf-plugin-api/Cargo.toml', 'crates/tf-plugin-guest/Cargo.toml')]
                  dependency_changed = False
                  if cargo_changed:
                      manifest_path = str((path.parent / 'backend' / 'Cargo.toml').resolve())
                      frontier = [manifests[manifest_path]]
                      seen = set()
                      while frontier:
                          ident = frontier.pop()
                          if ident in seen:
                              continue
                          seen.add(ident)
                          frontier.extend(dep['pkg'] for dep in nodes[ident]['deps'])
                      dependency_changed = bool(seen & cargo_changed)
                  if inputs or dependency_changed:
                      raise SystemExit(f'{slug}: published plugin inputs changed without a version bump')
          (root / 'pending.txt').write_text(''.join(slug + '\n' for slug in pending))
          PY
      - name: Build unsigned candidates without signing credentials
        run: |
          mkdir -p dist/unsigned
          while IFS= read -r slug; do
            target/release/xtask plugin pack "$slug" --unsigned --out dist/unsigned
          done < dist/plugins/pending.txt
      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: plugin-candidates-${{ github.run_id }}
          path: |
            dist/unsigned/
            dist/plugins/
          if-no-files-found: error
          retention-days: 7
          overwrite: true

  publish:
    needs: build
    if: github.ref == 'refs/heads/dev'
    runs-on: ubuntu-24.04
    environment: plugins-production
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          persist-credentials: false
      - name: Build the trusted signing tool on a clean runner
        run: |
          cargo build --locked --release -p xtask
          cargo test --locked -p xtask
          shellcheck scripts/cos-setup.sh scripts/publish-plugins.sh scripts/migrate-plugin-assets.sh
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
          name: plugin-candidates-${{ github.run_id }}
          path: dist
      - name: Install COSCLI and write its temporary credentials configuration
        env:
          COS_SECRET_ID: ${{ secrets.COS_SECRET_ID }}
          COS_SECRET_KEY: ${{ secrets.COS_SECRET_KEY }}
          COS_SESSION_TOKEN: ${{ secrets.COS_SESSION_TOKEN }}
          COS_REGION: ${{ vars.COS_REGION || 'ap-singapore' }}
          COS_BUCKET: toolforge-1258585472
        run: scripts/cos-setup.sh
      - name: Independently read current history with the publishing identity
        env:
          GH_TOKEN: ${{ github.token }}
          BOOTSTRAP: ${{ inputs.bootstrap || false }}
          COS_BUCKET: toolforge-1258585472
        run: |
          scripts/migrate-plugin-assets.sh dist/verified-history
          cp dist/verified-history/previous-release.json dist/plugins/previous-release.json
          cp -a dist/verified-history/previous/. dist/plugins/previous/
          # The bucket is private: read through authenticated COSCLI, never anonymous HTTP.
          if "$RUNNER_TEMP/coscli" cp "cos://$COS_BUCKET/release-catalog.json" "$RUNNER_TEMP/prior-catalog.json" -c "$RUNNER_TEMP/cos.yaml" --disable-log --fail-output=false --process-log=false; then
            jq -e '.schemaVersion == 1 and (.sourceCommit | test("^[a-f0-9]{40}$"))' "$RUNNER_TEMP/prior-catalog.json" >/dev/null
          elif [[ $BOOTSTRAP == true ]]; then
            rm -f "$RUNNER_TEMP/prior-catalog.json"
          else
            echo '::error::Cannot read COS catalog through COSCLI; first publication requires a bootstrap run'; exit 1
          fi
      - name: Independently sign candidates using the precompiled tool
        env:
          TOOLFORGE_PLUGIN_KEY: ${{ secrets.TOOLFORGE_PLUGIN_KEY }}
        run: |
          set -euo pipefail
          shopt -s nullglob
          for package in dist/unsigned/*.tfp; do
            target/release/xtask plugin sign "$package" --out dist/plugins
          done
      - name: Validate complete inventory and generate registry and catalog
        run: |
          target/release/xtask plugin verify dist/plugins
          previous=()
          if [[ -f "$RUNNER_TEMP/prior-catalog.json" ]]; then
            previous=(--merge-catalog "$RUNNER_TEMP/prior-catalog.json")
          fi
          target/release/xtask plugin index dist/plugins "$PLUGIN_STATS_BASE_URL" --source-commit "$GITHUB_SHA" --generation "$GITHUB_RUN_NUMBER" "${previous[@]}"
      - name: Publish verified objects, import catalog, then publish registry
        env:
          GH_TOKEN: ${{ github.token }}
          COS_BUCKET: toolforge-1258585472
          PLUGIN_CATALOG_SSH_TARGET: ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}
          PLUGIN_CATALOG_SSH_KEY: ${{ secrets.SSH_KEY }}
        run: |
          set -euo pipefail
          umask 077
          task_dir=$(mktemp -d "$RUNNER_TEMP/plugin-publish.XXXXXX")
          trap 'rm -rf "$task_dir"' EXIT
          export COSCLI_BIN="$RUNNER_TEMP/coscli" COS_CONFIG_PATH="$RUNNER_TEMP/cos.yaml" XTASK_BIN=target/release/xtask
          export PLUGIN_SSH_KEY_PATH="$task_dir/id"
          python3 - <<'PY'
          import os, pathlib
          if not os.environ.get('PLUGIN_CATALOG_SSH_KEY'):
              raise SystemExit('Missing protected publishing configuration: PLUGIN_CATALOG_SSH_KEY')
          pathlib.Path(os.environ['PLUGIN_SSH_KEY_PATH']).write_text(os.environ['PLUGIN_CATALOG_SSH_KEY'] + '\n')
          PY
          unset PLUGIN_CATALOG_SSH_KEY
          scripts/publish-plugins.sh dist/plugins