Implement --snapshot-url

.github/scripts/batch_index.sh

@@ -5,6 +5,11 @@ set -e
 # total time reasonable.
 batch_index=$1
 
+url=""
+if [[ -n "$2" ]]; then
+  url="--snapshot-url=$2"
+fi
+
 source .github/scripts/install.sh
 
 if [[ -d output ]]; then
@@ -19,7 +24,8 @@ clc-stackage \
   --batch 200 \
   --batch-index $batch_index \
   --cabal-options="--semaphore" \
-  --cleanup off
+  --cleanup off \
+  $url
 
 ec=$?
 

.github/scripts/dry_run.sh

@@ -1,3 +1,8 @@
+url=""
+if [[ -n "$1" ]]; then
+  url="--snapshot-url=$1"
+fi
+
 source .github/scripts/install.sh
 
 if [[ -d output ]]; then
@@ -7,7 +12,7 @@ fi
 echo "*** Building all with --dry-run ***"
 
 set +e
-clc-stackage --batch 200 --cabal-options="--dry-run"
+clc-stackage --batch 200 --cabal-options="--dry-run" $url
 
 ec=$?
 

.github/workflows/build.yaml

@@ -0,0 +1,60 @@
+name: build
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+    branches:
+      - master
+
+  schedule:
+    # Once a week.
+    #
+    # https://crontab.guru
+    - cron: '0 0 * * 0'
+
+  workflow_dispatch:
+
+jobs:
+  cabal:
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - "macos-latest"
+          - "ubuntu-latest"
+          - "windows-latest"
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: haskell-actions/setup@v2
+        with:
+          # Should be the current stackage nightly, though this will likely go
+          # out-of-date eventually, until a problem is reported.
+          ghc-version: "9.12"
+      - name: Configure
+        run: |
+          cabal configure --enable-tests --ghc-options -Werror
+
+      - name: Build executable
+        run: cabal build exe:clc-stackage
+
+      - name: Unit Tests
+        id: unit
+        run: cabal test unit
+
+      - name: Functional Tests
+        id: functional
+        # We want to run these tests even if the unit tests fail, because
+        # it is useful to know if e.g. the unit tests fail due to one
+        # stackage endpoint failing, but the functional tests pass due to
+        # a backup working.
+        if: always()
+        shell: bash
+        run: NO_CLEANUP=1 cabal test functional
+
+      - name: Print functional failures
+        if: ${{ failure() && steps.functional.conclusion == 'failure' }}
+        shell: bash
+        run: .github/scripts/print_logs.sh

.github/workflows/nightly.yaml

@@ -0,0 +1,22 @@
+name: nightly
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+    branches:
+      - master
+
+  schedule:
+    # Every day at 00:00 UTC.
+    #
+    # https://crontab.guru
+    - cron: '0 0 * * *'
+
+  workflow_dispatch:
+jobs:
+  trigger-build:
+    uses: ./.github/workflows/runner.yaml
+    with:
+      snapshot: 'nightly'

.github/workflows/ci.yaml → .github/workflows/runner.yaml

@@ -1,69 +1,15 @@
-name: ci
+name: runner
 on:
-  push:
-    branches:
-      - master
-
-  pull_request:
-    branches:
-      - master
-
-  schedule:
-    # Every day at 00:00 UTC.
-    #
-    # https://crontab.guru
-    - cron: '0 0 * * *'
+  workflow_call:
+    inputs:
+      snapshot:
+        required: true
+        type: string
 
   workflow_dispatch:
 jobs:
-  cabal:
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - "macos-latest"
-          - "ubuntu-latest"
-          - "windows-latest"
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v6
-      - uses: haskell-actions/setup@v2
-        with:
-          # Should be the current stackage nightly, though this will likely go
-          # out-of-date eventually, until a problem is reported.
-          ghc-version: "9.12"
-      - name: Configure
-        run: |
-          cabal configure --enable-tests --ghc-options -Werror
-
-      - name: Build executable
-        run: cabal build exe:clc-stackage
-
-      - name: Unit Tests
-        id: unit
-        run: cabal test unit
-
-      - name: Functional Tests
-        id: functional
-        # We want to run these tests even if the unit tests fail, because
-        # it is useful to know if e.g. the unit tests fail due to one
-        # stackage endpoint failing, but the functional tests pass due to
-        # a backup working.
-        if: always()
-        shell: bash
-        run: NO_CLEANUP=1 cabal test functional
-
-      - name: Print functional failures
-        if: ${{ failure() && steps.functional.conclusion == 'failure' }}
-        shell: bash
-        run: .github/scripts/print_logs.sh
   dry-run:
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-        - "ubuntu-latest"
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
     steps:
       - uses: actions/checkout@v6
 
@@ -73,8 +19,8 @@ jobs:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}
           nix_path: nixpkgs=channel:nixos-unstable
 
-      - name: Dry run
-        run: nix develop .#ci -Lv -c bash -c '.github/scripts/dry_run.sh'
+      - name: Dry run ${{ inputs.snapshot }}
+        run: nix develop .#ci -Lv -c bash -c '.github/scripts/dry_run.sh ${{ inputs.snapshot }}'
 
       # Upload installed binary so that build-batch does not need to re-install
       # it.
@@ -147,4 +93,4 @@ jobs:
           path: ./bin
 
       - name: Build
-        run: nix develop .#ci -Lv -c bash -c '.github/scripts/batch_index.sh ${{ matrix.index }}'
+        run: nix develop .#ci -Lv -c bash -c '.github/scripts/batch_index.sh ${{ matrix.index }} ${{ inputs.snapshot }}'

.github/workflows/stable.yaml

@@ -0,0 +1,22 @@
+name: stable
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+    branches:
+      - master
+
+  schedule:
+    # Once a week.
+    #
+    # https://crontab.guru
+    - cron: '0 0 * * 0'
+
+  workflow_dispatch:
+jobs:
+  trigger-build:
+    uses: ./.github/workflows/runner.yaml
+    with:
+      snapshot: 'nightly-2026-10-22'

README.md

@@ -1,6 +1,8 @@
 # `clc-stackage`
 
-[![ci](https://github.com/haskell/clc-stackage/actions/workflows/ci.yaml/badge.svg)](https://github.com/haskell/clc-stackage/actions/workflows/ci.yaml)
+[![build](https://github.com/haskell/clc-stackage/actions/workflows/build.yaml/badge.svg)](https://github.com/haskell/clc-stackage/actions/workflows/build.yaml)
+[![stable](https://github.com/haskell/clc-stackage/actions/workflows/stable.yaml/badge.svg)](https://github.com/haskell/clc-stackage/actions/workflows/stable.yaml)
+[![nightly](https://github.com/haskell/clc-stackage/actions/workflows/nightly.yaml/badge.svg)](https://github.com/haskell/clc-stackage/actions/workflows/nightly.yaml)
 
 ## How to?
 
@@ -14,7 +16,18 @@ An impact assessment is due when
 
 The procedure is as follows:
 
-1. Rebase changes, mandated by your proposal, atop the ghc branch (or tag) that corresponds to the current [stackage nightly](https://www.stackage.org/nightly). For example, if the latest snapshot ghc is `ghc-9.12.3`, we would want to rebase our changes on the `ghc-9.12.3-release` tag.
+1. Rebase changes, mandated by your proposal, atop the ghc branch (or tag) that corresponds to the snapshot you intend to use. The most natural snapshot choices are either the [current stable](./.github/workflows/stable.yaml):
+
+    ```yaml
+    # .github/workflows/stable.yaml
+    snapshot: 'nightly-<date>'
+    ```
+
+    Or just `nightly` for the latest. Ideally we would like to use `nightly`, as it is the most recent and likely easiest GHC to build. But `nightly` can sometimes bring in new, problematic packages, hence we also list a `stable`, as a last known "good" snapshot, for when `nightly` is broken.
+
+    Both snapshots have CI jobs that attempt to build them, so CI can be a useful indicator of the current snapshots' health.
+
+    As an example, if our chosen snapshot corresponds to `ghc-9.12.3`, we would want to rebase our changes on the `ghc-9.12.3-release` tag.
 
 2. Compile a patched GHC, say, `~/ghc/_build/stage1/bin/ghc`.
 
@@ -70,7 +83,17 @@ The procedure is as follows:
 
 ### Troubleshooting
 
-Because we build with `nightly` and are at the mercy of cabal's constraint solver, it is possible to run into solver / build issues that have nothing to do with our custom GHC. Some of the most common problems include:
+#### Stable
+
+While the stable snapshot is more stable than `nightly`, we can still run into problems:
+
+- Hackage revisions can make a build fail e.g. `base<0`.
+
+It may be useful to combine `stable` with `index-state` in [generated/cabal.project](./generated/cabal.project).
+
+#### Nightly
+
+Building with `nightly` leaves us at the mercy of cabal's constraint solver, hence it is possible to run into solver / build issues that have nothing to do with our custom GHC. Some of the most common problems include:
 
 - Nightly adds a new, problematic package `p` e.g.
 
@@ -133,7 +156,28 @@ In general, user mitigations for solver / build problems include:
 
 #### Querying stackage
 
-By default, `clc-stackage` queries https://www.stackage.org/ for snapshot information. In situations where this is not desirable (e.g. the server is not working, or we want to test a custom snapshot), the snapshot can be overridden:
+By default, `clc-stackage` queries `https://www.stackage.org/nightly` for snapshot information. In situations where this is not desirable (e.g. the server is not working, or we want to test a custom snapshot), the snapshot can be overridden in two ways.
+
+##### Supplying another url
+
+The `--snapshot-url` param can be used to override the default snapshot e.g.
+
+```sh
+# resolves to https://www.stackage.org/nightly-2026-04-23
+$ clc-stackage --snapshot-url=nightly-2026-04-23
+
+# https://www.stackage.org/lts-24.38
+$ clc-stackage --snapshot-url=lts-24.38
+
+# https://some-url
+$ clc-stackage --snapshot-url=https://some-url
+```
+
+Note that we expect the url to have the same API as stackage, and fall back to `<url>/cabal.config` if its json endpoint is not working.
+
+##### Supplying a manual snapshot
+
+We can also supply a snapshot manually:
 
 ```sh
 $ clc-stackage --snapshot-path=path/to/snapshot

dev.md

@@ -75,13 +75,20 @@ The executable that actually runs. This is a very thin wrapper over `runner`, wh
 
 ## Updating to a new snapshot
 
-`clc-stackage` is based on `nightly` -- which changes automatically -- meaning we do not necessarily have to do anything when a new (minor) snapshot is released. On the other hand, *major* snapshot updates will almost certainly bring in new packages that need to be excluded, so there are some general "update steps" we will want to take:
+While `clc-stackage` is based on `nightly` -- which changes automatically -- we also specify a `stable` snapshot as a backup for when `nightly` runs into build issues e.g. a new package is added that needs to be excluded or have a flag set. This occurs a few times a month. We therefore pick an arbitrary `nightly` snapshot that has many packages in it (generally 3000+). When we do want to update the snapshot -- generally, once per GHC major version update -- there are some general steps we will want to take:
 
-1. Modify [package_index.jsonc](package_index.jsonc) as needed. That is, updating the snapshot major version will probably bring in some new packages that we do not want. The update process is essentially trial-and-error i.e. run `clc-stackage` as normal, and later add any failing packages to `package_index.excluded` that should be excluded.
+1. Update the [stable snapshot date](./.github/workflows/stable.yaml):
 
-2. Update `ghc-version` in [.github/workflows/ci.yaml](.github/workflows/ci.yaml).
+    ```yaml
+    # .github/workflows/stable.yaml
+    snapshot: 'nightly-<date>'
+    ```
 
-3. Optional: Update nix:
+2. Modify [package_index.jsonc](package_index.jsonc) as needed. That is, updating the snapshot version will probably bring in some new packages that we do not want. The update process is essentially trial-and-error i.e. run `clc-stackage` as normal, and later add any failing packages to `package_index.excluded` that should be excluded.
+
+3. If this is a major ghc update, update the `ghc-version` in [.github/workflows/build.yaml](.github/workflows/build.yaml).
+
+4. Optional: Update nix:
 
     - Inputs (`nix flake update`).
     - GHC: Update the `compiler = pkgs.haskell.packages.ghc<vers>;` line.
@@ -115,4 +122,4 @@ Note that this only saves files from the _last_ test, so if you want to examine
 
 > [!TIP]
 >
-> CI has a job `build-batch` which actually builds the entire package set, hence it can be used in place of manual building / testing. Note it takes about an hour to run.
+> CI has workflows `nightly` and `stable` which build the entire package set, hence they can be used in place of manual building / testing. Note they each take about an hour to run.

src/CLC/Stackage/Parser.hs

@@ -45,9 +45,9 @@ import Text.ParserCombinators.ReadP qualified as ReadP
 
 -- | Retrieves the 'PackageSet', based on
 -- 'CLC.Stackage.Parser.API.stackageUrl'.
-getPackageList :: Logging.Handle -> Maybe OsPath -> IO PackageSet
-getPackageList hLogger msnapshotPath = do
-  response <- getStackageResponse hLogger msnapshotPath
+getPackageList :: Logging.Handle -> Maybe OsPath -> Maybe String -> IO PackageSet
+getPackageList hLogger msnapshotPath msnapshotUrl = do
+  response <- getStackageResponse hLogger msnapshotPath msnapshotUrl
   packageListToSet hLogger response.packages
 
 -- | Given a list of packages, returns a 'PackageSet' i.e. all constraints
@@ -57,9 +57,9 @@ packageListToSet hLogger packages =
   getPackageListByOs hLogger packages OS.currentOs
 
 -- | Prints the package list to a file.
-printPackageList :: Logging.Handle -> Maybe OsPath -> Maybe Os -> IO ()
-printPackageList hLogger msnapshotPath mOs = do
-  response <- getStackageResponse hLogger msnapshotPath
+printPackageList :: Logging.Handle -> Maybe OsPath -> Maybe String -> Maybe Os -> IO ()
+printPackageList hLogger msnapshotPath msnapshotUrl mOs = do
+  response <- getStackageResponse hLogger msnapshotPath msnapshotUrl
   case mOs of
     Just os -> printOsList response os
     Nothing -> for_ [minBound .. maxBound] (printOsList response)
@@ -134,10 +134,10 @@ getPackageListByOs hLogger packageList os = do
       Osx -> (.osx)
       Windows -> (.windows)
 
-getStackageResponse :: Logging.Handle -> Maybe OsPath -> IO StackageResponse
-getStackageResponse hLogger msnapshotPath = do
+getStackageResponse :: Logging.Handle -> Maybe OsPath -> Maybe String -> IO StackageResponse
+getStackageResponse hLogger msnapshotPath msnapshotUrl = do
   response <- case msnapshotPath of
-    Nothing -> API.getStackage hLogger
+    Nothing -> API.getStackage hLogger msnapshotUrl
     Just snapshotPath ->
       CabalConfig.parseCabalConfig
         <$> IO.readFileUtf8 snapshotPath