add-ml-dsa-signed-provider-checks

.github/workflows/signed-provider-tests.yml

@@ -0,0 +1,19 @@
+name: Signed provider checks
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - run: python -m unittest test_verify_provider

README.md

@@ -1,50 +1,80 @@
-# StackQL Provider Tests
-
-This repository contains a set of tests for StackQL providers. It is intended to be used by GitHub actions in the [stackql-provider-registry](https://github.com/stackql/stackql-provider-registry).  
-
-## Usage
-
-```bash
-test-provider.sh <provider_name> [<signed> <provider_path> <show_columns> <test_for_anonymous_cols>]
-```
-
-- `provider_name` - the name of the provider to test
-- `signed` - whether the provider is signed or not (default: false)
-- `provider_path` - the path to the provider (default: .)
-- `show_columns` - whether to show columns in output (default: false)
-- `test_for_anonymous_cols` - whether to test for anonymous columns (default: false)
-
-## Example
-
-```bash
-# test a local sumologic unsigned provider from the current directory
-sh test-provider.sh \
-sumologic
-false
-```
-
-```bash
-# test a local aws signed provider from another directory
-sh test-provider.sh \
-aws \
-true \
-/mnt/c/Users/javen/Documents/LocalGitRepos/stackql/local-registry
-```
-
-```bash
-# test a local google unsigned provider from another directory showing columns
-sh test-provider.sh \
-google \
-false \
-/mnt/c/Users/javen/Documents/LocalGitRepos/stackql/local-registry \
-true
-```
-
-```bash
-# test a local azure unsigned provider from the current directory showing columns
-sh test-provider.sh \
-azure \
-false \
-. \
-true
+# StackQL Provider Tests
+
+This repository contains a set of tests for StackQL providers. It is intended to be used by GitHub actions in the [stackql-provider-registry](https://github.com/stackql/stackql-provider-registry).
+
+The registry's main packaging workflow still invokes this repository.
+
+## Quantum-safe signed-provider checks
+
+Use a StackQL build containing the ML-DSA migration; older binaries understand
+only the retired `.sig` format. The `signed=true` mode in the existing runner
+continues to select `verifyConfig.nopVerify: false`.
+
+For a signature-specific check without virtualenv dependencies or a potentially
+pre-existing, unverified server:
+
+```sh
+python verify-provider.py aws --registry-root /path/to/provider-tests \
+  --stackql-bin /path/to/new/stackql
+```
+
+The root must contain `src/<provider>/<version>/provider.yaml` and service
+documents. The checker runs a fresh CLI to read provider metadata and every
+declared service. It fails on stderr even when StackQL reports a query failure
+with exit status zero. It neither starts nor stops a shared server.
+`--public-keys keys.json` supplies explicit `kid` to hex ML-DSA-65 public keys
+for private registries or ephemeral test fixtures; omit it for the embedded
+production keyset. Never provide private seeds.
+
+Unsigned or sidecar-only archives must be re-signed before this check can pass.
+Signature verification remains opt-in for normal StackQL usage. Archive
+extraction and freshness/rollback protection are outside these checks.
+
+Run the checker unit tests with `python -m unittest test_verify_provider`.
+
+## Usage
+
+```bash
+test-provider.sh <provider_name> [<signed> <provider_path> <show_columns> <test_for_anonymous_cols>]
+```
+
+- `provider_name` - the name of the provider to test
+- `signed` - whether the provider is signed or not (default: false)
+- `provider_path` - the path to the provider (default: .)
+- `show_columns` - whether to show columns in output (default: false)
+- `test_for_anonymous_cols` - whether to test for anonymous columns (default: false)
+
+## Example
+
+```bash
+# test a local sumologic unsigned provider from the current directory
+sh test-provider.sh \
+sumologic
+false
+```
+
+```bash
+# test a local aws signed provider from another directory
+sh test-provider.sh \
+aws \
+true \
+/mnt/c/Users/javen/Documents/LocalGitRepos/stackql/local-registry
+```
+
+```bash
+# test a local google unsigned provider from another directory showing columns
+sh test-provider.sh \
+google \
+false \
+/mnt/c/Users/javen/Documents/LocalGitRepos/stackql/local-registry \
+true
+```
+
+```bash
+# test a local azure unsigned provider from the current directory showing columns
+sh test-provider.sh \
+azure \
+false \
+. \
+true
 ```

test_verify_provider.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+import runpy
+import subprocess
+import unittest
+from unittest.mock import patch
+
+
+VERIFY = runpy.run_path(str(Path(__file__).with_name("verify-provider.py")))
+CONFIG = {"verifyConfig": {"nopVerify": False}}
+
+
+class SignedProviderTests(unittest.TestCase):
+    def test_reads_provider_and_every_service(self):
+        with patch("subprocess.run", side_effect=[
+            subprocess.CompletedProcess([], 0, "name\none\ntwo\n", ""),
+            subprocess.CompletedProcess([], 0, "name\nthing\n", ""),
+            subprocess.CompletedProcess([], 0, "name\nother\n", ""),
+        ]) as run:
+            self.assertEqual(VERIFY["verify_provider"]("stackql", "example", CONFIG), (2, 2))
+            self.assertEqual(run.call_count, 3)
+            self.assertIn("--registry=", run.call_args.args[0][3])
+
+    def test_zero_exit_status_does_not_hide_verification_failure(self):
+        with patch("subprocess.run", return_value=subprocess.CompletedProcess(
+            [], 0, "", "pqsig: signature verification failed"
+        )):
+            with self.assertRaisesRegex(RuntimeError, "signature verification failed"):
+                VERIFY["verify_provider"]("stackql", "example", CONFIG)
+
+    def test_rejects_disabled_verification(self):
+        with self.assertRaisesRegex(ValueError, "nopVerify"):
+            VERIFY["verify_provider"]("stackql", "example", {"verifyConfig": {"nopVerify": True}})
+
+    def test_empty_metadata_is_not_success(self):
+        with patch("subprocess.run", return_value=subprocess.CompletedProcess([], 0, "", "")):
+            with self.assertRaisesRegex(RuntimeError, "expected named metadata"):
+                VERIFY["verify_provider"]("stackql", "example", CONFIG)

verify-provider.py

@@ -0,0 +1,80 @@
+"""Verify provider metadata through a fresh StackQL CLI, without a shared server."""
+
+import argparse
+import csv
+import io
+import json
+from pathlib import Path
+import re
+import subprocess
+import sys
+import tempfile
+
+
+def identifier(name):
+    if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name):
+        return name
+    return "`" + name.replace("`", "``") + "`"
+
+
+def query(binary, registry, statement, app_root):
+    result = subprocess.run(
+        [str(binary), "exec", statement, "--registry=" + json.dumps(registry),
+         "--output=csv", "--approot=" + app_root],
+        capture_output=True,
+        text=True,
+        timeout=120,
+        check=False,
+    )
+    # StackQL currently reports some query errors on stderr with exit status zero.
+    if result.returncode or result.stderr.strip():
+        raise RuntimeError(f"{statement}: {result.stderr.strip() or result.stdout.strip()}")
+    rows = csv.DictReader(io.StringIO(result.stdout))
+    if not rows.fieldnames or "name" not in rows.fieldnames:
+        raise RuntimeError(f"{statement}: expected named metadata columns, got {result.stdout!r}")
+    return list(rows)
+
+
+def verify_provider(binary, provider, registry):
+    if registry.get("verifyConfig", {}).get("nopVerify") is not False:
+        raise ValueError("signed provider tests require explicit nopVerify: false")
+    with tempfile.TemporaryDirectory(prefix="stackql-verify-") as app_root:
+        services = query(binary, registry, f"SHOW SERVICES IN {identifier(provider)};", app_root)
+        if not services:
+            raise RuntimeError(f"no services found for {provider}")
+        resources = 0
+        for service in services:
+            address = f"{identifier(provider)}.{identifier(service['name'])}"
+            resources += len(query(binary, registry, f"SHOW RESOURCES IN {address};", app_root))
+    return len(services), resources
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("provider")
+    parser.add_argument("--registry-root", required=True, type=Path)
+    parser.add_argument("--stackql-bin", required=True, type=Path)
+    parser.add_argument("--public-keys", type=Path, help="optional kid-to-hex public key JSON for private/test registries")
+    args = parser.parse_args()
+    try:
+        root = args.registry_root.resolve(strict=True)
+        binary = args.stackql_bin.resolve(strict=True)
+        verify_config = {"nopVerify": False}
+        if args.public_keys:
+            keys = json.loads(args.public_keys.read_text(encoding="utf-8"))
+            if not isinstance(keys, dict) or not keys or not all(
+                isinstance(k, str) and isinstance(v, str) for k, v in keys.items()
+            ):
+                raise ValueError("--public-keys must contain a non-empty kid-to-hex object")
+            verify_config["publicKeys"] = keys
+        registry = {"url": root.as_uri(), "localDocRoot": str(root), "verifyConfig": verify_config}
+        services, resources = verify_provider(binary, args.provider, registry)
+    except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    print(f"Verified {services} services and {resources} resources in {args.provider}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())