diff --git a/.github/workflows/signed-provider-tests.yml b/.github/workflows/signed-provider-tests.yml new file mode 100644 index 0000000..43262a3 --- /dev/null +++ b/.github/workflows/signed-provider-tests.yml @@ -0,0 +1,19 @@ +name: Signed provider checks + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - run: python -m unittest test_verify_provider diff --git a/README.md b/README.md index 2d47c43..66a9bfe 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,80 @@ -# StackQL Provider Tests - -This repository contains a set of tests for StackQL providers. It is intended to be used by GitHub actions in the [stackql-provider-registry](https://github.com/stackql/stackql-provider-registry). - -## Usage - -```bash -test-provider.sh [ ] -``` - -- `provider_name` - the name of the provider to test -- `signed` - whether the provider is signed or not (default: false) -- `provider_path` - the path to the provider (default: .) -- `show_columns` - whether to show columns in output (default: false) -- `test_for_anonymous_cols` - whether to test for anonymous columns (default: false) - -## Example - -```bash -# test a local sumologic unsigned provider from the current directory -sh test-provider.sh \ -sumologic -false -``` - -```bash -# test a local aws signed provider from another directory -sh test-provider.sh \ -aws \ -true \ -/mnt/c/Users/javen/Documents/LocalGitRepos/stackql/local-registry -``` - -```bash -# test a local google unsigned provider from another directory showing columns -sh test-provider.sh \ -google \ -false \ -/mnt/c/Users/javen/Documents/LocalGitRepos/stackql/local-registry \ -true -``` - -```bash -# test a local azure unsigned provider from the current directory showing columns -sh test-provider.sh \ -azure \ -false \ -. \ -true +# StackQL Provider Tests + +This repository contains a set of tests for StackQL providers. It is intended to be used by GitHub actions in the [stackql-provider-registry](https://github.com/stackql/stackql-provider-registry). + +The registry's main packaging workflow still invokes this repository. + +## Quantum-safe signed-provider checks + +Use a StackQL build containing the ML-DSA migration; older binaries understand +only the retired `.sig` format. The `signed=true` mode in the existing runner +continues to select `verifyConfig.nopVerify: false`. + +For a signature-specific check without virtualenv dependencies or a potentially +pre-existing, unverified server: + +```sh +python verify-provider.py aws --registry-root /path/to/provider-tests \ + --stackql-bin /path/to/new/stackql +``` + +The root must contain `src///provider.yaml` and service +documents. The checker runs a fresh CLI to read provider metadata and every +declared service. It fails on stderr even when StackQL reports a query failure +with exit status zero. It neither starts nor stops a shared server. +`--public-keys keys.json` supplies explicit `kid` to hex ML-DSA-65 public keys +for private registries or ephemeral test fixtures; omit it for the embedded +production keyset. Never provide private seeds. + +Unsigned or sidecar-only archives must be re-signed before this check can pass. +Signature verification remains opt-in for normal StackQL usage. Archive +extraction and freshness/rollback protection are outside these checks. + +Run the checker unit tests with `python -m unittest test_verify_provider`. + +## Usage + +```bash +test-provider.sh [ ] +``` + +- `provider_name` - the name of the provider to test +- `signed` - whether the provider is signed or not (default: false) +- `provider_path` - the path to the provider (default: .) +- `show_columns` - whether to show columns in output (default: false) +- `test_for_anonymous_cols` - whether to test for anonymous columns (default: false) + +## Example + +```bash +# test a local sumologic unsigned provider from the current directory +sh test-provider.sh \ +sumologic +false +``` + +```bash +# test a local aws signed provider from another directory +sh test-provider.sh \ +aws \ +true \ +/mnt/c/Users/javen/Documents/LocalGitRepos/stackql/local-registry +``` + +```bash +# test a local google unsigned provider from another directory showing columns +sh test-provider.sh \ +google \ +false \ +/mnt/c/Users/javen/Documents/LocalGitRepos/stackql/local-registry \ +true +``` + +```bash +# test a local azure unsigned provider from the current directory showing columns +sh test-provider.sh \ +azure \ +false \ +. \ +true ``` \ No newline at end of file diff --git a/test_verify_provider.py b/test_verify_provider.py new file mode 100644 index 0000000..144687e --- /dev/null +++ b/test_verify_provider.py @@ -0,0 +1,37 @@ +from pathlib import Path +import runpy +import subprocess +import unittest +from unittest.mock import patch + + +VERIFY = runpy.run_path(str(Path(__file__).with_name("verify-provider.py"))) +CONFIG = {"verifyConfig": {"nopVerify": False}} + + +class SignedProviderTests(unittest.TestCase): + def test_reads_provider_and_every_service(self): + with patch("subprocess.run", side_effect=[ + subprocess.CompletedProcess([], 0, "name\none\ntwo\n", ""), + subprocess.CompletedProcess([], 0, "name\nthing\n", ""), + subprocess.CompletedProcess([], 0, "name\nother\n", ""), + ]) as run: + self.assertEqual(VERIFY["verify_provider"]("stackql", "example", CONFIG), (2, 2)) + self.assertEqual(run.call_count, 3) + self.assertIn("--registry=", run.call_args.args[0][3]) + + def test_zero_exit_status_does_not_hide_verification_failure(self): + with patch("subprocess.run", return_value=subprocess.CompletedProcess( + [], 0, "", "pqsig: signature verification failed" + )): + with self.assertRaisesRegex(RuntimeError, "signature verification failed"): + VERIFY["verify_provider"]("stackql", "example", CONFIG) + + def test_rejects_disabled_verification(self): + with self.assertRaisesRegex(ValueError, "nopVerify"): + VERIFY["verify_provider"]("stackql", "example", {"verifyConfig": {"nopVerify": True}}) + + def test_empty_metadata_is_not_success(self): + with patch("subprocess.run", return_value=subprocess.CompletedProcess([], 0, "", "")): + with self.assertRaisesRegex(RuntimeError, "expected named metadata"): + VERIFY["verify_provider"]("stackql", "example", CONFIG) diff --git a/verify-provider.py b/verify-provider.py new file mode 100644 index 0000000..302a857 --- /dev/null +++ b/verify-provider.py @@ -0,0 +1,80 @@ +"""Verify provider metadata through a fresh StackQL CLI, without a shared server.""" + +import argparse +import csv +import io +import json +from pathlib import Path +import re +import subprocess +import sys +import tempfile + + +def identifier(name): + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + return name + return "`" + name.replace("`", "``") + "`" + + +def query(binary, registry, statement, app_root): + result = subprocess.run( + [str(binary), "exec", statement, "--registry=" + json.dumps(registry), + "--output=csv", "--approot=" + app_root], + capture_output=True, + text=True, + timeout=120, + check=False, + ) + # StackQL currently reports some query errors on stderr with exit status zero. + if result.returncode or result.stderr.strip(): + raise RuntimeError(f"{statement}: {result.stderr.strip() or result.stdout.strip()}") + rows = csv.DictReader(io.StringIO(result.stdout)) + if not rows.fieldnames or "name" not in rows.fieldnames: + raise RuntimeError(f"{statement}: expected named metadata columns, got {result.stdout!r}") + return list(rows) + + +def verify_provider(binary, provider, registry): + if registry.get("verifyConfig", {}).get("nopVerify") is not False: + raise ValueError("signed provider tests require explicit nopVerify: false") + with tempfile.TemporaryDirectory(prefix="stackql-verify-") as app_root: + services = query(binary, registry, f"SHOW SERVICES IN {identifier(provider)};", app_root) + if not services: + raise RuntimeError(f"no services found for {provider}") + resources = 0 + for service in services: + address = f"{identifier(provider)}.{identifier(service['name'])}" + resources += len(query(binary, registry, f"SHOW RESOURCES IN {address};", app_root)) + return len(services), resources + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("provider") + parser.add_argument("--registry-root", required=True, type=Path) + parser.add_argument("--stackql-bin", required=True, type=Path) + parser.add_argument("--public-keys", type=Path, help="optional kid-to-hex public key JSON for private/test registries") + args = parser.parse_args() + try: + root = args.registry_root.resolve(strict=True) + binary = args.stackql_bin.resolve(strict=True) + verify_config = {"nopVerify": False} + if args.public_keys: + keys = json.loads(args.public_keys.read_text(encoding="utf-8")) + if not isinstance(keys, dict) or not keys or not all( + isinstance(k, str) and isinstance(v, str) for k, v in keys.items() + ): + raise ValueError("--public-keys must contain a non-empty kid-to-hex object") + verify_config["publicKeys"] = keys + registry = {"url": root.as_uri(), "localDocRoot": str(root), "verifyConfig": verify_config} + services, resources = verify_provider(binary, args.provider, registry) + except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print(f"Verified {services} services and {resources} resources in {args.provider}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())