diff --git a/.gitignore b/.gitignore index 68550868..531661e7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ ed25519tool scripts/*/node_modules/ scripts/*/package-lock.json stackql*.sh +/stackql_mcp_server_*.log /stackql stackql-zip .stackql/ diff --git a/README.md b/README.md index 6d4816f1..90cf3d7a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,85 @@ A repository of `provider` interface documents supporting [stackql](https://stackql.io/) ([stackql GitHub repo](https://github.com/stackql/stackql)). +## Table of Contents + + +- [Provider Catalog](#provider-catalog) + - [Updating the Catalog](#updating-the-catalog) +- [Context](#context) +- [Contributing](#contributing) +- [Developing a Provider](#developing-a-provider) +- [Build and Deployment Workflow](#build-and-deployment-workflow) +- [Testing a Provider using the `dev` Registry](#testing-a-provider-using-the-dev-registry) + + +## Provider Catalog + +The following StackQL providers are available in the public StackQL Provider Registry. Use `REGISTRY LIST` to discover providers and `REGISTRY PULL ` to install one. + + +| Icon | Provider | Full Name | Latest Version | Date First Published | Date Last Published | +| --- | --- | --- | --- | --- | --- | +| Anthropic | `anthropic` | Anthropic | `v26.09.00461` | 2024-10-25 | 2026-09-15 | +| Anthropic Admin | `anthropic_admin` | Anthropic Admin | `v26.07.00416` | 2026-07-09 | 2026-07-09 | +| Amazon Web Services | `aws` | Amazon Web Services | `v26.08.00444` | 2022-08-19 | 2026-08-20 | +| AWS Cloud Control | `awscc` | AWS Cloud Control | `v26.07.00426` | 2024-03-07 | 2026-07-28 | +| Microsoft Azure | `azure` | Microsoft Azure | `v26.07.00420` | 2022-08-20 | 2026-07-14 | +| Azure Extras | `azure_extras` | Azure Extras | `v26.07.00420` | 2022-08-20 | 2026-07-14 | +| Azure ISV | `azure_isv` | Azure ISV | `v26.07.00420` | 2024-01-25 | 2026-07-14 | +| Azure Stack | `azure_stack` | Azure Stack | `v26.07.00420` | 2024-01-25 | 2026-07-14 | +| ClickHouse | `clickhouse` | ClickHouse | `v26.08.00442` | 2026-08-17 | 2026-08-17 | +| Cloudflare | `cloudflare` | Cloudflare | `v26.07.00424` | 2023-03-24 | 2026-07-21 | +| Confluent | `confluent` | Confluent | `v26.05.00403` | 2024-11-01 | 2026-05-31 | +| Databricks Account | `databricks_account` | Databricks Account | `v26.07.00430` | 2024-12-20 | 2026-07-30 | +| Databricks Workspace | `databricks_workspace` | Databricks Workspace | `v26.07.00430` | 2024-12-20 | 2026-07-30 | +| DataDog | `datadog` | DataDog | `v26.08.00455` | 2023-12-28 | 2026-08-26 | +| Deno Deploy | `deno` | Deno Deploy | `v26.09.00459` | 2025-09-09 | 2026-09-15 | +| Digital Ocean | `digitalocean` | Digital Ocean | `v26.08.00434` | 2023-03-24 | 2026-08-03 | +| Microsoft Entra ID | `entra_id` | Microsoft Entra ID | `v26.07.00412` | 2026-06-02 | 2026-07-06 | +| Firebase | `firebase` | Firebase | `v26.08.00446` | 2022-07-04 | 2026-08-20 | +| Fivetran | `fivetran` | Fivetran | `v26.09.00467` | 2023-03-17 | 2026-09-28 | +| GitHub | `github` | GitHub | `v26.08.00448` | 2022-05-02 | 2026-08-20 | +| GitLab | `gitlab` | GitLab | `v26.09.00465` | 2026-09-26 | 2026-09-26 | +| GoDaddy | `godaddy` | GoDaddy | `v26.09.00463` | 2023-12-16 | 2026-09-16 | +| Google Admin | `googleadmin` | Google Admin | `v26.07.00432` | 2023-07-07 | 2026-07-31 | +| Google Cloud Platform | `googleapis.com` | Google Cloud Platform | `v26.08.00446` | 2022-03-15 | 2026-08-20 | +| Google Workspace | `googleworkspace` | Google Workspace | `v26.07.00432` | 2023-01-08 | 2026-07-31 | +| Homebrew | `homebrew` | Homebrew | `v25.01.00292` | 2023-12-28 | 2025-01-27 | +| Kubernetes | `k8s` | Kubernetes | `v26.08.00453` | 2022-06-19 | 2026-08-24 | +| Kafka | `kafka` | Kafka | `v26.05.00403` | 2026-05-31 | 2026-05-31 | +| Lambda Cloud | `lambda` | Lambda Cloud | `v26.10.00481` | 2026-10-21 | 2026-10-21 | +| Linode | `linode` | Linode | `v26.08.00440` | 2023-05-12 | 2026-08-04 | +| Netlify | `netlify` | Netlify | `v26.08.00457` | 2022-05-11 | 2026-08-28 | +| NVIDIA NGC | `nvidia` | NVIDIA NGC | `v26.10.00479` | 2026-10-19 | 2026-10-19 | +| Oracle Cloud Infrastructure | `oci` | Oracle Cloud Infrastructure | `v26.10.00475` | 2026-10-05 | 2026-10-05 | +| Okta | `okta` | Okta | `v26.08.00436` | 2022-03-15 | 2026-08-04 | +| OpenAI | `openai` | OpenAI | `v26.07.00422` | 2024-10-25 | 2026-07-15 | +| OpenAI Admin | `openai_admin` | OpenAI Admin | `v26.07.00422` | 2026-07-15 | 2026-07-15 | +| PagerDuty | `pagerduty` | PagerDuty | `v26.09.00459` | 2023-12-20 | 2026-09-15 | +| Railway | `railway` | Railway | `v26.09.00469` | 2026-09-28 | 2026-09-28 | +| Snowflake | `snowflake` | Snowflake | `v26.08.00438` | 2025-05-20 | 2026-08-04 | +| SumoLogic | `sumologic` | SumoLogic | `v26.09.00459` | 2022-12-10 | 2026-09-15 | +| Supabase | `supabase` | Supabase | `v26.10.00477` | 2026-10-12 | 2026-10-12 | +| TypeSafe AI | `typesafe` | TypeSafe AI | `v26.10.00473` | 2026-10-04 | 2026-10-04 | +| Vercel | `vercel` | Vercel | `v26.09.00459` | 2023-12-14 | 2026-09-15 | + + +Latest versions come from the [production registry manifest](https://registry.stackql.app/providers/dist/providers.yaml). Dates are UTC, inferred from production Git history: **Date First Published** is the earliest commit on the production branch that introduced the provider's source or packaged artifacts; **Date Last Published** is the release commit corresponding to its latest published version. These are Git-based publication estimates, not deployment completion timestamps. `googleapis.com` is the source directory for the registry provider named `google`. + +### Updating the Catalog + +With Node.js 18 or later, internet access, and a full Git checkout, run: + +```bash +git fetch origin main +node scripts/docs/update_provider_listings.js +``` + +The script updates the catalog and table of contents in this README. It reads published versions from the production registry and dates from `origin/main`; use `--git-ref ` to select another local production-history ref, or `--check` to check for stale content without writing. It fails explicitly if metadata, local icons, or release history are missing. When adding a provider, add its display name and icon mapping to the script and its favicon to `images/`. + +Favicons and display names were initially copied from the [StackQL website provider tiles](https://stackql.io/providers); Lambda Cloud's favicon comes from [Lambda](https://lambda.ai/favicon.ico), as it did not yet have a website tile. Brand spelling and capitalization are preserved. + ## Context diff --git a/images/anthropic.png b/images/anthropic.png new file mode 100644 index 00000000..f0b5d7c5 Binary files /dev/null and b/images/anthropic.png differ diff --git a/images/aws.ico b/images/aws.ico new file mode 100644 index 00000000..0c5c49c1 Binary files /dev/null and b/images/aws.ico differ diff --git a/images/azure.ico b/images/azure.ico new file mode 100644 index 00000000..f1fe5051 Binary files /dev/null and b/images/azure.ico differ diff --git a/images/clickhouse.ico b/images/clickhouse.ico new file mode 100644 index 00000000..f8fced34 Binary files /dev/null and b/images/clickhouse.ico differ diff --git a/images/cloudflare.ico b/images/cloudflare.ico new file mode 100644 index 00000000..3c174306 Binary files /dev/null and b/images/cloudflare.ico differ diff --git a/images/confluent.ico b/images/confluent.ico new file mode 100644 index 00000000..ec498520 Binary files /dev/null and b/images/confluent.ico differ diff --git a/images/databricks.ico b/images/databricks.ico new file mode 100644 index 00000000..50e597ba Binary files /dev/null and b/images/databricks.ico differ diff --git a/images/datadog.ico b/images/datadog.ico new file mode 100644 index 00000000..e04da3ed Binary files /dev/null and b/images/datadog.ico differ diff --git a/images/deno.ico b/images/deno.ico new file mode 100644 index 00000000..e8f14aa2 Binary files /dev/null and b/images/deno.ico differ diff --git a/images/digitalocean.png b/images/digitalocean.png new file mode 100644 index 00000000..d3254bb4 Binary files /dev/null and b/images/digitalocean.png differ diff --git a/images/entra_id.svg b/images/entra_id.svg new file mode 100644 index 00000000..0ed35fb7 --- /dev/null +++ b/images/entra_id.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/images/firebase.png b/images/firebase.png new file mode 100644 index 00000000..c64e8bd0 Binary files /dev/null and b/images/firebase.png differ diff --git a/images/fivetran.png b/images/fivetran.png new file mode 100644 index 00000000..aaf09efe Binary files /dev/null and b/images/fivetran.png differ diff --git a/images/github.ico b/images/github.ico new file mode 100644 index 00000000..a59308e2 Binary files /dev/null and b/images/github.ico differ diff --git a/images/gitlab.ico b/images/gitlab.ico new file mode 100644 index 00000000..2a26aa10 Binary files /dev/null and b/images/gitlab.ico differ diff --git a/images/godaddy.png b/images/godaddy.png new file mode 100644 index 00000000..7068fef2 Binary files /dev/null and b/images/godaddy.png differ diff --git a/images/google.ico b/images/google.ico new file mode 100644 index 00000000..82339b3b Binary files /dev/null and b/images/google.ico differ diff --git a/images/googleadmin.ico b/images/googleadmin.ico new file mode 100644 index 00000000..82339b3b Binary files /dev/null and b/images/googleadmin.ico differ diff --git a/images/googleworkspace.ico b/images/googleworkspace.ico new file mode 100644 index 00000000..82339b3b Binary files /dev/null and b/images/googleworkspace.ico differ diff --git a/images/homebrew.ico b/images/homebrew.ico new file mode 100644 index 00000000..2d068503 Binary files /dev/null and b/images/homebrew.ico differ diff --git a/images/k8s.png b/images/k8s.png new file mode 100644 index 00000000..f2bd3d64 Binary files /dev/null and b/images/k8s.png differ diff --git a/images/kafka.png b/images/kafka.png new file mode 100644 index 00000000..8dc6df2c Binary files /dev/null and b/images/kafka.png differ diff --git a/images/lambda.svg b/images/lambda.svg new file mode 100644 index 00000000..67fcc4da --- /dev/null +++ b/images/lambda.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/images/linode.ico b/images/linode.ico new file mode 100644 index 00000000..f2c712fd Binary files /dev/null and b/images/linode.ico differ diff --git a/images/netlify.ico b/images/netlify.ico new file mode 100644 index 00000000..862a39e3 Binary files /dev/null and b/images/netlify.ico differ diff --git a/images/nvidia.ico b/images/nvidia.ico new file mode 100644 index 00000000..d4ad598f Binary files /dev/null and b/images/nvidia.ico differ diff --git a/images/oci.ico b/images/oci.ico new file mode 100644 index 00000000..c59864ba Binary files /dev/null and b/images/oci.ico differ diff --git a/images/okta.png b/images/okta.png new file mode 100644 index 00000000..06a64fc9 Binary files /dev/null and b/images/okta.png differ diff --git a/images/openai.ico b/images/openai.ico new file mode 100644 index 00000000..1634901a Binary files /dev/null and b/images/openai.ico differ diff --git a/images/pagerduty.svg b/images/pagerduty.svg new file mode 100644 index 00000000..352e8941 --- /dev/null +++ b/images/pagerduty.svg @@ -0,0 +1,4 @@ + + + + diff --git a/images/railway.ico b/images/railway.ico new file mode 100644 index 00000000..0002f720 Binary files /dev/null and b/images/railway.ico differ diff --git a/images/snowflake.ico b/images/snowflake.ico new file mode 100644 index 00000000..880448d9 Binary files /dev/null and b/images/snowflake.ico differ diff --git a/images/sumologic.png b/images/sumologic.png new file mode 100644 index 00000000..347c4f0a Binary files /dev/null and b/images/sumologic.png differ diff --git a/images/supabase.ico b/images/supabase.ico new file mode 100644 index 00000000..4a970cd4 Binary files /dev/null and b/images/supabase.ico differ diff --git a/images/typesafe.png b/images/typesafe.png new file mode 100644 index 00000000..45107c7d Binary files /dev/null and b/images/typesafe.png differ diff --git a/images/vercel.ico b/images/vercel.ico new file mode 100644 index 00000000..4ddd8fff Binary files /dev/null and b/images/vercel.ico differ diff --git a/scripts/docs/update_provider_listings.js b/scripts/docs/update_provider_listings.js new file mode 100644 index 00000000..3cd12762 --- /dev/null +++ b/scripts/docs/update_provider_listings.js @@ -0,0 +1,243 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +const ROOT = path.resolve(__dirname, '..', '..'); +const MANIFEST_URL = 'https://registry.stackql.app/providers/dist/providers.yaml'; +const TOC_MARKER = 'TABLE OF CONTENTS'; +const CATALOG_MARKER = 'PROVIDER CATALOG'; + +// Display names and icons from stackql.io's provider tiles, keyed by source directory. +const PROVIDERS = { + anthropic: ['Anthropic', 'anthropic.png'], + anthropic_admin: ['Anthropic Admin', 'anthropic.png'], + aws: ['Amazon Web Services', 'aws.ico'], + awscc: ['AWS Cloud Control', 'aws.ico'], + azure: ['Microsoft Azure', 'azure.ico'], + azure_extras: ['Azure Extras', 'azure.ico'], + azure_isv: ['Azure ISV', 'azure.ico'], + azure_stack: ['Azure Stack', 'azure.ico'], + clickhouse: ['ClickHouse', 'clickhouse.ico'], + cloudflare: ['Cloudflare', 'cloudflare.ico'], + confluent: ['Confluent', 'confluent.ico'], + databricks_account: ['Databricks Account', 'databricks.ico'], + databricks_workspace: ['Databricks Workspace', 'databricks.ico'], + datadog: ['DataDog', 'datadog.ico'], + deno: ['Deno Deploy', 'deno.ico'], + digitalocean: ['Digital Ocean', 'digitalocean.png'], + entra_id: ['Microsoft Entra ID', 'entra_id.svg'], + firebase: ['Firebase', 'firebase.png'], + fivetran: ['Fivetran', 'fivetran.png'], + github: ['GitHub', 'github.ico'], + gitlab: ['GitLab', 'gitlab.ico'], + godaddy: ['GoDaddy', 'godaddy.png'], + googleadmin: ['Google Admin', 'googleadmin.ico'], + 'googleapis.com': ['Google Cloud Platform', 'google.ico'], + googleworkspace: ['Google Workspace', 'googleworkspace.ico'], + homebrew: ['Homebrew', 'homebrew.ico'], + k8s: ['Kubernetes', 'k8s.png'], + kafka: ['Kafka', 'kafka.png'], + lambda: ['Lambda Cloud', 'lambda.svg'], + linode: ['Linode', 'linode.ico'], + netlify: ['Netlify', 'netlify.ico'], + nvidia: ['NVIDIA NGC', 'nvidia.ico'], + oci: ['Oracle Cloud Infrastructure', 'oci.ico'], + okta: ['Okta', 'okta.png'], + openai: ['OpenAI', 'openai.ico'], + openai_admin: ['OpenAI Admin', 'openai.ico'], + pagerduty: ['PagerDuty', 'pagerduty.svg'], + railway: ['Railway', 'railway.ico'], + snowflake: ['Snowflake', 'snowflake.ico'], + sumologic: ['SumoLogic', 'sumologic.png'], + supabase: ['Supabase', 'supabase.ico'], + typesafe: ['TypeSafe AI', 'typesafe.png'], + vercel: ['Vercel', 'vercel.ico'], +}; + +function git(args) { + return execFileSync('git', ['--no-pager', ...args], { + cwd: ROOT, + encoding: 'utf8', + maxBuffer: 16 * 1024 * 1024, + }).trim(); +} + +// Accept only the registry's provider -> versions-list YAML schema, not arbitrary YAML. +function parseManifest(text) { + const providers = new Map(); + let provider; + let expectingVersions = false; + let rootSeen = false; + for (const line of text.replace(/^\uFEFF/, '').split(/\r?\n/)) { + if (!line.trim()) continue; + if (line === 'providers:' && !rootSeen) { + rootSeen = true; + continue; + } + let match = /^ ([a-z0-9_.-]+):$/.exec(line); + if (rootSeen && match && !expectingVersions) { + provider = match[1]; + if (providers.has(provider)) throw new Error(`Duplicate provider: ${provider}`); + providers.set(provider, []); + expectingVersions = true; + continue; + } + if (provider && expectingVersions && line === ' versions:') { + expectingVersions = false; + continue; + } + match = /^ (?: )?- (v\d+\.\d+\.\d+)$/.exec(line); + if (provider && !expectingVersions && match) { + providers.get(provider).push(match[1]); + continue; + } + throw new Error(`Unsupported registry manifest line: ${JSON.stringify(line)}`); + } + if (!providers.size || expectingVersions) throw new Error('Incomplete registry manifest'); + for (const [name, versions] of providers) { + if (!versions.length) throw new Error(`No published versions for ${name}`); + } + return providers; +} + +function compareVersions(left, right) { + const a = left.slice(1).split('.').map(Number); + const b = right.slice(1).split('.').map(Number); + return a[0] - b[0] || a[1] - b[1] || a[2] - b[2]; +} + +function utcDate(timestamp) { + const date = new Date(timestamp); + if (Number.isNaN(date.getTime())) throw new Error(`Invalid Git date: ${timestamp}`); + return date.toISOString().slice(0, 10); +} + +function releaseDates(log) { + const dates = new Map(); + for (const line of log.split(/\r?\n/)) { + if (!line) continue; + const [timestamp, subject] = line.split('\t'); + const match = /^Merge pull request #(\d+) from /.exec(subject) + || /\(#(\d+)\)\s*$/.exec(subject); + if (match) { + const date = utcDate(timestamp); + const version = `v${date.slice(2, 4)}.${date.slice(5, 7)}.${match[1].padStart(5, '0')}`; + if (dates.has(version)) throw new Error(`Ambiguous release history for ${version}`); + dates.set(version, date); + } + } + return dates; +} + +function replaceBlock(readme, marker, content) { + const begin = ``; + const end = ``; + const start = readme.indexOf(begin); + const finish = readme.indexOf(end); + if (start < 0 || finish < start || readme.indexOf(begin, start + 1) !== -1 + || readme.indexOf(end, finish + 1) !== -1) { + throw new Error(`Expected exactly one ${marker} marker pair in README.md`); + } + return `${readme.slice(0, start + begin.length)}\n${content}\n${readme.slice(finish)}`; +} + +function tableOfContents(readme) { + let fence; + const anchors = new Map(); + const entries = []; + for (const line of readme.split(/\r?\n/)) { + const delimiter = /^\s*(`{3,}|~{3,})/.exec(line); + if (delimiter) { + if (!fence) fence = delimiter[1]; + else if (delimiter[1][0] === fence[0] && delimiter[1].length >= fence.length) fence = undefined; + continue; + } + if (fence) continue; + const heading = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line); + if (!heading) continue; + const title = heading[2]; + const slug = title.toLowerCase().replace(/[`]/g, '').replace(/[^\w -]/g, '').replace(/ /g, '-'); + const count = anchors.get(slug) || 0; + anchors.set(slug, count + 1); + if (heading[1].length === 1 || title === 'Table of Contents') continue; + const anchor = count ? `${slug}-${count}` : slug; + entries.push(`${' '.repeat(heading[1].length - 2)}- [${title}](#${anchor})`); + } + return entries.join('\n'); +} + +function buildCatalog(manifest, dates, firstPublished, root = ROOT) { + const rows = [ + '| Icon | Provider | Full Name | Latest Version | Date First Published | Date Last Published |', + '| --- | --- | --- | --- | --- | --- |', + ]; + const names = [...manifest.keys()].map((name) => name === 'google' ? 'googleapis.com' : name).sort(); + for (const name of names) { + if (!Object.hasOwn(PROVIDERS, name)) throw new Error(`Add display name and icon metadata for ${name}`); + const [fullName, icon] = PROVIDERS[name]; + if (!fs.statSync(path.join(root, 'images', icon)).isFile()) { + throw new Error(`Missing favicon for ${name}: images/${icon}`); + } + const registryName = name === 'googleapis.com' ? 'google' : name; + const latest = [...manifest.get(registryName)].sort(compareVersions).at(-1); + const last = dates.get(latest); + if (!last) throw new Error(`No production release commit for ${name} ${latest}; fetch production Git history`); + const first = firstPublished(name, registryName); + if (!/^\d{4}-\d{2}-\d{2}$/.test(first) || first > last) { + throw new Error(`Invalid first publication date for ${name}: ${first}`); + } + rows.push(`| ${fullName} | \`${name}\` | ${fullName} | \`${latest}\` | ${first} | ${last} |`); + } + return rows.join('\n'); +} + +async function main(args = process.argv.slice(2)) { + let ref = 'origin/main'; + let check = false; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--check') check = true; + else if (args[i] === '--git-ref' && args[i + 1] && !args[i + 1].startsWith('-')) ref = args[++i]; + else throw new Error(`Unknown or incomplete argument: ${args[i]}`); + } + if (git(['rev-parse', '--is-shallow-repository']) !== 'false') { + throw new Error('Full Git history is required; run git fetch --unshallow'); + } + const commit = git(['rev-parse', '--verify', `${ref}^{commit}`]); + const dates = releaseDates(git(['log', '--first-parent', '--format=%cI%x09%s', commit])); + const response = await fetch(MANIFEST_URL, { signal: AbortSignal.timeout(30000) }); + if (!response.ok) throw new Error(`Registry request failed: HTTP ${response.status}`); + const manifest = parseManifest(await response.text()); + const catalog = buildCatalog(manifest, dates, (name, registryName) => { + const history = git([ + 'log', '--first-parent', '--reverse', '--format=%cI', commit, '--', + `providers/src/${name}`, `providers/dist/${name}`, `providers/dist/${registryName}`, + ]); + if (!history) throw new Error(`No production history for ${name}`); + return utcDate(history.split(/\r?\n/)[0]); + }); + const readmePath = path.join(ROOT, 'README.md'); + const original = fs.readFileSync(readmePath, 'utf8'); + const newline = original.includes('\r\n') ? '\r\n' : '\n'; + let updated = replaceBlock(original.replace(/\r\n/g, '\n'), CATALOG_MARKER, catalog); + updated = replaceBlock(updated, TOC_MARKER, tableOfContents(updated)).replace(/\n/g, newline); + if (updated === original) { + console.log(`Provider catalog is current (${manifest.size} providers).`); + } else if (check) { + throw new Error('README.md provider catalog or table of contents is stale; run this script without --check'); + } else { + fs.writeFileSync(readmePath, updated); + console.log(`Updated README.md (${manifest.size} providers).`); + } +} + +if (require.main === module) { + main().catch((error) => { + console.error(`Provider catalog update failed: ${error.message}`); + process.exitCode = 1; + }); +} + +module.exports = { parseManifest, compareVersions, utcDate, releaseDates, replaceBlock, tableOfContents, buildCatalog }; diff --git a/scripts/docs/update_provider_listings.test.js b/scripts/docs/update_provider_listings.test.js new file mode 100644 index 00000000..6d5176cb --- /dev/null +++ b/scripts/docs/update_provider_listings.test.js @@ -0,0 +1,133 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); +const { + parseManifest, compareVersions, utcDate, releaseDates, replaceBlock, tableOfContents, buildCatalog, +} = require('./update_provider_listings'); + +const ROOT = path.resolve(__dirname, '..', '..'); + +test('parses the registry schema with LF, CRLF and both list indentations', () => { + const yaml = 'providers:\n aws:\n versions:\n - v26.08.00444\n google:\n versions:\n - v26.08.00446\n'; + const expected = [['aws', ['v26.08.00444']], ['google', ['v26.08.00446']]]; + assert.deepEqual([...parseManifest(yaml)], expected); + assert.deepEqual([...parseManifest(`\uFEFF${yaml.replace(/\n/g, '\r\n')}`)], expected); +}); + +test('rejects incomplete, duplicate and unsupported manifest data', () => { + for (const yaml of [ + '', 'providers:', 'providers:\n aws:', + 'providers:\n aws:\n versions:', + 'providers:\n aws:\n versions:\n - invalid', + 'providers:\n aws:\n versions:\n - v26.08.00444\n aws:', + 'providers:\n aws:\n versions:\n - v26.08.00444\nunexpected: true', + ]) { + assert.throws(() => parseManifest(yaml)); + } +}); + +test('selects versions numerically rather than lexicographically or by list order', () => { + assert.deepEqual( + ['v9.12.99999', 'v26.9.00461', 'v26.10.00481', 'v26.09.00459'].sort(compareVersions), + ['v9.12.99999', 'v26.09.00459', 'v26.9.00461', 'v26.10.00481'], + ); +}); + +test('derives UTC release dates from merge and squash commits, ignoring source commits', () => { + const dates = releaseDates([ + '2026-10-21T16:15:29+11:00\tMerge pull request #481 from stackql/dev', + '2026-07-01T00:15:00+10:00\tProvider update (#405)', + '2026-10-19T17:22:55+11:00\tlambda provider', + ].join('\n')); + assert.deepEqual([...dates], [['v26.10.00481', '2026-10-21'], ['v26.06.00405', '2026-06-30']]); + assert.equal(utcDate('2022-08-20T00:15:24+10:00'), '2022-08-19'); + assert.throws(() => utcDate('not a date'), /Invalid Git date/); + assert.throws(() => releaseDates([ + '2026-10-21T05:15:29Z\tMerge pull request #481 from stackql/dev', + '2026-10-22T05:15:29Z\tAnother release (#481)', + ].join('\n')), /Ambiguous release history/); +}); + +test('replaces only generated blocks and rejects missing, reversed or duplicate markers', () => { + const readme = 'before\n\nold\n\nafter\n'; + assert.equal( + replaceBlock(readme, 'TEST', 'new'), + 'before\n\nnew\n\nafter\n', + ); + for (const invalid of [ + '', '', + `${readme}`, `${readme}`, + ]) { + assert.throws(() => replaceBlock(invalid, 'TEST', 'new'), /marker pair/); + } +}); + +test('generates nested TOC links, ignoring fenced code and handling duplicate headings', () => { + assert.equal(tableOfContents([ + '# Project', '## Table of Contents', '## Provider Catalog', '### Updating the Catalog', + '```bash', '## Not a heading', '```', '~~~', '## Also code', '~~~', + '## Testing using the `dev` Registry', '## Provider Catalog', + ].join('\n')), [ + '- [Provider Catalog](#provider-catalog)', + ' - [Updating the Catalog](#updating-the-catalog)', + '- [Testing using the `dev` Registry](#testing-using-the-dev-registry)', + '- [Provider Catalog](#provider-catalog-1)', + ].join('\n')); +}); + +test('builds six-column catalog sorted by source directory, with the Google alias and latest release', () => { + const manifest = new Map([ + ['google', ['v26.08.00446']], + ['aws', ['v26.08.00444', 'v26.07.00414']], + ['anthropic_admin', ['v26.07.00416']], + ]); + const dates = new Map([ + ['v26.08.00446', '2026-08-20'], + ['v26.08.00444', '2026-08-20'], + ['v26.07.00416', '2026-07-09'], + ]); + const names = []; + const catalog = buildCatalog(manifest, dates, (name, registryName) => { + names.push([name, registryName]); + return '2024-01-01'; + }); + assert.deepEqual(names, [ + ['anthropic_admin', 'anthropic_admin'], ['aws', 'aws'], ['googleapis.com', 'google'], + ]); + assert.match(catalog, /`googleapis\.com` \| Google Cloud Platform \| `v26\.08\.00446`/); + assert.match(catalog, /`aws` \| Amazon Web Services \| `v26\.08\.00444`/); + assert.doesNotMatch(catalog, /v26\.07\.00414/); + for (const row of catalog.split('\n')) assert.equal(row.split('|').length, 8); +}); + +test('fails explicitly on missing metadata, release history, icons or invalid publication dates', () => { + const manifest = new Map([['aws', ['v26.08.00444']]]); + const dates = new Map([['v26.08.00444', '2026-08-20']]); + assert.throws(() => buildCatalog(new Map([['unknown', ['v26.08.00444']]]), dates, () => '2022-01-01'), /metadata/); + assert.throws(() => buildCatalog(manifest, new Map(), () => '2022-01-01'), /release commit/); + assert.throws(() => buildCatalog(manifest, dates, () => '2022-01-01', __dirname), /ENOENT/); + assert.throws(() => buildCatalog(manifest, dates, () => ''), /first publication date/); + assert.throws(() => buildCatalog(manifest, dates, () => '2027-01-01'), /first publication date/); +}); + +test('README lists every current provider directory exactly once, sorted, with six columns and local icons', () => { + const readme = fs.readFileSync(path.join(ROOT, 'README.md'), 'utf8'); + const catalog = readme.split('')[1].split('')[0]; + const rows = catalog.split(/\r?\n/).filter((line) => line.startsWith('| /\| `([^`]+)` \|/.exec(line)[1]); + const expected = fs.readdirSync(path.join(ROOT, 'providers', 'src'), { withFileTypes: true }) + .filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort(); + assert.deepEqual(actual, expected); + for (const row of rows) { + assert.equal(row.split('|').length, 8); + const icon = /src="([^"]+)"/.exec(row)[1]; + assert.ok(fs.statSync(path.join(ROOT, ...icon.split('/'))).isFile()); + assert.match(row, /\| `v\d+\.\d+\.\d+` \| \d{4}-\d{2}-\d{2} \| \d{4}-\d{2}-\d{2} \|$/); + } + const toc = readme.split('')[1].split('')[0].trim(); + assert.equal(toc, tableOfContents(readme)); + assert.equal(replaceBlock(readme, 'PROVIDER CATALOG', catalog.trim()), readme); +});