diff --git a/.github/workflows/sandbox-backup-evidence.yml b/.github/workflows/sandbox-backup-evidence.yml index cbee0425d..4eb6e37f0 100644 --- a/.github/workflows/sandbox-backup-evidence.yml +++ b/.github/workflows/sandbox-backup-evidence.yml @@ -55,7 +55,7 @@ jobs: run: | set -uo pipefail failed=0 - for suffix in knowledge-backup-stale knowledge-worker-failure knowledge-mirror-stale knowledge-mirror-failed recurring-cron-pass-missed recurring-cron-failure recurring-cron-errors; do + for suffix in knowledge-backup-stale knowledge-repo-backup-stale knowledge-worker-failure knowledge-mirror-stale knowledge-mirror-failed recurring-cron-pass-missed recurring-cron-failure recurring-cron-errors; do name="dataops-v1-$suffix" if ! aws cloudwatch describe-alarms --alarm-names "$name" \ --query 'MetricAlarms' --output json > "$RUNNER_TEMP/$suffix.json"; then @@ -83,21 +83,23 @@ jobs: done exit "$failed" - - name: Prove the backup freshness alarm reaches a subscriber + - name: Prove the backup freshness alarms reach a subscriber if: ${{ !cancelled() }} shell: bash run: | set -euo pipefail - alarm="$RUNNER_TEMP/knowledge-backup-stale.json" - # Success=1 per daily backup; alarm when the 2-day sum is <1 or missing. - jq -e '.[0] | .Namespace == "DataOps/Knowledge" and .MetricName == "Success" - and .Threshold == 1 and .ComparisonOperator == "LessThanThreshold" - and .TreatMissingData == "breaching"' "$alarm" >/dev/null - for topic in $(jq -r '.[0].AlarmActions[]' "$alarm"); do - confirmed=$(aws sns list-subscriptions-by-topic --topic-arn "$topic" --output json \ - | jq '[.Subscriptions[] | select(.SubscriptionArn != "PendingConfirmation")] | length') - echo "$topic confirmed subscriptions: $confirmed" - [ "$confirmed" -ge 1 ] + # Success=1 per daily backup/check; alarm when the 2-day sum is <1 or missing. + for suffix in knowledge-backup-stale knowledge-repo-backup-stale; do + alarm="$RUNNER_TEMP/$suffix.json" + jq -e '.[0] | .Namespace == "DataOps/Knowledge" and .MetricName == "Success" + and .Threshold == 1 and .ComparisonOperator == "LessThanThreshold" + and .TreatMissingData == "breaching"' "$alarm" >/dev/null + for topic in $(jq -r '.[0].AlarmActions[]' "$alarm"); do + confirmed=$(aws sns list-subscriptions-by-topic --topic-arn "$topic" --output json \ + | jq '[.Subscriptions[] | select(.SubscriptionArn != "PendingConfirmation")] | length') + echo "$suffix -> $topic confirmed subscriptions: $confirmed" + [ "$confirmed" -ge 1 ] + done done - name: Read actual knowledge success datapoints diff --git a/backend/src/auth/knowledgePublisher.ts b/backend/src/auth/knowledgePublisher.ts index 5dbc12d83..d1e9acb97 100644 --- a/backend/src/auth/knowledgePublisher.ts +++ b/backend/src/auth/knowledgePublisher.ts @@ -17,7 +17,7 @@ import { createPublicKey, verify as verifySignature, type JsonWebKey } from 'nod import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; import { seedDeploymentRuntime } from '../deploymentSeeds'; -import { handleDocsRoutes, isDocsDomainEnabled, regenerateKnowledgeForTemplates } from '../docs/contentApi'; +import { docsKnowledgeStore, handleDocsRoutes, isDocsDomainEnabled, regenerateKnowledgeForTemplates } from '../docs/contentApi'; import { listTemplates } from '../db/templates'; import { parseMirrorHeartbeat, recordMirrorHeartbeat } from '../knowledge/mirrorHeartbeat'; import type { LambdaEvent, LambdaResponse } from '../types'; @@ -131,7 +131,7 @@ export async function handleKnowledgePublisher( if (method === 'POST' && path === '/knowledge/mirror-heartbeat') { const heartbeat = parseMirrorHeartbeat(event.body); if (!heartbeat) return response(400, { error: 'outcome must be success or failure' }); - recordMirrorHeartbeat(heartbeat); + await recordMirrorHeartbeat(docsKnowledgeStore(), heartbeat); return response(200, { recorded: heartbeat.outcome }); } const allowed = (method === 'GET' && (path === '/knowledge/manifest' || path === '/knowledge/history' || path === '/knowledge/version' || path === '/knowledge/download')) diff --git a/backend/src/automations/registry.ts b/backend/src/automations/registry.ts index 56e36a556..e3730d4c2 100644 --- a/backend/src/automations/registry.ts +++ b/backend/src/automations/registry.ts @@ -204,7 +204,7 @@ export const AUTOMATIONS: readonly AutomationDefinition[] = Object.freeze([ { id: 'knowledge-repo-backup', name: 'Knowledge repository backup', - description: 'Zips the knowledge repository and uploads it to S3 when the commit changed since the last backup.', + description: 'Zips the knowledge repository and uploads it to S3 when the commit changed since the last backup; a daily freshness check pages the operator topic when repository commits stop being backed up.', runsIn: 'GitHub Actions, dataops-knowledge', system: true, schedule: { kind: 'cron', expression: '17 3 * * *' }, diff --git a/backend/src/docs/contentApi.ts b/backend/src/docs/contentApi.ts index 58590253f..0ace83540 100644 --- a/backend/src/docs/contentApi.ts +++ b/backend/src/docs/contentApi.ts @@ -230,6 +230,11 @@ export function resetDocsRuntime(): void { sharedRecent = null; } +/** The configured runtime's knowledge store, or one built from the environment. */ +export function docsKnowledgeStore(): KnowledgeStore { + return runtime ? runtime.store : createKnowledgeStore(knowledgeStoreConfigFromEnv()); +} + interface KnowledgePlan { mutations: Mutation[]; regenerated: string[]; diff --git a/backend/src/knowledge/mirrorHeartbeat.ts b/backend/src/knowledge/mirrorHeartbeat.ts index d3ed41059..ed0d09589 100644 --- a/backend/src/knowledge/mirrorHeartbeat.ts +++ b/backend/src/knowledge/mirrorHeartbeat.ts @@ -11,8 +11,16 @@ * Failure, and on any UTC day without a Success (the dead-man check: the * schedule stopped, the workflow was disabled, or it never reached the * report step), and send the recovery notice when the next success lands. + * + * The report also persists the mirrored Git HEAD to + * jobs/knowledge-mirror/last-report.json: the repo-backup freshness check + * compares it against the S3 backup manifest's commit to tell a quiet + * repository (backed up, nothing new) from one whose backups stopped. */ +import {PutObjectCommand} from '@aws-sdk/client-s3'; +import type {KnowledgeStore} from '../docs/knowledgeStore'; import { writeEmbeddedMetric } from '../embeddedMetrics'; +import { MIRROR_LAST_REPORT_KEY } from './repoBackupFreshness'; export const MIRROR_JOB = 'knowledge-mirror'; @@ -28,7 +36,7 @@ export function parseMirrorHeartbeat(body: string | null | undefined): MirrorHea return { outcome, revision: text(revision), commit: text(commit) }; } -export function recordMirrorHeartbeat(heartbeat: MirrorHeartbeat): void { +export async function recordMirrorHeartbeat(store: Pick, heartbeat: MirrorHeartbeat): Promise { const success = heartbeat.outcome === 'success'; writeEmbeddedMetric({ _aws: { @@ -41,4 +49,12 @@ export function recordMirrorHeartbeat(heartbeat: MirrorHeartbeat): void { revision: heartbeat.revision, commit: heartbeat.commit, }); + // The freshness check only reads the commit; keep the whole report for + // operators inspecting the object. + await store.client.send(new PutObjectCommand({ + Bucket: store.bucket, + Key: MIRROR_LAST_REPORT_KEY, + Body: JSON.stringify({ ...heartbeat, at: new Date().toISOString() }), + ContentType: 'application/json', + })); } diff --git a/backend/src/knowledge/repoBackupFreshness.ts b/backend/src/knowledge/repoBackupFreshness.ts new file mode 100644 index 000000000..1cc2260a2 --- /dev/null +++ b/backend/src/knowledge/repoBackupFreshness.ts @@ -0,0 +1,86 @@ +/** + * Dead-man check for the daily dataops-knowledge repository backup. + * + * The backup runs in the knowledge repository's GitHub Actions workflow + * (backup-to-s3.yml) and uploads straight to the private backups bucket, so + * nothing on the AWS side sees the workflow run — and the workflow's own + * success also covers skip runs that upload nothing because HEAD did not + * change. The truthful signal is the backup's `{prefix}/latest/manifest.json`: + * it is re-uploaded whenever the workflow finds a commit it has not backed up + * yet, and its `commit_sha` names the newest backed-up repository commit. + * + * The scheduled check turns that object into the `DataOps/Knowledge` + * Job=knowledge-repo-backup Success metric that the KnowledgeRepoBackupStaleAlarm + * in the infra template pages the operator topic from. A backup is fresh when + * its commit matches the last mirror-reported repository HEAD + * (jobs/knowledge-mirror/last-report.json, written by the heartbeat route) or + * when it landed within the last two days. A quiet repository therefore stays + * green indefinitely, while a workflow that silently stops scheduling pages as + * soon as a real commit goes unbacked — the 2026-10-02 -> 2026-11-15 outage + * ran 44 days without any alert. + */ +import {GetObjectCommand} from '@aws-sdk/client-s3'; +import type {KnowledgeStore} from '../docs/knowledgeStore'; +import {writeEmbeddedMetric} from '../embeddedMetrics'; + +export const REPO_BACKUP_JOB='knowledge-repo-backup'; +/** Backups are daily; two days of silence means the latest upload is behind. */ +export const FRESH_BACKUP_MAX_AGE_MS=48*60*60*1000; +/** Where recordMirrorHeartbeat persists the last reported mirror commit. */ +export const MIRROR_LAST_REPORT_KEY='jobs/knowledge-mirror/last-report.json'; + +export interface RepoBackupManifest {commit:string|null; lastModified:Date|null} +export interface RepoBackupFreshness {fresh:boolean; reason:string; backup:RepoBackupManifest; mirrorCommit:string|null} + +/** Fresh when the backup contains the newest mirror-reported HEAD, or a backup landed recently. */ +export function evaluateRepoBackupFreshness(input:{backup:RepoBackupManifest; mirrorCommit:string|null; now:Date}):RepoBackupFreshness { + const {backup,mirrorCommit,now}=input; + if(!backup.lastModified) return {fresh:false,reason:'missing-manifest',backup,mirrorCommit}; + const age=now.getTime()-backup.lastModified.getTime(); + if(mirrorCommit && backup.commit && backup.commit===mirrorCommit) return {fresh:true,reason:'commit-match',backup,mirrorCommit}; + if(age<=FRESH_BACKUP_MAX_AGE_MS) return {fresh:true,reason:'recent-upload',backup,mirrorCommit}; + return {fresh:false,reason:backup.commit?'divergent-commit':'manifest-incomplete',backup,mirrorCommit}; +} + +/** One Success datapoint per day keeps the stale alarm green; 0 starts the dead-man count. */ +export function recordRepoBackupFreshness(freshness:RepoBackupFreshness):void { + writeEmbeddedMetric({ + _aws:{ + Timestamp:Date.now(), + CloudWatchMetrics:[{Namespace:'DataOps/Knowledge',Dimensions:[['Job']],Metrics:[{Name:'Success',Unit:'Count'}]}], + }, + Job:REPO_BACKUP_JOB, + Success:freshness.fresh?1:0, + reason:freshness.reason, + backup_commit:freshness.backup.commit||'', + backup_last_modified:freshness.backup.lastModified?.toISOString()||'', + mirror_commit:freshness.mirrorCommit||'', + }); +} + +async function readJsonObject(client:{send(command:unknown):Promise},bucket:string,key:string):Promise<{value:any;lastModified:Date|null}|null> { + let response:any; + try { + response=await client.send(new GetObjectCommand({Bucket:bucket,Key:key})); + } catch(err) { + if(['NotFound','NoSuchKey','NoSuchBucket'].includes((err as {name?:string}).name||'')) return null; + throw err; + } + const text=await response.Body.transformToString(); + return {value:JSON.parse(text),lastModified:response.LastModified instanceof Date?response.LastModified:null}; +} + +export interface RepoBackupCheckDeps {store:KnowledgeStore; backupBucket:string; backupPrefix:string; now?:Date} + +export async function runRepoBackupFreshnessCheck(deps:RepoBackupCheckDeps):Promise { + if(!deps.backupBucket) throw new Error('KNOWLEDGE_REPO_BACKUP_BUCKET is not configured'); + const prefix=deps.backupPrefix.replace(/^\/+|\/+$/g,''); + const manifestKey=`${prefix}/latest/manifest.json`; + const manifest=await readJsonObject(deps.store.client,deps.backupBucket,manifestKey); + const backup:RepoBackupManifest=manifest?{commit:typeof manifest.value.commit_sha==='string'?manifest.value.commit_sha:null,lastModified:manifest.lastModified}:{commit:null,lastModified:null}; + const report=await readJsonObject(deps.store.client,deps.store.bucket,MIRROR_LAST_REPORT_KEY); + const mirrorCommit=report&&typeof report.value.commit==='string'?report.value.commit:null; + const freshness=evaluateRepoBackupFreshness({backup,mirrorCommit,now:deps.now||new Date()}); + recordRepoBackupFreshness(freshness); + return freshness; +} diff --git a/backend/src/knowledge/worker.ts b/backend/src/knowledge/worker.ts index e6e9128f2..95fbcd3c7 100644 --- a/backend/src/knowledge/worker.ts +++ b/backend/src/knowledge/worker.ts @@ -14,6 +14,7 @@ import {join} from 'node:path'; import {CopyObjectCommand,HeadObjectCommand,PutObjectCommand} from '@aws-sdk/client-s3'; import {KnowledgeStore,bounded,type Publication} from '../docs/knowledgeStore'; import {writeEmbeddedMetric} from '../embeddedMetrics'; +import {runRepoBackupFreshnessCheck} from './repoBackupFreshness'; import {withKnowledgeJobLease} from './lease'; export interface RecoverySnapshot {schemaVersion:1; revision:string; createdAt:string; publications:Publication[]; payloads:{sha:string;size:number;key:string}[];} @@ -71,7 +72,15 @@ function metric(job:string,success:number):void { writeEmbeddedMetric({_aws:{Timestamp:Date.now(),CloudWatchMetrics:[{Namespace:'DataOps/Knowledge',Dimensions:[['Job']],Metrics:[{Name:'Success',Unit:'Count'},{Name:'Failure',Unit:'Count'}]}]},Job:job,Success:success,Failure:success?0:1}); } export async function handler(event:{detail?:{dataopsAction?:string}}):Promise { - if(event.detail?.dataopsAction!=='knowledge-backup') throw new Error('Unknown knowledge job'); + const action=event.detail?.dataopsAction; + if(action==='knowledge-repo-backup-check') { + return runRepoBackupFreshnessCheck({ + store:new KnowledgeStore(), + backupBucket:process.env.KNOWLEDGE_REPO_BACKUP_BUCKET||'', + backupPrefix:process.env.KNOWLEDGE_REPO_BACKUP_PREFIX||'dataops-knowledge', + }); + } + if(action!=='knowledge-backup') throw new Error('Unknown knowledge job'); const store=new KnowledgeStore(); try { const result=await withKnowledgeJobLease(store,'knowledge-backup',async signal=>{ diff --git a/backend/tests/helpers/knowledge.ts b/backend/tests/helpers/knowledge.ts index d737a9c64..e3d5cf5b8 100644 --- a/backend/tests/helpers/knowledge.ts +++ b/backend/tests/helpers/knowledge.ts @@ -2,17 +2,20 @@ import {randomUUID} from 'node:crypto'; import {KnowledgeStore,checksum,type KnowledgeConfig,type Publication} from '../../src/docs/knowledgeStore'; /** Faithful conditional-object semantics, injected at the SDK boundary; no GitHub calls. */ export class MemoryS3 { - objects=new Map(); etags=new Map();calls:{name:string;input:any}[]=[]; + objects=new Map(); etags=new Map();modified=new Map();calls:{name:string;input:any}[]=[]; failKey=''; + /** LastModified is fixed per object the way S3 returns it, or defaults to this stamp. */ + now=new Date('2026-11-18T06:00:00Z'); + lastModified(key:string):Date {return this.modified.get(key)||this.now;} send=async(command:any):Promise=>{ const i=command.input;const key=`${i.Bucket}/${i.Key}`;const name=command.constructor.name;this.calls.push({name,input:i}); if(i.Key===this.failKey) throw new Error('Injected storage failure'); const old=this.objects.get(key);const etag=this.etags.get(key); if(name==='GetObjectCommand') { if(!old) throw Object.assign(new Error('Missing object'),{name:'NoSuchKey'}); - return {ETag:etag,Body:{transformToString:async()=>old.toString(),transformToByteArray:async()=>new Uint8Array(old)}}; + return {ETag:etag,LastModified:this.lastModified(key),Body:{transformToString:async()=>old.toString(),transformToByteArray:async()=>new Uint8Array(old)}}; } - if(name==='HeadObjectCommand') {if(!old) throw Object.assign(new Error('Missing object'),{name:'NotFound'});return{ETag:etag,ContentLength:old.length};} + if(name==='HeadObjectCommand') {if(!old) throw Object.assign(new Error('Missing object'),{name:'NotFound'});return{ETag:etag,LastModified:this.lastModified(key),ContentLength:old.length};} if(name==='PutObjectCommand' || name==='CopyObjectCommand') { if((i.IfNoneMatch==='*' && old) || (i.IfMatch && i.IfMatch!==etag)) throw Object.assign(new Error('Conditional conflict'),{name:'PreconditionFailed'}); let bytes:Buffer|undefined; @@ -20,7 +23,7 @@ export class MemoryS3 { else if(i.Body && typeof i.Body[Symbol.asyncIterator]==='function') { const chunks:Buffer[]=[];for await(const chunk of i.Body) chunks.push(Buffer.from(chunk));bytes=Buffer.concat(chunks); } else bytes=Buffer.from(i.Body); - if(!bytes) throw new Error('Missing copy source');this.objects.set(key,bytes);const next=`"${randomUUID()}"`;this.etags.set(key,next);return{ETag:next}; + if(!bytes) throw new Error('Missing copy source');this.objects.set(key,bytes);this.modified.set(key,this.now);const next=`"${randomUUID()}"`;this.etags.set(key,next);return{ETag:next}; } throw new Error(`Unexpected command ${name}`); }; diff --git a/backend/tests/knowledge-publisher-auth.test.ts b/backend/tests/knowledge-publisher-auth.test.ts index b1fe88651..0807d1431 100644 --- a/backend/tests/knowledge-publisher-auth.test.ts +++ b/backend/tests/knowledge-publisher-auth.test.ts @@ -6,7 +6,7 @@ import { resolve } from 'node:path'; import { handler } from '../src/handler'; import { configureDocsRuntime, handleDocsRoutes, resetDocsRuntime } from '../src/docs/contentApi'; import { resetKnowledgePublisherCaches } from '../src/auth/knowledgePublisher'; -import { syntheticKnowledge } from './helpers/knowledge'; +import { syntheticKnowledge, MemoryS3 } from './helpers/knowledge'; import { truncateTestTables, useTestDatabase } from './helpers/db'; import { listTemplates } from '../src/db/templates'; import { setDeploymentTemplateLoaderForTest } from '../src/deploymentSeeds'; @@ -50,6 +50,7 @@ const request = (method: string, path: string, credential: string, body: unknown describe('GitHub Actions OIDC knowledge publisher', () => { let dir: string; let revision: string; + let s3: MemoryS3; before(async () => { Object.assign(process.env, config); await useTestDatabase(); @@ -69,6 +70,7 @@ describe('GitHub Actions OIDC knowledge publisher', () => { dir = mkdtempSync(resolve('../.tmp/publisher-')); const fixture = syntheticKnowledge({ 'content/a.md': 'old' }, dir, async () => 'https://example.invalid/synthetic-download'); configureDocsRuntime(fixture.store); + s3 = fixture.s3; revision = fixture.publication.revision; resetKnowledgePublisherCaches(); }); @@ -119,6 +121,13 @@ describe('GitHub Actions OIDC knowledge publisher', () => { } const mirror = records.filter((record) => record.Job === 'knowledge-mirror'); assert.deepEqual(mirror.map((record) => [record.Success, record.Failure]), [[0, 1], [1, 0]]); + // The last report persists for the repo-backup freshness check: it compares + // the mirrored HEAD against the S3 backup manifest's commit. The success + // heartbeat above (without a commit) overwrote the failure report. + const stored = JSON.parse(s3.objects.get(`test-knowledge/jobs/knowledge-mirror/last-report.json`)!.toString()); + assert.equal(stored.outcome, 'success'); + assert.equal(stored.revision, 'rev1'); + assert.ok(stored.at); }); it('syncs authored templates on request and reports failures', async () => { diff --git a/backend/tests/knowledge-repo-backup-freshness.test.ts b/backend/tests/knowledge-repo-backup-freshness.test.ts new file mode 100644 index 000000000..777ef0dd4 --- /dev/null +++ b/backend/tests/knowledge-repo-backup-freshness.test.ts @@ -0,0 +1,133 @@ +import {it} from 'node:test'; +import assert from 'node:assert/strict'; + +import {evaluateRepoBackupFreshness, recordRepoBackupFreshness, runRepoBackupFreshnessCheck, FRESH_BACKUP_MAX_AGE_MS, MIRROR_LAST_REPORT_KEY, REPO_BACKUP_JOB} from '../src/knowledge/repoBackupFreshness'; +import {syntheticKnowledge} from './helpers/knowledge'; +import type {RepoBackupManifest} from '../src/knowledge/repoBackupFreshness'; + +const NOW = new Date('2026-11-18T05:30:00Z'); +const day = (days: number) => new Date(NOW.getTime() - days * 24 * 60 * 60 * 1000); + +/** Capture the raw stdout lines the embedded metric is written to. */ +function captureStdout(action: () => void): string[] { + const originalWrite = process.stdout.write; + const lines: string[] = []; + process.stdout.write = ((chunk: unknown) => { lines.push(String(chunk)); return true; }) as typeof process.stdout.write; + try { action(); } finally { process.stdout.write = originalWrite; } + return lines; +} + +function backup(commit: string | null, lastModified: Date | null): RepoBackupManifest { + return { commit, lastModified }; +} + +it('counts a backup as fresh while it contains the mirrored HEAD, however old the upload is', () => { + // A quiet repository keeps HEAD stable for weeks; the backup of that HEAD + // never changes and must not page. + const freshness = evaluateRepoBackupFreshness({ backup: backup('a'.repeat(40), day(40)), mirrorCommit: 'a'.repeat(40), now: NOW }); + assert.equal(freshness.fresh, true); + assert.equal(freshness.reason, 'commit-match'); +}); + +it('counts a recent upload as fresh even when the mirror already moved on', () => { + // The mirror reports the new HEAD at 04:00; the next backup lands at 03:17 + // the following day, so a one-day-old manifest of an older commit is normal. + const freshness = evaluateRepoBackupFreshness({ backup: backup('b'.repeat(40), day(1)), mirrorCommit: 'c'.repeat(40), now: NOW }); + assert.equal(freshness.fresh, true); + assert.equal(freshness.reason, 'recent-upload'); +}); + +it('counts exactly-two-day-old uploads as fresh on the boundary', () => { + const freshness = evaluateRepoBackupFreshness({ backup: backup('b'.repeat(40), new Date(NOW.getTime() - FRESH_BACKUP_MAX_AGE_MS)), mirrorCommit: 'c'.repeat(40), now: NOW }); + assert.equal(freshness.fresh, true); +}); + +it('pages only when an unbacked commit is older than the freshness window', () => { + // The incident shape: the backup workflow stopped scheduling while the + // repository kept receiving commits - nothing alerted for 44 days. + const freshness = evaluateRepoBackupFreshness({ backup: backup('b'.repeat(40), day(44)), mirrorCommit: 'c'.repeat(40), now: NOW }); + assert.equal(freshness.fresh, false); + assert.equal(freshness.reason, 'divergent-commit'); +}); + +it('pages when the backup manifest is missing', () => { + const freshness = evaluateRepoBackupFreshness({ backup: backup(null, null), mirrorCommit: 'c'.repeat(40), now: NOW }); + assert.equal(freshness.fresh, false); + assert.equal(freshness.reason, 'missing-manifest'); +}); + +it('judges by upload age when the mirror has never reported', () => { + const freshness = evaluateRepoBackupFreshness({ backup: backup('b'.repeat(40), day(44)), mirrorCommit: '', now: NOW }); + assert.equal(freshness.fresh, false); + assert.equal(freshness.reason, 'divergent-commit'); + const oldUnidentified = evaluateRepoBackupFreshness({ backup: backup(null, day(44)), mirrorCommit: '', now: NOW }); + assert.equal(oldUnidentified.fresh, false); + assert.equal(oldUnidentified.reason, 'manifest-incomplete'); +}); + +it('writes the freshness metric as a bare stdout EMF record', () => { + const lines = captureStdout(() => recordRepoBackupFreshness({ + fresh: false, reason: 'divergent-commit', + backup: { commit: 'b'.repeat(40), lastModified: day(44) }, + mirrorCommit: 'c'.repeat(40), + })); + assert.equal(lines.length, 1); + assert.ok(lines[0].endsWith('\n')); + const record = JSON.parse(lines[0]); + assert.equal(record._aws.CloudWatchMetrics[0].Namespace, 'DataOps/Knowledge'); + assert.equal(record.Job, REPO_BACKUP_JOB); + assert.equal(record.Success, 0); + assert.equal(record.reason, 'divergent-commit'); + assert.equal(record.mirror_commit, 'c'.repeat(40)); +}); + +it('reads the backup manifest and mirror report from S3 and emits the stale verdict', async () => { + const { store, s3 } = syntheticKnowledge({}); + const backupBucket = 'dataops-knowledge-backups-1-eu-west-1'; + s3.objects.set(`${backupBucket}/dataops-knowledge/latest/manifest.json`, Buffer.from(JSON.stringify({ commit_sha: 'b'.repeat(40) }))); + s3.modified.set(`${backupBucket}/dataops-knowledge/latest/manifest.json`, day(44)); + s3.objects.set(`${store.bucket}/${MIRROR_LAST_REPORT_KEY}`, Buffer.from(JSON.stringify({ outcome: 'success', revision: 'rev1', commit: 'c'.repeat(40), at: NOW.toISOString() }))); + + let stdout: string[] = []; + const originalWrite = process.stdout.write; + process.stdout.write = ((chunk: unknown) => { stdout.push(String(chunk)); return true; }) as typeof process.stdout.write; + let freshness; + try { + freshness = await runRepoBackupFreshnessCheck({ store, backupBucket, backupPrefix: 'dataops-knowledge', now: NOW }); + } finally { + process.stdout.write = originalWrite; + } + assert.equal(freshness.fresh, false); + assert.equal(freshness.reason, 'divergent-commit'); + assert.equal(freshness.backup.commit, 'b'.repeat(40)); + assert.equal(freshness.backup.lastModified!.toISOString(), day(44).toISOString()); + assert.equal(freshness.mirrorCommit, 'c'.repeat(40)); + assert.equal(stdout.length, 1); + assert.equal(JSON.parse(stdout[0]).Success, 0); +}); + +it('reports fresh for a matching mirror commit even after a 44-day upload gap', async () => { + const { store, s3 } = syntheticKnowledge({}); + const backupBucket = 'dataops-knowledge-backups-2-eu-west-1'; + s3.objects.set(`${backupBucket}/dataops-knowledge/latest/manifest.json`, Buffer.from(JSON.stringify({ commit_sha: 'q'.repeat(40) }))); + s3.modified.set(`${backupBucket}/dataops-knowledge/latest/manifest.json`, day(44)); + s3.objects.set(`${store.bucket}/${MIRROR_LAST_REPORT_KEY}`, Buffer.from(JSON.stringify({ outcome: 'success', revision: 'rev2', commit: 'q'.repeat(40), at: NOW.toISOString() }))); + const freshness = await runRepoBackupFreshnessCheck({ store, backupBucket, backupPrefix: 'dataops-knowledge/', now: NOW }); + assert.equal(freshness.fresh, true); + assert.equal(freshness.reason, 'commit-match'); +}); + +it('reports fresh for a day-old upload of a not-yet-backed-up HEAD', async () => { + const { store, s3 } = syntheticKnowledge({}); + const backupBucket = 'dataops-knowledge-backups-3-eu-west-1'; + s3.objects.set(`${backupBucket}/dataops-knowledge/latest/manifest.json`, Buffer.from(JSON.stringify({ commit_sha: 'q'.repeat(40) }))); + s3.modified.set(`${backupBucket}/dataops-knowledge/latest/manifest.json`, day(1)); + const freshness = await runRepoBackupFreshnessCheck({ store, backupBucket, backupPrefix: 'dataops-knowledge', now: NOW }); + assert.equal(freshness.fresh, true); + assert.equal(freshness.reason, 'recent-upload'); +}); + +it('fails loudly when the backup bucket is not configured', async () => { + const { store } = syntheticKnowledge({}); + await assert.rejects(() => runRepoBackupFreshnessCheck({ store, backupBucket: '', backupPrefix: 'dataops-knowledge', now: NOW }), /KNOWLEDGE_REPO_BACKUP_BUCKET is not configured/); +}); diff --git a/infra/template.full.yaml b/infra/template.full.yaml index 6162ccfa6..72324f2b7 100644 --- a/infra/template.full.yaml +++ b/infra/template.full.yaml @@ -89,6 +89,16 @@ Parameters: Type: String Default: arn:aws:sns:eu-west-1:817685572750:dapier-AlarmTopic-6LjEWSJwA6Ya Description: Shared operator alarm SNS topic (delivers via AlarmNotifyFunction to dev@dtcdev.click). Paged by the daily PITR guard, the knowledge backup/mirror alarms and store failures. + KnowledgeRepoBackupBucketName: + Type: String + Default: dataops-knowledge-backups-817685572750-eu-west-1 + AllowedPattern: "^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$" + Description: The aws-infra knowledge-backups bucket the dataops-knowledge repository's backup workflow uploads to; read for the freshness dead-man check. + KnowledgeRepoBackupPrefix: + Type: String + Default: dataops-knowledge + AllowedPattern: "^[a-zA-Z0-9_.\\/-]+$" + Description: Key prefix the backup workflow uploads under (aws-infra BackupPrefix). DynamoPitrAlertEmail: Type: String Default: dev@dtcdev.click @@ -1131,6 +1141,10 @@ Resources: Effect: Allow Action: [s3:GetObject] Resource: !Sub arn:${AWS::Partition}:s3:::${KnowledgeBucketName}/* + - Statement: + Effect: Allow + Action: [s3:PutObject] + Resource: !Sub arn:${AWS::Partition}:s3:::${KnowledgeBucketName}/jobs/* - Statement: Effect: Allow Action: [s3:PutObject] @@ -1560,6 +1574,8 @@ Resources: Variables: KNOWLEDGE_BUCKET: !Ref KnowledgeBucketName KNOWLEDGE_BACKUP_BUCKET: !Ref KnowledgeBackupBucketName + KNOWLEDGE_REPO_BACKUP_BUCKET: !Ref KnowledgeRepoBackupBucketName + KNOWLEDGE_REPO_BACKUP_PREFIX: !Ref KnowledgeRepoBackupPrefix Policies: - Statement: Effect: Allow @@ -1581,6 +1597,11 @@ Resources: Effect: Allow Action: [s3:PutObject] Resource: !Sub arn:${AWS::Partition}:s3:::${KnowledgeBucketName}/jobs/* + # Repo-backup freshness check reads only the latest backup manifest. + - Statement: + Effect: Allow + Action: [s3:GetObject] + Resource: !Sub arn:${AWS::Partition}:s3:::${KnowledgeRepoBackupBucketName}/${KnowledgeRepoBackupPrefix}/latest/* Events: DailyKnowledgeBackup: Type: Schedule @@ -1588,6 +1609,14 @@ Resources: Schedule: cron(0 3 * * ? *) Enabled: true Input: '{"detail":{"dataopsAction":"knowledge-backup"}}' + # After the 04:00 UTC mirror (which reports the repository HEAD this + # check compares the backup manifest against) and the 03:17 backup. + DailyKnowledgeRepoBackupCheck: + Type: Schedule + Properties: + Schedule: cron(30 5 * * ? *) + Enabled: true + Input: '{"detail":{"dataopsAction":"knowledge-repo-backup-check"}}' KnowledgeWorkerFailureAlarm: Type: AWS::CloudWatch::Alarm Properties: @@ -1622,6 +1651,28 @@ Resources: TreatMissingData: breaching AlarmActions: [!Ref DynamoPitrAlertTopicArn] OKActions: [!Ref DynamoPitrAlertTopicArn] + # Repo-backup freshness: the worker reads the backup workflow's latest S3 + # manifest daily and writes Success=1 while it contains the mirrored HEAD or + # landed within two days. Pages when repository commits stop being backed up + # (the 2026-10-02 -> 2026-11-15 scheduling outage ran 44 days unalerted); a + # quiet repository with a current backup stays green. + KnowledgeRepoBackupStaleAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ${AWS::StackName}-knowledge-repo-backup-stale + AlarmDescription: The dataops-knowledge repository backup has not captured the mirrored HEAD for two days; the backup-to-s3 workflow may have stopped running. Run history is in the dataops-knowledge Actions tab. + Namespace: DataOps/Knowledge + MetricName: Success + Dimensions: [{Name: Job, Value: knowledge-repo-backup}] + Statistic: Sum + Period: 86400 + EvaluationPeriods: 2 + DatapointsToAlarm: 2 + Threshold: 1 + ComparisonOperator: LessThanThreshold + TreatMissingData: breaching + AlarmActions: [!Ref DynamoPitrAlertTopicArn] + OKActions: [!Ref DynamoPitrAlertTopicArn] # The daily DataOps -> Git mirror runs in GitHub Actions and reports through # POST /knowledge/mirror-heartbeat (see backend/src/knowledge/mirrorHeartbeat.ts).