Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 14 additions & 12 deletions .github/workflows/sandbox-backup-evidence.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
run: |
set -uo pipefail
failed=0
for suffix in knowledge-backup-stale knowledge-worker-failure knowledge-mirror-stale knowledge-mirror-failed recurring-cron-pass-missed recurring-cron-failure recurring-cron-errors; do
for suffix in knowledge-backup-stale knowledge-repo-backup-stale knowledge-worker-failure knowledge-mirror-stale knowledge-mirror-failed recurring-cron-pass-missed recurring-cron-failure recurring-cron-errors; do
name="dataops-v1-$suffix"
if ! aws cloudwatch describe-alarms --alarm-names "$name" \
--query 'MetricAlarms' --output json > "$RUNNER_TEMP/$suffix.json"; then
Expand Down Expand Up @@ -83,21 +83,23 @@ jobs:
done
exit "$failed"

- name: Prove the backup freshness alarm reaches a subscriber
- name: Prove the backup freshness alarms reach a subscriber
if: ${{ !cancelled() }}
shell: bash
run: |
set -euo pipefail
alarm="$RUNNER_TEMP/knowledge-backup-stale.json"
# Success=1 per daily backup; alarm when the 2-day sum is <1 or missing.
jq -e '.[0] | .Namespace == "DataOps/Knowledge" and .MetricName == "Success"
and .Threshold == 1 and .ComparisonOperator == "LessThanThreshold"
and .TreatMissingData == "breaching"' "$alarm" >/dev/null
for topic in $(jq -r '.[0].AlarmActions[]' "$alarm"); do
confirmed=$(aws sns list-subscriptions-by-topic --topic-arn "$topic" --output json \
| jq '[.Subscriptions[] | select(.SubscriptionArn != "PendingConfirmation")] | length')
echo "$topic confirmed subscriptions: $confirmed"
[ "$confirmed" -ge 1 ]
# Success=1 per daily backup/check; alarm when the 2-day sum is <1 or missing.
for suffix in knowledge-backup-stale knowledge-repo-backup-stale; do
alarm="$RUNNER_TEMP/$suffix.json"
jq -e '.[0] | .Namespace == "DataOps/Knowledge" and .MetricName == "Success"
and .Threshold == 1 and .ComparisonOperator == "LessThanThreshold"
and .TreatMissingData == "breaching"' "$alarm" >/dev/null
for topic in $(jq -r '.[0].AlarmActions[]' "$alarm"); do
confirmed=$(aws sns list-subscriptions-by-topic --topic-arn "$topic" --output json \
| jq '[.Subscriptions[] | select(.SubscriptionArn != "PendingConfirmation")] | length')
echo "$suffix -> $topic confirmed subscriptions: $confirmed"
[ "$confirmed" -ge 1 ]
done
done

- name: Read actual knowledge success datapoints
Expand Down
4 changes: 2 additions & 2 deletions backend/src/auth/knowledgePublisher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { createPublicKey, verify as verifySignature, type JsonWebKey } from 'nod
import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';

import { seedDeploymentRuntime } from '../deploymentSeeds';
import { handleDocsRoutes, isDocsDomainEnabled, regenerateKnowledgeForTemplates } from '../docs/contentApi';
import { docsKnowledgeStore, handleDocsRoutes, isDocsDomainEnabled, regenerateKnowledgeForTemplates } from '../docs/contentApi';
import { listTemplates } from '../db/templates';
import { parseMirrorHeartbeat, recordMirrorHeartbeat } from '../knowledge/mirrorHeartbeat';
import type { LambdaEvent, LambdaResponse } from '../types';
Expand Down Expand Up @@ -131,7 +131,7 @@ export async function handleKnowledgePublisher(
if (method === 'POST' && path === '/knowledge/mirror-heartbeat') {
const heartbeat = parseMirrorHeartbeat(event.body);
if (!heartbeat) return response(400, { error: 'outcome must be success or failure' });
recordMirrorHeartbeat(heartbeat);
await recordMirrorHeartbeat(docsKnowledgeStore(), heartbeat);
return response(200, { recorded: heartbeat.outcome });
}
const allowed = (method === 'GET' && (path === '/knowledge/manifest' || path === '/knowledge/history' || path === '/knowledge/version' || path === '/knowledge/download'))
Expand Down
2 changes: 1 addition & 1 deletion backend/src/automations/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ export const AUTOMATIONS: readonly AutomationDefinition[] = Object.freeze([
{
id: 'knowledge-repo-backup',
name: 'Knowledge repository backup',
description: 'Zips the knowledge repository and uploads it to S3 when the commit changed since the last backup.',
description: 'Zips the knowledge repository and uploads it to S3 when the commit changed since the last backup; a daily freshness check pages the operator topic when repository commits stop being backed up.',
runsIn: 'GitHub Actions, dataops-knowledge',
system: true,
schedule: { kind: 'cron', expression: '17 3 * * *' },
Expand Down
5 changes: 5 additions & 0 deletions backend/src/docs/contentApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,11 @@ export function resetDocsRuntime(): void {
sharedRecent = null;
}

/** The configured runtime's knowledge store, or one built from the environment. */
export function docsKnowledgeStore(): KnowledgeStore {
return runtime ? runtime.store : createKnowledgeStore(knowledgeStoreConfigFromEnv());
}

interface KnowledgePlan {
mutations: Mutation[];
regenerated: string[];
Expand Down
18 changes: 17 additions & 1 deletion backend/src/knowledge/mirrorHeartbeat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,16 @@
* Failure, and on any UTC day without a Success (the dead-man check: the
* schedule stopped, the workflow was disabled, or it never reached the
* report step), and send the recovery notice when the next success lands.
*
* The report also persists the mirrored Git HEAD to
* jobs/knowledge-mirror/last-report.json: the repo-backup freshness check
* compares it against the S3 backup manifest's commit to tell a quiet
* repository (backed up, nothing new) from one whose backups stopped.
*/
import {PutObjectCommand} from '@aws-sdk/client-s3';
import type {KnowledgeStore} from '../docs/knowledgeStore';
import { writeEmbeddedMetric } from '../embeddedMetrics';
import { MIRROR_LAST_REPORT_KEY } from './repoBackupFreshness';

export const MIRROR_JOB = 'knowledge-mirror';

Expand All @@ -28,7 +36,7 @@ export function parseMirrorHeartbeat(body: string | null | undefined): MirrorHea
return { outcome, revision: text(revision), commit: text(commit) };
}

export function recordMirrorHeartbeat(heartbeat: MirrorHeartbeat): void {
export async function recordMirrorHeartbeat(store: Pick<KnowledgeStore, 'bucket' | 'client'>, heartbeat: MirrorHeartbeat): Promise<void> {
const success = heartbeat.outcome === 'success';
writeEmbeddedMetric({
_aws: {
Expand All @@ -41,4 +49,12 @@ export function recordMirrorHeartbeat(heartbeat: MirrorHeartbeat): void {
revision: heartbeat.revision,
commit: heartbeat.commit,
});
// The freshness check only reads the commit; keep the whole report for
// operators inspecting the object.
await store.client.send(new PutObjectCommand({
Bucket: store.bucket,
Key: MIRROR_LAST_REPORT_KEY,
Body: JSON.stringify({ ...heartbeat, at: new Date().toISOString() }),
ContentType: 'application/json',
}));
}
86 changes: 86 additions & 0 deletions backend/src/knowledge/repoBackupFreshness.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/**
* Dead-man check for the daily dataops-knowledge repository backup.
*
* The backup runs in the knowledge repository's GitHub Actions workflow
* (backup-to-s3.yml) and uploads straight to the private backups bucket, so
* nothing on the AWS side sees the workflow run — and the workflow's own
* success also covers skip runs that upload nothing because HEAD did not
* change. The truthful signal is the backup's `{prefix}/latest/manifest.json`:
* it is re-uploaded whenever the workflow finds a commit it has not backed up
* yet, and its `commit_sha` names the newest backed-up repository commit.
*
* The scheduled check turns that object into the `DataOps/Knowledge`
* Job=knowledge-repo-backup Success metric that the KnowledgeRepoBackupStaleAlarm
* in the infra template pages the operator topic from. A backup is fresh when
* its commit matches the last mirror-reported repository HEAD
* (jobs/knowledge-mirror/last-report.json, written by the heartbeat route) or
* when it landed within the last two days. A quiet repository therefore stays
* green indefinitely, while a workflow that silently stops scheduling pages as
* soon as a real commit goes unbacked — the 2026-10-02 -> 2026-11-15 outage
* ran 44 days without any alert.
*/
import {GetObjectCommand} from '@aws-sdk/client-s3';
import type {KnowledgeStore} from '../docs/knowledgeStore';
import {writeEmbeddedMetric} from '../embeddedMetrics';

export const REPO_BACKUP_JOB='knowledge-repo-backup';
/** Backups are daily; two days of silence means the latest upload is behind. */
export const FRESH_BACKUP_MAX_AGE_MS=48*60*60*1000;
/** Where recordMirrorHeartbeat persists the last reported mirror commit. */
export const MIRROR_LAST_REPORT_KEY='jobs/knowledge-mirror/last-report.json';

export interface RepoBackupManifest {commit:string|null; lastModified:Date|null}
export interface RepoBackupFreshness {fresh:boolean; reason:string; backup:RepoBackupManifest; mirrorCommit:string|null}

/** Fresh when the backup contains the newest mirror-reported HEAD, or a backup landed recently. */
export function evaluateRepoBackupFreshness(input:{backup:RepoBackupManifest; mirrorCommit:string|null; now:Date}):RepoBackupFreshness {
const {backup,mirrorCommit,now}=input;
if(!backup.lastModified) return {fresh:false,reason:'missing-manifest',backup,mirrorCommit};
const age=now.getTime()-backup.lastModified.getTime();
if(mirrorCommit && backup.commit && backup.commit===mirrorCommit) return {fresh:true,reason:'commit-match',backup,mirrorCommit};
if(age<=FRESH_BACKUP_MAX_AGE_MS) return {fresh:true,reason:'recent-upload',backup,mirrorCommit};
return {fresh:false,reason:backup.commit?'divergent-commit':'manifest-incomplete',backup,mirrorCommit};
}

/** One Success datapoint per day keeps the stale alarm green; 0 starts the dead-man count. */
export function recordRepoBackupFreshness(freshness:RepoBackupFreshness):void {
writeEmbeddedMetric({
_aws:{
Timestamp:Date.now(),
CloudWatchMetrics:[{Namespace:'DataOps/Knowledge',Dimensions:[['Job']],Metrics:[{Name:'Success',Unit:'Count'}]}],
},
Job:REPO_BACKUP_JOB,
Success:freshness.fresh?1:0,
reason:freshness.reason,
backup_commit:freshness.backup.commit||'',
backup_last_modified:freshness.backup.lastModified?.toISOString()||'',
mirror_commit:freshness.mirrorCommit||'',
});
}

async function readJsonObject(client:{send(command:unknown):Promise<any>},bucket:string,key:string):Promise<{value:any;lastModified:Date|null}|null> {
let response:any;
try {
response=await client.send(new GetObjectCommand({Bucket:bucket,Key:key}));
} catch(err) {
if(['NotFound','NoSuchKey','NoSuchBucket'].includes((err as {name?:string}).name||'')) return null;
throw err;
}
const text=await response.Body.transformToString();
return {value:JSON.parse(text),lastModified:response.LastModified instanceof Date?response.LastModified:null};
}

export interface RepoBackupCheckDeps {store:KnowledgeStore; backupBucket:string; backupPrefix:string; now?:Date}

export async function runRepoBackupFreshnessCheck(deps:RepoBackupCheckDeps):Promise<RepoBackupFreshness> {
if(!deps.backupBucket) throw new Error('KNOWLEDGE_REPO_BACKUP_BUCKET is not configured');
const prefix=deps.backupPrefix.replace(/^\/+|\/+$/g,'');
const manifestKey=`${prefix}/latest/manifest.json`;
const manifest=await readJsonObject(deps.store.client,deps.backupBucket,manifestKey);
const backup:RepoBackupManifest=manifest?{commit:typeof manifest.value.commit_sha==='string'?manifest.value.commit_sha:null,lastModified:manifest.lastModified}:{commit:null,lastModified:null};
const report=await readJsonObject(deps.store.client,deps.store.bucket,MIRROR_LAST_REPORT_KEY);
const mirrorCommit=report&&typeof report.value.commit==='string'?report.value.commit:null;
const freshness=evaluateRepoBackupFreshness({backup,mirrorCommit,now:deps.now||new Date()});
recordRepoBackupFreshness(freshness);
return freshness;
}
11 changes: 10 additions & 1 deletion backend/src/knowledge/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {join} from 'node:path';
import {CopyObjectCommand,HeadObjectCommand,PutObjectCommand} from '@aws-sdk/client-s3';
import {KnowledgeStore,bounded,type Publication} from '../docs/knowledgeStore';
import {writeEmbeddedMetric} from '../embeddedMetrics';
import {runRepoBackupFreshnessCheck} from './repoBackupFreshness';
import {withKnowledgeJobLease} from './lease';

export interface RecoverySnapshot {schemaVersion:1; revision:string; createdAt:string; publications:Publication[]; payloads:{sha:string;size:number;key:string}[];}
Expand Down Expand Up @@ -71,7 +72,15 @@ function metric(job:string,success:number):void {
writeEmbeddedMetric({_aws:{Timestamp:Date.now(),CloudWatchMetrics:[{Namespace:'DataOps/Knowledge',Dimensions:[['Job']],Metrics:[{Name:'Success',Unit:'Count'},{Name:'Failure',Unit:'Count'}]}]},Job:job,Success:success,Failure:success?0:1});
}
export async function handler(event:{detail?:{dataopsAction?:string}}):Promise<unknown> {
if(event.detail?.dataopsAction!=='knowledge-backup') throw new Error('Unknown knowledge job');
const action=event.detail?.dataopsAction;
if(action==='knowledge-repo-backup-check') {
return runRepoBackupFreshnessCheck({
store:new KnowledgeStore(),
backupBucket:process.env.KNOWLEDGE_REPO_BACKUP_BUCKET||'',
backupPrefix:process.env.KNOWLEDGE_REPO_BACKUP_PREFIX||'dataops-knowledge',
});
}
if(action!=='knowledge-backup') throw new Error('Unknown knowledge job');
const store=new KnowledgeStore();
try {
const result=await withKnowledgeJobLease(store,'knowledge-backup',async signal=>{
Expand Down
11 changes: 7 additions & 4 deletions backend/tests/helpers/knowledge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,28 @@ import {randomUUID} from 'node:crypto';
import {KnowledgeStore,checksum,type KnowledgeConfig,type Publication} from '../../src/docs/knowledgeStore';
/** Faithful conditional-object semantics, injected at the SDK boundary; no GitHub calls. */
export class MemoryS3 {
objects=new Map<string,Buffer>(); etags=new Map<string,string>();calls:{name:string;input:any}[]=[];
objects=new Map<string,Buffer>(); etags=new Map<string,string>();modified=new Map<string,Date>();calls:{name:string;input:any}[]=[];
failKey='';
/** LastModified is fixed per object the way S3 returns it, or defaults to this stamp. */
now=new Date('2026-11-18T06:00:00Z');
lastModified(key:string):Date {return this.modified.get(key)||this.now;}
send=async(command:any):Promise<any>=>{
const i=command.input;const key=`${i.Bucket}/${i.Key}`;const name=command.constructor.name;this.calls.push({name,input:i});
if(i.Key===this.failKey) throw new Error('Injected storage failure');
const old=this.objects.get(key);const etag=this.etags.get(key);
if(name==='GetObjectCommand') {
if(!old) throw Object.assign(new Error('Missing object'),{name:'NoSuchKey'});
return {ETag:etag,Body:{transformToString:async()=>old.toString(),transformToByteArray:async()=>new Uint8Array(old)}};
return {ETag:etag,LastModified:this.lastModified(key),Body:{transformToString:async()=>old.toString(),transformToByteArray:async()=>new Uint8Array(old)}};
}
if(name==='HeadObjectCommand') {if(!old) throw Object.assign(new Error('Missing object'),{name:'NotFound'});return{ETag:etag,ContentLength:old.length};}
if(name==='HeadObjectCommand') {if(!old) throw Object.assign(new Error('Missing object'),{name:'NotFound'});return{ETag:etag,LastModified:this.lastModified(key),ContentLength:old.length};}
if(name==='PutObjectCommand' || name==='CopyObjectCommand') {
if((i.IfNoneMatch==='*' && old) || (i.IfMatch && i.IfMatch!==etag)) throw Object.assign(new Error('Conditional conflict'),{name:'PreconditionFailed'});
let bytes:Buffer|undefined;
if(name==='CopyObjectCommand') bytes=this.objects.get(i.CopySource);
else if(i.Body && typeof i.Body[Symbol.asyncIterator]==='function') {
const chunks:Buffer[]=[];for await(const chunk of i.Body) chunks.push(Buffer.from(chunk));bytes=Buffer.concat(chunks);
} else bytes=Buffer.from(i.Body);
if(!bytes) throw new Error('Missing copy source');this.objects.set(key,bytes);const next=`"${randomUUID()}"`;this.etags.set(key,next);return{ETag:next};
if(!bytes) throw new Error('Missing copy source');this.objects.set(key,bytes);this.modified.set(key,this.now);const next=`"${randomUUID()}"`;this.etags.set(key,next);return{ETag:next};
}
throw new Error(`Unexpected command ${name}`);
};
Expand Down
11 changes: 10 additions & 1 deletion backend/tests/knowledge-publisher-auth.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { resolve } from 'node:path';
import { handler } from '../src/handler';
import { configureDocsRuntime, handleDocsRoutes, resetDocsRuntime } from '../src/docs/contentApi';
import { resetKnowledgePublisherCaches } from '../src/auth/knowledgePublisher';
import { syntheticKnowledge } from './helpers/knowledge';
import { syntheticKnowledge, MemoryS3 } from './helpers/knowledge';
import { truncateTestTables, useTestDatabase } from './helpers/db';
import { listTemplates } from '../src/db/templates';
import { setDeploymentTemplateLoaderForTest } from '../src/deploymentSeeds';
Expand Down Expand Up @@ -50,6 +50,7 @@ const request = (method: string, path: string, credential: string, body: unknown
describe('GitHub Actions OIDC knowledge publisher', () => {
let dir: string;
let revision: string;
let s3: MemoryS3;
before(async () => {
Object.assign(process.env, config);
await useTestDatabase();
Expand All @@ -69,6 +70,7 @@ describe('GitHub Actions OIDC knowledge publisher', () => {
dir = mkdtempSync(resolve('../.tmp/publisher-'));
const fixture = syntheticKnowledge({ 'content/a.md': 'old' }, dir, async () => 'https://example.invalid/synthetic-download');
configureDocsRuntime(fixture.store);
s3 = fixture.s3;
revision = fixture.publication.revision;
resetKnowledgePublisherCaches();
});
Expand Down Expand Up @@ -119,6 +121,13 @@ describe('GitHub Actions OIDC knowledge publisher', () => {
}
const mirror = records.filter((record) => record.Job === 'knowledge-mirror');
assert.deepEqual(mirror.map((record) => [record.Success, record.Failure]), [[0, 1], [1, 0]]);
// The last report persists for the repo-backup freshness check: it compares
// the mirrored HEAD against the S3 backup manifest's commit. The success
// heartbeat above (without a commit) overwrote the failure report.
const stored = JSON.parse(s3.objects.get(`test-knowledge/jobs/knowledge-mirror/last-report.json`)!.toString());
assert.equal(stored.outcome, 'success');
assert.equal(stored.revision, 'rev1');
assert.ok(stored.at);
});

it('syncs authored templates on request and reports failures', async () => {
Expand Down
Loading