diff --git a/.vscode/settings.json b/.vscode/settings.json index 009aae3..8644b0a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "typescript.tsdk": "node_modules/typescript/lib", + "js/ts.tsdk.path": "node_modules/typescript/lib", "files.eol": "\n", "editor.defaultFormatter": "esbenp.prettier-vscode", "files.insertFinalNewline": true diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 11aaa93..1ba7871 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -73,24 +73,49 @@ "__format": "boolean" } }, - "s3": { - "endpoint": "S3_ENDPOINT", - "accessKeyId": "S3_ACCESS_KEY_ID", - "secretAccessKey": "S3_SECRET_ACCESS_KEY", - "sslEnabled": { - "__name": "S3_SSL_ENABLED", - "__format": "boolean" + "storage": { + "cleanupStorageProviders": { + "__name": "CLEANUP_STORAGE_PROVIDERS", + "__format": "json" }, - "forcePathStyle": { - "__name": "S3_FORCE_PATH_STYLE", - "__format": "boolean" + "s3": { + "delete": { + "batchSize": { + "__name": "S3_DELETE_BATCH_SIZE", + "__format": "number" + } + }, + "endpoint": "S3_ENDPOINT", + "accessKeyId": "S3_ACCESS_KEY_ID", + "secretAccessKey": "S3_SECRET_ACCESS_KEY", + "sslEnabled": { + "__name": "S3_SSL_ENABLED", + "__format": "boolean" + }, + "forcePathStyle": { + "__name": "S3_FORCE_PATH_STYLE", + "__format": "boolean" + }, + "region": "S3_REGION" }, - "region": "S3_REGION" + "fs": { + "delete": { + "batchSize": { + "__name": "FS_DELETE_BATCH_SIZE", + "__format": "number" + } + }, + "basePath": "FS_BASE_PATH", + "subPaths": { + "__name": "FS_SUB_PATHS", + "__format": "json" + } + } }, "strategies": { "tilesDeletion": { "s3Bucket": "TILES_DELETION_S3_BUCKET", - "fsBasePath": "TILES_DELETION_FS_BASE_PATH", + "fsSubPath": "TILES_DELETION_FS_SUB_PATH", "batchSize": { "__name": "TILES_DELETION_BATCH_SIZE", "__format": "number" @@ -98,10 +123,6 @@ "concurrency": { "__name": "TILES_DELETION_CONCURRENCY", "__format": "number" - }, - "failureSampleSize": { - "__name": "TILES_DELETION_FAILURE_SAMPLE_SIZE", - "__format": "number" } } } diff --git a/config/default.json b/config/default.json index 901d821..4d97d6d 100644 --- a/config/default.json +++ b/config/default.json @@ -9,9 +9,7 @@ "level": "info", "prettyPrint": false, "opentelemetryOptions": { - "enabled": false, - "url": "", - "resourceAttributes": {} + "enabled": false } } }, @@ -28,18 +26,26 @@ { "job": "Ingestion_Swap_Update", "task": "tiles-deletion" + }, + { + "job": "Delete_Layer", + "task": "tiles-deletion" + }, + { + "job": "Delete_Layer", + "task": "artifacts-deletion" } ] } }, "queue": { - "jobManagerBaseUrl": "http://localhost:8080", - "heartbeatBaseUrl": "http://localhost:8081", + "jobManagerBaseUrl": "http://localhost:8081", + "heartbeatBaseUrl": "http://localhost:8082", "heartbeatIntervalMs": 1000, "dequeueIntervalMs": 3000 }, "servicesUrl": { - "jobTracker": "http://localhost:8082" + "jobTracker": "http://localhost:8083" }, "disableHttpClientLogs": false, "jobDefinitions": { @@ -49,12 +55,23 @@ }, "swapUpdate": { "type": "Ingestion_Swap_Update" + }, + "deleteLayer": { + "type": "Delete_Layer" } }, "tasks": { "tilesDeletion": { "type": "tiles-deletion", "maxAttempts": 3 + }, + "layerDeletion": { + "type": "tiles-deletion", + "maxAttempts": 3 + }, + "artifactsDeletion": { + "type": "artifacts-deletion", + "maxAttempts": 3 } } }, @@ -63,21 +80,35 @@ "delay": "exponential", "shouldResetTimeout": true }, - "s3": { - "endpoint": "http://localhost:9000", - "accessKeyId": "minioadmin", - "secretAccessKey": "minioadmin", - "sslEnabled": false, - "forcePathStyle": true, - "region": "us-east-1" + "storage": { + "cleanupStorageProviders": ["FS", "S3"], + "s3": { + "delete": { + "batchSize": 1000 + }, + "endpoint": "http://localhost:9000", + "accessKeyId": "minioadmin", + "secretAccessKey": "minioadmin", + "sslEnabled": false, + "forcePathStyle": true, + "region": "us-east-1" + }, + "fs": { + "delete": { + "batchSize": 1000 + }, + "basePath": "/data", + "subPaths": { + "tilesSubPath": "tiles" + } + } }, "strategies": { "tilesDeletion": { "s3Bucket": "", - "fsBasePath": "/tiles", + "fsSubPath": "tiles", "batchSize": 1000, - "concurrency": 10, - "failureSampleSize": 3 + "concurrency": 10 } } } diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 855dbaa..27a5aff 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -2,8 +2,10 @@ {{- $serviceUrls := fromYaml (include "common.serviceUrls.merged" .) -}} {{- $storage := fromYaml (include "common.storage.merged" .) -}} {{- $s3 := ($storage.s3) | default dict -}} -{{- $internalPvc := (($storage.fs).internalPvc) | default dict -}} -{{- $tilesFSBasePath := clean (printf "/%s/%s" $internalPvc.mountPath ($internalPvc.tilesSubPath | default "tiles")) -}} +{{- $fs := ($storage.fs) | default dict -}} +{{- $internalPvc := (($fs).internalPvc) | default dict -}} +{{- $fsBasePath := clean (printf "/%s" $internalPvc.mountPath) -}} +{{- $fsTilesDeletionSubPath := clean (printf "%s" ($internalPvc.tilesSubPath | default "tiles")) -}} {{- if .Values.enabled -}} apiVersion: v1 kind: ConfigMap @@ -54,15 +56,29 @@ data: {{- with .Values.env.jobnik.worker }} JOBNIK_WORKER_CONCURRENCY: {{ .concurrency | default 1 | quote }} {{- end }} + CLEANUP_STORAGE_PROVIDERS: {{ $storage.cleanupStorageProviders | toJson | quote }} + {{- if has "S3" $storage.cleanupStorageProviders }} + S3_DELETE_BATCH_SIZE: {{ $s3.delete.batchSize | default 1000 | quote }} S3_ENDPOINT: {{ if $s3.endpointUrl }}{{ printf "%s://%s" ($s3.sslEnabled | ternary "https" "http") $s3.endpointUrl | quote }}{{ else }}{{ "" | quote }}{{ end }} S3_FORCE_PATH_STYLE: {{ $s3.forcePathStyle | default false | quote }} S3_SSL_ENABLED: {{ $s3.sslEnabled | default false | quote }} S3_REGION: {{ $s3.region | default "us-east-1" | quote }} + {{- end }} + {{- if has "FS" $storage.cleanupStorageProviders }} + FS_DELETE_BATCH_SIZE: {{ $fs.delete.batchSize | default 1000 | quote }} + FS_BASE_PATH: {{ $fsBasePath | quote }} + {{- $subPaths := dict -}} + {{- range $key, $val := $internalPvc -}} + {{- if and (hasSuffix "SubPath" $key) $val -}} + {{- $_ := set $subPaths (trimSuffix "SubPath" $key) (clean $val) -}} + {{- end -}} + {{- end }} + FS_SUB_PATHS: {{ $subPaths | toJson | quote }} + TILES_DELETION_FS_SUB_PATH: {{ $fsTilesDeletionSubPath | quote }} + {{- end }} TILES_DELETION_S3_BUCKET: {{ $s3.tilesBucket | default "" | quote }} - TILES_DELETION_FS_BASE_PATH: {{ $tilesFSBasePath | quote }} {{- with .Values.env.strategies.tilesDeletion }} TILES_DELETION_BATCH_SIZE: {{ .batchSize | default 1000 | quote }} TILES_DELETION_CONCURRENCY: {{ .concurrency | default 10 | quote }} - TILES_DELETION_FAILURE_SAMPLE_SIZE: {{ .failureSampleSize | default 3 | quote }} {{- end }} {{- end }} diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index ec4d9ff..0b8be5f 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -54,10 +54,10 @@ spec: imagePullPolicy: {{ .pullPolicy | default "IfNotPresent" }} {{- end }} {{- if .Values.command }} - command: + command: {{- toYaml .Values.command | nindent 12 }} {{- if .Values.args }} - args: + args: {{- toYaml .Values.args | nindent 12 }} {{- end }} {{- end }} @@ -122,7 +122,7 @@ spec: httpGet: path: {{ .Values.readinessProbe.path }} port: {{ .Values.env.targetPort }} - {{- end }} + {{- end }} {{- if .Values.resources.enabled }} resources: {{- toYaml .Values.resources.value | nindent 12 }} diff --git a/helm/values.yaml b/helm/values.yaml index be16b49..fe496fb 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -13,7 +13,12 @@ serviceUrls: jobTracker: "" storage: + cleanupStorageProviders: + - FS + - S3 s3: + delete: + batchSize: 1000 endpointUrl: "" forcePathStyle: true secretName: "" @@ -21,12 +26,13 @@ storage: region: "" tilesBucket: "" fs: + delete: + batchSize: 1000 internalPvc: enabled: false name: "" mountPath: "" - tilesSubPath: "" - + tilesSubPath: "" # e.g. folder/tiles mclabels: component: backend @@ -127,7 +133,6 @@ env: tilesDeletion: batchSize: 1000 concurrency: 10 - failureSampleSize: 3 resources: enabled: true diff --git a/package-lock.json b/package-lock.json index 4d16ca5..958df41 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@map-colonies/js-logger": "^5.0.0", "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", - "@map-colonies/raster-shared": "^8.1.0-alpha.3", + "@map-colonies/raster-shared": "^8.3.0-alpha.2", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.20.0", "@map-colonies/telemetry": "^10.0.1", @@ -2543,9 +2543,9 @@ "license": "ISC" }, "node_modules/@map-colonies/raster-shared": { - "version": "8.1.0-alpha.3", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.1.0-alpha.3.tgz", - "integrity": "sha512-igF8yhnSeXaUdpx6gW3C5gi4AM9J56jVwgqhh7+Zv0mYb/yBhxv0xsv4Mhyv+41UpFCVduYf90DJZRrY29kLpQ==", + "version": "8.3.0-alpha.2", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.3.0-alpha.2.tgz", + "integrity": "sha512-P2p0MddLonzxsG/MSkcKopfI05JsaWxe9BpfYPrVllZ7wCbrzyBUnsYsfP3Rl6jHeNlqWWPCJdHkpzvgTi0FOQ==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^9.1.0", diff --git a/package.json b/package.json index d71fedb..ecb938c 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "prebuild": "npm run clean", "build": "tsc --project tsconfig.build.json && tsc-alias -p tsconfig.build.json && npm run assets:copy", "start": "npm run build && cd dist && node --import ./instrumentation.mjs ./index.js", - "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true node --enable-source-maps --import ./instrumentation.mjs ./index.js", + "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true node --enable-source-maps --import ./instrumentation.mjs ./index.js", "assets:copy": "copyfiles -f ./config/* ./dist/config && copyfiles ./package.json dist", "clean": "rimraf dist", "prepare": "node .husky/install.mjs" @@ -37,7 +37,7 @@ "@map-colonies/js-logger": "^5.0.0", "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", - "@map-colonies/raster-shared": "^8.1.0-alpha.3", + "@map-colonies/raster-shared": "^8.3.0-alpha.2", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.20.0", "@map-colonies/telemetry": "^10.0.1", diff --git a/scripts/simulate-deletion.mjs b/scripts/simulate-deletion.mjs index 59ef463..5d8a19a 100644 --- a/scripts/simulate-deletion.mjs +++ b/scripts/simulate-deletion.mjs @@ -1,5 +1,5 @@ /** - * Simulation script for tiles-deletion tasks. + * Simulation script for tiles-deletion and artifacts-deletion tasks. * * Usage: * node scripts/simulate-deletion.mjs --provider S3 @@ -9,6 +9,15 @@ * node scripts/simulate-deletion.mjs --provider S3 --partial # multi-zoom partial deletion * node scripts/simulate-deletion.mjs --provider FS --real-tiles --source-tile scripts/tile_deletion_test.jpeg * node scripts/simulate-deletion.mjs --provider S3 --real-tiles --source-tile scripts/tile_deletion_test.jpeg --zooms 17,18,19,20 --tile-count 400 + * node scripts/simulate-deletion.mjs --provider S3 --artifacts-deletion + * node scripts/simulate-deletion.mjs --provider FS --artifacts-deletion --paths config,gpkg,reports + * + * --artifacts-deletion: simulate the Delete_Layer / artifacts-deletion task + * (DeleteStoredResourcesStrategy), which recursively deletes whole resource + * trees (e.g. gpkg, config, validation reports) rather than a tile range. + * --paths : comma-separated relative paths to seed & delete (default: config,gpkg,reports) + * Each path is seeded with a couple of nested fake files to prove the whole + * subtree — not just its top-level file — gets removed. * * --real-tiles: seed a real local tile file replicated across a multi-zoom grid. * --source-tile : local file to use as tile content for every seeded tile (required) @@ -42,7 +51,7 @@ * * Override defaults with env vars: * QUEUE_JOB_MANAGER_BASE_URL, S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, - * TILES_DELETION_S3_BUCKET, TILES_DELETION_FS_BASE_PATH, TILES_PATH, ZOOM, MIN_X, MAX_X, MIN_Y, MAX_Y, + * TILES_DELETION_S3_BUCKET, FS_BASE_PATH, TILES_PATH, ARTIFACTS_PATH, ZOOM, MIN_X, MAX_X, MIN_Y, MAX_Y, * SEED_CONCURRENCY (default: 50) */ @@ -69,6 +78,7 @@ Modes (mutually exclusive): --partial Seed tiles across 3 zoom levels; task only deletes a subset --real-tiles Use a real tile file instead of fake content --skip-seed Skip seeding; go straight to job creation (idempotent delete) + --artifacts-deletion Simulate the Delete_Layer / artifacts-deletion task (whole-resource deletion) Real-tiles options (require --real-tiles): --source-tile Local tile file to replicate across the grid (required) @@ -76,14 +86,18 @@ Real-tiles options (require --real-tiles): --zooms Zoom levels, comma-separated (default: 17,18,19,20) --tile-count Total tiles to seed across all zoom levels (default: 400) +Artifacts-deletion options (require --artifacts-deletion): + --paths Relative resource paths to seed & delete, comma-separated (default: config,gpkg,reports) + Env vars (all optional — pod ConfigMap values are used automatically): QUEUE_JOB_MANAGER_BASE_URL Job manager endpoint S3_ENDPOINT S3 endpoint URL S3_ACCESS_KEY_ID S3 access key S3_SECRET_ACCESS_KEY S3 secret key - TILES_DELETION_S3_BUCKET S3 bucket name - TILES_DELETION_FS_BASE_PATH FS base path for tile files + TILES_DELETION_S3_BUCKET S3 bucket name (tiles-deletion) / S3 bucket for artifacts-deletion + FS_BASE_PATH FS base path for stored files (tiles and artifacts) TILES_PATH Relative path prefix for tiles (default: simulate/layer/v1) + ARTIFACTS_PATH Relative base path for artifacts-deletion resources (default: simulate/artifacts) ZOOM Zoom level (default: 10) MIN_X, MAX_X X tile range (default: 0..3) MIN_Y, MAX_Y Y tile range (default: 0..3) @@ -94,6 +108,7 @@ Examples: node scripts/simulate-deletion.mjs --provider FS --partial node scripts/simulate-deletion.mjs --provider S3 --real-tiles --source-tile scripts/tile_deletion_test.jpeg node scripts/simulate-deletion.mjs --provider FS --real-tiles --source-tile scripts/tile_deletion_test.jpeg + node scripts/simulate-deletion.mjs --provider S3 --artifacts-deletion MAX_X=999 MAX_Y=999 ZOOM=18 node scripts/simulate-deletion.mjs --provider S3 --skip-seed `); process.exit(0); @@ -105,6 +120,7 @@ if (!providerFlag || !['S3', 'FS'].includes(providerFlag)) { console.error( ' node scripts/simulate-deletion.mjs --provider --real-tiles --source-tile [--zooms ] [--tile-count ]' ); + console.error(' node scripts/simulate-deletion.mjs --provider --artifacts-deletion [--paths ]'); console.error('\nRun with --help for full usage information.'); process.exit(1); } @@ -112,12 +128,16 @@ const PROVIDER = providerFlag; const SKIP_SEED = args.includes('--skip-seed'); const PARTIAL = args.includes('--partial'); const REAL_TILES = args.includes('--real-tiles'); +const ARTIFACTS_DELETION = args.includes('--artifacts-deletion'); // real-tiles specific args const SOURCE_TILE = args.includes('--source-tile') ? args[args.indexOf('--source-tile') + 1] : null; const ZOOMS_INPUT = args.includes('--zooms') ? args[args.indexOf('--zooms') + 1] : '17,18,19,20'; const TILE_COUNT = args.includes('--tile-count') ? Number(args[args.indexOf('--tile-count') + 1]) : 400; +// artifacts-deletion specific args +const ARTIFACT_PATHS_INPUT = args.includes('--paths') ? args[args.indexOf('--paths') + 1] : 'config,gpkg,reports'; + if (REAL_TILES) { if (!SOURCE_TILE) { console.error('--real-tiles requires --source-tile '); @@ -129,6 +149,11 @@ if (REAL_TILES) { } } +if (ARTIFACTS_DELETION && (PARTIAL || REAL_TILES)) { + console.error('--artifacts-deletion cannot be combined with --partial or --real-tiles'); + process.exit(1); +} + // ─── Read local.json as config base (env vars override) ────────────────────── let localConfig = {}; @@ -141,7 +166,8 @@ try { } const cfg = { - s3: localConfig.s3 ?? {}, + s3: localConfig.storage?.s3 ?? {}, + fs: localConfig.storage?.fs ?? {}, strategies: localConfig.strategies?.tilesDeletion ?? {}, queue: localConfig.queue ?? {}, }; @@ -154,7 +180,7 @@ const S3_ENDPOINT = process.env.S3_ENDPOINT ?? cfg.s3.endpoint ?? 'http://localh const S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID ?? cfg.s3.accessKeyId ?? 'minioadmin'; const S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY ?? cfg.s3.secretAccessKey ?? 'minioadmin'; const S3_BUCKET = process.env.TILES_DELETION_S3_BUCKET ?? cfg.strategies.s3Bucket ?? ''; -const FS_BASE_PATH = process.env.TILES_DELETION_FS_BASE_PATH ?? cfg.strategies.fsBasePath ?? '/tiles'; +const FS_BASE_PATH = process.env.FS_BASE_PATH ?? cfg.fs.basePath ?? '/tiles'; const SEED_CONCURRENCY = Number(process.env.SEED_CONCURRENCY ?? 200); // Tile range to seed + delete @@ -166,6 +192,11 @@ const MIN_Y = Number(process.env.MIN_Y ?? 0); const MAX_Y = Number(process.env.MAX_Y ?? 3); const FILE_EXTENSION = 'jpeg'; +// Artifacts-deletion resource paths to seed + delete +const ARTIFACTS_PATH = process.env.ARTIFACTS_PATH ?? 'simulate/artifacts'; +const ARTIFACT_SUBPATHS = ARTIFACT_PATHS_INPUT.split(',').map((s) => s.trim()); +const ARTIFACT_PATHS = ARTIFACT_SUBPATHS.map((subPath) => `${ARTIFACTS_PATH}/${subPath}`); + // ─── Partial-deletion scenario definition ──────────────────────────────────── // // Three zoom levels are seeded identically (4×4 grid by default). @@ -527,9 +558,125 @@ async function runRealTilesMode() { await createRealTilesJob(zooms, grid, TILES_PATH, ext); } +// ─── Artifacts-deletion mode (Delete_Layer / artifacts-deletion task) ───────── +// +// Unlike tiles-deletion (which deletes a tile-range grid), artifacts-deletion +// recursively removes whole resource trees given their root paths (e.g. a +// layer's gpkg, config and validation-report directories). Each seeded path +// gets a couple of nested files to prove the whole subtree is removed, not +// just its top-level entry. + +function* artifactFilesForPath(rootPath) { + yield `${rootPath}/file.dat`; + yield `${rootPath}/nested/file.dat`; +} + +async function seedS3Artifacts() { + if (!S3_BUCKET) { + throw new Error('S3_BUCKET env var is required for S3 provider (or set it in config/local.json)'); + } + + const client = new S3Client({ + endpoint: S3_ENDPOINT, + credentials: { accessKeyId: S3_ACCESS_KEY_ID, secretAccessKey: S3_SECRET_ACCESS_KEY }, + forcePathStyle: true, + region: 'us-east-1', + tls: false, + }); + + console.log(`[S3] Seeding artifacts under s3://${S3_BUCKET}/{${ARTIFACT_PATHS.join(', ')}}/...`); + for (const rootPath of ARTIFACT_PATHS) { + for (const key of artifactFilesForPath(rootPath)) { + await client.send( + new PutObjectCommand({ Bucket: S3_BUCKET, Key: key, Body: Buffer.from('fake-artifact'), ContentType: 'application/octet-stream' }) + ); + } + } + const firstKey = [...artifactFilesForPath(ARTIFACT_PATHS[0])][0]; + await client.send(new HeadObjectCommand({ Bucket: S3_BUCKET, Key: firstKey })); + console.log(`[S3] Verified: s3://${S3_BUCKET}/${firstKey} exists`); + console.log(`[S3] Done — seeded ${ARTIFACT_PATHS.length} artifact path(s).\n`); +} + +function seedFsArtifacts() { + console.log(`[FS] Seeding artifacts under ${FS_BASE_PATH}/{${ARTIFACT_PATHS.join(', ')}}/...`); + for (const rootPath of ARTIFACT_PATHS) { + for (const relativePath of artifactFilesForPath(rootPath)) { + const fullPath = join(FS_BASE_PATH, relativePath); + mkdirSync(join(fullPath, '..'), { recursive: true }); + writeFileSync(fullPath, 'fake-artifact'); + } + } + const firstPath = join(FS_BASE_PATH, [...artifactFilesForPath(ARTIFACT_PATHS[0])][0]); + if (!existsSync(firstPath)) throw new Error(`Seeding failed — ${firstPath} not found`); + console.log(`[FS] Verified: ${firstPath} exists`); + console.log(`[FS] Done — seeded ${ARTIFACT_PATHS.length} artifact path(s).\n`); +} + +async function createArtifactsDeletionJob() { + const taskParameters = { + storageProvider: PROVIDER, + paths: ARTIFACT_PATHS, + ...(PROVIDER === 'S3' && { bucket: S3_BUCKET }), + }; + + const body = { + resourceId: `simulate-artifacts-${PROVIDER.toLowerCase()}-${Date.now()}`, + version: '1.0.0', + type: 'Delete_Layer', + parameters: {}, + domain: 'RASTER', + tasks: [{ type: 'artifacts-deletion', parameters: taskParameters }], + }; + + console.log('[Job] Creating job with task parameters:'); + console.log(JSON.stringify(taskParameters, null, 2)); + + const res = await fetch(`${JOB_MANAGER_URL}/jobs`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`Job creation failed [${res.status}]: ${text}`); + } + + const { id: jobId, taskIds } = await res.json(); + + console.log(`\n[Job] Created successfully:`); + console.log(` Job ID : ${jobId}`); + console.log(` Task ID : ${taskIds[0]}`); + console.log(`\nStart the cleaner — it will pick up task "${taskIds[0]}" and delete ${ARTIFACT_PATHS.length} artifact path(s).`); + console.log(`Track progress: GET ${JOB_MANAGER_URL}/jobs/${jobId}?shouldReturnTasks=true`); +} + +async function runArtifactsDeletionMode() { + if (!SKIP_SEED) { + if (PROVIDER === 'S3') { + await seedS3Artifacts(); + } else { + seedFsArtifacts(); + } + } else { + console.log(`[seed] Skipped — missing paths are treated as success (idempotent delete).\n`); + } + + await createArtifactsDeletionJob(); +} + // ─── Main ───────────────────────────────────────────────────────────────────── async function main() { + if (ARTIFACTS_DELETION) { + console.log( + `\n=== Simulating artifacts-deletion | storageProvider: ${PROVIDER} | paths: ${ARTIFACT_PATHS.length} | seed: ${SKIP_SEED ? 'skipped' : 'yes'} ===\n` + ); + await runArtifactsDeletionMode(); + return; + } + if (REAL_TILES) { const zooms = parseZoomList(ZOOMS_INPUT); const ext = extname(SOURCE_TILE).slice(1).toLowerCase() || FILE_EXTENSION; diff --git a/src/cleaner/errors/errors.ts b/src/cleaner/errors/errors.ts index a3efe73..5c10557 100644 --- a/src/cleaner/errors/errors.ts +++ b/src/cleaner/errors/errors.ts @@ -89,8 +89,8 @@ export class ValidationError extends UnrecoverableError { * This is always unrecoverable since the strategy won't appear on retry. */ export class StrategyNotFoundError extends UnrecoverableError { - public constructor(taskType: string) { - super(`No strategy registered for task type: ${taskType}`); + public constructor({ jobType, taskType }: { jobType: string; taskType: string }) { + super(`No strategy registered for job type: ${jobType} and task type: ${taskType}`); this.name = StrategyNotFoundError.name; Error.captureStackTrace(this, this.constructor); } diff --git a/src/cleaner/errors/index.ts b/src/cleaner/errors/index.ts index 0315ef3..330ee2f 100644 --- a/src/cleaner/errors/index.ts +++ b/src/cleaner/errors/index.ts @@ -1,2 +1,2 @@ -export { toError, describeError, RecoverableError, UnrecoverableError, ConfigurationError, ValidationError, StrategyNotFoundError } from './errors'; export { ErrorHandler } from './errorHandler'; +export { ConfigurationError, describeError, RecoverableError, StrategyNotFoundError, toError, UnrecoverableError, ValidationError } from './errors'; diff --git a/src/cleaner/storageProviders/deleteFailureSummary.ts b/src/cleaner/storageProviders/deleteFailureSummary.ts deleted file mode 100644 index 2cd7aad..0000000 --- a/src/cleaner/storageProviders/deleteFailureSummary.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { DeleteFailure } from './iStorageProvider'; - -/** - * Aggregated view of a batch of delete failures — designed to be embedded in a - * task rejection reason or log line without further post-processing. - */ -export interface DeleteFailureSummary { - /** Raw count per reason string, e.g. `{ ENOENT: 150, EACCES: 3 }`. */ - counts: Record; - /** Reasons formatted descending by count, e.g. `'ENOENT=150, EACCES=3'`. */ - summary: string; - /** Up to `sampleSize` `path (reason)` strings, preserving input order. */ - sample: string[]; -} - -/** - * Reduces a list of provider delete failures into a compact, log-friendly - * shape. Lives alongside `IStorageProvider` because it operates purely on - * `DeleteFailure[]` — any caller of `provider.delete()` can use it, regardless - * of which storage backend produced the failures. - */ -export function summarizeDeleteFailures(failures: DeleteFailure[], sampleSize: number): DeleteFailureSummary { - const counts: Record = {}; - for (const { reason } of failures) { - counts[reason] = (counts[reason] ?? 0) + 1; - } - const summary = Object.entries(counts) - .sort(([, a], [, b]) => b - a) - .map(([reason, count]) => `${reason}=${count}`) - .join(', '); - const sample = failures.slice(0, sampleSize).map((f) => `${f.path} (${f.reason})`); - return { counts, summary, sample }; -} diff --git a/src/cleaner/storageProviders/failuresHandling.ts b/src/cleaner/storageProviders/failuresHandling.ts new file mode 100644 index 0000000..19c1d1d --- /dev/null +++ b/src/cleaner/storageProviders/failuresHandling.ts @@ -0,0 +1,43 @@ +import type { DeleteFailure, DeleteResult } from './iStorageProvider'; + +/** + * Aggregated view of a batch of delete failures — designed to be embedded in a + * task rejection reason or log line without further post-processing. + */ +export interface DeleteFailureSummary { + /** Count of all failures */ + failuresCount: number; + /** Reasons formatted ordered by descending by count, e.g. `'ENOENT=150, EACCES=3'`. */ + summary: string; + /** Samples of failing resources ordered by descending by count. */ + samples: string[]; +} + +export const mergeFailures = ({ source, target }: { source: DeleteFailure; target: DeleteFailure }): DeleteFailure => { + const failures: DeleteFailure = structuredClone(target); + + source.forEach((failed, reason) => { + const failure = failures.get(reason); + failures.set(reason, { count: (failure?.count ?? 0) + failed.count, sample: failure?.sample ?? failed.sample }); + }); + + return failures; +}; + +/** + * Reduces a list of provider delete failures into a compact, log-friendly + * shape. Lives alongside `IStorageProvider` because it operates purely on + * `DeleteResult` — any caller of `provider.delete()` can use it, regardless + * of which storage backend produced the failures. + */ +export function summarizeDeleteFailures({ failures }: DeleteResult): DeleteFailureSummary { + let failuresCount = 0; + for (const { count } of failures.values()) { + failuresCount += count; + } + + const sortedFailures = Array.from(failures.entries()).sort(([, { count: a }], [, { count: b }]) => b - a); + const summary = sortedFailures.map(([reason, { count }]) => `${reason}=${count}`).join(', '); + const samples = sortedFailures.map(([reason, { sample }]) => `${sample} (${reason})`); + return { failuresCount, summary, samples }; +} diff --git a/src/cleaner/storageProviders/fsStorageProvider.ts b/src/cleaner/storageProviders/fsStorageProvider.ts index 87734d5..f91e69b 100644 --- a/src/cleaner/storageProviders/fsStorageProvider.ts +++ b/src/cleaner/storageProviders/fsStorageProvider.ts @@ -1,15 +1,29 @@ +import { rm, rmdir, stat, unlink } from 'node:fs/promises'; import { join } from 'node:path'; -import { stat, unlink, rmdir } from 'node:fs/promises'; import type { Logger } from '@map-colonies/js-logger'; -import { describeError } from '../errors'; -import type { DeleteFailure, IStorageProvider } from './iStorageProvider'; +import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; +import { inject, injectable } from 'tsyringe'; +import { mergeFailures, type DeleteFailure, type DeleteResult, type IStorageProvider, type StorageProvider } from '@src/cleaner/storageProviders'; +import { getChunk, normalizeFolderPath, resolveAbsolutePath } from '@src/cleaner/utils'; +import { SERVICES } from '@common/constants'; +import { describeError, UnrecoverableError } from '../errors'; +import type { FsStorageConfig } from './storageConfig'; -export class FsStorageProvider implements IStorageProvider { - public constructor(private readonly logger: Logger) {} +type FSStorageProviderType = Extract; - public async targetExists(storageTarget: string, relativePath: string): Promise { +@injectable() +export class FsStorageProvider implements IStorageProvider<'FS'> { + public constructor( + @inject(SERVICES.FS_STORAGE_CONFIG) private readonly fsConfig: FsStorageConfig, + @inject(SERVICES.LOGGER) private readonly logger: Logger + ) { + this.logger.debug({ msg: 'Loaded FS storage provider', basePath: this.fsConfig.basePath }); + } + + public async targetExists(basePath: string, relativePath: string): Promise { + this.logger.debug({ msg: 'Checking if target resource exists', basePath, path: relativePath }); try { - await stat(join(storageTarget, relativePath)); + await stat(join(basePath, relativePath)); return true; } catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; @@ -17,33 +31,104 @@ export class FsStorageProvider implements IStorageProvider { } } - public async delete(paths: string[], storageTarget: string): Promise { - if (paths.length === 0) { - return []; - } - - this.logger.info({ msg: 'Deleting files from filesystem', basePath: storageTarget, count: paths.length }); + public async delete(paths: string[], basePath: string): Promise { + this.logger.debug({ msg: 'Deleting files from filesystem', basePath, pathsCount: paths.length }); + let failures: DeleteFailure = new Map(); const results = await Promise.allSettled( paths.map(async (relativePath) => { - await unlink(join(storageTarget, relativePath)); + await unlink(join(basePath, relativePath)); }) ); - const failures: DeleteFailure[] = []; + const chunkFailures: DeleteFailure = new Map(); for (const [idx, result] of results.entries()) { if (result.status === 'rejected') { const relativePath = paths[idx]!; const error: unknown = result.reason; const reason = describeError(error); - this.logger.debug({ msg: 'Failed to delete file', path: join(storageTarget, relativePath), reason, error }); - failures.push({ path: relativePath, reason }); + this.logger.debug({ msg: 'Failed to delete file', path: join(basePath, relativePath), reason, error }); + const chunkFailure = chunkFailures.get(reason); + chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? relativePath }); + } + } + failures = mergeFailures({ source: chunkFailures, target: failures }); + + await this.cleanupEmptyDirs(paths, basePath); + + return { failures }; + } + + public async deleteResources({ + paths, + subPath, + }: Extract): Promise { + this.logger.debug({ msg: 'Starting FS files/dirs deletion', subPath, pathsCount: paths.length }); + let totalDeletedPathsCount = 0, + totalFailedPathsCount = 0; + + const relativePaths = paths.map((path) => join(subPath, path)); + + if (!this.arePathsValid(relativePaths)) + throw new UnrecoverableError( + 'Cannot delete files/folders outside base path or subpath as well as base path or subpath itself. paths must also match a valid configured path.' + ); + + let failures: DeleteFailure = new Map(); + + for (const relativePathsChunk of getChunk(relativePaths, this.fsConfig.batchSize)) { + const results = await Promise.allSettled( + relativePathsChunk.map(async (relativePath) => { + await rm(join(this.fsConfig.basePath, relativePath), { recursive: true, force: true }); + }) + ); + + const chunkFailures: DeleteFailure = new Map(); + for (const [idx, result] of results.entries()) { + if (result.status === 'rejected') { + const fullPath = join(this.fsConfig.basePath, relativePathsChunk[idx]!); + const reason = describeError(result.reason); + this.logger.error({ msg: 'Failed to delete file/folder', fullPath, reason, err: result.reason }); + const chunkFailure = chunkFailures.get(reason); + chunkFailures.set(reason, { count: (chunkFailure?.count ?? 0) + 1, sample: chunkFailure?.sample ?? fullPath }); + } } + failures = mergeFailures({ source: chunkFailures, target: failures }); + + let failedPathsCount = 0; + chunkFailures.forEach((chunkFailure) => (failedPathsCount += chunkFailure.count)); + const deletedPathsCount = relativePathsChunk.length - failedPathsCount; + totalDeletedPathsCount += deletedPathsCount; + totalFailedPathsCount += failedPathsCount; + this.logger.debug({ + msg: 'Completed processing current chunk of paths', + deletedPathsCount, + totalDeletedPathsCount, + failedPathsCount, + totalFailedPathsCount, + }); } - await this.cleanupEmptyDirs(paths, storageTarget); + return { failures }; + } - return failures; + /** + * Preforms several checks on input `paths`. + * Includes a check for path traversal (i.e. accessing folders above root folder) + * @param paths + * @returns boolean whether `paths` are valid and pass all checks + */ + private arePathsValid(paths: string[]): boolean { + this.logger.debug({ msg: 'Checking paths validity', paths }); + const badPaths = paths.filter((path) => { + const startsWithAllowedSubPath = this.fsConfig.subPaths.some((subPath) => path.startsWith(normalizeFolderPath(subPath))); + const absolutePath = resolveAbsolutePath(join(this.fsConfig.basePath, path)); + const startsWithBasePath = absolutePath.startsWith(normalizeFolderPath(this.fsConfig.basePath)); + return !(startsWithAllowedSubPath && startsWithBasePath); + }); + const areValid = badPaths.length === 0; + this.logger.debug({ msg: `Paths validity check ${areValid ? 'succeeded' : 'failed'}`, ...(!areValid && { badPaths }) }); + return areValid; } // Attempts to remove any directories that became empty after file deletion. @@ -59,6 +144,7 @@ export class FsStorageProvider implements IStorageProvider { // levelIdx 2 → { storageTarget/layer/v1 } // levelIdx 3 → { storageTarget/layer } (root ancestor, tried last) private async cleanupEmptyDirs(relativePaths: string[], storageTarget: string): Promise { + this.logger.debug({ msg: 'Deleting empty directories', pathsCount: relativePaths.length }); // Map from levelIdx → unique absolute dir paths at that depth. // Using a Set per level deduplicates dirs shared by multiple deleted files // (e.g. a shared parent directory when multiple files within it are deleted at once). diff --git a/src/cleaner/storageProviders/iStorageProvider.ts b/src/cleaner/storageProviders/iStorageProvider.ts index e392459..481a529 100644 --- a/src/cleaner/storageProviders/iStorageProvider.ts +++ b/src/cleaner/storageProviders/iStorageProvider.ts @@ -1,22 +1,33 @@ +import type { DeleteStoredResourcesParams, Storage } from '@map-colonies/raster-shared'; + /** - * A single failed deletion paired with a short reason string (e.g. 'ENOENT', - * 'AccessDenied', 'NoSuchKey'). + * A storage for failures with additional metadata. */ -export interface DeleteFailure { - path: string; - reason: string; +export type DeleteFailure = Map; + +export interface DeleteResult { + failures: DeleteFailure; } -export interface IStorageProvider { +export type StorageProvider = Storage['storageProvider']; + +export interface IStorageProvider { /** * Deletes a batch of relative file paths within the given storage target. * - S3: storageTarget = bucket name; paths are object keys * - FS: storageTarget = base directory; full path = join(storageTarget, path) * - * Returns one entry per failed deletion. "Not found" is reported as a failure - * (with reason 'ENOENT' / 'NoSuchKey'). + * Returns an object including delete failures aggregation with one entry per failed reason. + * "Not found" is reported as a failure with additional metadata on failure - count and sample */ - delete: (paths: string[], storageTarget: string) => Promise; + delete: (paths: string[], storageTarget: string) => Promise; + + /** + * Deletes ALL objects/files under the given paths. + * - S3: storageTarget = bucket name; paths are root paths to resource(s) + * - FS: storageTarget = base directory; paths are relative paths from mounted dir + */ + deleteResources: (deleteStoredResourcesParams: Extract) => Promise; /** * Returns true if relativePath exists within storageTarget and contains data. @@ -25,3 +36,7 @@ export interface IStorageProvider { */ targetExists: (storageTarget: string, relativePath: string) => Promise; } + +export type StorageProviders = { + [T in StorageProvider]?: IStorageProvider; +}; diff --git a/src/cleaner/storageProviders/index.ts b/src/cleaner/storageProviders/index.ts index 54b39c1..43f390e 100644 --- a/src/cleaner/storageProviders/index.ts +++ b/src/cleaner/storageProviders/index.ts @@ -1,4 +1,12 @@ -export type { IStorageProvider, DeleteFailure } from './iStorageProvider'; -export { S3StorageProvider } from './s3StorageProvider'; +export { mergeFailures, summarizeDeleteFailures, type DeleteFailureSummary } from './failuresHandling'; export { FsStorageProvider } from './fsStorageProvider'; -export { summarizeDeleteFailures, type DeleteFailureSummary } from './deleteFailureSummary'; +export type { DeleteFailure, DeleteResult, IStorageProvider, StorageProvider, StorageProviders } from './iStorageProvider'; +export { S3StorageProvider } from './s3StorageProvider'; +export { + buildFsStorageConfig, + buildS3StorageConfig, + type FsConfig, + type FsStorageConfig, + type S3Config, + type S3StorageConfig, +} from './storageConfig'; diff --git a/src/cleaner/storageProviders/s3StorageProvider.ts b/src/cleaner/storageProviders/s3StorageProvider.ts index d554a20..57be934 100644 --- a/src/cleaner/storageProviders/s3StorageProvider.ts +++ b/src/cleaner/storageProviders/s3StorageProvider.ts @@ -1,29 +1,37 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { S3Client, DeleteObjectsCommand, ListObjectsV2Command, NoSuchBucket } from '@aws-sdk/client-s3'; +import { + DeleteObjectsCommand, + HeadBucketCommand, + HeadObjectCommand, + ListObjectsV2Command, + NoSuchBucket, + NotFound, + paginateListObjectsV2, + S3Client, + S3ServiceException, + type _Error, + type _Object, +} from '@aws-sdk/client-s3'; import type { Logger } from '@map-colonies/js-logger'; -import type { ConfigType } from '@common/config'; -import { describeError } from '../errors'; -import type { DeleteFailure, IStorageProvider } from './iStorageProvider'; - -const S3_MAX_DELETE_BATCH = 1000; - -interface S3Config { - endpoint: string; - accessKeyId: string; - secretAccessKey: string; - sslEnabled: boolean; - forcePathStyle: boolean; - region: string; -} +import type { DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; +import { inject, injectable } from 'tsyringe'; +import { SERVICES } from '@common/constants'; +import { mergeFailures, type DeleteFailure, type DeleteResult, type IStorageProvider, type StorageProvider } from '@src/cleaner/storageProviders'; +import { getChunk, normalizeFolderPath } from '@src/cleaner/utils'; +import { describeError, UnrecoverableError } from '../errors'; +import type { S3StorageConfig } from './storageConfig'; + +type S3StorageProviderType = Extract; -export class S3StorageProvider implements IStorageProvider { +@injectable() +export class S3StorageProvider implements IStorageProvider { private readonly s3Client: S3Client; public constructor( - config: ConfigType, - private readonly logger: Logger + @inject(SERVICES.S3_STORAGE_CONFIG) private readonly s3Config: S3StorageConfig, + @inject(SERVICES.LOGGER) private readonly logger: Logger ) { - const s3Config = config.get('s3') as S3Config; + // TODO: move client to a singleton resolution since this.s3Client = new S3Client({ endpoint: s3Config.endpoint, credentials: { @@ -34,35 +42,47 @@ export class S3StorageProvider implements IStorageProvider { region: s3Config.region, tls: s3Config.sslEnabled, }); + this.logger.debug({ msg: 'Loaded S3 storage provider', endpoint: s3Config.endpoint, batchSize: this.s3Config.batchSize }); } - public async delete(paths: string[], storageTarget: string): Promise { - if (paths.length === 0) { - return []; + public async delete(paths: string[], bucket: string): Promise { + this.logger.debug({ msg: 'Deleting objects from S3', bucket, pathsCount: paths.length }); + let failures: DeleteFailure = new Map(); + + for (const chunk of getChunk(paths, this.s3Config.batchSize)) { + const chunkFailures = await this.deleteObjects(chunk, bucket); + failures = mergeFailures({ source: chunkFailures, target: failures }); } - this.logger.debug({ msg: 'Deleting objects from S3', bucket: storageTarget, count: paths.length }); + return { failures }; + } - const failures: DeleteFailure[] = []; + public async deleteResources({ + bucket, + paths, + }: Extract): Promise { + this.logger.debug({ msg: `Starting S3 resources deletion`, bucket, pathsCount: paths.length }); + let failures: DeleteFailure = new Map(); - for (const chunk of this.chunk(paths, S3_MAX_DELETE_BATCH)) { - try { - const failed = await this.deleteChunk(chunk, storageTarget); - failures.push(...failed); - } catch (error) { - // Whole chunk failed (network/auth/etc) — mark every path in it with the same reason - // so the caller still gets a per-path failure list and a human-readable cause. - const reason = describeError(error); - this.logger.error({ msg: 'S3 batch request failed', bucket: storageTarget, reason, error }); - failures.push(...chunk.map((path) => ({ path, reason }))); - } + if (paths.length === 0) return { failures }; + if (paths.some((path) => path.length === 0)) throw new UnrecoverableError('Cannot delete resources directly under root path of the bucket'); // Prevent root deletion + + const exists = await this.bucketExists(bucket); + if (!exists) { + throw new UnrecoverableError(`Bucket does not exist: ${bucket}`); } - return failures; + for (const path of paths) { + const pathFailures = await this.deleteResource({ bucket, path }); + failures = mergeFailures({ source: pathFailures, target: failures }); + } + + return { failures }; } - public async targetExists(bucket: string, relativePath: string): Promise { - const prefix = relativePath.endsWith('/') ? relativePath : `${relativePath}/`; + public async targetExists(bucket: string, path: string): Promise { + this.logger.debug({ msg: 'Checking if target resource exists', bucket, path }); + const prefix = normalizeFolderPath(path); try { const result = await this.s3Client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, MaxKeys: 1 })); return (result.KeyCount ?? 0) > 0; @@ -73,21 +93,193 @@ export class S3StorageProvider implements IStorageProvider { } } - private async deleteChunk(paths: string[], bucket: string): Promise { - const command = new DeleteObjectsCommand({ - Bucket: bucket, - Delete: { Objects: paths.map((Key) => ({ Key })) }, + private async bucketExists(bucket: string): Promise { + try { + this.logger.debug({ msg: 'Checking bucket exists', bucket }); + const command = new HeadBucketCommand({ Bucket: bucket }); + await this.s3Client.send(command); // If it resolves, the bucket exists and you have permission to access it + this.logger.debug({ msg: 'Bucket exists', bucket }); + return true; + } catch (err) { + if (err instanceof NotFound) { + this.logger.error({ msg: 'Bucket does not exist', bucket, err }); + return false; + } + const reason = describeError(err); + this.logger.error({ msg: 'Failed to check if bucket exists', bucket, reason, err }); + throw err; + } + } + + private async deleteObjects(paths: string[], bucket: string): Promise { + try { + const command = new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: paths.map((Key) => ({ Key })) }, + }); + + const response = await this.s3Client.send(command); + const failures: DeleteFailure = new Map(); + let totalFailuresCount = 0; + (response.Errors ?? []) + .filter((error): error is _Error & Required> => error.Key !== undefined) // Only include entries with a Key + .forEach((error) => { + const reason = error.Code ?? error.Message ?? 'Unknown'; + const failure = failures.get(reason); + const failuresCount = (failure?.count ?? 0) + 1; + totalFailuresCount += failuresCount; + failures.set(reason, { count: failuresCount, sample: failure?.sample ?? error.Key }); + }); + + if (failures.size > 0) + this.logger.warn({ + msg: 'Failed to delete some objects', + totalFailuresCount, + uniqueFailureTypesCount: failures.size, + failureTypes: Array.from(failures.keys()), + }); + return failures; + } catch (err) { + const reason = describeError(err); + this.logger.error({ msg: 'S3 delete objects request failed', bucket, reason, err }); + return new Map([[reason, { count: paths.length, sample: paths[0]! }]]); + } + } + + private async deleteResource({ bucket, path }: { bucket: string; path: string }): Promise { + this.logger.debug({ msg: 'Deleting a resource', bucket, path }); + let failures: DeleteFailure = new Map(); + let totalDeletedObjectsCount = 0, + totalFailedObjectsCount = 0; + + const s3Objects = this.getS3Objects({ + bucket, + prefix: path, + pageSize: this.s3Config.batchSize, }); - const response = await this.s3Client.send(command); - return (response.Errors ?? []) - .filter((e): e is typeof e & { Key: string } => Boolean(e.Key)) // Only include entries with a Key so every failure maps to a specific path. - .map((e) => ({ path: e.Key, reason: e.Code ?? e.Message ?? 'Unknown' })); + try { + for await (const pageOfObjects of s3Objects) { + this.logger.debug({ + msg: 'Received a page of objects to delete', + bucket, + path, + keysSize: pageOfObjects.length, + pageSize: this.s3Config.batchSize, + ...(pageOfObjects.length > 0 && { samplePageResponse: pageOfObjects[0] }), + }); + const keys = pageOfObjects.map((obj) => obj.Key).filter((key): key is string => key !== undefined && this.matchesTarget(key, path)); + + if (keys.length === 0) { + continue; + } + + const chunkFailures = await this.deleteObjects(keys, bucket); + failures = mergeFailures({ source: chunkFailures, target: failures }); + + let failedObjectsCount = 0; + chunkFailures.forEach((chunkFailure) => (failedObjectsCount += chunkFailure.count)); + const deletedObjectsCount = keys.length - failedObjectsCount; + totalDeletedObjectsCount += deletedObjectsCount; + totalFailedObjectsCount += failedObjectsCount; + + this.logger.debug({ + msg: 'Completed processing current page of objects', + deletedObjectsCount, + totalDeletedObjectsCount, + failedObjectsCount, + totalFailedObjectsCount, + }); + } + this.logger.debug({ msg: 'Resource deletion completed', path, totalDeletedObjectsCount, totalFailedObjectsCount }); + return failures; + } catch (err) { + this.logger.error({ + msg: 'Stream of objects for deletion was interrupted by an error', + path, + totalDeletedObjectsCount, + totalFailedObjectsCount, + err, + }); + throw err; + } + } + + private async *getS3Objects({ + bucket, + prefix, + pageSize, + }: { + bucket: string; + prefix?: string; + pageSize?: number; + }): AsyncGenerator<_Object[], void, unknown> { + try { + this.logger.debug({ msg: 'Starting iteration over matching objects', bucket, prefix, pageSize }); + // First, if object exists it is removed. This is to mitigate an issue in MinIO that shadows paths sharing common path with an object. + // Second, objects having this path are iterated and removed + if (prefix !== undefined && (await this.resourceExists({ bucket, path: prefix }))) { + this.logger.debug({ msg: 'Matched an exact object', bucket, prefix }); + yield [{ Key: prefix }]; + } + + const paginatorConfig = { + client: this.s3Client, + pageSize, + }; + + const commandInput = { + Bucket: bucket, + Prefix: prefix !== undefined ? normalizeFolderPath(prefix) : prefix, + }; + + let pageNumber = 0; + const paginator = paginateListObjectsV2(paginatorConfig, commandInput); + for await (const page of paginator) { + pageNumber++; + this.logger.debug({ + msg: 'Got a page of objects', + bucket, + prefix, + pageNumber, + ...(page.KeyCount !== undefined && { pathsCount: page.KeyCount }), + }); + yield page.Contents ?? []; + } + } catch (err) { + if (err instanceof NoSuchBucket) { + this.logger.error({ msg: `S3 Error [${err.name}] no such bucket: ${err.message}`, err }); + } else if (err instanceof S3ServiceException) { + this.logger.error({ msg: `S3 Error [${err.name}] occured during pagination: ${err.message}`, err }); + } else { + this.logger.error({ msg: 'Unexpected error occurred during pagination', err, bucket, prefix }); + } + throw err; + } + } + + // A key belongs to the target if it IS the target object or lives under 'target/'. + // The 'target/' guard prevents matching sibling keys that merely share the prefix + // (e.g. target 'photos' must not match 'photos_old/img.jpg', target 'metadata.txt' + // must not match 'metadata.txt.bak'). + private matchesTarget(key: string, target: string): boolean { + return key === target || key.startsWith(normalizeFolderPath(target)); } - private *chunk(paths: string[], size: number): Generator { - for (let i = 0; i < paths.length; i += size) { - yield paths.slice(i, i + size); + private async resourceExists({ bucket, path }: { bucket: string; path: string }): Promise { + try { + await this.s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: path })); + return true; + } catch (err) { + if (err instanceof NotFound) { + return false; + } else if (err instanceof NoSuchBucket) { + this.logger.warn({ msg: `S3 Error [${err.name}] no such bucket: ${err.message}`, err }); + return false; + } else { + this.logger.error({ msg: 'resourceExists object check failed', err, bucket, path }); + throw err; + } } } } diff --git a/src/cleaner/storageProviders/storageConfig.ts b/src/cleaner/storageProviders/storageConfig.ts new file mode 100644 index 0000000..8bd2f8b --- /dev/null +++ b/src/cleaner/storageProviders/storageConfig.ts @@ -0,0 +1,92 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { assertCanDeleteFromFolder, resolveAbsolutePath } from '@src/cleaner/utils'; +import type { ConfigType } from '@src/common/config'; +import { ConfigurationError } from '../errors'; + +// S3/MinIO reject DeleteObjects requests with more than 1000 keys, regardless of configured batch size. +const S3_DELETE_OBJECTS_MAX_KEYS = 1000; + +export interface FsConfig { + delete: { + batchSize: number; + }; + basePath: string; + subPaths: Record; +} + +export interface S3Config { + delete: { + batchSize: number; + }; + endpoint: string; + accessKeyId: string; + secretAccessKey: string; + sslEnabled?: boolean; + forcePathStyle?: boolean; + region?: string; +} + +export interface FsStorageConfig { + basePath: string; + subPaths: string[]; + batchSize: number; +} + +export interface S3StorageConfig { + endpoint: string; + accessKeyId: string; + secretAccessKey: string; + sslEnabled?: boolean; + forcePathStyle?: boolean; + region?: string; + batchSize: number; +} + +/** + * Reads and validates `storage.fs`. + * @returns {FsStorageConfig} FS configuration for FsStorageProvider + * @throws {ConfigurationError} if the config is unusable or the base path is not a writable directory + */ +export function buildFsStorageConfig(config: ConfigType, logger: Logger): FsStorageConfig { + //TODO: when we create a worker config schema the shape checks below can be dropped along with the cast + const fsConfig = config.get('storage.fs') as unknown as FsConfig; + + const { + delete: { batchSize: deleteBatchSize }, + ...fsStorageConfig + } = fsConfig; + + const subPaths = Object.values(fsStorageConfig.subPaths); + if (subPaths.length === 0) throw new ConfigurationError('Deletion subpaths must have at least 1 entry'); + if (deleteBatchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); + + const basePath = resolveAbsolutePath(fsStorageConfig.basePath); + assertCanDeleteFromFolder(basePath, logger); + + logger.info({ msg: 'Validated FS storage config', basePath, subPaths, batchSize: deleteBatchSize }); + return { basePath, subPaths, batchSize: deleteBatchSize }; +} + +/** + * Reads and validates `storage.s3`. + * @returns {S3StorageConfig} S3 configuration for S3StorageProvider + * @throws {ConfigurationError} if the config is unusable + */ +export function buildS3StorageConfig(config: ConfigType, logger: Logger): S3StorageConfig { + //TODO: when we create a worker config schema the shape check below can be dropped along with the cast + const s3Config = config.get('storage.s3') as unknown as S3Config; + + const { + delete: { batchSize: deleteBatchSize }, + ...s3StorageConfig + } = s3Config; + + if (deleteBatchSize <= 0) throw new ConfigurationError('Deletion batch size must be greater than 0'); + const batchSize = Math.min(deleteBatchSize, S3_DELETE_OBJECTS_MAX_KEYS); + + logger.debug({ msg: 'Validated S3 storage config', endpoint: s3StorageConfig.endpoint, batchSize }); + return { + ...s3StorageConfig, + batchSize, + }; +} diff --git a/src/cleaner/strategies/deleteStoredResourcesStrategy.ts b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts new file mode 100644 index 0000000..157b371 --- /dev/null +++ b/src/cleaner/strategies/deleteStoredResourcesStrategy.ts @@ -0,0 +1,69 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { deleteStoredResourcesParamsSchema, type DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; +import { inject, injectable } from 'tsyringe'; +import type { ConfigType } from '@common/config'; +import { SERVICES } from '@common/constants'; +import { summarizeDeleteFailures, type IStorageProvider, type StorageProvider, type StorageProviders } from '@src/cleaner/storageProviders'; +import { RecoverableError, UnrecoverableError } from '../errors'; +import { validateSchema } from '../utils'; +import type { ITaskStrategy } from './taskStrategy'; + +@injectable() +export class DeleteStoredResourcesStrategy implements ITaskStrategy { + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(SERVICES.STORAGE_PROVIDERS) private readonly storageProviders: StorageProviders + ) {} + + public validate(params: unknown): DeleteStoredResourcesParams { + this.logger.debug({ msg: `Validating input parameters` }); + return validateSchema(deleteStoredResourcesParamsSchema, params, this.logger); + } + + public async execute(params: DeleteStoredResourcesParams): Promise { + const { paths } = params; + const provider = this.resolveStorageProvider(params); + + this.logger.info({ + msg: 'Starting deletion', + count: paths.length, + paths, + provider: params.storageProvider, + ...(params.storageProvider === 'S3' && { bucket: params.bucket }), + ...(params.storageProvider === 'FS' && { subPath: params.subPath }), + }); + + const { failures } = await provider.deleteResources(params); + + if (failures.size > 0) { + const { failuresCount, samples, summary } = summarizeDeleteFailures({ failures }); + this.logger.error({ + msg: 'Deletion failed', + provider: params.storageProvider, + paths, + failuresCount, + summary, + samples, + }); + throw new RecoverableError(`Failed to delete ${failuresCount} objects. Reasons: ${summary}. Samples: ${samples.join(', ')}`); + } + + this.logger.info({ + msg: 'Deletion completed successfully', + provider: params.storageProvider, + paths, + }); + } + + private resolveStorageProvider( + params: Extract + ): IStorageProvider { + if (!(params.storageProvider in this.storageProviders)) throw new UnrecoverableError(`Unsupported storage provider ${params.storageProvider}`); + // eslint-disable-next-line @typescript-eslint/naming-convention + const storageProvider = this.storageProviders[params.storageProvider]; + if (storageProvider === undefined) throw new UnrecoverableError(`Unsupported storage provider ${params.storageProvider}`); + this.logger.debug({ msg: `Using ${params.storageProvider} provider` }); + return storageProvider; + } +} diff --git a/src/cleaner/strategies/index.ts b/src/cleaner/strategies/index.ts index f3895d4..2bcd2c4 100644 --- a/src/cleaner/strategies/index.ts +++ b/src/cleaner/strategies/index.ts @@ -1,3 +1,4 @@ -export { type ITaskStrategy } from './taskStrategy'; +export { DeleteStoredResourcesStrategy } from './deleteStoredResourcesStrategy'; export { StrategyFactory, type TaskContext } from './strategyFactory'; +export type { ITaskStrategy } from './taskStrategy'; export { TilesDeletionStrategy } from './tilesDeletionStrategy'; diff --git a/src/cleaner/strategies/strategyFactory.ts b/src/cleaner/strategies/strategyFactory.ts index 00da2be..c56f541 100644 --- a/src/cleaner/strategies/strategyFactory.ts +++ b/src/cleaner/strategies/strategyFactory.ts @@ -1,8 +1,9 @@ -import { container, inject, injectable } from 'tsyringe'; import type { Logger } from '@map-colonies/js-logger'; +import { container, inject, injectable } from 'tsyringe'; import { SERVICES } from '@common/constants'; -import { StrategyNotFoundError } from '../errors'; -import type { ITaskStrategy } from './taskStrategy'; +import { StrategyNotFoundError } from '@src/cleaner/errors'; +import type { ITaskStrategy } from '@src/cleaner/strategies'; +import { getJobAndTaskToken } from '@src/common/dependencyRegistration'; export interface TaskContext { jobId: string; @@ -27,8 +28,10 @@ export class StrategyFactory { public resolveWithContext(taskContext: TaskContext): ITaskStrategy { this.logger.debug({ msg: 'Resolving strategy with task context', ...taskContext }); - if (!container.isRegistered(taskContext.taskType)) { - throw new StrategyNotFoundError(taskContext.taskType); + const jobTaskToken = getJobAndTaskToken(taskContext); + + if (!container.isRegistered(jobTaskToken)) { + throw new StrategyNotFoundError({ jobType: taskContext.jobType, taskType: taskContext.taskType }); } const taskContainer = container.createChildContainer(); @@ -39,7 +42,7 @@ export class StrategyFactory { taskContainer.register(SERVICES.LOGGER, { useValue: taskLogger }); taskContainer.register(SERVICES.TASK_CONTEXT, { useValue: taskContext }); - const strategy = taskContainer.resolve(taskContext.taskType); + const strategy = taskContainer.resolve(jobTaskToken); taskLogger.debug({ msg: 'Strategy resolved successfully with task context' }); diff --git a/src/cleaner/strategies/tilesDeletionStrategy.ts b/src/cleaner/strategies/tilesDeletionStrategy.ts index c0b1b6c..792b884 100644 --- a/src/cleaner/strategies/tilesDeletionStrategy.ts +++ b/src/cleaner/strategies/tilesDeletionStrategy.ts @@ -1,13 +1,22 @@ -import { inject, injectable } from 'tsyringe'; +import { join } from 'node:path'; +import { NoSuchKey } from '@aws-sdk/client-s3'; import type { Logger } from '@map-colonies/js-logger'; -import { SourceType, TileRange, TilesDeletionParams, tilesDeletionParamsSchema } from '@map-colonies/raster-shared'; import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; -import { NoSuchKey } from '@aws-sdk/client-s3'; -import { PERCENTAGE_COMPLETE, SERVICES } from '@common/constants'; +import { SourceType, TileRange, TilesDeletionParams, tilesDeletionParamsSchema } from '@map-colonies/raster-shared'; +import { inject, injectable } from 'tsyringe'; import type { ConfigType } from '@common/config'; -import { validateSchema } from '../utils'; +import { PERCENTAGE_COMPLETE, SERVICES } from '@common/constants'; +import { + mergeFailures, + summarizeDeleteFailures, + type DeleteFailure, + type FsConfig, + type IStorageProvider, + type StorageProvider, + type StorageProviders, +} from '@src/cleaner/storageProviders'; import { RecoverableError, UnrecoverableError, describeError } from '../errors'; -import { summarizeDeleteFailures, type DeleteFailure, type IStorageProvider } from '../storageProviders'; +import { validateSchema } from '../utils'; import type { TaskContext } from './strategyFactory'; import type { ITaskStrategy } from './taskStrategy'; @@ -17,30 +26,31 @@ const NOT_FOUND_REASONS = new Set([NoSuchKey.name, 'ENOENT']); export class TilesDeletionStrategy implements ITaskStrategy { private readonly batchSize: number; private readonly concurrency: number; - private readonly failureSampleSize: number; private readonly s3Bucket: string; private readonly fsBasePath: string; + private readonly fsTilesDeletionSubPath: string; public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.CONFIG) config: ConfigType, - @inject(SERVICES.STORAGE_PROVIDERS) private readonly storageProviders: Map, + @inject(SERVICES.STORAGE_PROVIDERS) private readonly storageProviders: StorageProviders, @inject(SERVICES.QUEUE_CLIENT) private readonly queueClient: QueueClient, @inject(SERVICES.TASK_CONTEXT) private readonly taskContext: TaskContext ) { this.batchSize = config.get('strategies.tilesDeletion.batchSize') as unknown as number; this.concurrency = config.get('strategies.tilesDeletion.concurrency') as unknown as number; - this.failureSampleSize = config.get('strategies.tilesDeletion.failureSampleSize') as unknown as number; this.s3Bucket = config.get('strategies.tilesDeletion.s3Bucket') as unknown as string; - this.fsBasePath = config.get('strategies.tilesDeletion.fsBasePath') as unknown as string; + this.fsBasePath = config.get('storage.fs.basePath') as unknown as FsConfig['basePath']; + this.fsTilesDeletionSubPath = config.get('strategies.tilesDeletion.fsSubPath') as unknown as string; } public validate(params: unknown): TilesDeletionParams { + this.logger.debug({ msg: `Validating input parameters` }); return validateSchema(tilesDeletionParamsSchema, params, this.logger); } public async execute(params: TilesDeletionParams): Promise { - const { provider, storageTarget } = this.resolveProvider(params); + const { provider, storageTarget } = this.resolveStorageProvider(params); if (!(await provider.targetExists(storageTarget, params.tilesPath))) { throw new UnrecoverableError(`${params.sourceProvider} storage target does not exist: ${storageTarget}/${params.tilesPath}`); @@ -67,36 +77,41 @@ export class TilesDeletionStrategy implements ITaskStrategy * matches the desired end-state) but counted separately for visibility. * Terminal progress to 100% is handled by the queue's task-ack — no explicit call needed here. */ - private reportOutcome(failures: DeleteFailure[], totalTiles: number): void { - const retryable: DeleteFailure[] = []; - const notFound: DeleteFailure[] = []; + private reportOutcome(failures: DeleteFailure, totalTiles: number): void { + const retryable: DeleteFailure = new Map(); + const notFound: DeleteFailure = new Map(); for (const failure of failures) { - (NOT_FOUND_REASONS.has(failure.reason) ? notFound : retryable).push(failure); + (NOT_FOUND_REASONS.has(failure[0]) ? notFound : retryable).set(failure[0], failure[1]); } - const deletedCount = totalTiles - retryable.length - notFound.length; + let retryableCount = 0; + retryable.forEach((retryableFailure) => (retryableCount += retryableFailure.count)); + let notFoundCount = 0; + notFound.forEach((notFoundFailure) => (notFoundCount += notFoundFailure.count)); + const deletedCount = totalTiles - retryableCount - notFoundCount; - if (retryable.length > 0) { - const { counts, summary, sample } = summarizeDeleteFailures(retryable, this.failureSampleSize); + if (retryable.size > 0) { + const { failuresCount, samples, summary } = summarizeDeleteFailures({ failures: retryable }); this.logger.error({ msg: 'Tiles deletion partially failed', totalTiles, - failedCount: retryable.length, - notFoundCount: notFound.length, + uniqueFailureTypesCount: retryable.size, + notFoundCount, deletedCount, - reasonCounts: counts, - sample, + totalFailuresCount: failuresCount, + summary, + samples, }); - throw new RecoverableError(`Failed to delete ${retryable.length} tiles. Reasons: ${summary}. Sample: ${sample.join(', ')}`); + throw new RecoverableError(`Failed to delete ${failuresCount} tiles. Reasons: ${summary}. Samples: ${samples.join(', ')}`); } - if (notFound.length > 0) { + if (notFound.size > 0) { this.logger.warn({ msg: 'Tiles deletion completed with missing tiles', totalTiles, - notFoundCount: notFound.length, + notFoundCount: notFound.size, deletedCount, - allTilesMissing: notFound.length === totalTiles, + allTilesMissing: notFound.size === totalTiles, }); return; } @@ -104,13 +119,16 @@ export class TilesDeletionStrategy implements ITaskStrategy this.logger.info({ msg: 'Tiles deletion completed successfully', deletedCount: totalTiles }); } - private resolveProvider(params: TilesDeletionParams): { provider: IStorageProvider; storageTarget: string } { - const provider = this.storageProviders.get(params.sourceProvider); - if (provider === undefined) { - throw new UnrecoverableError(`Unknown storage provider: ${params.sourceProvider}`); - } - const storageTarget = params.sourceProvider === SourceType.S3 ? this.s3Bucket : this.fsBasePath; - return { provider, storageTarget }; + private resolveStorageProvider( + params: Extract + ): { provider: IStorageProvider; storageTarget: string } { + if (!(params.sourceProvider in this.storageProviders)) throw new UnrecoverableError(`Unsupported storage provider ${params.sourceProvider}`); + // eslint-disable-next-line @typescript-eslint/naming-convention + const storageProvider = this.storageProviders[params.sourceProvider]; + if (storageProvider === undefined) throw new UnrecoverableError(`Unsupported storage provider ${params.sourceProvider}`); + const storageTarget = params.sourceProvider === SourceType.S3 ? this.s3Bucket : join(this.fsBasePath, this.fsTilesDeletionSubPath); + this.logger.debug({ msg: `Using ${params.sourceProvider} provider` }); + return { provider: storageProvider, storageTarget }; } private async deleteAllTiles( @@ -118,9 +136,9 @@ export class TilesDeletionStrategy implements ITaskStrategy storageTarget: string, params: TilesDeletionParams, totalTiles: number - ): Promise { + ): Promise { const { jobId, taskId } = this.taskContext; - const failures: DeleteFailure[] = []; + let failures: DeleteFailure = new Map(); const pendingBatches: string[][] = []; let batch: string[] = []; let processedTiles = 0; @@ -131,10 +149,12 @@ export class TilesDeletionStrategy implements ITaskStrategy pendingBatches.push(batch); batch = []; if (pendingBatches.length === this.concurrency) { - processedTiles += await this.flushBatches(provider, storageTarget, pendingBatches, failures); + const { batchFailures, processedTilesCount } = await this.flushBatches(provider, storageTarget, pendingBatches); + processedTiles += processedTilesCount; + failures = mergeFailures({ source: batchFailures, target: failures }); const percentage = Math.round((processedTiles / totalTiles) * PERCENTAGE_COMPLETE); await this.queueClient.updateProgress(jobId, taskId, percentage); - this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.length }); + this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.size }); } } } @@ -143,8 +163,10 @@ export class TilesDeletionStrategy implements ITaskStrategy pendingBatches.push(batch); } if (pendingBatches.length > 0) { - await this.flushBatches(provider, storageTarget, pendingBatches, failures); - this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.length }); + const { batchFailures, processedTilesCount } = await this.flushBatches(provider, storageTarget, pendingBatches); + processedTiles += processedTilesCount; + failures = mergeFailures({ source: batchFailures, target: failures }); + this.logger.info({ msg: 'Tiles deletion progress', deletionProgress: `${processedTiles}/${totalTiles}`, failedTiles: failures.size }); } return failures; @@ -163,24 +185,26 @@ export class TilesDeletionStrategy implements ITaskStrategy private async flushBatches( provider: IStorageProvider, storageTarget: string, - pendingBatches: string[][], - failures: DeleteFailure[] - ): Promise { - const flushedCount = pendingBatches.reduce((sum, b) => sum + b.length, 0); + pendingBatches: string[][] + ): Promise<{ batchFailures: DeleteFailure; processedTilesCount: number }> { + let failures: DeleteFailure = new Map(); + + const processedTilesCount = pendingBatches.reduce((sum, b) => sum + b.length, 0); const results = await Promise.allSettled(pendingBatches.map(async (batch) => provider.delete(batch, storageTarget))); for (const [index, result] of results.entries()) { if (result.status === 'fulfilled') { - failures.push(...result.value); + failures = mergeFailures({ source: result.value.failures, target: failures }); } else { const error: unknown = result.reason; const reason = describeError(error); this.logger.error({ msg: 'Batch delete threw unexpectedly', reason, error }); const batch = pendingBatches[index] ?? []; - failures.push(...batch.map((path) => ({ path, reason }))); + const failure = failures.get(reason); + failures.set(reason, { count: (failure?.count ?? 0) + batch.length, sample: failure?.sample ?? batch[0]! }); } } pendingBatches.length = 0; - return flushedCount; + return { batchFailures: failures, processedTilesCount }; } /** diff --git a/src/cleaner/utils/chunk.ts b/src/cleaner/utils/chunk.ts new file mode 100644 index 0000000..d419943 --- /dev/null +++ b/src/cleaner/utils/chunk.ts @@ -0,0 +1,5 @@ +export function* getChunk(items: T[], size: number): Generator { + for (let i = 0; i < items.length; i += size) { + yield items.slice(i, i + size); + } +} diff --git a/src/cleaner/utils/fs.ts b/src/cleaner/utils/fs.ts new file mode 100644 index 0000000..18a6c2a --- /dev/null +++ b/src/cleaner/utils/fs.ts @@ -0,0 +1,28 @@ +import { accessSync, constants, statSync } from 'node:fs'; +import type { Logger } from '@map-colonies/js-logger'; +import { ConfigurationError, describeError } from '../errors'; + +export const assertCanDeleteFromFolder = (path: string, logger: Logger): void => { + try { + accessSync(path, constants.F_OK | constants.R_OK | constants.W_OK); + logger.debug({ msg: 'Able to delete from directory', path }); + } catch (err) { + if (err instanceof Error && 'code' in err && err.code === 'ENOENT') { + throw new ConfigurationError(`FS path does not exist: ${path}`); + } else if (err instanceof Error && 'code' in err && (err.code === 'EACCES' || err.code === 'EPERM')) { + throw new ConfigurationError(`FS path permission denied for path: ${path}`); + } else { + throw new ConfigurationError(`An unexpected error occurred on FS path accessibility check: ${describeError(err)}`); + } + } + + try { + const pathStat = statSync(path); + if (!pathStat.isDirectory()) { + throw new ConfigurationError(`FS path exists but it is a file, not a directory: ${path}`); + } + } catch (err) { + if (err instanceof ConfigurationError) throw err; + throw new ConfigurationError(`An unexpected error occurred on FS info check: ${describeError(err)}`); + } +}; diff --git a/src/cleaner/utils/index.ts b/src/cleaner/utils/index.ts index fc203c9..49d0a4f 100644 --- a/src/cleaner/utils/index.ts +++ b/src/cleaner/utils/index.ts @@ -1,2 +1,5 @@ +export { getChunk } from './chunk'; +export { assertCanDeleteFromFolder } from './fs'; export { buildPollingPairs } from './pairBuilder'; +export { normalizeFolderPath, resolveAbsolutePath } from './path'; export { validateSchema } from './validationHelper'; diff --git a/src/cleaner/utils/path.ts b/src/cleaner/utils/path.ts new file mode 100644 index 0000000..c3049ee --- /dev/null +++ b/src/cleaner/utils/path.ts @@ -0,0 +1,16 @@ +import { resolve, sep } from 'node:path/posix'; + +export const normalizeFolderPath = (path: string): string => { + return path.endsWith(sep) ? path : `${path}${sep}`; +}; + +/** + * Resolves a file system path to an absolute path. + * Ensures the path is resolved as an absolute path and properly formatted + * with a leading separator if not already present. + * @param path - The input path string to normalize + * @returns An absolute path with proper path separators + */ +export const resolveAbsolutePath = (path: string): string => { + return resolve(`${path.startsWith(sep) ? '' : sep}${path}`); +}; diff --git a/src/common/constants.ts b/src/common/constants.ts index 01c8bfd..a9be61f 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -19,6 +19,8 @@ export const SERVICES = { TASK_VALIDATOR: Symbol('TaskValidator'), POLLING_PAIRS: Symbol('PollingPairs'), STORAGE_PROVIDERS: Symbol('StorageProviders'), + FS_STORAGE_CONFIG: Symbol('FsStorageConfig'), + S3_STORAGE_CONFIG: Symbol('S3StorageConfig'), TASK_CONTEXT: Symbol('TaskContext'), JOB_TRACKER_CLIENT: Symbol('JobTrackerClient'), // ============================================================================= diff --git a/src/common/dependencyRegistration.ts b/src/common/dependencyRegistration.ts index a591ff9..29f34e1 100644 --- a/src/common/dependencyRegistration.ts +++ b/src/common/dependencyRegistration.ts @@ -8,6 +8,8 @@ export interface InjectionObject { provider: Providers; } +export const getJobAndTaskToken = ({ jobType, taskType }: { jobType: string; taskType: string }): string => `${jobType}-${taskType}`; + export const registerDependencies = ( dependencies: InjectionObject[], override?: InjectionObject[], diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 586613b..db8c797 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,22 +1,23 @@ -import { getOtelMixin } from '@map-colonies/telemetry'; +import { IWorker, JobnikSDK } from '@map-colonies/jobnik-sdk'; +import { jsLogger, type Logger } from '@map-colonies/js-logger'; +import { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import { SourceType } from '@map-colonies/raster-shared'; +import { getOtelMixin } from '@map-colonies/telemetry'; import { trace } from '@opentelemetry/api'; import { Registry } from 'prom-client'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; -import { jsLogger, type Logger } from '@map-colonies/js-logger'; -import { IWorker, JobnikSDK } from '@map-colonies/jobnik-sdk'; -import { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; -import { InjectionObject, registerDependencies } from '@common/dependencyRegistration'; -import { SERVICES, SERVICE_NAME } from '@common/constants'; +import { SERVICE_NAME, SERVICES } from '@common/constants'; +import { getJobAndTaskToken, InjectionObject, registerDependencies } from '@common/dependencyRegistration'; import { getTracing } from '@common/tracing'; +import type { StorageProviders } from '@src/cleaner/storageProviders'; +import { ErrorHandler } from './cleaner/errors'; +import { JobTrackerClient } from './cleaner/httpClients'; +import { buildFsStorageConfig, buildS3StorageConfig, FsStorageProvider, S3StorageProvider } from './cleaner/storageProviders'; +import { DeleteStoredResourcesStrategy, StrategyFactory, TilesDeletionStrategy } from './cleaner/strategies'; import type { QueueConfig } from './cleaner/types'; import { ConfigType, getConfig } from './common/config'; import { workerBuilder } from './worker'; -import { StrategyFactory, TilesDeletionStrategy } from './cleaner/strategies'; -import { ErrorHandler } from './cleaner/errors'; -import { S3StorageProvider, FsStorageProvider, type IStorageProvider } from './cleaner/storageProviders'; -import { JobTrackerClient } from './cleaner/httpClients'; export interface RegisterOptions { override?: InjectionObject[]; @@ -34,6 +35,11 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const metricsRegistry = new Registry(); configInstance.initializeMetrics(metricsRegistry); + // Startup validations + const cleanupStorageProviders = configInstance.get('storage.cleanupStorageProviders') as unknown as string[]; + const fsStorageConfig = cleanupStorageProviders.includes(SourceType.FS) ? buildFsStorageConfig(configInstance, logger) : undefined; + const s3StorageConfig = cleanupStorageProviders.includes(SourceType.S3) ? buildS3StorageConfig(configInstance, logger) : undefined; + const dependencies: InjectionObject[] = [ { token: SERVICES.CONFIG, provider: { useValue: configInstance } }, { token: SERVICES.LOGGER, provider: { useValue: logger } }, @@ -97,25 +103,60 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise useClass: JobTrackerClient, }, }, + ...(fsStorageConfig ? [{ token: SERVICES.FS_STORAGE_CONFIG, provider: { useValue: fsStorageConfig } }] : []), + ...(s3StorageConfig ? [{ token: SERVICES.S3_STORAGE_CONFIG, provider: { useValue: s3StorageConfig } }] : []), { token: SERVICES.STORAGE_PROVIDERS, provider: { - useFactory: instancePerContainerCachingFactory((container) => { - const config = container.resolve(SERVICES.CONFIG); - const logger = container.resolve(SERVICES.LOGGER); - return new Map([ - [SourceType.S3, new S3StorageProvider(config, logger)], - [SourceType.FS, new FsStorageProvider(logger)], - ]); + useFactory: instancePerContainerCachingFactory((container) => { + const providers = { + ...(s3StorageConfig && { [SourceType.S3]: container.resolve(S3StorageProvider) }), + ...(fsStorageConfig && { [SourceType.FS]: container.resolve(FsStorageProvider) }), + }; + return providers; }), }, }, { - token: configInstance.get('jobDefinitions.tasks.tilesDeletion.type') as unknown as string, //TODO: when we create worker config schema we can move this to a constant and remove the cast + token: getJobAndTaskToken({ + //TODO: when we create worker config schema we can move this to a constant and remove the cast + jobType: configInstance.get('jobDefinitions.jobs.update.type') as unknown as string, + taskType: configInstance.get('jobDefinitions.tasks.tilesDeletion.type') as unknown as string, + }), + provider: { + useClass: TilesDeletionStrategy, + }, + }, + { + token: getJobAndTaskToken({ + //TODO: when we create worker config schema we can move this to a constant and remove the cast + jobType: configInstance.get('jobDefinitions.jobs.swapUpdate.type') as unknown as string, + taskType: configInstance.get('jobDefinitions.tasks.tilesDeletion.type') as unknown as string, + }), provider: { useClass: TilesDeletionStrategy, }, }, + { + token: getJobAndTaskToken({ + //TODO: when we create worker config schema we can move this to a constant and remove the cast + jobType: configInstance.get('jobDefinitions.jobs.deleteLayer.type') as unknown as string, + taskType: configInstance.get('jobDefinitions.tasks.layerDeletion.type') as unknown as string, + }), + provider: { + useClass: DeleteStoredResourcesStrategy, + }, + }, + { + token: getJobAndTaskToken({ + //TODO: when we create worker config schema we can move this to a constant and remove the cast + jobType: configInstance.get('jobDefinitions.jobs.deleteLayer.type') as unknown as string, + taskType: configInstance.get('jobDefinitions.tasks.artifactsDeletion.type') as unknown as string, + }), + provider: { + useClass: DeleteStoredResourcesStrategy, + }, + }, { token: 'onSignal', provider: { diff --git a/tests/helpers/mocks.ts b/tests/helpers/mocks.ts index 079851f..949b565 100644 --- a/tests/helpers/mocks.ts +++ b/tests/helpers/mocks.ts @@ -1,12 +1,13 @@ -import { vi } from 'vitest'; import type { Logger } from '@map-colonies/js-logger'; import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; -import type { ConfigType } from '../../src/common/config'; -import type { ITaskStrategy, StrategyFactory } from '../../src/cleaner/strategies'; -import type { IStorageProvider } from '../../src/cleaner/storageProviders'; +import { vi } from 'vitest'; +import type { IStorageProvider, StorageProvider } from '@src/cleaner/storageProviders/iStorageProvider'; +import type { FsConfig, FsStorageConfig, S3Config, S3StorageConfig } from '@src/cleaner/storageProviders/storageConfig'; import type { ErrorHandler } from '../../src/cleaner/errors'; -import type { ErrorDecision, PollingPairConfig } from '../../src/cleaner/types'; import type { JobTrackerClient } from '../../src/cleaner/httpClients'; +import type { ITaskStrategy, StrategyFactory } from '../../src/cleaner/strategies'; +import type { ErrorDecision, PollingPairConfig } from '../../src/cleaner/types'; +import type { ConfigType } from '../../src/common/config'; import { TaskPoller } from '../../src/worker/taskPoller'; // ─── Logger ────────────────────────────────────────────────────────────────── @@ -66,9 +67,10 @@ export function createMockErrorHandler(defaultDecision: ErrorDecision = { should // ─── StorageProvider ───────────────────────────────────────────────────────── -export function createMockStorageProvider(): IStorageProvider { +export function createMockStorageProvider(): IStorageProvider { return { - delete: vi.fn().mockResolvedValue([]), + delete: vi.fn().mockResolvedValue({ failures: new Map() }), + deleteResources: vi.fn().mockResolvedValue({ failures: new Map() }), targetExists: vi.fn().mockResolvedValue(true), }; } @@ -78,18 +80,29 @@ export function createMockStorageProvider(): IStorageProvider { export const TILES_DELETION_CONFIG_DEFAULTS = { batchSize: 100, concurrency: 2, - failureSampleSize: 3, s3Bucket: 'test-bucket', - fsBasePath: '/test/tiles', + fsBasePath: '/test', + fsSubPath: 'tiles', } as const; export function createMockStrategyConfig(overrides: Record = {}): ConfigType { const values: Record = { 'strategies.tilesDeletion.batchSize': TILES_DELETION_CONFIG_DEFAULTS.batchSize, 'strategies.tilesDeletion.concurrency': TILES_DELETION_CONFIG_DEFAULTS.concurrency, - 'strategies.tilesDeletion.failureSampleSize': TILES_DELETION_CONFIG_DEFAULTS.failureSampleSize, 'strategies.tilesDeletion.s3Bucket': TILES_DELETION_CONFIG_DEFAULTS.s3Bucket, - 'strategies.tilesDeletion.fsBasePath': TILES_DELETION_CONFIG_DEFAULTS.fsBasePath, + 'strategies.tilesDeletion.fsSubPath': TILES_DELETION_CONFIG_DEFAULTS.fsSubPath, + 'storage.fs.basePath': FS_STORAGE_CONFIG_DEFAULTS.basePath, + ...overrides, + }; + return { get: vi.fn().mockImplementation((key: string) => values[key]) } as unknown as ConfigType; +} + +// ─── Strategy Config (DeleteStoredResourcesStrategy) ──────────────────────────── + +export const STORED_RESOURCES_DELETION_CONFIG_DEFAULTS = {} as const; + +export function createMockStoredResourcesDeletionStrategyConfig(overrides: Record = {}): ConfigType { + const values: Record = { ...overrides, }; return { get: vi.fn().mockImplementation((key: string) => values[key]) } as unknown as ConfigType; @@ -98,20 +111,69 @@ export function createMockStrategyConfig(overrides: Record = {} // ─── S3 Storage Config (S3StorageProvider) ─────────────────────────────────── export const S3_STORAGE_CONFIG_DEFAULTS = { + delete: { + batchSize: 100, + }, endpoint: 'http://localhost:9000', accessKeyId: 'test-key', secretAccessKey: 'test-secret', sslEnabled: false, forcePathStyle: true, region: 'us-east-1', -} as const; +} as const satisfies S3Config; -export function createMockS3Config(): ConfigType { +export function createMockS3Config(overrides: Record = {}): ConfigType { return { - get: vi.fn().mockReturnValue({ ...S3_STORAGE_CONFIG_DEFAULTS }), + get: vi.fn().mockReturnValue({ ...S3_STORAGE_CONFIG_DEFAULTS, ...overrides }), } as unknown as ConfigType; } +// ─── S3 Validated Storage Config ─────────────────────────────────── + +export const S3_VALIDATED_CONFIG_DEFAULTS = { + endpoint: S3_STORAGE_CONFIG_DEFAULTS.endpoint, + accessKeyId: S3_STORAGE_CONFIG_DEFAULTS.accessKeyId, + secretAccessKey: S3_STORAGE_CONFIG_DEFAULTS.secretAccessKey, + sslEnabled: S3_STORAGE_CONFIG_DEFAULTS.sslEnabled, + forcePathStyle: S3_STORAGE_CONFIG_DEFAULTS.forcePathStyle, + region: S3_STORAGE_CONFIG_DEFAULTS.region, + batchSize: S3_STORAGE_CONFIG_DEFAULTS.delete.batchSize, +} as const satisfies S3StorageConfig; + +export function createS3StorageConfig(overrides: Partial = {}): S3StorageConfig { + return { ...S3_VALIDATED_CONFIG_DEFAULTS, ...overrides }; +} + +// ─── FS Storage Config (FsStorageProvider) ─────────────────────────────────── + +export const FS_STORAGE_CONFIG_DEFAULTS = { + delete: { + batchSize: 3, + }, + basePath: '/test', + subPaths: { + tiles: 'artifacts/tiles', + }, +} as const satisfies FsConfig; + +export function createMockFsConfig(overrides: Record = {}): ConfigType { + return { + get: vi.fn().mockReturnValue({ ...FS_STORAGE_CONFIG_DEFAULTS, ...overrides }), + } as unknown as ConfigType; +} + +// ─── FS Validated Storage Config ─────────────────────────────────── + +export const FS_VALIDATED_CONFIG_DEFAULTS = { + basePath: FS_STORAGE_CONFIG_DEFAULTS.basePath, + subPaths: Object.values(FS_STORAGE_CONFIG_DEFAULTS.subPaths), + batchSize: FS_STORAGE_CONFIG_DEFAULTS.delete.batchSize, +} as const satisfies FsStorageConfig; + +export function createFsStorageConfig(overrides: Partial = {}): FsStorageConfig { + return { ...FS_VALIDATED_CONFIG_DEFAULTS, ...overrides }; +} + // ─── JobTrackerClient ───────────────────────────────────────────────────────── export function createMockJobTrackerClient(): JobTrackerClient { diff --git a/tests/storageProviders/deleteFailureSummary.spec.ts b/tests/storageProviders/deleteFailureSummary.spec.ts deleted file mode 100644 index 6f669cc..0000000 --- a/tests/storageProviders/deleteFailureSummary.spec.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { describe, it, expect } from 'vitest'; -import { summarizeDeleteFailures } from '@src/cleaner/storageProviders'; -import type { DeleteFailure } from '@src/cleaner/storageProviders'; - -describe('summarizeDeleteFailures', () => { - it('should return zero counts, empty summary and empty sample for an empty input', () => { - const result = summarizeDeleteFailures([], 5); - expect(result).toEqual({ counts: {}, summary: '', sample: [] }); - }); - - it('should count occurrences per reason', () => { - const failures: DeleteFailure[] = [ - { path: 'a', reason: 'ENOENT' }, - { path: 'b', reason: 'ENOENT' }, - { path: 'c', reason: 'EACCES' }, - ]; - - const { counts } = summarizeDeleteFailures(failures, 3); - - expect(counts).toEqual({ ENOENT: 2, EACCES: 1 }); - }); - - it('should format the summary with reasons sorted by descending count', () => { - const failures: DeleteFailure[] = [ - { path: 'a', reason: 'EACCES' }, - { path: 'b', reason: 'ENOENT' }, - { path: 'c', reason: 'ENOENT' }, - { path: 'd', reason: 'ENOENT' }, - ]; - - const { summary } = summarizeDeleteFailures(failures, 3); - - expect(summary).toBe('ENOENT=3, EACCES=1'); - }); - - it('should annotate each sampled path with its reason', () => { - const failures: DeleteFailure[] = [ - { path: 'tile/1.png', reason: 'ENOENT' }, - { path: 'tile/2.png', reason: 'EACCES' }, - ]; - - const { sample } = summarizeDeleteFailures(failures, 3); - - expect(sample).toEqual(['tile/1.png (ENOENT)', 'tile/2.png (EACCES)']); - }); - - it('should cap the sample to sampleSize and preserve input order', () => { - const failures: DeleteFailure[] = [ - { path: 'a', reason: 'ENOENT' }, - { path: 'b', reason: 'ENOENT' }, - { path: 'c', reason: 'ENOENT' }, - { path: 'd', reason: 'ENOENT' }, - ]; - - const { sample } = summarizeDeleteFailures(failures, 2); - - expect(sample).toEqual(['a (ENOENT)', 'b (ENOENT)']); - expect(sample).length(2); - }); - - it('should return all failures in the sample when sampleSize exceeds input length', () => { - const failures: DeleteFailure[] = [{ path: 'a', reason: 'ENOENT' }]; - - const { sample } = summarizeDeleteFailures(failures, 10); - - expect(sample).toEqual(['a (ENOENT)']); - }); - - it('should return an empty sample when sampleSize is 0', () => { - const failures: DeleteFailure[] = [{ path: 'a', reason: 'ENOENT' }]; - - const { sample } = summarizeDeleteFailures(failures, 0); - - expect(sample).toEqual([]); - }); -}); diff --git a/tests/storageProviders/failuresHandling.spec.ts b/tests/storageProviders/failuresHandling.spec.ts new file mode 100644 index 0000000..016af25 --- /dev/null +++ b/tests/storageProviders/failuresHandling.spec.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from 'vitest'; +import { mergeFailures, summarizeDeleteFailures, type DeleteFailure } from '@src/cleaner/storageProviders'; + +describe('failuresHandling', () => { + describe('#mergeFailures', () => { + it('should return an empty map when both source and target are empty', () => { + expect(mergeFailures({ source: new Map(), target: new Map() })).toEqual(new Map()); + }); + + it('should return the target entries unchanged for an empty source', () => { + const target: DeleteFailure = new Map([['ENOENT', { count: 2, sample: 'a' }]]); + + expect(mergeFailures({ source: new Map(), target })).toEqual(target); + }); + + it('should keep entries of both maps when reasons do not overlap', () => { + const source: DeleteFailure = new Map([['EACCES', { count: 1, sample: 'b' }]]); + const target: DeleteFailure = new Map([['ENOENT', { count: 2, sample: 'a' }]]); + + expect(mergeFailures({ source, target })).toEqual( + new Map([ + ['ENOENT', { count: 2, sample: 'a' }], + ['EACCES', { count: 1, sample: 'b' }], + ]) + ); + }); + + it('should sum counts and keep the target sample when a reason appears in both maps', () => { + const source: DeleteFailure = new Map([['ENOENT', { count: 3, sample: 'source-sample' }]]); + const target: DeleteFailure = new Map([['ENOENT', { count: 2, sample: 'target-sample' }]]); + + expect(mergeFailures({ source, target })).toEqual(new Map([['ENOENT', { count: 5, sample: 'target-sample' }]])); + }); + + it('should not mutate the source or the target', () => { + const source: DeleteFailure = new Map([['ENOENT', { count: 3, sample: 'b' }]]); + const target: DeleteFailure = new Map([['ENOENT', { count: 2, sample: 'a' }]]); + + mergeFailures({ source, target }); + + expect(target).toEqual(new Map([['ENOENT', { count: 2, sample: 'a' }]])); + expect(source).toEqual(new Map([['ENOENT', { count: 3, sample: 'b' }]])); + }); + }); + + describe('#summarizeDeleteFailures', () => { + it('should return zero counts, empty summary for an empty input', () => { + const failures = { failures: new Map() }; + + const result = summarizeDeleteFailures(failures); + + expect(result).toEqual({ failuresCount: 0, summary: '', samples: [] }); + }); + + it('should format the summary with reasons and samples sorted by descending count', () => { + const failures = { + failures: new Map([ + ['EACCES', { count: 1, sample: 'a' }], + ['ENOENT', { count: 3, sample: 'b' }], + ]), + }; + + const result = summarizeDeleteFailures(failures); + + expect(result).toStrictEqual({ failuresCount: 4, summary: 'ENOENT=3, EACCES=1', samples: ['b (ENOENT)', 'a (EACCES)'] }); + }); + + it('should annotate each sampled path with its reason', () => { + const failures = { + failures: new Map([ + ['ENOENT', { count: 1, sample: 'tile/1.png' }], + ['EACCES', { count: 1, sample: 'tile/2.png' }], + ]), + }; + + const { samples } = summarizeDeleteFailures(failures); + + expect(samples).toEqual(['tile/1.png (ENOENT)', 'tile/2.png (EACCES)']); + }); + + it('should order samples by descending count', () => { + const failures = { + failures: new Map([ + ['EACCES', { count: 1, sample: 'a' }], + ['ENOENT', { count: 4, sample: 'b' }], + ]), + }; + + const { samples } = summarizeDeleteFailures(failures); + + expect(samples).toEqual(['b (ENOENT)', 'a (EACCES)']); + }); + }); +}); diff --git a/tests/storageProviders/fsStorageProvider.spec.ts b/tests/storageProviders/fsStorageProvider.spec.ts index cdca3f4..09603c6 100644 --- a/tests/storageProviders/fsStorageProvider.spec.ts +++ b/tests/storageProviders/fsStorageProvider.spec.ts @@ -1,30 +1,44 @@ -import { stat, unlink, rmdir } from 'node:fs/promises'; +import type { Stats } from 'node:fs'; +import { rm, rmdir, stat, unlink } from 'node:fs/promises'; import { join } from 'node:path'; -import { Stats } from 'node:fs'; -import { describe, it, expect, beforeEach, vi } from 'vitest'; +import type { Logger } from '@map-colonies/js-logger'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { UnrecoverableError } from '@src/cleaner/errors'; import { FsStorageProvider } from '@src/cleaner/storageProviders/fsStorageProvider'; -import { createMockLogger } from '../helpers/mocks'; +import { createFsStorageConfig, createMockLogger, FS_VALIDATED_CONFIG_DEFAULTS } from '../helpers/mocks'; vi.mock('node:fs/promises', () => ({ stat: vi.fn(), unlink: vi.fn(), rmdir: vi.fn(), + rm: vi.fn(), })); -const BASE_PATH = '/tiles/test'; +const BASE_PATH = FS_VALIDATED_CONFIG_DEFAULTS.basePath; describe('FsStorageProvider', () => { let provider: FsStorageProvider; + let mockLogger: Logger; beforeEach(() => { vi.clearAllMocks(); vi.mocked(stat).mockResolvedValue({} as Stats); vi.mocked(unlink).mockResolvedValue(undefined); vi.mocked(rmdir).mockResolvedValue(undefined); - provider = new FsStorageProvider(createMockLogger()); + vi.mocked(rm).mockResolvedValue(undefined); + mockLogger = createMockLogger(); + provider = new FsStorageProvider(createFsStorageConfig(), mockLogger); }); - describe('targetExists', () => { + describe('#constructor', () => { + it('should return an instance of the class', () => { + const provider = new FsStorageProvider(createFsStorageConfig({ basePath: '/other/base' }), mockLogger); + + expect(provider).toBeInstanceOf(FsStorageProvider); + }); + }); + + describe('#targetExists', () => { const RELATIVE_PATH = 'layer/v1'; it('should call stat with full target path', async () => { @@ -50,10 +64,10 @@ describe('FsStorageProvider', () => { }); }); - describe('delete', () => { - it('should return empty array for empty input', async () => { + describe('#delete', () => { + it('should return empty failures map for empty input', async () => { const result = await provider.delete([], BASE_PATH); - expect(result).toEqual([]); + expect(result).toEqual({ failures: new Map() }); expect(unlink).not.toHaveBeenCalled(); }); @@ -74,9 +88,9 @@ describe('FsStorageProvider', () => { } }); - it('should return empty array when all unlinks succeed', async () => { + it('should return empty failures map when all unlinks succeed', async () => { const result = await provider.delete(['tile/10/0/0.png', 'tile/10/0/1.png'], BASE_PATH); - expect(result).toEqual([]); + expect(result).toEqual({ failures: new Map() }); }); it('should treat ENOENT as a failed deletion tagged with ENOENT reason', async () => { @@ -85,7 +99,7 @@ describe('FsStorageProvider', () => { const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'ENOENT' }]); + expect(result).toEqual({ failures: new Map([['ENOENT', { count: 1, sample: 'tile/10/0/0.png' }]]) }); }); it('should return failed path with reason for non-ENOENT errors', async () => { @@ -94,7 +108,7 @@ describe('FsStorageProvider', () => { const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'EACCES' }]); + expect(result).toEqual({ failures: new Map([['EACCES', { count: 1, sample: 'tile/10/0/0.png' }]]) }); }); it('should fall back to error message when error has no errno code', async () => { @@ -102,7 +116,7 @@ describe('FsStorageProvider', () => { const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'disk on fire' }]); + expect(result).toEqual({ failures: new Map([['disk on fire', { count: 1, sample: 'tile/10/0/0.png' }]]) }); }); it('should fall back to "Unknown" when error has neither errno code nor message', async () => { @@ -110,7 +124,56 @@ describe('FsStorageProvider', () => { const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); - expect(result).toEqual([{ path: 'tile/10/0/0.png', reason: 'Unknown' }]); + expect(result).toEqual({ failures: new Map([['Unknown', { count: 1, sample: 'tile/10/0/0.png' }]]) }); + }); + + it('should tag failures with the stringified value when a non-Error is thrown', async () => { + vi.mocked(unlink).mockRejectedValue('raw string failure'); + + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + + expect(result).toEqual({ failures: new Map([['raw string failure', { count: 1, sample: 'tile/10/0/0.png' }]]) }); + }); + + it('should fall back to "Unknown" when a non-Error empty value is thrown', async () => { + vi.mocked(unlink).mockRejectedValue(''); + + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + + expect(result).toEqual({ failures: new Map([['Unknown', { count: 1, sample: 'tile/10/0/0.png' }]]) }); + }); + + it('should fall back to a generic reason when the thrown value cannot be stringified', async () => { + vi.mocked(unlink).mockRejectedValue({ + toString: () => { + throw new Error('toString failed'); + }, + }); + + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + + expect(result).toEqual({ failures: new Map([['non-serializable thrown value', { count: 1, sample: 'tile/10/0/0.png' }]]) }); + }); + + it('should unlink every path of a large input', async () => { + const paths = Array.from({ length: 7 }, (_, i) => `tile/10/0/${i}.png`); + + await provider.delete(paths, BASE_PATH); + + expect(unlink).toHaveBeenCalledTimes(7); + for (const path of paths) { + expect(unlink).toHaveBeenCalledWith(join(BASE_PATH, path)); + } + }); + + it('should aggregate failures of the same reason keeping the first sample', async () => { + const permError = Object.assign(new Error('EACCES'), { code: 'EACCES' }); + vi.mocked(unlink).mockRejectedValue(permError); + const paths = Array.from({ length: 7 }, (_, i) => `tile/10/0/${i}.png`); + + const result = await provider.delete(paths, BASE_PATH); + + expect(result).toEqual({ failures: new Map([['EACCES', { count: 7, sample: 'tile/10/0/0.png' }]]) }); }); it('should handle mixed success, ENOENT and real errors', async () => { @@ -125,10 +188,12 @@ describe('FsStorageProvider', () => { const paths = ['tile/10/0/0.png', 'tile/10/0/1.png', 'tile/10/0/2.png']; const result = await provider.delete(paths, BASE_PATH); - expect(result).toEqual([ - { path: 'tile/10/0/1.png', reason: 'ENOENT' }, - { path: 'tile/10/0/2.png', reason: 'EACCES' }, - ]); + expect(result).toEqual({ + failures: new Map([ + ['ENOENT', { count: 1, sample: 'tile/10/0/1.png' }], + ['EACCES', { count: 1, sample: 'tile/10/0/2.png' }], + ]), + }); }); it('should use relative path (not the full absolute path) in failure entries', async () => { @@ -138,8 +203,8 @@ describe('FsStorageProvider', () => { const relativePath = 'layer/v1/10/5/3.png'; const result = await provider.delete([relativePath], BASE_PATH); - expect(result).toEqual([{ path: relativePath, reason: 'EACCES' }]); - expect(result[0]!.path).not.toContain(BASE_PATH); + expect(result).toEqual({ failures: new Map([['EACCES', { count: 1, sample: relativePath }]]) }); + expect(Array.from(result.failures.values())[0]?.sample).not.toMatch(`^${BASE_PATH}*`); }); describe('cleanupEmptyDirs', () => { @@ -172,14 +237,198 @@ describe('FsStorageProvider', () => { const enotempty = Object.assign(new Error('ENOTEMPTY'), { code: 'ENOTEMPTY' }); vi.mocked(rmdir).mockRejectedValue(enotempty); + const result = await provider.delete(['tile/10/0/0.png'], BASE_PATH); + // Should not throw and should return correct failed paths - await expect(provider.delete(['tile/10/0/0.png'], BASE_PATH)).resolves.toEqual([]); + expect(result).toEqual({ failures: new Map() }); }); it('should not call rmdir when input is empty', async () => { await provider.delete([], BASE_PATH); expect(rmdir).not.toHaveBeenCalled(); }); + + it('should not call rmdir for a path that has no directory segments', async () => { + await provider.delete(['0.png'], BASE_PATH); + + expect(unlink).toHaveBeenCalledWith(join(BASE_PATH, '0.png')); + expect(rmdir).not.toHaveBeenCalled(); + }); + + it('should attempt to rmdir ancestors of paths from every batch', async () => { + // batchSize is 3 → cleanup runs once for all paths, after the last batch + const paths = Array.from({ length: 4 }, (_, i) => `tile/10/${i}/0.png`); + + await provider.delete(paths, BASE_PATH); + + for (let i = 0; i < 4; i++) { + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, `tile/10/${i}`)); + } + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, 'tile/10')); + expect(rmdir).toHaveBeenCalledWith(join(BASE_PATH, 'tile')); + }); + + it('should attempt to rmdir deeper directories before their ancestors', async () => { + await provider.delete(['layer/v1/10/0/0.png'], BASE_PATH); + + const order = vi.mocked(rmdir).mock.calls.map(([path]) => path); + expect(order).toEqual([ + join(BASE_PATH, 'layer/v1/10/0'), + join(BASE_PATH, 'layer/v1/10'), + join(BASE_PATH, 'layer/v1'), + join(BASE_PATH, 'layer'), + ]); + }); + }); + }); + + describe('#deleteResources', () => { + const FS_SUB_PATH = FS_VALIDATED_CONFIG_DEFAULTS.subPaths[0]!; + const RELATIVE_PATH = 'layer/v1'; + + it('should successfully return without failures for empty paths', async () => { + const result = await provider.deleteResources({ paths: [], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ failures: new Map() }); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should successfully call delete all files and return without failures', async () => { + const result = await provider.deleteResources({ paths: [RELATIVE_PATH], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ failures: new Map() }); + expect(rm).toHaveBeenCalledWith(join(BASE_PATH, FS_SUB_PATH, RELATIVE_PATH), { recursive: true, force: true }); + }); + + it('should successfully call delete all files and return without failures for multiple paths', async () => { + const result = await provider.deleteResources({ paths: [RELATIVE_PATH, RELATIVE_PATH], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ failures: new Map() }); + expect(rm).toHaveBeenCalledWith(join(BASE_PATH, FS_SUB_PATH, RELATIVE_PATH), { recursive: true, force: true }); + }); + + it('should successfully call delete all files and return without failures for multiple paths', async () => { + const result = await provider.deleteResources({ paths: [RELATIVE_PATH, `${RELATIVE_PATH}/old`], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ failures: new Map() }); + expect(rm).toHaveBeenCalledWith(join(BASE_PATH, FS_SUB_PATH, RELATIVE_PATH), { recursive: true, force: true }); + }); + + it('should throw UnrecoverableError when a path escapes the base path via traversal', async () => { + const result = provider.deleteResources({ paths: ['../../../../etc/passwd'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when only one of several paths escapes the base path', async () => { + const result = provider.deleteResources({ paths: [RELATIVE_PATH, '../../../../escape'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when a path resolves to the base path itself', async () => { + const result = provider.deleteResources({ paths: ['../../../'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when a path resolves to the base path itself via "."', async () => { + const result = provider.deleteResources({ paths: ['../../../.'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should return failures entry when rm rejects', async () => { + vi.mocked(rm).mockRejectedValue(new Error('EACCES')); + + const result = await provider.deleteResources({ paths: [RELATIVE_PATH], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ + failures: new Map([['EACCES', { count: 1, sample: join(BASE_PATH, FS_SUB_PATH, RELATIVE_PATH) }]]), + }); + }); + + it('should not throw when rm rejects', async () => { + vi.mocked(rm).mockRejectedValue(new Error('Permission denied')); + + const result = provider.deleteResources({ paths: [RELATIVE_PATH], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).resolves.not.toThrow(); + }); + + it('should rm every path when the input spans multiple batches', async () => { + // batchSize is 3 → 7 paths span 3 batches (3 + 3 + 1) + const paths = Array.from({ length: 7 }, (_, i) => `layer/v${i}`); + + await provider.deleteResources({ paths, subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(rm).toHaveBeenCalledTimes(7); + for (const path of paths) { + expect(rm).toHaveBeenCalledWith(join(BASE_PATH, FS_SUB_PATH, path), { recursive: true, force: true }); + } + }); + + it('should accumulate failures of the same reason across batches keeping the first sample', async () => { + vi.mocked(rm).mockRejectedValue(Object.assign(new Error('EACCES'), { code: 'EACCES' })); + const paths = Array.from({ length: 7 }, (_, i) => `layer/v${i}`); + + const result = await provider.deleteResources({ paths, subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ + failures: new Map([['EACCES', { count: 7, sample: join(BASE_PATH, FS_SUB_PATH, 'layer/v0') }]]), + }); + }); + + it('should group failures by reason across batches', async () => { + const paths = ['layer/v0', 'layer/v1', 'layer/v2', 'layer/v3']; + vi.mocked(rm) + .mockRejectedValueOnce(Object.assign(new Error('EACCES'), { code: 'EACCES' })) + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(Object.assign(new Error('EBUSY'), { code: 'EBUSY' })) + // 4th path lands in the second batch + .mockRejectedValueOnce(Object.assign(new Error('EACCES'), { code: 'EACCES' })); + + const result = await provider.deleteResources({ paths, subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + expect(result).toEqual({ + failures: new Map([ + ['EACCES', { count: 2, sample: join(BASE_PATH, FS_SUB_PATH, 'layer/v0') }], + ['EBUSY', { count: 1, sample: join(BASE_PATH, FS_SUB_PATH, 'layer/v2') }], + ]), + }); + }); + + it('should throw UnrecoverableError when a path does not sit under any configured subPath', async () => { + const result = provider.deleteResources({ paths: [RELATIVE_PATH], subPath: 'unconfigured/subPath', storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when a path resolves to the configured subPath itself', async () => { + const result = provider.deleteResources({ paths: [''], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when a path escapes into a sibling that shares the subPath prefix', async () => { + // resolves to '/artifacts/tiles-backup' — under the base path, but outside the configured subPath + const result = provider.deleteResources({ paths: ['../tiles-backup'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when only one of several paths escapes the configured subPath', async () => { + const result = provider.deleteResources({ paths: [RELATIVE_PATH, '../tiles-backup'], subPath: FS_SUB_PATH, storageProvider: 'FS' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(rm).not.toHaveBeenCalled(); }); }); }); diff --git a/tests/storageProviders/s3StorageProvider.spec.ts b/tests/storageProviders/s3StorageProvider.spec.ts index 1dde516..50c9cd7 100644 --- a/tests/storageProviders/s3StorageProvider.spec.ts +++ b/tests/storageProviders/s3StorageProvider.spec.ts @@ -1,23 +1,42 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { S3Client, DeleteObjectsCommand, ListObjectsV2Command, NoSuchBucket } from '@aws-sdk/client-s3'; +import { + DeleteObjectsCommand, + ListObjectsV2Command, + NoSuchBucket, + NotFound, + paginateListObjectsV2, + S3Client, + S3ServiceException, + type DeleteObjectsCommandOutput, +} from '@aws-sdk/client-s3'; +import { faker } from '@faker-js/faker'; +import type { Logger } from '@map-colonies/js-logger'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { UnrecoverableError } from '@src/cleaner/errors'; import { S3StorageProvider } from '@src/cleaner/storageProviders/s3StorageProvider'; -import { createMockLogger, createMockS3Config, S3_STORAGE_CONFIG_DEFAULTS } from '../helpers/mocks'; +import { createMockLogger, createS3StorageConfig, S3_VALIDATED_CONFIG_DEFAULTS } from '../helpers/mocks'; const mockSend = vi.fn(); +const mockPaginateListObjectsV2Next = vi.fn(); +const mockPaginateListObjectsV2Return = vi.fn(); +const mockPaginateListObjectsV2Throw = vi.fn(); +const mockPaginateListObjectsV2Iterator = vi.fn().mockReturnValue({ + next: mockPaginateListObjectsV2Next, + return: mockPaginateListObjectsV2Return, + throw: mockPaginateListObjectsV2Throw, +}); +const mockPaginateListObjectsV2 = { + [Symbol.asyncIterator]: mockPaginateListObjectsV2Iterator, +}; -vi.mock('@aws-sdk/client-s3', () => { - class NoSuchBucket extends Error { - public constructor() { - super('NoSuchBucket'); - this.name = 'NoSuchBucket'; - } - } +vi.mock(import('@aws-sdk/client-s3'), async (importOriginal) => { + const originModule = await importOriginal(); return { - S3Client: vi.fn(() => ({ send: mockSend })), - DeleteObjectsCommand: vi.fn((input: unknown) => input), - ListObjectsV2Command: vi.fn((input: unknown) => input), - NoSuchBucket, + ...originModule, + S3Client: vi.fn(() => ({ send: mockSend })) as unknown as typeof S3Client, + DeleteObjectsCommand: vi.fn((input: unknown) => input) as unknown as typeof DeleteObjectsCommand, + ListObjectsV2Command: vi.fn((input: unknown) => input) as unknown as typeof ListObjectsV2Command, + paginateListObjectsV2: vi.fn(() => mockPaginateListObjectsV2) as unknown as typeof paginateListObjectsV2, }; }); @@ -25,17 +44,41 @@ const BUCKET = 'test-bucket'; describe('S3StorageProvider', () => { let provider: S3StorageProvider; + let mockLogger: Logger; beforeEach(() => { vi.clearAllMocks(); - mockSend.mockResolvedValue({ Errors: [] }); - provider = new S3StorageProvider(createMockS3Config(), createMockLogger()); + mockLogger = createMockLogger(); + provider = new S3StorageProvider(createS3StorageConfig(), mockLogger); }); - describe('delete', () => { - it('should return empty array for empty input', async () => { + describe('#constructor', () => { + it('should construct S3Client with config values', () => { + const provider = new S3StorageProvider(createS3StorageConfig(), mockLogger); + expect(S3Client).toHaveBeenCalledWith( + expect.objectContaining({ + credentials: { + accessKeyId: S3_VALIDATED_CONFIG_DEFAULTS.accessKeyId, + secretAccessKey: S3_VALIDATED_CONFIG_DEFAULTS.secretAccessKey, + }, + endpoint: S3_VALIDATED_CONFIG_DEFAULTS.endpoint, + forcePathStyle: S3_VALIDATED_CONFIG_DEFAULTS.forcePathStyle, + region: S3_VALIDATED_CONFIG_DEFAULTS.region, + tls: S3_VALIDATED_CONFIG_DEFAULTS.sslEnabled, + }) + ); + expect(provider).toBeInstanceOf(S3StorageProvider); + }); + }); + + describe('#delete', () => { + beforeEach(() => { + mockSend.mockResolvedValue({ Errors: [] }); + }); + + it('should return empty failures map for empty input', async () => { const result = await provider.delete([], BUCKET); - expect(result).toEqual([]); + expect(result).toEqual({ failures: new Map() }); expect(mockSend).not.toHaveBeenCalled(); }); @@ -51,12 +94,12 @@ describe('S3StorageProvider', () => { expect(mockSend).toHaveBeenCalledTimes(1); }); - it('should return empty array when all deletes succeed', async () => { + it('should return empty failures map when all deletes succeed', async () => { const paths = ['a.txt', 'b.txt']; const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([]); + expect(result).toEqual({ failures: new Map() }); }); it('should return failed paths tagged with the S3 error Code', async () => { @@ -67,7 +110,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([{ path: 'a.txt', reason: 'AccessDenied' }]); + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 1, sample: 'a.txt' }]]) }); }); it('should treat NoSuchKey as a failed deletion tagged with NoSuchKey reason', async () => { @@ -78,7 +121,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([{ path: 'missing.txt', reason: 'NoSuchKey' }]); + expect(result).toEqual({ failures: new Map([['NoSuchKey', { count: 1, sample: 'missing.txt' }]]) }); }); it('should fall back to Message when error has no Code', async () => { @@ -88,7 +131,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(['a.txt'], BUCKET); - expect(result).toEqual([{ path: 'a.txt', reason: 'Something bad' }]); + expect(result).toEqual({ failures: new Map([['Something bad', { count: 1, sample: 'a.txt' }]]) }); }); it('should fall back to "Unknown" when error has neither Code nor Message', async () => { @@ -98,7 +141,7 @@ describe('S3StorageProvider', () => { const result = await provider.delete(['a.txt'], BUCKET); - expect(result).toEqual([{ path: 'a.txt', reason: 'Unknown' }]); + expect(result).toEqual({ failures: new Map([['Unknown', { count: 1, sample: 'a.txt' }]]) }); }); it('should return all errors including NoSuchKey with their codes', async () => { @@ -113,16 +156,18 @@ describe('S3StorageProvider', () => { const result = await provider.delete(['a.txt', 'b.txt', 'c.txt', 'd.txt'], BUCKET); - expect(result).toEqual([ - { path: 'a.txt', reason: 'NoSuchKey' }, - { path: 'b.txt', reason: 'AccessDenied' }, - { path: 'c.txt', reason: 'NoSuchKey' }, - { path: 'd.txt', reason: 'InternalError' }, - ]); + expect(result).toEqual({ + failures: new Map([ + ['NoSuchKey', { count: 2, sample: 'a.txt' }], + ['AccessDenied', { count: 1, sample: 'b.txt' }], + ['InternalError', { count: 1, sample: 'd.txt' }], + ]), + }); }); - it('should batch paths into chunks of 1000 (S3 limit)', async () => { + it('should batch paths into chunks of the configured batch size', async () => { const paths = Array.from({ length: 1500 }, (_, i) => `object-${i}.txt`); + provider = new S3StorageProvider(createS3StorageConfig({ batchSize: 1000 }), mockLogger); await provider.delete(paths, BUCKET); @@ -137,6 +182,18 @@ describe('S3StorageProvider', () => { expect(secondCallInput.Delete.Objects).toHaveLength(500); }); + it('should never exceed the S3 max keys limit per request for the maximum allowed batch size', async () => { + const paths = Array.from({ length: 2500 }, (_, i) => `object-${i}.txt`); + provider = new S3StorageProvider(createS3StorageConfig({ batchSize: 1000 }), mockLogger); + + await provider.delete(paths, BUCKET); + + expect(mockSend).toHaveBeenCalledTimes(3); + const callInputs = vi.mocked(DeleteObjectsCommand).mock.calls.map((call) => call[0] as { Delete: { Objects: { Key: string }[] } }); + expect(callInputs.every((input) => input.Delete.Objects.length <= 1000)).toBe(true); + expect(callInputs.map((input) => input.Delete.Objects.length)).toEqual([1000, 1000, 500]); + }); + it('should accumulate failures across multiple chunks', async () => { const paths = Array.from({ length: 1500 }, (_, i) => `object-${i}.txt`); mockSend @@ -145,10 +202,33 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([ - { path: 'object-0.txt', reason: 'AccessDenied' }, - { path: 'object-1000.txt', reason: 'AccessDenied' }, - ]); + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 2, sample: 'object-0.txt' }]]) }); + }); + + it('should ignore returned errors that carry no Key', async () => { + mockSend.mockResolvedValue({ + Errors: [{ Code: 'InternalError' }, { Key: 'b.txt', Code: 'AccessDenied' }], + }); + + const result = await provider.delete(['a.txt', 'b.txt'], BUCKET); + + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 1, sample: 'b.txt' }]]) }); + }); + + it('should return an empty failures map when every returned error carries no Key', async () => { + mockSend.mockResolvedValue({ Errors: [{ Code: 'InternalError' }] }); + + const result = await provider.delete(['a.txt'], BUCKET); + + expect(result).toEqual({ failures: new Map() }); + }); + + it('should tag the chunk with the stringified value when send rejects with a non-Error', async () => { + mockSend.mockRejectedValue('connection reset'); + + const result = await provider.delete(['a.txt', 'b.txt'], BUCKET); + + expect(result).toEqual({ failures: new Map([['connection reset', { count: 2, sample: 'a.txt' }]]) }); }); it('should add entire chunk to failures tagged with the thrown error when send rejects', async () => { @@ -157,14 +237,15 @@ describe('S3StorageProvider', () => { const result = await provider.delete(paths, BUCKET); - expect(result).toEqual([ - { path: 'a.txt', reason: 'Network error' }, - { path: 'b.txt', reason: 'Network error' }, - ]); + expect(result).toEqual({ failures: new Map([['Network error', { count: 2, sample: 'a.txt' }]]) }); }); }); - describe('targetExists', () => { + describe('#targetExists', () => { + beforeEach(() => { + mockSend.mockResolvedValue({ Errors: [] }); + }); + const PREFIX = 'some/prefix'; it('should list objects with bucket and relativePath prefix', async () => { @@ -215,16 +296,510 @@ describe('S3StorageProvider', () => { }); }); - describe('constructor', () => { - it('should construct S3Client with config values', () => { - expect(S3Client).toHaveBeenCalledWith( - expect.objectContaining({ - endpoint: S3_STORAGE_CONFIG_DEFAULTS.endpoint, - forcePathStyle: S3_STORAGE_CONFIG_DEFAULTS.forcePathStyle, - region: S3_STORAGE_CONFIG_DEFAULTS.region, - tls: S3_STORAGE_CONFIG_DEFAULTS.sslEnabled, - }) - ); + describe('#deleteResources', () => { + const PATH = 'layer/v1'; + const NORMALIZED_PATH = `${PATH}/`; + + it('should return empty result when input paths is empty', async () => { + const result = await provider.deleteResources({ paths: [], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(0); + }); + + it('should return empty result when listing returns no keys', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should not double-add trailing slash when prefix already ends with one', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); + + await provider.deleteResources({ paths: [`${PATH}/`], bucket: BUCKET, storageProvider: 'S3' }); + + expect(paginateListObjectsV2).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ Prefix: NORMALIZED_PATH })); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should list and delete a single object', async () => { + const path = 'layer/v1/0/0.png'; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [path], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should list and delete a single page of objects', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [] }); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should list and delete a single page of objects - no matching single object (bucket does not exists)', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NoSuchBucket({ $metadata: {}, message: 'no bucket' })) // no bucket found for single object + .mockResolvedValueOnce({ Errors: [] }); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should list and delete multiple pages of objects', async () => { + const page1Keys = ['layer/v1/0/0.png']; + const page2Keys = ['layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [] }) // delete page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(4); + }); + + it('should list and delete multiple pages of objects skipping deletion of empty keys', async () => { + const page1Keys = ['layer/v1/0/0.png']; + const page2Keys = ['layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map(() => ({ Key: undefined })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should list and delete while skipping a single object deletion of a similar key', async () => { + const objectKey = 'layer/v1/0/0.png'; + const page1Keys = [`${objectKey}8`]; + const page2Keys = [objectKey]; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [objectKey], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should list and delete multiple paths and their objects', async () => { + const PATH1 = 'layer/v1'; + const PATH2 = 'layer/v2'; + const path1Page1Keys = ['layer/v1/0/0.png']; + const path1Page2Keys = ['layer/v1/0/1.png']; + const path2Page1Keys = ['layer/v2/0/0.png']; + const path2Page2Keys = ['layer/v2/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object for path 1 + .mockResolvedValueOnce({ Errors: [] }) // delete path 1 page 1 + .mockResolvedValueOnce({ Errors: [] }) // delete path 1 page 2 + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object for path 2 + .mockResolvedValueOnce({ Errors: [] }) // delete path 2 page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete path 2 page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: path1Page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: path1Page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }) + .mockResolvedValueOnce({ done: false, value: { Contents: path2Page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: path2Page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH1, PATH2], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(6); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(7); + }); + + it('should list and delete a single object and matching paths and their objects', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockResolvedValueOnce(undefined) // single object exists + .mockResolvedValueOnce({ Errors: [] }) // delete single object + .mockResolvedValueOnce({ Errors: [] }); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(4); + }); + + it('should handle empty page Contents response', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: false, value: {} }).mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should handle a page that reports its KeyCount', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [] }); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { KeyCount: keys.length, Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(DeleteObjectsCommand).toHaveBeenCalledWith({ Bucket: BUCKET, Delete: { Objects: keys.map((Key) => ({ Key })) } }); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should skip an entire page whose keys all belong to a sibling sharing the prefix', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v10/0/0.png' }, { Key: 'layer/v1x/0/0.png' }] } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(DeleteObjectsCommand).not.toHaveBeenCalled(); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should accumulate failures of the same reason across pages keeping the first sample', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v1/0/0.png', Code: 'AccessDenied' }] }) // delete page 1 + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v1/0/1.png', Code: 'AccessDenied' }] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v1/0/0.png' }] } }) + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v1/0/1.png' }] } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 2, sample: 'layer/v1/0/0.png' }]]) }); + }); + + it('should accumulate failures of the same reason across paths keeping the first sample', async () => { + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for path 1 + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v1/0/0.png', Code: 'AccessDenied' }] }) // delete path 1 page 1 + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for path 2 + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v2/0/0.png', Code: 'AccessDenied' }] }); // delete path 2 page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v1/0/0.png' }] } }) + .mockResolvedValueOnce({ done: true, value: undefined }) + .mockResolvedValueOnce({ done: false, value: { Contents: [{ Key: 'layer/v2/0/0.png' }] } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: ['layer/v1', 'layer/v2'], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 2, sample: 'layer/v1/0/0.png' }]]) }); + }); + + it('should request pages sized by the configured batch size', async () => { + provider = new S3StorageProvider(createS3StorageConfig({ batchSize: 500 }), mockLogger); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object + mockPaginateListObjectsV2Next.mockResolvedValueOnce({ done: true, value: undefined }); + + await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(paginateListObjectsV2).toHaveBeenCalledWith(expect.objectContaining({ pageSize: 500 }), expect.anything()); + }); + + it('should handle empty delete page Errors response', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ $metadata: {} } satisfies DeleteObjectsCommandOutput); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should handle delete page Errors response without elements', async () => { + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ $metadata: {}, Errors: [] } satisfies DeleteObjectsCommandOutput); // delete page 1 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map() }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it('should throw UnrecoverableError when bucket does not exist', async () => { + mockSend.mockRejectedValueOnce(new NotFound({ $metadata: {}, message: '' })); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(mockSend).toHaveBeenCalledTimes(1); + }); + + it('should throw an error when storage existence check failing', async () => { + const expectedError = new Error('error'); + mockSend.mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockSend).toHaveBeenCalledTimes(1); + }); + + it('should throw UnrecoverableError when a path is empty (root deletion guard)', async () => { + const result = provider.deleteResources({ paths: [''], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(paginateListObjectsV2).not.toHaveBeenCalled(); + expect(mockSend).toHaveBeenCalledTimes(0); + }); + + it('should throw UnrecoverableError when any path is empty even if others are valid', async () => { + const result = provider.deleteResources({ paths: [PATH, ''], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(mockSend).toHaveBeenCalledTimes(0); + }); + + it('should throw an error when checking for matching single object is failing (generic error)', async () => { + const expectedError = new Error('NetworkError'); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(expectedError); // error thrown for single object lookup + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should paginate and return all delete errors returned by S3 per object', async () => { + const page1Keys = ['layer/v1/0/0.png']; + const page2Keys = ['layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockResolvedValueOnce({ Errors: [{ Key: 'layer/v1/0/0.png', Code: 'AccessDenied' }] }) // delete page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map([['AccessDenied', { count: 1, sample: 'layer/v1/0/0.png' }]]) }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(4); + }); + + it('should paginate and return all delete errors thrown and unhandled by S3', async () => { + const page1Keys = ['layer/v1/0/0.png']; + const page2Keys = ['layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })) // no listing found for single object + .mockRejectedValueOnce(new Error('NetworkError')) // delete page 1 + .mockResolvedValueOnce({ Errors: [] }); // delete page 2 + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: page1Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: false, value: { Contents: page2Keys.map((Key) => ({ Key })) } }) + .mockResolvedValueOnce({ done: true, value: undefined }); + + const result = await provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + expect(result).toEqual({ failures: new Map([['NetworkError', { count: 1, sample: 'layer/v1/0/0.png' }]]) }); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(3); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(4); + }); + + it('should stop pagination and throw an error when bucket does not exist', async () => { + const expectedError = new NoSuchBucket({ $metadata: { requestId: faker.string.uuid() }, message: 'msg' }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object + mockPaginateListObjectsV2Next.mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: '', storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should stop pagination and throw an error when S3 throws a service error', async () => { + const expectedError = new S3ServiceException({ $fault: 'server', $metadata: { requestId: faker.string.uuid() }, name: 'msg' }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object + mockPaginateListObjectsV2Next.mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should stop pagination and throw an error when S3 throws a bucket does not exists error', async () => { + const expectedError = new NoSuchBucket({ $metadata: {}, message: '' }); + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object + mockPaginateListObjectsV2Next.mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(1); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it('should stop pagination and return failure when list call throws', async () => { + const expectedError = new Error('NetworkError'); + const keys = ['layer/v1/0/0.png', 'layer/v1/0/1.png']; + mockSend + .mockResolvedValueOnce(undefined) // bucket exists + .mockRejectedValueOnce(new NotFound({ $metadata: {}, message: 'not found' })); // no listing found for single object + mockPaginateListObjectsV2Next + .mockResolvedValueOnce({ done: false, value: { Contents: keys.map((Key) => ({ Key })) } }) + .mockRejectedValueOnce(expectedError); + + const result = provider.deleteResources({ paths: [PATH], bucket: BUCKET, storageProvider: 'S3' }); + + await expect(result).rejects.toThrow(expectedError); + expect(mockPaginateListObjectsV2Next).toHaveBeenCalledTimes(2); + expect(mockPaginateListObjectsV2Return).toHaveBeenCalledTimes(0); + expect(mockPaginateListObjectsV2Throw).toHaveBeenCalledTimes(0); + expect(mockSend).toHaveBeenCalledTimes(3); }); }); }); diff --git a/tests/storageProviders/storageConfig.spec.ts b/tests/storageProviders/storageConfig.spec.ts new file mode 100644 index 0000000..a427b7e --- /dev/null +++ b/tests/storageProviders/storageConfig.spec.ts @@ -0,0 +1,136 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { faker } from '@faker-js/faker'; +import { ConfigurationError } from '@src/cleaner/errors'; +import { buildFsStorageConfig, buildS3StorageConfig, type FsConfig } from '@src/cleaner/storageProviders/storageConfig'; +import { assertCanDeleteFromFolder } from '@src/cleaner/utils/fs'; +import type { ConfigType } from '@src/common/config'; +import { + createFsStorageConfig, + createMockFsConfig, + createMockLogger, + createMockS3Config, + createS3StorageConfig, + FS_STORAGE_CONFIG_DEFAULTS, +} from '../helpers/mocks'; + +vi.mock('@src/cleaner/utils/fs', () => ({ + assertCanDeleteFromFolder: vi.fn(), +})); + +describe('storageConfig', () => { + let mockLogger: Logger; + + beforeEach(() => { + vi.clearAllMocks(); + mockLogger = createMockLogger(); + }); + + describe('#buildFsStorageConfig', () => { + let mockConfig: ConfigType; + + beforeEach(() => { + mockConfig = createMockFsConfig(); + }); + + it('should return the validated config', () => { + const result = buildFsStorageConfig(mockConfig, mockLogger); + + expect(result).toEqual(createFsStorageConfig()); + }); + + it('should assert the resolved base path is deletable from', () => { + buildFsStorageConfig(mockConfig, mockLogger); + + expect(assertCanDeleteFromFolder).toHaveBeenCalledWith(FS_STORAGE_CONFIG_DEFAULTS.basePath, mockLogger); + }); + + it('should flatten every configured subPath into the returned list', () => { + const config = createMockFsConfig({ subPaths: { tiles: 'artifacts/tiles', gpkg: 'artifacts/gpkg' } satisfies FsConfig['subPaths'] }); + + const result = buildFsStorageConfig(config, mockLogger); + + expect(result.subPaths).toEqual(['artifacts/tiles', 'artifacts/gpkg']); + }); + + it('should resolve a relative base path (without a leading separator) to an absolute path', () => { + const config = createMockFsConfig({ basePath: 'relative/tiles' } satisfies Pick); + + const result = buildFsStorageConfig(config, mockLogger); + + expect(result.basePath).toBe('/relative/tiles'); + // the assertion must run against the resolved path, not the raw configured one + expect(assertCanDeleteFromFolder).toHaveBeenCalledWith('/relative/tiles', mockLogger); + }); + + it('should normalize a base path containing traversal segments before asserting it', () => { + const config = createMockFsConfig({ basePath: '/test/tiles/../tiles' } satisfies Pick); + + const result = buildFsStorageConfig(config, mockLogger); + + expect(result.basePath).toBe('/test/tiles'); + expect(assertCanDeleteFromFolder).toHaveBeenCalledWith('/test/tiles', mockLogger); + }); + + it('should propagate a ConfigurationError raised by the base path assertion', () => { + const expectedError = new ConfigurationError('FS path does not exist: /test'); + vi.mocked(assertCanDeleteFromFolder).mockImplementation(() => { + throw expectedError; + }); + + expect(() => buildFsStorageConfig(mockConfig, mockLogger)).toThrow(expectedError); + }); + + it('should throw ConfigurationError when no subPaths are configured', () => { + const config = createMockFsConfig({ subPaths: {} satisfies FsConfig['subPaths'] }); + + expect(() => buildFsStorageConfig(config, mockLogger)).toThrow(ConfigurationError); + }); + + it('should throw ConfigurationError when batchSize is less than or equal to 0', () => { + const config = createMockFsConfig({ delete: { batchSize: faker.number.int({ max: 0, min: -Number.MAX_SAFE_INTEGER }) } }); + + expect(() => buildFsStorageConfig(config, mockLogger)).toThrow(ConfigurationError); + }); + }); + + describe('#buildS3StorageConfig', () => { + it('should return the validated config', () => { + const config = createMockS3Config(); + + const result = buildS3StorageConfig(config, mockLogger); + + expect(result).toStrictEqual(createS3StorageConfig()); + }); + + it('should keep a configured batchSize that is below the S3 max keys limit', () => { + const config = createMockS3Config({ delete: { batchSize: 250 } }); + + const result = buildS3StorageConfig(config, mockLogger); + + expect(result.batchSize).toBe(250); + }); + + it('should clamp a configured batchSize above the S3 max keys limit to 1000', () => { + const config = createMockS3Config({ delete: { batchSize: 2000 } }); + + const result = buildS3StorageConfig(config, mockLogger); + + expect(result.batchSize).toBe(1000); + }); + + it('should keep a configured batchSize that is exactly the S3 max keys limit', () => { + const config = createMockS3Config({ delete: { batchSize: 1000 } }); + + const result = buildS3StorageConfig(config, mockLogger); + + expect(result.batchSize).toBe(1000); + }); + + it('should throw ConfigurationError when batchSize is less than or equal to 0', () => { + const config = createMockS3Config({ delete: { batchSize: faker.number.int({ max: 0, min: -Number.MAX_SAFE_INTEGER }) } }); + + expect(() => buildS3StorageConfig(config, mockLogger)).toThrow(ConfigurationError); + }); + }); +}); diff --git a/tests/strategies/deleteStoredResourcesStrategy.spec.ts b/tests/strategies/deleteStoredResourcesStrategy.spec.ts new file mode 100644 index 0000000..1ff0a63 --- /dev/null +++ b/tests/strategies/deleteStoredResourcesStrategy.spec.ts @@ -0,0 +1,166 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { SourceType, type DeleteStoredResourcesParams } from '@map-colonies/raster-shared'; +import { beforeEach, describe, expect, it, type vi } from 'vitest'; +import { RecoverableError, UnrecoverableError, ValidationError } from '@src/cleaner/errors'; +import type { IStorageProvider, StorageProviders } from '@src/cleaner/storageProviders'; +import { DeleteStoredResourcesStrategy } from '@src/cleaner/strategies/deleteStoredResourcesStrategy'; +import type { ConfigType } from '@src/common/config'; +import { createMockStoredResourcesDeletionStrategyConfig, createMockLogger, createMockStorageProvider } from '../helpers/mocks'; + +const S3_BUCKET = 'test-bucket'; +const FS_SUB_PATH = 'test/artifacts/tiles'; + +const s3Params: DeleteStoredResourcesParams = { storageProvider: SourceType.S3, paths: ['layer1'], bucket: S3_BUCKET }; +const fsParams: DeleteStoredResourcesParams = { storageProvider: SourceType.FS, paths: ['layer2'], subPath: FS_SUB_PATH }; + +describe('DeleteStoredResourcesStrategy', () => { + let strategy: DeleteStoredResourcesStrategy; + // eslint-disable-next-line @typescript-eslint/naming-convention + let mockS3Provider: IStorageProvider<'S3'>; + // eslint-disable-next-line @typescript-eslint/naming-convention + let mockFsProvider: IStorageProvider<'FS'>; + let mockLogger: Logger; + let mockConfig: ConfigType; + + beforeEach(() => { + mockS3Provider = createMockStorageProvider(); + mockFsProvider = createMockStorageProvider(); + mockLogger = createMockLogger(); + mockConfig = createMockStoredResourcesDeletionStrategyConfig(); + + const storageProviders: StorageProviders = { + [SourceType.FS]: mockFsProvider, + [SourceType.S3]: mockS3Provider, + }; + + strategy = new DeleteStoredResourcesStrategy(mockLogger, mockConfig, storageProviders); + }); + + describe('#validate', () => { + it('should validate and return S3 params', () => { + const result = strategy.validate(s3Params); + + expect(result).toEqual(s3Params); + }); + + it('should validate and return FS params', () => { + const result = strategy.validate(fsParams); + + expect(result).toEqual(fsParams); + }); + + it('should throw ValidationError when storageProvider is missing', () => { + expect(() => strategy.validate({ catalogId: 'layer1' })).toThrow(ValidationError); + }); + + it('should throw ValidationError when storageProvider is unknown', () => { + expect(() => strategy.validate({ storageProvider: 'GCS', catalogId: 'layer1' })).toThrow(ValidationError); + }); + + it('should throw ValidationError when tilesPath is empty string', () => { + expect(() => strategy.validate({ storageProvider: SourceType.S3, catalogId: '' })).toThrow(ValidationError); + }); + + it('should throw ValidationError when tilesPath is missing', () => { + expect(() => strategy.validate({ storageProvider: SourceType.S3 })).toThrow(ValidationError); + }); + + it('should throw ValidationError for null params', () => { + expect(() => strategy.validate(null)).toThrow(ValidationError); + }); + }); + + describe('#execute', () => { + it('should call delete all resources on S3 provider', async () => { + await strategy.execute(s3Params); + + expect(mockS3Provider.deleteResources).toHaveBeenCalledWith({ paths: s3Params.paths, bucket: S3_BUCKET, storageProvider: 'S3' }); + expect(mockFsProvider.deleteResources).not.toHaveBeenCalled(); + }); + + it('should call delete all resources on FS provider', async () => { + await strategy.execute(fsParams); + + expect(mockFsProvider.deleteResources).toHaveBeenCalledWith({ paths: fsParams.paths, subPath: FS_SUB_PATH, storageProvider: 'FS' }); + expect(mockS3Provider.deleteResources).not.toHaveBeenCalled(); + }); + + it('should resolve without throwing when deleteResources returns no failures', async () => { + (mockS3Provider.deleteResources as ReturnType).mockResolvedValueOnce({ failures: [] }); + + const result = strategy.execute(s3Params); + + await expect(result).resolves.toBeUndefined(); + }); + + it('should throw UnrecoverableError for unknown provider', async () => { + const unknownParams = { ...s3Params, storageProvider: 'UNKNOWN' } as unknown as DeleteStoredResourcesParams; + + const result = strategy.execute(unknownParams); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(mockS3Provider.deleteResources).not.toHaveBeenCalled(); + }); + + it('should throw UnrecoverableError when the provider is registered but resolves to undefined', async () => { + const storageProviders = { + [SourceType.FS]: mockFsProvider, + [SourceType.S3]: undefined, + } as unknown as StorageProviders; + strategy = new DeleteStoredResourcesStrategy(mockLogger, mockConfig, storageProviders); + + const result = strategy.execute(s3Params); + + await expect(result).rejects.toThrow(UnrecoverableError); + expect(mockS3Provider.deleteResources).not.toHaveBeenCalled(); + }); + + it('should throw RecoverableError when deleteResources returns failures', async () => { + (mockS3Provider.deleteResources as ReturnType).mockResolvedValueOnce({ + failures: new Map([['AccessDenied', { count: 1, sample: 'layer1/0/0.png' }]]), + }); + + const result = strategy.execute(s3Params); + + await expect(result).rejects.toThrow(RecoverableError); + }); + + it('should include the failures count, grouped reasons and samples in the RecoverableError message', async () => { + (mockS3Provider.deleteResources as ReturnType).mockResolvedValueOnce({ + failures: new Map([ + ['AccessDenied', { count: 1, sample: 'layer1/0/0.png' }], + ['InternalError', { count: 4, sample: 'layer1/0/1.png' }], + ]), + }); + + // reasons and samples are ordered by descending count + await expect(strategy.execute(s3Params)).rejects.toThrow( + 'Failed to delete 5 objects. Reasons: InternalError=4, AccessDenied=1. Samples: layer1/0/1.png (InternalError), layer1/0/0.png (AccessDenied)' + ); + }); + + it('should pass every path through to the provider', async () => { + const params: DeleteStoredResourcesParams = { ...s3Params, paths: ['layer1', 'layer2', 'layer3'] }; + + await strategy.execute(params); + + expect(mockS3Provider.deleteResources).toHaveBeenCalledWith({ paths: params.paths, bucket: S3_BUCKET, storageProvider: 'S3' }); + }); + + it('should resolve without throwing for an empty paths list', async () => { + const result = strategy.execute({ ...s3Params, paths: [] }); + + await expect(result).resolves.toBeUndefined(); + expect(mockS3Provider.deleteResources).toHaveBeenCalledWith({ paths: [], bucket: S3_BUCKET, storageProvider: 'S3' }); + }); + + it('should rethrow error thrown by deleteResources', async () => { + const expectedError = new Error('Custom'); + (mockS3Provider.deleteResources as ReturnType).mockRejectedValueOnce(expectedError); + + const result = strategy.execute(s3Params); + + await expect(result).rejects.toThrow(expectedError); + }); + }); +}); diff --git a/tests/tilesDeletionStrategy.spec.ts b/tests/strategies/tilesDeletionStrategy.spec.ts similarity index 57% rename from tests/tilesDeletionStrategy.spec.ts rename to tests/strategies/tilesDeletionStrategy.spec.ts index 7669a80..d6dfc10 100644 --- a/tests/tilesDeletionStrategy.spec.ts +++ b/tests/strategies/tilesDeletionStrategy.spec.ts @@ -1,14 +1,15 @@ -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { type TilesDeletionParams, SourceType } from '@map-colonies/raster-shared'; -import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; +import { join } from 'node:path'; import { faker } from '@faker-js/faker'; -import { TilesDeletionStrategy } from '@src/cleaner/strategies/tilesDeletionStrategy'; +import type { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; +import { type TilesDeletionParams, SourceType } from '@map-colonies/raster-shared'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { RecoverableError, UnrecoverableError, ValidationError } from '@src/cleaner/errors'; +import type { IStorageProvider, StorageProviders } from '@src/cleaner/storageProviders'; import type { TaskContext } from '@src/cleaner/strategies/strategyFactory'; -import { ValidationError, RecoverableError, UnrecoverableError } from '@src/cleaner/errors'; -import type { IStorageProvider } from '@src/cleaner/storageProviders'; -import { createMockLogger, createMockStorageProvider, createMockStrategyConfig, TILES_DELETION_CONFIG_DEFAULTS } from './helpers/mocks'; +import { TilesDeletionStrategy } from '@src/cleaner/strategies/tilesDeletionStrategy'; +import { createMockLogger, createMockStorageProvider, createMockStrategyConfig, TILES_DELETION_CONFIG_DEFAULTS } from '../helpers/mocks'; -const { s3Bucket: S3_BUCKET, fsBasePath: FS_BASE_PATH } = TILES_DELETION_CONFIG_DEFAULTS; +const { s3Bucket: S3_BUCKET, fsBasePath: FS_BASE_PATH, fsSubPath: FS_SUB_PATH } = TILES_DELETION_CONFIG_DEFAULTS; const JOB_ID = faker.string.uuid(); const TASK_ID = faker.string.uuid(); @@ -28,8 +29,8 @@ const tilePath = (z: number, x: number, y: number): string => `${s3Params.tilesP describe('TilesDeletionStrategy', () => { let strategy: TilesDeletionStrategy; - let MockS3Provider: IStorageProvider; - let MockFsProvider: IStorageProvider; + let MockS3Provider: IStorageProvider<'S3'>; + let MockFsProvider: IStorageProvider<'FS'>; let mockUpdateProgress: ReturnType; beforeEach(() => { @@ -37,16 +38,16 @@ describe('TilesDeletionStrategy', () => { MockFsProvider = createMockStorageProvider(); mockUpdateProgress = vi.fn().mockResolvedValue(undefined); - const storageProviders = new Map([ - [SourceType.S3, MockS3Provider], - [SourceType.FS, MockFsProvider], - ]); + const storageProviders: StorageProviders = { + [SourceType.FS]: MockFsProvider, + [SourceType.S3]: MockS3Provider, + }; const queueClient = { updateProgress: mockUpdateProgress } as unknown as QueueClient; strategy = new TilesDeletionStrategy(createMockLogger(), createMockStrategyConfig(), storageProviders, queueClient, TASK_CONTEXT); }); - describe('validate', () => { + describe('#validate', () => { it('should validate and return S3 params', () => { expect(strategy.validate(s3Params)).toEqual(s3Params); }); @@ -89,7 +90,7 @@ describe('TilesDeletionStrategy', () => { }); }); - describe('execute', () => { + describe('#execute', () => { describe('target validation', () => { it('should throw UnrecoverableError when S3 storage target does not exist', async () => { vi.mocked(MockS3Provider.targetExists).mockResolvedValue(false); @@ -114,7 +115,15 @@ describe('TilesDeletionStrategy', () => { it('should check targetExists with FS base path and tilesPath as relativePath', async () => { await strategy.execute(fsParams); - expect(MockFsProvider.targetExists).toHaveBeenCalledWith(FS_BASE_PATH, fsParams.tilesPath); + expect(MockFsProvider.targetExists).toHaveBeenCalledWith(join(FS_BASE_PATH, FS_SUB_PATH), fsParams.tilesPath); + }); + + it('should propagate an error thrown by the target existence check', async () => { + const expectedError = new Error('EACCES'); + vi.mocked(MockS3Provider.targetExists).mockRejectedValue(expectedError); + + await expect(strategy.execute(s3Params)).rejects.toThrow(expectedError); + expect(MockS3Provider.delete).not.toHaveBeenCalled(); }); }); @@ -129,7 +138,7 @@ describe('TilesDeletionStrategy', () => { it('should call FS provider with fsBasePath as storage target', async () => { await strategy.execute(fsParams); - expect(MockFsProvider.delete).toHaveBeenCalledWith(expect.any(Array), FS_BASE_PATH); + expect(MockFsProvider.delete).toHaveBeenCalledWith(expect.any(Array), join(FS_BASE_PATH, FS_SUB_PATH)); expect(MockS3Provider.delete).not.toHaveBeenCalled(); }); @@ -138,6 +147,24 @@ describe('TilesDeletionStrategy', () => { await expect(strategy.execute(unknownParams)).rejects.toThrow(UnrecoverableError); }); + + it('should throw UnrecoverableError when the provider is registered but resolves to undefined', async () => { + const storageProviders = { + [SourceType.FS]: MockFsProvider, + [SourceType.S3]: undefined, + } satisfies StorageProviders; + strategy = new TilesDeletionStrategy( + createMockLogger(), + createMockStrategyConfig(), + storageProviders, + { updateProgress: mockUpdateProgress } as unknown as QueueClient, + TASK_CONTEXT + ); + + await expect(strategy.execute(s3Params)).rejects.toThrow(UnrecoverableError); + expect(MockS3Provider.targetExists).not.toHaveBeenCalled(); + expect(MockS3Provider.delete).not.toHaveBeenCalled(); + }); }); describe('tile path generation', () => { @@ -201,6 +228,52 @@ describe('TilesDeletionStrategy', () => { expect(mockUpdateProgress).not.toHaveBeenCalledWith(JOB_ID, TASK_ID, 100); }); + it('should report the percentage of tiles processed so far', async () => { + // batchSize=100, concurrency=2 → flush + progress report after the first 200 of 210 tiles + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 13, minY: 0, maxY: 14 }], + }; + + await strategy.execute(params); + + expect(mockUpdateProgress).toHaveBeenCalledWith(JOB_ID, TASK_ID, Math.round((200 / 210) * 100)); + }); + + it('should report progress once per completed concurrency window', async () => { + // 420 tiles → two full windows of 200, then a trailing batch of 20 + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 20, minY: 0, maxY: 19 }], + }; + + await strategy.execute(params); + + expect(mockUpdateProgress).toHaveBeenCalledTimes(2); + expect(mockUpdateProgress).toHaveBeenNthCalledWith(1, JOB_ID, TASK_ID, Math.round((200 / 420) * 100)); + expect(mockUpdateProgress).toHaveBeenNthCalledWith(2, JOB_ID, TASK_ID, Math.round((400 / 420) * 100)); + }); + + it('should not report progress for a tile set smaller than one concurrency window', async () => { + await strategy.execute(s3Params); + + expect(mockUpdateProgress).not.toHaveBeenCalled(); + }); + + it('should not flush an empty trailing batch when the tile count divides evenly', async () => { + // 200 tiles = exactly batchSize (100) × concurrency (2) → one window, no remainder + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 9, minY: 0, maxY: 19 }], + }; + + await strategy.execute(params); + + expect(MockS3Provider.delete).toHaveBeenCalledTimes(2); + expect(mockUpdateProgress).toHaveBeenCalledTimes(1); + expect(mockUpdateProgress).toHaveBeenCalledWith(JOB_ID, TASK_ID, 100); + }); + it('should pass the correct jobId and taskId on mid-stream updates', async () => { const params: TilesDeletionParams = { ...s3Params, @@ -213,7 +286,7 @@ describe('TilesDeletionStrategy', () => { }); it('should not call updateProgress when retryable failures occur', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'AccessDenied' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['AccessDenied', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).rejects.toThrow(RecoverableError); @@ -221,7 +294,7 @@ describe('TilesDeletionStrategy', () => { }); it('should not call updateProgress when only not-found failures occur', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'NoSuchKey' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['NoSuchKey', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).resolves.toBeUndefined(); expect(mockUpdateProgress).not.toHaveBeenCalled(); @@ -230,62 +303,61 @@ describe('TilesDeletionStrategy', () => { describe('failure handling', () => { it('should throw RecoverableError when provider returns fatal failed paths', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'AccessDenied' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['AccessDenied', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).rejects.toThrow(RecoverableError); }); it('should include fatal failed count in RecoverableError message', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([ - { path: tilePath(10, 0, 0), reason: 'AccessDenied' }, - { path: tilePath(10, 0, 1), reason: 'AccessDenied' }, - ]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['AccessDenied', { count: 2, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).rejects.toThrow(/Failed to delete 2/); }); it('should include grouped reason counts in RecoverableError message', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([ - { path: tilePath(10, 0, 0), reason: 'EACCES' }, - { path: tilePath(10, 0, 1), reason: 'EACCES' }, - { path: tilePath(10, 1, 0), reason: 'AccessDenied' }, - ]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ + failures: new Map([ + ['EACCES', { count: 2, sample: tilePath(10, 0, 0) }], + ['AccessDenied', { count: 1, sample: tilePath(10, 1, 0) }], + ]), + }); await expect(strategy.execute(s3Params)).rejects.toThrow(/Reasons: EACCES=2, AccessDenied=1/); }); it('should exclude not-found reasons from the RecoverableError reason summary', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([ - { path: tilePath(10, 0, 0), reason: 'NoSuchKey' }, - { path: tilePath(10, 0, 1), reason: 'AccessDenied' }, - ]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ + failures: new Map([ + ['NoSuchKey', { count: 1, sample: tilePath(10, 0, 0) }], + ['AccessDenied', { count: 1, sample: tilePath(10, 0, 1) }], + ]), + }); await expect(strategy.execute(s3Params)).rejects.toThrow(/Failed to delete 1.*Reasons: AccessDenied=1/); }); it('should include path and reason in the failure sample', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'AccessDenied' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['AccessDenied', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).rejects.toThrow(/layer\/v1\/10\/0\/0\.png \(AccessDenied\)/); }); it('should resolve successfully when all failures are not-found (ENOENT)', async () => { - vi.mocked(MockFsProvider.delete).mockResolvedValue([ - { path: tilePath(10, 0, 0), reason: 'ENOENT' }, - { path: tilePath(10, 0, 1), reason: 'ENOENT' }, - ]); + vi.mocked(MockFsProvider.delete).mockResolvedValue({ + failures: new Map([['ENOENT', { count: 1, sample: tilePath(10, 0, 0) }]]), + }); await expect(strategy.execute(fsParams)).resolves.toBeUndefined(); }); it('should resolve successfully when all failures are not-found (NoSuchKey)', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([{ path: tilePath(10, 0, 0), reason: 'NoSuchKey' }]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map([['NoSuchKey', { count: 1, sample: tilePath(10, 0, 0) }]]) }); await expect(strategy.execute(s3Params)).resolves.toBeUndefined(); }); it('should resolve successfully when provider returns no failed paths', async () => { - vi.mocked(MockS3Provider.delete).mockResolvedValue([]); + vi.mocked(MockS3Provider.delete).mockResolvedValue({ failures: new Map() }); await expect(strategy.execute(s3Params)).resolves.toBeUndefined(); }); @@ -308,6 +380,55 @@ describe('TilesDeletionStrategy', () => { await expect(strategy.execute(s3Params)).rejects.toThrow(/S3 connection lost/); }); + + it('should aggregate hard failures of the same reason across concurrent batches', async () => { + // 200 tiles → two batches of 100, both rejecting with the same error + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 9, minY: 0, maxY: 19 }], + }; + vi.mocked(MockS3Provider.delete).mockRejectedValue(new Error('S3 connection lost')); + + // count is the sum of both batches, sample comes from the first batch that failed + await expect(strategy.execute(params)).rejects.toThrow( + `Failed to delete 200 tiles. Reasons: S3 connection lost=200. Samples: ${tilePath(5, 0, 0)} (S3 connection lost)` + ); + }); + + it('should aggregate hard failures of different reasons across concurrent batches', async () => { + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 9, minY: 0, maxY: 19 }], + }; + vi.mocked(MockS3Provider.delete).mockRejectedValueOnce(new Error('first down')).mockRejectedValueOnce(new Error('second down')); + + await expect(strategy.execute(params)).rejects.toThrow(/Reasons: first down=100, second down=100/); + }); + + it('should surface both soft failures and hard rejections from the same flush', async () => { + const params: TilesDeletionParams = { + ...s3Params, + ranges: [{ zoom: 5, minX: 0, maxX: 9, minY: 0, maxY: 19 }], + }; + vi.mocked(MockS3Provider.delete) + .mockResolvedValueOnce({ failures: new Map([['AccessDenied', { count: 3, sample: tilePath(5, 0, 0) }]]) }) + .mockRejectedValueOnce(new Error('S3 connection lost')); + + // reasons are ordered by descending count + await expect(strategy.execute(params)).rejects.toThrow(/Failed to delete 103 tiles\. Reasons: S3 connection lost=100, AccessDenied=3/); + }); + + it('should tag hard-rejected batches with the errno code when the thrown error carries one', async () => { + vi.mocked(MockFsProvider.delete).mockRejectedValue(Object.assign(new Error('permission denied'), { code: 'EACCES' })); + + await expect(strategy.execute(fsParams)).rejects.toThrow(/Reasons: EACCES=4/); + }); + + it('should treat a hard-rejected batch tagged ENOENT as missing tiles rather than a retryable failure', async () => { + vi.mocked(MockFsProvider.delete).mockRejectedValue(Object.assign(new Error('no such file'), { code: 'ENOENT' })); + + await expect(strategy.execute(fsParams)).resolves.toBeUndefined(); + }); }); }); }); diff --git a/tests/strategyFactory.spec.ts b/tests/strategyFactory.spec.ts index 2daa995..027a1f1 100644 --- a/tests/strategyFactory.spec.ts +++ b/tests/strategyFactory.spec.ts @@ -1,11 +1,13 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { container } from 'tsyringe'; import { faker } from '@faker-js/faker'; import type { Logger } from '@map-colonies/js-logger'; -import { SERVICES } from '../src/common/constants'; -import { StrategyFactory, TilesDeletionStrategy, type ITaskStrategy, type TaskContext } from '../src/cleaner/strategies'; +import { SourceType } from '@map-colonies/raster-shared'; +import { container } from 'tsyringe'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { IStorageProvider, StorageProviders } from '@src/cleaner/storageProviders'; import { StrategyNotFoundError } from '../src/cleaner/errors'; -import { createMockLogger, createMockConfig, createMockQueueClient } from './helpers/mocks'; +import { StrategyFactory, TilesDeletionStrategy, type ITaskStrategy, type TaskContext } from '../src/cleaner/strategies'; +import { SERVICES } from '../src/common/constants'; +import { createMockConfig, createMockLogger, createMockQueueClient, createMockStorageProvider } from './helpers/mocks'; class MockStrategy implements ITaskStrategy { public validate(params: unknown): Record { @@ -20,13 +22,25 @@ class MockStrategy implements ITaskStrategy { describe('StrategyFactory', () => { let strategyFactory: StrategyFactory; let mockLogger: Logger; + // eslint-disable-next-line @typescript-eslint/naming-convention + let mockS3Provider: IStorageProvider<'S3'>; + // eslint-disable-next-line @typescript-eslint/naming-convention + let mockFsProvider: IStorageProvider<'FS'>; beforeEach(() => { mockLogger = createMockLogger(); + mockS3Provider = createMockStorageProvider(); + mockFsProvider = createMockStorageProvider(); + + const storageProviders: StorageProviders = { + [SourceType.FS]: mockFsProvider, + [SourceType.S3]: mockS3Provider, + }; + container.register(SERVICES.LOGGER, { useValue: mockLogger }); container.register(SERVICES.CONFIG, { useValue: createMockConfig() }); - container.register(SERVICES.STORAGE_PROVIDERS, { useValue: new Map() }); + container.register(SERVICES.STORAGE_PROVIDERS, { useValue: storageProviders }); container.register(SERVICES.QUEUE_CLIENT, { useValue: createMockQueueClient() }); strategyFactory = new StrategyFactory(mockLogger); @@ -37,15 +51,16 @@ describe('StrategyFactory', () => { container.clearInstances(); }); - describe('resolveWithContext', () => { + describe('#resolveWithContext', () => { it('should resolve registered strategy with enriched logger context', () => { + const jobType = 'Ingestion_Update'; const taskType = 'tiles-deletion'; - container.register(taskType, { useClass: TilesDeletionStrategy }); + container.register(`${jobType}-${taskType}`, { useClass: TilesDeletionStrategy }); const taskContext: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), - jobType: 'Ingestion_Update', + jobType, taskType, }; @@ -63,13 +78,14 @@ describe('StrategyFactory', () => { }); it('should create child logger with task context', () => { + const jobType = 'Ingestion_Swap_Update'; const taskType = 'tiles-deletion'; - container.register(taskType, { useClass: TilesDeletionStrategy }); + container.register(`${jobType}-${taskType}`, { useClass: TilesDeletionStrategy }); const taskContext: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), - jobType: 'Ingestion_Swap_Update', + jobType, taskType, }; @@ -96,11 +112,12 @@ describe('StrategyFactory', () => { }); it('should create separate instances for different tasks (child container isolation)', () => { + const jobType = 'Ingestion_Update'; const taskType = 'tiles-deletion'; - container.register(taskType, { useClass: TilesDeletionStrategy }); + container.register(`${jobType}-${taskType}`, { useClass: TilesDeletionStrategy }); - const context1: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType: 'Ingestion_Update', taskType }; - const context2: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType: 'Ingestion_Update', taskType }; + const context1: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType, taskType }; + const context2: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType, taskType }; const strategy1 = strategyFactory.resolveWithContext(context1); const strategy2 = strategyFactory.resolveWithContext(context2); @@ -109,14 +126,15 @@ describe('StrategyFactory', () => { }); it('should resolve different strategies for different task types', () => { + const jobType = 'Ingestion_Update'; const taskType1 = 'tiles-deletion'; const taskType2 = 'files-deletion'; - container.register(taskType1, { useClass: TilesDeletionStrategy }); - container.register(taskType2, { useClass: MockStrategy }); + container.register(`${jobType}-${taskType1}`, { useClass: TilesDeletionStrategy }); + container.register(`${jobType}-${taskType2}`, { useClass: MockStrategy }); - const context1: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType: 'Ingestion_Update', taskType: taskType1 }; - const context2: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType: 'Ingestion_Update', taskType: taskType2 }; + const context1: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType, taskType: taskType1 }; + const context2: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), jobType, taskType: taskType2 }; const strategy1 = strategyFactory.resolveWithContext(context1); const strategy2 = strategyFactory.resolveWithContext(context2); @@ -127,13 +145,14 @@ describe('StrategyFactory', () => { }); it('should handle special characters in task type', () => { + const jobType = 'CustomJob'; const taskType = 'task-with-special_chars.v2'; - container.register(taskType, { useClass: MockStrategy }); + container.register(`${jobType}-${taskType}`, { useClass: MockStrategy }); const taskContext: TaskContext = { jobId: faker.string.uuid(), taskId: faker.string.uuid(), - jobType: 'CustomJob', + jobType, taskType, }; diff --git a/tests/utils/fs.spec.ts b/tests/utils/fs.spec.ts new file mode 100644 index 0000000..0a39473 --- /dev/null +++ b/tests/utils/fs.spec.ts @@ -0,0 +1,97 @@ +import { accessSync, constants, statSync, type Stats } from 'node:fs'; +import type { Logger } from '@map-colonies/js-logger'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { ConfigurationError } from '@src/cleaner/errors'; +import { assertCanDeleteFromFolder } from '@src/cleaner/utils/fs'; +import { createMockLogger } from '../helpers/mocks'; + +vi.mock(import('node:fs'), async (importOriginal) => { + const originModule = await importOriginal(); + return { + ...originModule, + accessSync: vi.fn(), + statSync: vi.fn(), + }; +}); + +const PATH = '/test/tiles'; + +describe('fs', () => { + describe('#assertCanDeleteFromFolder', () => { + let mockLogger: Logger; + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(accessSync).mockReturnValue(undefined); + vi.mocked(statSync).mockReturnValue({ isDirectory: () => true } as Stats); + mockLogger = createMockLogger(); + }); + + it('should not throw when the path is an accessible directory', () => { + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).not.toThrow(); + }); + + it('should check the path for existence, read and write access', () => { + assertCanDeleteFromFolder(PATH, mockLogger); + + expect(accessSync).toHaveBeenCalledWith(PATH, constants.F_OK | constants.R_OK | constants.W_OK); + }); + + it('should check that the path is a directory', () => { + assertCanDeleteFromFolder(PATH, mockLogger); + + expect(statSync).toHaveBeenCalledWith(PATH); + }); + + it('should throw ConfigurationError when the path does not exist (ENOENT)', () => { + vi.mocked(accessSync).mockImplementation(() => { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + }); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow(new ConfigurationError(`FS path does not exist: ${PATH}`)); + }); + + it('should throw ConfigurationError when access is denied (EACCES)', () => { + vi.mocked(accessSync).mockImplementation(() => { + throw Object.assign(new Error('EACCES'), { code: 'EACCES' }); + }); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow(new ConfigurationError(`FS path permission denied for path: ${PATH}`)); + }); + + it('should throw ConfigurationError describing an unexpected accessibility error', () => { + vi.mocked(accessSync).mockImplementation(() => { + throw Object.assign(new Error('too many open files'), { code: 'EMFILE' }); + }); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow( + new ConfigurationError('An unexpected error occurred on FS path accessibility check: EMFILE') + ); + }); + + it('should throw ConfigurationError when the path exists but is a file', () => { + vi.mocked(statSync).mockReturnValue({ isDirectory: () => false } satisfies Partial as Stats); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow( + new ConfigurationError(`FS path exists but it is a file, not a directory: ${PATH}`) + ); + }); + + it('should throw ConfigurationError when the info check fails unexpectedly', () => { + // e.g. the directory is removed between the accessSync and statSync calls + vi.mocked(statSync).mockImplementation(() => { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + }); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).toThrow( + new ConfigurationError('An unexpected error occurred on FS info check: ENOENT') + ); + }); + + it('should not re-wrap the not-a-directory ConfigurationError as an info check failure', () => { + vi.mocked(statSync).mockReturnValue({ isDirectory: () => false } satisfies Partial as Stats); + + expect(() => assertCanDeleteFromFolder(PATH, mockLogger)).not.toThrow(/An unexpected error occurred on FS info check/); + }); + }); +});