diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index b4c3341..d31b3a3 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -51,3 +51,40 @@ jobs: with: name: Test Reporters ${{ matrix.node }} path: ./reports/** + + upload_package: + needs: [eslint, tests] + continue-on-error: true + runs-on: ubuntu-latest + steps: + - name: Check out TS Project Git repository + uses: actions/checkout@v6 + - name: Init nodejs + uses: ./.github/actions/init-npm + + - name: Pack the package + run: npm pack + - name: get properties + id: json_properties + uses: ActionsTools/read-json-action@main + with: + file_path: 'package.json' + - uses: azure/CLI@v2 + with: + inlineScript: | + az storage blob upload --name raster-shared-${{ github.sha }}.tgz --account-name ghatmpstorage --container-name=npm-packages --file map-colonies-raster-shared-${{steps.json_properties.outputs.version}}.tgz --sas-token "${{ secrets.SAS_TOKEN }}" + - name: Comment on PR + uses: thollander/actions-comment-pull-request@v3 + with: + message: | + Hi it is me, your friendly bot helper! :wave: + I've packed this commit for you :blush: + You can install it like this: + ```json + { + "dependencies": { + "@map-colonies/raster-shared": "https://ghatmpstorage.blob.core.windows.net/npm-packages/raster-shared-${{ github.sha }}.tgz" + } + } + ``` + The link will expire in a week :hourglass: \ No newline at end of file diff --git a/src/constants/core/constants.ts b/src/constants/core/constants.ts index e0296c1..5bbc3df 100644 --- a/src/constants/core/constants.ts +++ b/src/constants/core/constants.ts @@ -67,6 +67,15 @@ export const SourceType = { export type SourceType = (typeof SourceType)[keyof typeof SourceType]; +/* eslint-disable @typescript-eslint/naming-convention */ +export const StorageProvider = { + ...pickEnum(SourceType, ['FS', 'S3']), + REDIS: 'REDIS', +} as const; +/* eslint-enable @typescript-eslint/naming-convention */ + +export type StorageProvider = (typeof StorageProvider)[keyof typeof StorageProvider]; + /* eslint-disable @typescript-eslint/naming-convention */ export const InstanceType = { INGESTION: 'ingestion', diff --git a/src/constants/deletion/constants.ts b/src/constants/deletion/constants.ts index 67b29d3..ace0c9b 100644 --- a/src/constants/deletion/constants.ts +++ b/src/constants/deletion/constants.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ export const DeletionJobTypes = { Delete_Layer: 'Delete_Layer', + Cache_Deletion: 'Cache_Deletion', } as const; export type DeletionJobTypes = (typeof DeletionJobTypes)[keyof typeof DeletionJobTypes]; @@ -9,7 +10,7 @@ export const DeletionTaskTypes = { Delete: 'delete', LayerTilesDeletion: 'tiles-deletion', ArtifactsDeletion: 'artifacts-deletion', + CacheDeletion: 'cache-deletion', } as const; export type DeletionTaskTypes = (typeof DeletionTaskTypes)[keyof typeof DeletionTaskTypes]; -/* eslint-enable @typescript-eslint/naming-convention */ diff --git a/src/constants/export/constants.ts b/src/constants/export/constants.ts index 26c58b4..bedc881 100644 --- a/src/constants/export/constants.ts +++ b/src/constants/export/constants.ts @@ -1,5 +1,5 @@ import { ArtifactRasterType } from '@map-colonies/types'; -import { pickEnum } from '../../utils'; +import { pickEnum } from '../../utils/helpers.utils'; /* eslint-disable @typescript-eslint/naming-convention */ export const ExportArtifactType = pickEnum(ArtifactRasterType, ['GPKG', 'METADATA']); diff --git a/src/schemas/core/storage.schema.ts b/src/schemas/core/storage.schema.ts index 0b45ff1..43d5796 100644 --- a/src/schemas/core/storage.schema.ts +++ b/src/schemas/core/storage.schema.ts @@ -1,14 +1,34 @@ import { z } from 'zod'; -import { SourceType } from '../../constants/core/constants'; +import { StorageProvider } from '../../constants/core/constants'; -export const fsStorageSchema = z.object({ - storageProvider: z.literal(SourceType.FS), - subPath: z.string().min(1), +export const s3BucketSchema = z.object({ + bucket: z.string().min(1), }); -export const s3StorageSchema = z.object({ - storageProvider: z.literal(SourceType.S3), - bucket: z.string().min(1), +export const redisPrefixSchema = z.object({ + prefix: z.string().min(1), // Full Redis key prefix, e.g. `myLayer-redis_WorldCRS84` }); -export const storageSchema = z.discriminatedUnion('storageProvider', [fsStorageSchema, s3StorageSchema]); +export const fsSubPathSchema = z.object({ + subPath: z.string().min(1), +}); + +export const fsStorageSchema = z + .object({ + storageProvider: z.literal(StorageProvider.FS), + }) + .merge(fsSubPathSchema); + +export const s3StorageSchema = z + .object({ + storageProvider: z.literal(StorageProvider.S3), + }) + .merge(s3BucketSchema); + +export const redisStorageSchema = z + .object({ + storageProvider: z.literal(StorageProvider.REDIS), + }) + .merge(redisPrefixSchema); + +export const storageSchema = z.discriminatedUnion('storageProvider', [fsStorageSchema, s3StorageSchema, redisStorageSchema]); diff --git a/src/schemas/core/tile.schema.ts b/src/schemas/core/tile.schema.ts index 3e46911..f912c7b 100644 --- a/src/schemas/core/tile.schema.ts +++ b/src/schemas/core/tile.schema.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { MAX_ZOOM_LEVEL, SourceType, TileOutputFormat } from '../../constants'; +import { MAX_ZOOM_LEVEL, TileOutputFormat } from '../../constants/core/constants'; export const tileRangeSchema = z.object({ zoom: z.number().int().min(0).max(MAX_ZOOM_LEVEL), @@ -9,18 +9,14 @@ export const tileRangeSchema = z.object({ maxY: z.number().int().min(0), }); -export const tilesDeletionParamsBaseSchema = z.object({ - tilesPath: z.string().min(1), // Base path for the tiles to be deleted +export const tileRangesSchema = z.object({ ranges: z.array(tileRangeSchema).min(1), // Array of tile ranges to be deleted - fileExtension: z.literal(TileOutputFormat.PNG.toLowerCase()).or(z.literal(TileOutputFormat.JPEG.toLowerCase())), // File extension of the tiles (e.g., 'png', 'jpeg') }); -export const s3TilesDeletionParamsSchema = tilesDeletionParamsBaseSchema.extend({ - sourceProvider: z.literal(SourceType.S3), +/** Fields that only make sense for path-based tile stores. */ +export const tilesInfoSchema = z.object({ + tilesRelativePath: z.string().min(1), // Base path for the tiles to be deleted + fileExtension: z.literal(TileOutputFormat.PNG.toLowerCase()).or(z.literal(TileOutputFormat.JPEG.toLowerCase())), // e.g. 'png', 'jpeg' }); -export const fsTilesDeletionParamsSchema = tilesDeletionParamsBaseSchema.extend({ - sourceProvider: z.literal(SourceType.FS), -}); - -export const tilesDeletionParamsSchema = z.discriminatedUnion('sourceProvider', [s3TilesDeletionParamsSchema, fsTilesDeletionParamsSchema]); +export const tilePyramidSchema = tileRangesSchema.merge(tilesInfoSchema).describe('tilePyramidSchema'); diff --git a/src/schemas/deletion/task.schema.ts b/src/schemas/deletion/task.schema.ts index 27c89dc..f8bf01b 100644 --- a/src/schemas/deletion/task.schema.ts +++ b/src/schemas/deletion/task.schema.ts @@ -1,6 +1,16 @@ import { z } from 'zod'; -import { storageSchema } from '../core'; +import { fsStorageSchema, redisStorageSchema, s3StorageSchema } from '../core/storage.schema'; +import { tilePyramidSchema, tileRangesSchema } from '../core/tile.schema'; +/** + * Shared by every Redis deletion shape: the key prefix that locates the entries, plus an + * optional wait for a mapproxy reload to settle before the keys are removed. + */ +const redisDeletionBaseSchema = redisStorageSchema.extend({ + delaySeconds: z.number().int().nonnegative().optional(), +}); + +//#region DeleteTaskParams export const deleteTaskParamsSchema = z .object({ deleteFromCatalog: z.boolean().default(false), @@ -9,11 +19,39 @@ export const deleteTaskParamsSchema = z deletePolygonParts: z.boolean().default(false), }) .describe('deleteTaskParamsSchema'); +//#endregion DeleteTaskParams + +//#region TilesDeletionParams +export const s3TilesDeletionParamsSchema = s3StorageSchema.merge(tilePyramidSchema); + +export const fsTilesDeletionParamsSchema = fsStorageSchema.merge(tilePyramidSchema); + +export const redisTilesDeletionParamsSchema = redisDeletionBaseSchema.merge(tileRangesSchema).strict(); // Reject path-store fields outright: a prefix store has no tilesPath -export const deleteStoredResourcesParamsBaseSchema = z.object({ +export const tilesDeletionParamsSchema = z + .discriminatedUnion('storageProvider', [s3TilesDeletionParamsSchema, fsTilesDeletionParamsSchema, redisTilesDeletionParamsSchema]) + .describe('tilesDeletionParamsSchema'); +//#endregion TilesDeletionParams + +//#region DeleteStoredResourcesParams +/** Paths to delete recursively. */ +export const resourcePathsSchema = z.object({ paths: z.array(z.string().min(1)).min(1), }); -export const deleteStoredResourcesParamsSchema = deleteStoredResourcesParamsBaseSchema - .and(storageSchema) +export const s3DeleteStoredResourcesParamsSchema = s3StorageSchema.merge(resourcePathsSchema); + +export const fsDeleteStoredResourcesParamsSchema = fsStorageSchema.merge(resourcePathsSchema); + +// The prefix IS the locator for a key-value store, so `paths` is meaningless here +// and is rejected outright rather than silently stripped. +export const redisDeleteStoredResourcesParamsSchema = redisDeletionBaseSchema.strict(); + +export const deleteStoredResourcesParamsSchema = z + .discriminatedUnion('storageProvider', [ + s3DeleteStoredResourcesParamsSchema, + fsDeleteStoredResourcesParamsSchema, + redisDeleteStoredResourcesParamsSchema, + ]) .describe('deleteStoredResourcesParamsSchema'); +//#endregion DeleteStoredResourcesParams diff --git a/src/types/core/storage.type.ts b/src/types/core/storage.type.ts index ebb6bb6..577b49d 100644 --- a/src/types/core/storage.type.ts +++ b/src/types/core/storage.type.ts @@ -1,6 +1,7 @@ import type { z } from 'zod'; -import type { fsStorageSchema, s3StorageSchema, storageSchema } from '../../schemas'; +import type { fsStorageSchema, redisStorageSchema, s3StorageSchema, storageSchema } from '../../schemas'; export type FsStorage = z.infer; export type S3Storage = z.infer; +export type RedisStorage = z.infer; export type Storage = z.infer; diff --git a/src/types/core/tile.type.ts b/src/types/core/tile.type.ts index 15570f7..7c833a6 100644 --- a/src/types/core/tile.type.ts +++ b/src/types/core/tile.type.ts @@ -1,5 +1,4 @@ import z from 'zod'; -import { tileRangeSchema, tilesDeletionParamsSchema } from '../../schemas/core/tile.schema'; +import { tileRangeSchema } from '../../schemas/core/tile.schema'; -export type TilesDeletionParams = z.infer; export type TileRange = z.infer; diff --git a/src/types/deletion/task.type.ts b/src/types/deletion/task.type.ts index 0a1e256..a803893 100644 --- a/src/types/deletion/task.type.ts +++ b/src/types/deletion/task.type.ts @@ -1,5 +1,30 @@ import z from 'zod'; -import { deleteTaskParamsSchema, deleteStoredResourcesParamsSchema } from '../../schemas/deletion/task.schema'; +import { + deleteStoredResourcesParamsSchema, + deleteTaskParamsSchema, + fsDeleteStoredResourcesParamsSchema, + fsTilesDeletionParamsSchema, + redisDeleteStoredResourcesParamsSchema, + redisTilesDeletionParamsSchema, + s3DeleteStoredResourcesParamsSchema, + s3TilesDeletionParamsSchema, + tilesDeletionParamsSchema, +} from '../../schemas/deletion/task.schema'; +//#region DeleteTaskParams export type DeleteTaskParams = z.infer; +//#endregion DeleteTaskParams + +//#region TilesDeletionParams +export type TilesDeletionParams = z.infer; +export type S3TilesDeletionParams = z.infer; +export type FsTilesDeletionParams = z.infer; +export type RedisTilesDeletionParams = z.infer; +//#endregion TilesDeletionParams + +//#region DeleteStoredResourcesParams export type DeleteStoredResourcesParams = z.infer; +export type S3DeleteStoredResourcesParams = z.infer; +export type FsDeleteStoredResourcesParams = z.infer; +export type RedisDeleteStoredResourcesParams = z.infer; +//#endregion DeleteStoredResourcesParams