diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..b1b900394 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,323 @@ +# AGENTS.md + +This file provides guidance to Codex (Codex.ai/code) when working with code in this repository. + +Ocean Node is the all-in-one backend for the Ocean Protocol stack. A single Node process +replaces three legacy components: **Provider** (data access / encryption / compute), +**Aquarius** (metadata cache) and the **subgraph** (on-chain event indexing). It is a +TypeScript ESM project (Node 22) that exposes an HTTP API and a libp2p P2P interface, both +of which dispatch to the same set of command handlers. + +--- + +## 1. Environment & tooling prerequisites + +- **Node.js ≥ 22.13 is required** (`.nvmrc` pins `22.22.2`, matching the Dockerfile and CI; + `package.json` `engines` requires `>=22.13.0`). Always run `nvm use` (or `source ~/.nvm/nvm.sh && nvm use`) before + any `npm`, build, or test command. The wrong Node version fails with errors like + `Unexpected token 'with'`, missing `GLIBC_2.38`, or — since the SQLite layer uses the + built-in `node:sqlite` module — `ERR_UNKNOWN_BUILTIN_MODULE: node:sqlite` on Node < 22.13. + This is enforced by `.cursor/rules/tests-nvm.mdc` and the in-repo `AGENTS.md`. +- **`postinstall` runs `scripts/fix-libp2p-http-utils.js`** — a patch applied to a libp2p + dependency. Expect it to run on every `npm install`; don't remove it. +- **Docker + docker-compose** are needed for the metadata database (Typesense or + Elasticsearch) and for C2D (Compute-to-Data) via the local Docker socket. +- TypeScript config: ESM (`module: esnext`, `target: ES2022`, `moduleResolution: node`), + `experimentalDecorators` + `emitDecoratorMetadata` enabled, `rootDir: ./src`, + `outDir: ./dist`. All local imports use the `.js` extension (compiled ESM convention). + +### Only-mandatory config: `PRIVATE_KEY` + +`PRIVATE_KEY` (with the `0x` prefix) is the **only** required environment variable — it +seeds the node identity (libp2p peerId), the EVM signer, and encryption keys. Everything +else has defaults. Generate one with: + +```bash +node dist/helpers/scripts/generatePK.js # prints a key/address to stdout +node dist/helpers/scripts/generatePK.js --save # writes .pk.out / .wallet.out +``` + +Or use the interactive helper `./src/helpers/scripts/setupNodeEnv.sh` (npm: `setupEnv`), +which generates a key, configures DB, and writes `.env`. `npm run quickstart` +(`scripts/ocean-node-quickstart.sh`) walks through a full Docker deployment and auto-detects +GPUs into `DOCKER_COMPUTE_ENVIRONMENTS`. + +--- + +## 2. Common commands + +All commands assume `nvm use` has been run first. + +### Build / run + +```bash +npm run build # clean ./dist then tsc (build:tsc, emits sourcemaps) +npm run type-check # tsc --noEmit (also runs as part of `npm run lint`) +npm run start # node dist/index.js (needs a prior `npm run build`) +``` + +`start` runs with `--max-old-space-size=28784 --experimental-specifier-resolution=node`. +The Dockerfile uses the identical CMD. The compiled entry point is `dist/index.js` +(source `src/index.ts`). + +### Lint / format + +```bash +npm run lint # eslint (.ts,.tsx) + type-check +npm run lint:fix # eslint --fix +npm run format # prettier --write '**/*.{js,jsx,ts,tsx}' +``` + +ESLint extends `oceanprotocol` + `prettier/recommended`. Notable rules: `require-await` +is an **error**, `no-unused-vars` is an **error**, empty catch blocks are allowed. Prettier: +no semicolons, single quotes, `printWidth: 90`, no trailing commas, 2-space tabs. + +### Tests (important build quirk) + +**Tests run against compiled JS in `dist/test/`, not the TypeScript source.** The +`test:*` scripts all call `npm run build-tests` first, which: + +- compiles `src/` (incl. `src/test`) into `dist/`, +- copies `src/test/.env.test` and `.env.test2` into `dist/test`, +- copies `src/test/config.json` to `$HOME/config.json`. + +Mocha config (`.mocharc.json`): `bail: true` (stops on first failure), `timeout: 20000`, +`exit: true`, and it **requires `./dist/test/utils/hooks.js`** (global setup/teardown). The +runner is `npm run mocha` = `mocha --node-env=test --config .mocharc.json `. + +```bash +npm run test # full CI gate: lint + unit(+coverage) + integration(+coverage) +npm run test:unit # build-tests + mocha ./dist/test/unit/**/*.test.js +npm run test:integration # build-tests + mocha ./dist/test/integration/**/*.test.js +npm run test:integration:light # integration minus the heavy compute.test.js +npm run test:computeunit # unit compute tests only (fast; used for quick checks) +npm run test:computeintegration # integration compute tests +npm run test:servicesintegration # service-on-demand integration +npm run test:indexer # indexer integration +``` + +**Running a single test / file.** Build once, then invoke mocha directly on a compiled +file (or filter by name with `--grep`): + +```bash +npm run build-tests +npx mocha --node-env=test --config .mocharc.json "./dist/test/unit/crypt.test.js" +npx mocha --node-env=test --config .mocharc.json "./dist/test/unit/**/*.test.js" --grep "nonce" +``` + +Remember to re-run `build-tests` after editing source or test files, since mocha only sees +the compiled output. In tests, do **not** mutate `process.env` directly — use the +`setupEnvironment()` / `tearDownEnvironment()` helpers in `before()`/`after()` (see +`docs/testing.md`) so config changes are reverted and don't leak between suites. + +**Integration tests require a running local chain + services (Barge).** Clone +`oceanprotocol/barge`, `git checkout feature/nodes`, then `./start_ocean.sh -with-c2d` +(see `docs/testing.md` and `docs/database.md` for the exact flags per DB type). The default +`config.json` points the DB at Typesense on `http://localhost:8108`. + +Other useful scripts: `npm run client` (2-node download flow demo), `npm run check-nonce` +(nonce tracking; needs DB), `npm run logs` (tail logs), and k6 perf tests +`test:smoke` / `test:load` / `test:stress` / `test:request:rate` (require k6 installed and +a running node). + +### Databases for local dev + +```bash +docker-compose -f typesense-compose.yml up -d # Typesense (default in config.json) +docker-compose -f elasticsearch-compose.yml up -d # Elasticsearch alternative +``` + +--- + +## 3. Configuration mechanism + +Config is resolved by `getConfiguration()` in `src/utils/config/builder.ts` (re-exported +via `src/utils/config/index.ts` and `src/utils/index.ts`). Key facts: + +- **Two config sources.** Either environment variables, or a JSON file. `getConfigFilePath()` + resolves the file from `CONFIG_PATH` env var, else `./config.json` in cwd. The repo ships a + ready `config.json` (Typesense DB, chain `8996`/development, single Docker C2D env). +- **`INTERFACES`** env var (JSON array like `["HTTP","P2P"]`) toggles `hasHttp` / `hasP2P`. + Omitted = both enabled. `DB_URL`/`DB_TYPE` presence drives `hasIndexer` — without a valid + DB config the Indexer is disabled and only the SQLite nonce DB is available. +- The full catalog of env vars is defined as `ENVIRONMENT_VARIABLES` in + `src/utils/constants.ts` and documented in `docs/env.md` (long file — the authoritative + reference). `.env.example` lists them grouped as core / p2p / compute. +- `getConfiguration(true)` is called once at startup with a verbose flag to reduce repeated + logging (config is read from many places; there's a known TODO to centralize access on the + `OceanNode` class). + +Selected env vars worth knowing: `RPCS` (per-chain RPC map, JSON), `DB_URL`/`DB_TYPE`, +`ALLOWED_ADMINS`(+`_LIST`) (gate admin commands), `AUTHORIZED_DECRYPTERS`/`_PUBLISHERS`/ +`ALLOWED_VALIDATORS` (+ their access-list variants), `MAX_REQ_PER_MINUTE` (rate limit), +`RATE_DENY_LIST`, `HTTP_API_PORT`, `JWT_SECRET`, `OPERATOR_SERVICE_URL` (external C2D +clusters), `POLICY_SERVER_URL`, `DOCKER_COMPUTE_ENVIRONMENTS` (C2D env + GPU/resource +definitions), and the `P2P_*` family (bind addresses/ports, bootstrap nodes, NAT/relay +toggles, announce address filtering). + +--- + +## 4. High-level architecture + +### Layered model (see `docs/Arhitecture.md`) + +1. **Network layer** — libp2p (peer-to-peer) + Express HTTP API. Both are entry points that + normalize a request into a `Command` object and hand it to the components layer. +2. **Components layer** — Indexer, Provider, C2D, Database, P2P, Auth, KeyManager, + BlockchainRegistry, Escrow, PersistentStorage, PolicyServer. +3. **Modules / handlers layer** — the concrete command handlers under + `src/components/core/` that execute the actual work. + +### Startup (`src/index.ts` → `OceanNode`) + +`src/index.ts` bootstraps in this order: load config → compute `codeHash` of the codebase → +`Database.init(config.dbConfig)` → create `KeyManager` and `BlockchainRegistry` → optionally +start `OceanP2P` (if `hasP2P`), `OceanIndexer` (if `hasIndexer` + DB), `OceanProvider` (if +DB) → build the **`OceanNode` singleton** via `OceanNode.getInstance(...)` → +`addC2DEngines()` → if `hasHttp`, wire up the Express app (CORS, a middleware that attaches +`req.oceanNode` + caller IP, `requestValidator`, then mounts `httpRoutes`) and start +HTTP/HTTPS → `scheduleCronJobs()`. + +`OceanNode` (`src/OceanNode.ts`) is the central singleton wiring everything together. It +holds the `CoreHandlersRegistry`, `C2DEngines`, `Escrow`, `Auth`, `PersistentStorage`, +`Database`, `KeyManager`, `BlockchainRegistry`, and the per-caller rate-limit `requestMap`. +`handleDirectProtocolCommand(message)` is the single choke point for dispatch: it JSON-parses +the command, looks up the handler by `task.command`, and calls `handler.handle(task)`. + +### Request / command flow + +Two front doors, one dispatcher: + +- **HTTP `POST /directCommand`** (`src/components/httpRoutes/commands.ts`): validates the + body, then decides _local vs remote_. If the command targets this node (or no P2P), it + calls `oceanNode.handleDirectProtocolCommand(...)`. If it targets another peer and P2P is + enabled, it forwards via `oceanNode.getP2PNode().sendTo(node, msg, multiAddrs)`. Responses + are streamed back to the client (binary or text). +- **P2P inbound** (`src/components/P2P/handleProtocolCommands.ts`): a libp2p protocol handler + reads a length-prefixed command frame off the stream, applies connection/request rate + limits, and dispatches through the same `CoreHandlersRegistry`. +- **RESTful routes** (`src/components/httpRoutes/*`, e.g. `provider.ts`, `aquarius.ts`, + `compute.ts`, `auth.ts`, `escrow.ts`, `accessList.ts`, `persistentStorage.ts`) are + ergonomic wrappers that build the equivalent `Command` and invoke the corresponding + handler. Route mounting lives in `src/components/httpRoutes/index.ts` + (`getAllServiceEndpoints()` enumerates them for the status/root endpoint). + +So: **every capability is ultimately a command handler**, reachable identically over HTTP +`/directCommand`, over P2P, and (for most) via a dedicated REST route. + +### Command registry & handler pattern + +- **`src/utils/constants.ts`** defines `PROTOCOL_COMMANDS` (name → string) and the parallel + `SUPPORTED_PROTOCOL_COMMANDS` allow-list. Keep both in sync. +- **`src/components/core/handler/coreHandlersRegistry.ts`** — `CoreHandlersRegistry` is a + singleton that, in its constructor, instantiates and registers one handler per command + (`download`, `encrypt`, `getDDO`, `query`, `status`, `getFees`, `fileInfo`, the `compute*` + family, the `service*` family, the admin commands, `getP2P*`, auth tokens, persistent + storage, access lists, escrow events, …). `getHandler(command)` returns the instance. +- **`src/components/core/handler/handler.ts`** — `BaseHandler` (abstract) defines the + contract: `verifyParamsAndRateLimits(task)` and `handle(task)`, plus shared rate-limiting + (`checkRateLimit` / `checkRequestData` against `OceanNode.requestMap`). Handlers extend + `BaseHandler` / `CommandHandler` (admin handlers extend `AdminCommandHandler`). + +**To add a new command:** add the name to `PROTOCOL_COMMANDS` + `SUPPORTED_PROTOCOL_COMMANDS`; +create a handler extending `BaseHandler`/`CommandHandler` under `src/components/core/`; +register it in the `CoreHandlersRegistry` constructor; add param validation; optionally add a +REST route in `src/components/httpRoutes/` and mount it in `httpRoutes/index.ts`. + +Handler source is grouped under `src/components/core/`: + +- `handler/` — general handlers (ddo, download, encrypt, fees, nonce, query, status, p2p, + auth, accessList, escrow, fileInfo, persistentStorage, policyServer, getJobs). +- `compute/` — C2D command handlers: `initialize`, `startCompute` (paid), `freeStartCompute`, + `getStatus`, `getResults`, `getStreamableLogs`, `stopCompute`, `environments`. +- `service/` — Service-on-Demand handlers (`start`, `stop`, `restart`, `extend`, `getStatus`, + `getTemplates`, plus `templateLoader`). +- `admin/` — privileged handlers gated by `ALLOWED_ADMINS`: `stopNode`, `stopJob`, + `reindexTx`, `reindexChain`, `IndexingThreadHandler`, `collectFees`, `fetchConfig`, + `pushConfig`, `getLogs`. +- `utils/` — shared logic: `escrow`, `feesHandler`, `findDdoHandler`, `nonceHandler`, + `statusHandler`, `validateOrders`. + +### Components in `src/components/` + +- **P2P/** — `OceanP2P` (extends EventEmitter) builds a libp2p node: TCP + WebSockets + transports, Noise encryption, Yamux muxing, Kademlia DHT + mDNS peer discovery, + circuit-relay v2, AutoNAT, UPnP, identify/ping/dcutr, and auto-TLS. Handles the custom + Ocean protocol stream, DDO DHT caching (`FindDDOResponse`), peer/announce address filtering, + and cross-node request forwarding (`sendTo`). A LevelDB datastore persists at + `./databases/p2p-store`. +- **Indexer/** — `OceanIndexer` orchestrates one `ChainIndexer` per configured chain + (single-threaded, async/await concurrency; **no worker threads**). Each ChainIndexer polls + its chain, and a set of event **processors** (`src/components/Indexer/processors/`) handle + specific events: `MetadataCreated/Updated/State`, `OrderStarted/Reused`, exchange/dispenser + lifecycle, access-list changes, and Escrow events. It validates DDOs against SHACL schemas, + stores orders, supports version-based + admin-triggered reindexing, and a `purgatory`. + Communication uses module-level `EventEmitter`s (`INDEXER_DDO_EVENT_EMITTER`, + `INDEXER_CRAWLING_EVENT_EMITTER`). +- **Provider/** — deliberately thin (`OceanProvider` just wraps the `Database`). The real + "provider" behavior (download streaming, encrypt/decrypt, fees, initialize, nonce) lives in + the core handlers and the `providerRoutes`. +- **c2d/** — Compute-to-Data. `C2DEngines` builds engines from `config.c2dClusters`; the + implemented engine is `C2DEngineDocker` (`compute_engine_docker.ts`, base + `compute_engine_base.ts`) which orchestrates jobs via the host Docker socket. Compute + lifecycle: `initializeCompute` → `startCompute`/`freeStartCompute` → `getComputeStatus` → + `getComputeResult` (+ `getComputeStreamableLogs`) → `stopCompute`. Paid compute settles via + the `Escrow` component; `serviceResourceMatching.ts` maps requested cpu/ram/disk/gpu against + environment pools (dual-gate: per-env ceiling + engine-wide pool; GPUs tracked globally). + See `docs/compute.md`. +- **database/** — `Database.init()` factory (`index.ts`, `DatabaseFactory.ts`). The metadata + DB backend is pluggable: **Typesense or Elasticsearch** (chosen by `DB_TYPE`) for DDOs, + indexer state, logs, orders, ddoState, access lists, escrow events — behind the + `Abstract*Database` interfaces in `BaseDatabase.ts`. **SQLite** is always used for the + nonce DB, config DB, C2D job DB, and auth-token DB (works even with no metadata DB + configured) — via Node's built-in `node:sqlite` module (no native addon), wrapped by + `SqliteClient` in `src/components/database/sqliteClient.ts`. See `docs/database.md`. +- **KeyManager/** — provider-abstraction over the node key (`docs/KeyManager.md`). Currently + `RawPrivateKeyProvider` (from `PRIVATE_KEY`); derives the libp2p peerId/keys and the EVM + address, and caches the ethers signer. Designed to add KMS providers (GCP/AWS) later. +- **BlockchainRegistry/** — manages per-chain `Blockchain` instances (`src/utils/blockchain.ts`), + giving handlers RPC providers/signers keyed by `chainId` (`OceanNode.getBlockchain(chainId)`). +- **Auth/** — auth-token issuance/validation (JWT-based, `JWT_SECRET`) as an alternative to + per-request signatures. +- **persistentStorage/** — pluggable storage (S3 / IPFS, `PersistentStorageFactory`) for C2D + job outputs and user buckets. See `docs/persistentStorage.md`, `docs/Storage.md`. +- **policyServer/** — passthrough integration to an external policy server for access + decisions (`POLICY_SERVER_URL`, `docs/PolicyServer.md`). + +### Types, utils, and support directories + +- `src/@types/` — shared TypeScript types (`OceanNode.ts` for config/response shapes, + `commands.ts` for `Command`/handler interfaces, `blockchain.ts`, `C2D/`). +- `src/utils/` — cross-cutting helpers: `config/` (config builder + zod schemas), + `constants.ts`, `crypt.ts` (hashing/signing), `blockchain.ts`, `logging/` (winston, with a + DB transport in prod/staging), `cronjobs/`, `validators.ts`, `credentials.ts`, + `accessList.ts`, `asset.ts`, `attestation.ts`. +- Runtime data dirs: `databases/` (SQLite files + libp2p LevelDB store), `c2d_storage/` (C2D + job working data), `logs/`, `schemas/` (SHACL DDO validation schemas — shipped into the + Docker image). Service-on-demand templates are not shipped: the node reads + them from the folder given by `serviceTemplatesPath` / `SERVICE_TEMPLATES_PATH`, which the + operator mounts in at run time. +- `tsoa.json` configures OpenAPI spec generation from `src/components/httpRoutes/**`; the + actual routing is plain Express routers, not tsoa-generated. + +--- + +## 5. Docker & deployment + +Multi-stage `Dockerfile` (builder + slim runner) on `node:22`. The runner ships only +`dist/`, `node_modules`, `schemas/`, and `config.json` (`.dockerignore` excludes all of +`docs/`) — no service templates. It exposes P2P ports `9000-9003,9005` and +HTTP `8000`. `docker-entrypoint.sh` handles Docker socket group membership at runtime so C2D +can talk to `/var/run/docker.sock`. Deployment options (Docker, local Docker build via +`quickstart`, PM2, plain npm) are in `README.md`; production deployment details in +`docs/dockerDeployment.md`. + +--- + +## 6. Documentation map (`docs/`) + +`Arhitecture.md` (note the spelling), `API.md` (full HTTP API reference — very large, plus a +Postman collection), `env.md` (authoritative env-var reference), `database.md`, +`Storage.md` / `persistentStorage.md`, `KeyManager.md`, `PolicyServer.md`, `services.md` +(Service-on-Demand), `compute.md` (C2D configuration: resources, GPUs, constraints, pricing), `networking.md`, `Logs.md`, +`Publishing.md`, `testing.md`, `dockerDeployment.md`. diff --git a/docs/env.md b/docs/env.md index bc44a49bf..d0643039e 100644 --- a/docs/env.md +++ b/docs/env.md @@ -35,6 +35,8 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `VALIDATE_UNSIGNED_DDO`: If set to `false`, the node will not validate unsigned DDOs and will request a signed message with the publisher address, nonce and signature. Default is `true`. Example: `false` - `JWT_SECRET`: Secret used to sign JWT tokens. Default is `ocean-node-secret`. Example: `"my-secret-jwt-token"` - `PERSISTENT_STORAGE`: Persistent storage config. See [persistent storage](persistentStorage.md). +- `SERVICE_BUCKET_QUOTA_BYTES`: Quota, in bytes, of the output bucket `serviceStart` creates when the request has no `outputBucketId`. The quota is stored on the bucket when it is created, so a change only applies to new buckets. Must be an integer `>= 1`. Default is `5368709120` (5 GB). Example: `10737418240` +- `SERVICE_BUCKET_RETENTION_SECONDS`: How long, in seconds, such a bucket is kept after its service's paid window ends. It is applied when the bucket is created and again whenever `serviceExtend` or another service start pushes the date out. Must be an integer `>= 0`. Default is `604800` (1 week). Example: `259200` ## Database diff --git a/docs/persistentStorage.md b/docs/persistentStorage.md index b5e0afcfe..62a5b72af 100644 --- a/docs/persistentStorage.md +++ b/docs/persistentStorage.md @@ -240,6 +240,25 @@ Because results are regular bucket files, they can feed the next compute job wit --- +## Service default buckets + +When `serviceStart` has no `outputBucketId`, a `localfs` node creates a bucket for the service +(see [Results bucket](services.md#results-bucket)). These buckets differ from ones made with +`createBucket` in three ways, all shown by `getBuckets`: + +- `serviceId` — the service the bucket was created for. +- `quotaBytes` — `SERVICE_BUCKET_QUOTA_BYTES` at creation time, 5 GB by default. Uploads that + would go over it fail. The quota is soft: the service using the bucket is never stopped, and + its container's own writes are not capped. +- `expiresAt` — unix seconds, `SERVICE_BUCKET_RETENTION_SECONDS` (one week by default) after the + service's paid window ends (pushed out by + `serviceExtend` or by another service starting into the bucket). An hourly sweep then deletes + the bucket and its files. + +Buckets created with `createBucket` have `null` for all three: no quota, never deleted. + +--- + ## Limitations and notes - The bucket registry is local to the node (SQLite file). If you run multiple nodes, each node’s registry is independent unless you externalize/replicate it. diff --git a/docs/services.md b/docs/services.md index 00e68036e..41c0267b7 100644 --- a/docs/services.md +++ b/docs/services.md @@ -149,6 +149,42 @@ redeploy) cannot run conflicting operations on the same service. Leases are hear every 30 s while the operation runs; a lease not refreshed for 2 minutes belongs to a crashed process and is stolen automatically, so no manual cleanup is ever needed. +## Results bucket + +A service writes durable results to `/data/outputs`, which is bind-mounted from a +[persistent storage](persistentStorage.md) bucket: + +- **Your own bucket.** Pass `outputBucketId` in `serviceStart`. You must own the bucket or be on + its access list (`403` otherwise). +- **A default bucket.** Without `outputBucketId`, and when the node's persistent storage is + `localfs`, the node creates a bucket for the service: owned by the consumer, no access list, + a quota of `SERVICE_BUCKET_QUOTA_BYTES` (**5 GB** by default). Its id is returned as `outputBucketId` in the `serviceStart` response (and in + `serviceStatus`). It is created only after every other check has passed, so a refused start + leaves no bucket behind, and there is at most one per `serviceId`. Restarts keep the same + bucket. To relaunch into it — e.g. after editing a service — pass that id as `outputBucketId` + to the new `serviceStart`. +- If persistent storage is disabled (or not `localfs`), the service runs without a bucket, as + before, and nothing it writes outlives the container. + +**Quota.** The quota is soft and never stops a service. Once the bucket is full, uploads through +the persistent storage API that would go over the quota are rejected. The service keeps +running, and starts and restarts into the bucket are allowed. `serviceStatus` reports the fill +level of any bucket that has a quota: + +```json +"outputBucketUsage": { "quotaBytes": 5368709120, "usedBytes": 5368709120, "full": true } +``` + +The reading can be up to 30 s old. Uploads and deletes through the storage API refresh it at +once. To make room, delete files from the bucket. +A bind mount can't be size-capped, so writes the service container makes to `/data/outputs` +are **not** blocked and can take the bucket past its quota. + +**Expiry.** A default bucket is deleted, contents included, `SERVICE_BUCKET_RETENTION_SECONDS` +after its service's paid window ends (**one week** by default: `expiresAt` + 7 days). `serviceExtend` pushes that date out, and so does starting +another service into the bucket. An hourly sweep deletes buckets past their date. Buckets you +created with `createBucket` never expire and have no quota. + ## Configuration Service-on-demand is configured per Docker connection under `serviceOnDemand`: diff --git a/src/@types/C2D/ServiceOnDemand.ts b/src/@types/C2D/ServiceOnDemand.ts index 644cedcfe..edd373d26 100644 --- a/src/@types/C2D/ServiceOnDemand.ts +++ b/src/@types/C2D/ServiceOnDemand.ts @@ -182,6 +182,12 @@ export const SERVICE_START_PENDING_STATUSES: readonly ServiceStatusNumber[] = [ ServiceStatusNumber.Restarting ] +export interface ServiceOutputBucketUsage { + quotaBytes: number + usedBytes: number + full: boolean // usedBytes >= quotaBytes +} + export interface ServiceJob { serviceId: string // unique id for a running service — distinct from a compute jobId clusterHash: string diff --git a/src/@types/OceanNode.ts b/src/@types/OceanNode.ts index 196708031..b255f894b 100644 --- a/src/@types/OceanNode.ts +++ b/src/@types/OceanNode.ts @@ -189,6 +189,9 @@ export interface OceanNodeConfig { httpKeyPath?: string enableBenchmark?: boolean persistentStorage?: PersistentStorageConfig + // default SERVICE_START output bucket: quota (bytes) and retention past expiresAt (seconds) + serviceBucketQuotaBytes?: number + serviceBucketRetentionSeconds?: number } export interface P2PStatusResponse { diff --git a/src/components/core/service/extendService.ts b/src/components/core/service/extendService.ts index 15c952efa..83777953b 100644 --- a/src/components/core/service/extendService.ts +++ b/src/components/core/service/extendService.ts @@ -364,6 +364,19 @@ export class ServiceExtendHandler extends CommandHandler { freshJob.expiresAt += task.additionalDuration * 1000 freshJob.duration += task.additionalDuration await engine.db.updateServiceJob(freshJob) + // The results bucket must outlive the longer window too. Best-effort: the + // extension is already paid for, so a storage hiccup must not fail it. + if (freshJob.outputBucketId) { + await this.getOceanNode() + .getPersistentStorage() + ?.extendBucketRetention(freshJob.outputBucketId, freshJob.expiresAt) + .catch((e: any) => + CORE_LOGGER.error( + `Service ${task.serviceId}: could not extend retention of bucket ` + + `${freshJob.outputBucketId}: ${e.message}` + ) + ) + } CORE_LOGGER.logMessage( `Service ${task.serviceId} extended by ${task.additionalDuration}s, new expiresAt: ${freshJob.expiresAt}`, diff --git a/src/components/core/service/getStatus.ts b/src/components/core/service/getStatus.ts index 72def8e57..e0cfbdf4b 100644 --- a/src/components/core/service/getStatus.ts +++ b/src/components/core/service/getStatus.ts @@ -6,9 +6,39 @@ import { ValidateParams, validateCommandParameters } from '../../httpRoutes/validateCommands.js' -import type { ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' +import type { + ServiceJob, + ServiceOutputBucketUsage +} from '../../../@types/C2D/ServiceOnDemand.js' +import type { PersistentStorageFactory } from '../../persistentStorage/PersistentStorageFactory.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' import { toPublicServiceJob } from './utils.js' +// Sizing a bucket walks its folder, and clients poll status every few seconds, so a +// reading may be this old. Uploads and deletes through the storage API refresh it. +const BUCKET_USAGE_MAX_AGE_MS = 30_000 + +// Best-effort: a bucket that is gone or can't be sized just leaves the field off. +async function getOutputBucketUsage( + storage: PersistentStorageFactory | null, + job: ServiceJob +): Promise { + if (!storage || !job.outputBucketId) return undefined + try { + const usage = await storage.getBucketQuotaUsage( + job.outputBucketId, + BUCKET_USAGE_MAX_AGE_MS + ) + if (!usage) return undefined + return { ...usage, full: usage.usedBytes >= usage.quotaBytes } + } catch (e: any) { + CORE_LOGGER.debug( + `Service ${job.serviceId}: could not size bucket ${job.outputBucketId}: ${e.message}` + ) + return undefined + } +} + export class ServiceGetStatusHandler extends CommandHandler { validate(command: ServiceGetStatusCommand): ValidateParams { // consumerAddress is required: it is the owner scope AND the identity the @@ -47,14 +77,18 @@ export class ServiceGetStatusHandler extends CommandHandler { // Ownership is already proven above (this command is always authenticated), so runtime // metrics are included BY DEFAULT here — only an explicit includeMetrics=false opts out. + const storage = this.getOceanNode().getPersistentStorage() + const out = await Promise.all( + jobs.map(async (job) => { + const pub = toPublicServiceJob(job, { + includeMetrics: task.includeMetrics !== false + }) + const outputBucketUsage = await getOutputBucketUsage(storage, job) + return outputBucketUsage ? { ...pub, outputBucketUsage } : pub + }) + ) return { - stream: Readable.from( - JSON.stringify( - jobs.map((job) => - toPublicServiceJob(job, { includeMetrics: task.includeMetrics !== false }) - ) - ) - ), + stream: Readable.from(JSON.stringify(out)), status: { httpStatus: 200 } } } diff --git a/src/components/core/service/startService.ts b/src/components/core/service/startService.ts index 49856d6d4..6f4eba8ed 100644 --- a/src/components/core/service/startService.ts +++ b/src/components/core/service/startService.ts @@ -18,7 +18,11 @@ import type { import { generateUniqueID, validateOutputBucket } from '../compute/utils.js' import { validateAccess } from '../compute/startCompute.js' import { isJobMetadataSizeValid, INVALID_JOB_METADATA_MESSAGE } from '../../c2d/index.js' -import { decryptUserData, toPublicServiceJob } from './utils.js' +import { + decryptUserData, + resolveServiceOutputBucket, + toPublicServiceJob +} from './utils.js' export class ServiceStartHandler extends CommandHandler { validate(command: ServiceStartCommand): ValidateParams { @@ -247,6 +251,17 @@ export class ServiceStartHandler extends CommandHandler { cost } + // 6c. Results bucket, resolved last so a refused start never leaves one behind. Its + // id is on the returned job (outputBucketId), for the consumer to fetch results + // or hand to a later SERVICE_START. + const outputBucketId = await resolveServiceOutputBucket( + node, + task.consumerAddress, + serviceId, + Date.now() + task.duration * 1000, + task.outputBucketId + ) + // 7. Persist the Starting record and return immediately with the serviceId. The // engine's background loop (processServiceStart) then performs escrow lock → image // pull/build → claim/cancel → container start. Clients poll SERVICE_GET_STATUS to @@ -268,7 +283,7 @@ export class ServiceStartHandler extends CommandHandler { serviceId, task.userData, task.metadata, - task.outputBucketId + outputBucketId ) return { diff --git a/src/components/core/service/utils.ts b/src/components/core/service/utils.ts index 77de3d63c..8be7966d6 100644 --- a/src/components/core/service/utils.ts +++ b/src/components/core/service/utils.ts @@ -6,6 +6,7 @@ import type { C2DDatabase } from '../../database/C2DDatabase.js' import type { C2DEngine } from '../../c2d/compute_engine_base.js' import type { C2DEngines } from '../../c2d/compute_engines.js' import { sanitizePublicMetrics } from '../../c2d/index.js' +import type { OceanNode } from '../../../OceanNode.js' // Looks up a service job and resolves the engine that OWNS it (by clusterHash). Every // engine shares the same C2DDatabase, so any engine's db returns the job — taking the @@ -26,6 +27,32 @@ export async function findServiceJobAndEngine( return { job, engine } } +// Picks the bucket a new service writes its results (/data/outputs) to. A bucket the +// consumer named is kept, with its retention (if it expires at all) pushed out to cover +// this service. Without one, the node creates a default bucket for the service — only on +// localfs, the one backend that works today; otherwise the service runs without a bucket, +// exactly as before. +export async function resolveServiceOutputBucket( + node: OceanNode, + owner: string, + serviceId: string, + serviceExpiresAtMs: number, + requestedBucketId?: string +): Promise { + const storage = node.getPersistentStorage() + if (requestedBucketId) { + await storage.extendBucketRetention(requestedBucketId, serviceExpiresAtMs) + return requestedBucketId + } + if (!storage || node.getConfig().persistentStorage?.type !== 'localfs') return undefined + const bucket = await storage.getOrCreateServiceBucket( + owner, + serviceId, + serviceExpiresAtMs + ) + return bucket.bucketId +} + // Converts the decrypted userData object into a flat container env-var map (stringified values). export function userDataToEnv(userData: Record): Record { const env: Record = {} diff --git a/src/components/persistentStorage/PersistentStorageFactory.ts b/src/components/persistentStorage/PersistentStorageFactory.ts index 380ecd92e..bf23ad874 100644 --- a/src/components/persistentStorage/PersistentStorageFactory.ts +++ b/src/components/persistentStorage/PersistentStorageFactory.ts @@ -10,6 +10,11 @@ import { SqliteClient } from '../database/sqliteClient.js' import { getAddress } from 'ethers' import { OceanNode } from '../../OceanNode.js' import { checkAddressOnAccessList } from '../../utils/accessList.js' +import { CORE_LOGGER } from '../../utils/logging/common.js' +import { + DEFAULT_SERVICE_BUCKET_QUOTA_BYTES, + DEFAULT_SERVICE_BUCKET_RETENTION_SECONDS +} from '../../utils/config/constants.js' export class PersistentStorageAccessDeniedError extends Error { constructor(message = 'You are not allowed to access this bucket') { @@ -18,6 +23,22 @@ export class PersistentStorageAccessDeniedError extends Error { } } +export class PersistentStorageQuotaExceededError extends Error { + constructor(bucketId: string, usedBytes: number, quotaBytes: number) { + super( + `Bucket ${bucketId} is over its quota (${usedBytes} of ${quotaBytes} bytes used) — ` + + 'delete files from it to free space' + ) + this.name = 'PersistentStorageQuotaExceededError' + } +} + +export type CreateBucketOptions = { + serviceId?: string // the service the bucket was auto-created for (at most one bucket each) + quotaBytes?: number + expiresAt?: number // unix seconds; the expiry sweep deletes the bucket after this +} + function normalizeWeb3Address(addr: string): string { try { return getAddress(addr) @@ -41,6 +62,9 @@ export type BucketRow = { accessListJson: string createdAt: number label: string | null + serviceId: string | null + quotaBytes: number | null + expiresAt: number | null } export interface PersistentStorageFileInfo { @@ -55,6 +79,9 @@ export type CreateBucketResult = { owner: string accessList: AccessList[] label?: string | null + serviceId?: string | null + quotaBytes?: number | null + expiresAt?: number | null } /** Bucket metadata from registry (list APIs and internal filtering). */ @@ -64,13 +91,21 @@ export type PersistentStorageBucketRecord = { createdAt: number accessLists: AccessList[] label?: string | null + serviceId?: string | null + quotaBytes?: number | null + expiresAt?: number | null } +const BUCKET_COLUMNS = + 'bucketId, owner, accessListJson, createdAt, label, serviceId, quotaBytes, expiresAt' + export abstract class PersistentStorageFactory { private db: SqliteClient private node: OceanNode private dbReady = false private dbReadyPromise: Promise + // bucketId → last measured size, see getBucketQuotaUsage + private bucketUsageCache: Map = new Map() constructor(node: OceanNode) { this.node = node @@ -82,20 +117,35 @@ export abstract class PersistentStorageFactory { owner TEXT NOT NULL, accessListJson TEXT NOT NULL, createdAt INTEGER NOT NULL, - label TEXT + label TEXT, + serviceId TEXT, + quotaBytes INTEGER, + expiresAt INTEGER ); `) - // Migration: add the label column if it doesn't exist. A fresh table already has it, - // so ALTER throws "duplicate column name" — swallow only that; surface any other + // Migration: add columns missing from older databases. A fresh table already has + // them, so ALTER throws "duplicate column name" — swallow only that; surface any other // failure instead of starting with a broken schema. Schema setup is synchronous now, // so the DB is ready by the time the constructor returns. - try { - this.db.exec(`ALTER TABLE persistent_storage_buckets ADD COLUMN label TEXT`) - } catch (alterErr) { - if (!/duplicate column name/i.test(alterErr.message)) { - throw alterErr + for (const column of [ + 'label TEXT', + 'serviceId TEXT', + 'quotaBytes INTEGER', + 'expiresAt INTEGER' + ]) { + try { + this.db.exec(`ALTER TABLE persistent_storage_buckets ADD COLUMN ${column}`) + } catch (alterErr) { + if (!/duplicate column name/i.test(alterErr.message)) { + throw alterErr + } } } + // At most one auto-created bucket per service, so a retried start can't make two. + this.db.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS idx_persistent_storage_buckets_serviceId + ON persistent_storage_buckets (serviceId) WHERE serviceId IS NOT NULL; + `) this.dbReady = true this.dbReadyPromise = Promise.resolve() } @@ -127,9 +177,16 @@ export abstract class PersistentStorageFactory { public abstract createNewBucket( accessList: AccessList[], owner: string, - label?: string + label?: string, + options?: CreateBucketOptions ): Promise + /** Removes the bucket's contents and its registry row. */ + public abstract deleteBucket(bucketId: string): Promise + + /** Total bytes stored in the bucket, including nested folders a container wrote. */ + public abstract getBucketUsageBytes(bucketId: string): Promise + public abstract listFiles( bucketId: string, consumerAddress: string @@ -238,10 +295,119 @@ export abstract class PersistentStorageFactory { owner: row.owner, createdAt: row.createdAt, accessLists: parseBucketAccessListsJson(row.accessListJson), - label: row.label ?? null + label: row.label ?? null, + serviceId: row.serviceId ?? null, + quotaBytes: row.quotaBytes ?? null, + expiresAt: row.expiresAt ?? null })) } + /** + * Returns the output bucket auto-created for `serviceId`, creating it on first call: + * owned by `owner`, no access list, SERVICE_BUCKET_QUOTA_BYTES quota, and kept for + * SERVICE_BUCKET_RETENTION_SECONDS past the service's paid window (both env vars, 5 GB and + * 1 week by default). Idempotent per + * serviceId — a second call returns the same bucket with its retention extended. + */ + async getOrCreateServiceBucket( + owner: string, + serviceId: string, + serviceExpiresAtMs: number + ): Promise { + const existing = await this.dbGetBucketByServiceId(serviceId) + if (existing) { + await this.extendBucketRetention(existing.bucketId, serviceExpiresAtMs) + return (await this.dbGetBucket(existing.bucketId)) ?? existing + } + const { bucketId } = await this.createNewBucket([], owner, `service-${serviceId}`, { + serviceId, + quotaBytes: this.serviceBucketQuotaBytes(), + expiresAt: this.serviceBucketExpiryFor(serviceExpiresAtMs) + }) + return await this.dbGetBucket(bucketId) + } + + /** + * Keeps an expiring bucket alive until SERVICE_BUCKET_RETENTION_SECONDS after the given + * service window ends. Never shortens it, and leaves buckets without an expiry alone. + */ + async extendBucketRetention( + bucketId: string, + serviceExpiresAtMs: number + ): Promise { + const sql = `UPDATE persistent_storage_buckets SET expiresAt = MAX(expiresAt, ?) WHERE bucketId = ? AND expiresAt IS NOT NULL` + await this.ensureDbReady() + this.db.run(sql, [this.serviceBucketExpiryFor(serviceExpiresAtMs), bucketId]) + } + + // Default output bucket SERVICE_START creates when the request carries no outputBucketId. + // Buckets created any other way have neither a quota nor an expiry. The quota is stamped + // on the bucket at creation, so changing the env var only affects new buckets. + serviceBucketQuotaBytes(): number { + return ( + this.node.getConfig().serviceBucketQuotaBytes ?? DEFAULT_SERVICE_BUCKET_QUOTA_BYTES + ) + } + + // Retention counts from the END of the paid service window (expiresAt), not from creation: + // a Stopped service stays restartable until expiresAt, and its results must outlive it. + serviceBucketExpiryFor(serviceExpiresAtMs: number): number { + const retention = + this.node.getConfig().serviceBucketRetentionSeconds ?? + DEFAULT_SERVICE_BUCKET_RETENTION_SECONDS + return Math.floor(serviceExpiresAtMs / 1000) + retention + } + + /** + * Quota and current usage of a bucket, or null when the bucket has no quota. Sizing walks + * the whole bucket folder, so a caller that polls (service status) can accept a reading + * up to `maxAgeMs` old. Uploads and deletes through this API drop the cached reading; + * writes a service container makes show up once it ages out. + */ + async getBucketQuotaUsage( + bucketId: string, + maxAgeMs = 0 + ): Promise<{ quotaBytes: number; usedBytes: number } | null> { + const bucket = await this.getBucket(bucketId) + if (!bucket || bucket.quotaBytes === null || bucket.quotaBytes === undefined) { + return null + } + const cached = this.bucketUsageCache.get(bucketId) + if (maxAgeMs > 0 && cached && Date.now() - cached.at <= maxAgeMs) { + return { quotaBytes: bucket.quotaBytes, usedBytes: cached.usedBytes } + } + const usedBytes = await this.getBucketUsageBytes(bucketId) + this.bucketUsageCache.set(bucketId, { usedBytes, at: Date.now() }) + return { quotaBytes: bucket.quotaBytes, usedBytes } + } + + /** Drops the cached size of a bucket after its contents changed. */ + protected forgetBucketUsage(bucketId: string): void { + this.bucketUsageCache.delete(bucketId) + } + + /** Deletes every bucket whose expiry has passed. Returns how many were removed. */ + async deleteExpiredBuckets( + nowSeconds = Math.floor(Date.now() / 1000) + ): Promise { + const sql = `SELECT ${BUCKET_COLUMNS} FROM persistent_storage_buckets WHERE expiresAt IS NOT NULL AND expiresAt <= ?` + await this.ensureDbReady() + const expired = this.db.all(sql, [nowSeconds]) + let deleted = 0 + for (const bucket of expired) { + try { + await this.deleteBucket(bucket.bucketId) + deleted++ + } catch (e) { + // Left in place: the next sweep retries it. + CORE_LOGGER.error( + `Could not delete expired bucket ${bucket.bucketId}: ${e?.message ?? e}` + ) + } + } + return deleted + } + /* * NOTE: db* methods are intentionally gated on ensureDbReady() to avoid races * with constructor-time schema creation. @@ -252,27 +418,44 @@ export abstract class PersistentStorageFactory { owner: string, accessListJson: string, createdAt: number, - label: string | null + label: string | null, + options: CreateBucketOptions = {} ): Promise { // ON CONFLICT does not touch label, so a re-create never clobbers a rename. const sql = ` - INSERT INTO persistent_storage_buckets (bucketId, owner, accessListJson, createdAt, label) - VALUES (?, ?, ?, ?, ?) + INSERT INTO persistent_storage_buckets (${BUCKET_COLUMNS}) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(bucketId) DO UPDATE SET accessListJson=excluded.accessListJson; ` await this.ensureDbReady() - this.db.run(sql, [bucketId, owner, accessListJson, createdAt, label]) + this.db.run(sql, [ + bucketId, + owner, + accessListJson, + createdAt, + label, + options.serviceId ?? null, + options.quotaBytes ?? null, + options.expiresAt ?? null + ]) } async dbGetBucket(bucketId: string): Promise { - const sql = `SELECT bucketId, owner, accessListJson, createdAt, label FROM persistent_storage_buckets WHERE bucketId = ?` + const sql = `SELECT ${BUCKET_COLUMNS} FROM persistent_storage_buckets WHERE bucketId = ?` await this.ensureDbReady() const row = this.db.get(sql, [bucketId]) return row ?? null } + async dbGetBucketByServiceId(serviceId: string): Promise { + const sql = `SELECT ${BUCKET_COLUMNS} FROM persistent_storage_buckets WHERE serviceId = ?` + await this.ensureDbReady() + const row = this.db.get(sql, [serviceId]) + return row ?? null + } + async dbListBucketsByOwner(owner: string): Promise { - const sql = `SELECT bucketId, owner, accessListJson, createdAt, label FROM persistent_storage_buckets WHERE owner = ? ORDER BY createdAt ASC` + const sql = `SELECT ${BUCKET_COLUMNS} FROM persistent_storage_buckets WHERE owner = ? ORDER BY createdAt ASC` await this.ensureDbReady() return this.db.all(sql, [owner]) } diff --git a/src/components/persistentStorage/PersistentStorageLocalFS.ts b/src/components/persistentStorage/PersistentStorageLocalFS.ts index 92ac36698..00da606f9 100644 --- a/src/components/persistentStorage/PersistentStorageLocalFS.ts +++ b/src/components/persistentStorage/PersistentStorageLocalFS.ts @@ -1,6 +1,7 @@ import fs from 'fs' import fsp from 'fs/promises' import path from 'path' +import { Transform } from 'stream' import { pipeline } from 'stream/promises' import { createHash, randomUUID } from 'crypto' import { uniqueNamesGenerator, adjectives, animals } from 'unique-names-generator' @@ -13,14 +14,60 @@ import type { } from '../../@types/PersistentStorage.js' import { + CreateBucketOptions, CreateBucketResult, PersistentStorageBucketRecord, PersistentStorageFactory, - PersistentStorageFileInfo + PersistentStorageFileInfo, + PersistentStorageQuotaExceededError } from './PersistentStorageFactory.js' import { OceanNode } from '../../OceanNode.js' import { CORE_LOGGER } from '../../utils/logging/common.js' +/* eslint-disable security/detect-non-literal-fs-filename -- walks a bucket folder */ +// Sums file sizes below `dir`. Symlinks are counted as links, never followed, so a +// container can't point one outside the bucket to inflate or dodge its usage. +async function folderSizeBytes(dir: string): Promise { + let total = 0 + const entries = await fsp.readdir(dir, { withFileTypes: true }) + for (const ent of entries) { + const entryPath = path.join(dir, ent.name) + if (ent.isDirectory()) { + total += await folderSizeBytes(entryPath) + } else { + const st = await fsp.lstat(entryPath).catch((): null => null) + if (st) total += st.size + } + } + return total +} +/* eslint-enable security/detect-non-literal-fs-filename */ + +// Passes at most `maxBytes` through, failing the stream as soon as more arrive. +function limitBytes( + maxBytes: number, + bucketId: string, + usage: { quotaBytes: number; usedBytes: number } +): Transform { + let seen = 0 + return new Transform({ + transform(chunk: Buffer, _encoding, callback) { + seen += chunk.length + if (seen > maxBytes) { + callback( + new PersistentStorageQuotaExceededError( + bucketId, + usage.quotaBytes - maxBytes + seen, + usage.quotaBytes + ) + ) + return + } + callback(null, chunk) + } + }) +} + export class PersistentStorageLocalFS extends PersistentStorageFactory { /* eslint-disable security/detect-non-literal-fs-filename -- localfs backend operates on filesystem paths */ private baseFolder: string @@ -98,7 +145,8 @@ export class PersistentStorageLocalFS extends PersistentStorageFactory { async createNewBucket( accessList: AccessList[], owner: string, - label?: string + label?: string, + options: CreateBucketOptions = {} ): Promise { const bucketId = randomUUID() const createdAt = Math.floor(Date.now() / 1000) @@ -114,10 +162,33 @@ export class PersistentStorageLocalFS extends PersistentStorageFactory { owner, JSON.stringify(accessList ?? []), createdAt, - finalLabel + finalLabel, + options ) - return { bucketId, owner, accessList, label: finalLabel } + return { + bucketId, + owner, + accessList, + label: finalLabel, + serviceId: options.serviceId ?? null, + quotaBytes: options.quotaBytes ?? null, + expiresAt: options.expiresAt ?? null + } + } + + async deleteBucket(bucketId: string): Promise { + await this.ensureBucketExists(bucketId) + // Folder first: if the rm fails the row survives, so the bucket is still known and + // the expiry sweep retries it instead of leaking an orphan folder. + await fsp.rm(this.bucketPath(bucketId), { recursive: true, force: true }) + await super.dbDeleteBucket(bucketId) + this.forgetBucketUsage(bucketId) + } + + async getBucketUsageBytes(bucketId: string): Promise { + await this.ensureBucketExists(bucketId) + return await folderSizeBytes(this.bucketPath(bucketId)) } async listFiles( @@ -163,7 +234,39 @@ export class PersistentStorageLocalFS extends PersistentStorageFactory { await fsp.mkdir(targetDir, { recursive: true }) const targetPath = path.join(targetDir, fileName) - await pipeline(content, fs.createWriteStream(targetPath)) + const usage = await this.getBucketQuotaUsage(bucketId) + if (!usage) { + await pipeline(content, fs.createWriteStream(targetPath)) + } else { + // The upload replaces any file of the same name, so its current size is freed. + const replacedBytes = await fsp + .stat(targetPath) + .then((st) => (st.isFile() ? st.size : 0)) + .catch((): number => 0) + const allowedBytes = usage.quotaBytes - usage.usedBytes + replacedBytes + if (allowedBytes <= 0) { + throw new PersistentStorageQuotaExceededError( + bucketId, + usage.usedBytes, + usage.quotaBytes + ) + } + // Write to a temp name and rename on success, so a rejected upload neither leaves + // a partial file behind nor destroys the file it would have replaced. + const tmpPath = path.join(targetDir, `.upload-${randomUUID()}`) + try { + await pipeline( + content, + limitBytes(allowedBytes, bucketId, usage), + fs.createWriteStream(tmpPath) + ) + await fsp.rename(tmpPath, targetPath) + } catch (e) { + await fsp.rm(tmpPath, { force: true }) + throw e + } + this.forgetBucketUsage(bucketId) + } const st = await fsp.stat(targetPath) return { @@ -185,6 +288,7 @@ export class PersistentStorageLocalFS extends PersistentStorageFactory { const targetPath = path.join(this.bucketPath(bucketId), fileName) await fsp.rm(targetPath) + this.forgetBucketUsage(bucketId) } async getFileObject( diff --git a/src/components/persistentStorage/PersistentStorageS3.ts b/src/components/persistentStorage/PersistentStorageS3.ts index f5c9912b0..25f3475aa 100644 --- a/src/components/persistentStorage/PersistentStorageS3.ts +++ b/src/components/persistentStorage/PersistentStorageS3.ts @@ -1,4 +1,5 @@ import { + CreateBucketOptions, CreateBucketResult, PersistentStorageBucketRecord, PersistentStorageFactory, @@ -35,11 +36,22 @@ export class PersistentStorageS3 extends PersistentStorageFactory { async createNewBucket( accessList: AccessList[], _owner: string, - _label?: string + _label?: string, + _options?: CreateBucketOptions ): Promise { throw new Error('PersistentStorageS3 is not implemented yet') } + // eslint-disable-next-line require-await + async deleteBucket(_bucketId: string): Promise { + throw new Error('PersistentStorageS3 is not implemented yet') + } + + // eslint-disable-next-line require-await + async getBucketUsageBytes(_bucketId: string): Promise { + throw new Error('PersistentStorageS3 is not implemented yet') + } + // eslint-disable-next-line require-await async listFiles( _bucketId: string, diff --git a/src/test/unit/persistentStorage/serviceBuckets.test.ts b/src/test/unit/persistentStorage/serviceBuckets.test.ts new file mode 100644 index 000000000..945e022c4 --- /dev/null +++ b/src/test/unit/persistentStorage/serviceBuckets.test.ts @@ -0,0 +1,215 @@ +import { expect } from 'chai' +import fsp from 'fs/promises' +import os from 'os' +import path from 'path' +import { randomUUID } from 'crypto' +import { Readable } from 'stream' +import { PersistentStorageLocalFS } from '../../../components/persistentStorage/PersistentStorageLocalFS.js' +import { PersistentStorageQuotaExceededError } from '../../../components/persistentStorage/PersistentStorageFactory.js' +import { + DEFAULT_SERVICE_BUCKET_QUOTA_BYTES, + DEFAULT_SERVICE_BUCKET_RETENTION_SECONDS +} from '../../../utils/config/constants.js' + +const OWNER = '0x0000000000000000000000000000000000000aBc' + +describe('Service output buckets (localfs)', () => { + let folder: string + let storage: PersistentStorageLocalFS + + before(async () => { + folder = await fsp.mkdtemp(path.join(os.tmpdir(), 'ps-service-buckets-')) + const node: any = { + getConfig: () => ({ + persistentStorage: { enabled: true, type: 'localfs', options: { folder } } + }) + } + storage = new PersistentStorageLocalFS(node) + }) + + after(async () => { + await fsp.rm(folder, { recursive: true, force: true }) + }) + + // The registry DB is shared across runs, so every test uses a fresh serviceId. + const newServiceId = () => `svc-${randomUUID()}` + + it('creates a bucket owned by the consumer, with no access list, the 5 GB quota and a week of retention', async () => { + const serviceExpiresAt = Date.now() + 3600_000 + const bucket = await storage.getOrCreateServiceBucket( + OWNER, + newServiceId(), + serviceExpiresAt + ) + expect(bucket.owner).to.equal(OWNER) + expect(JSON.parse(bucket.accessListJson)).to.deep.equal([]) + expect(bucket.quotaBytes).to.equal(DEFAULT_SERVICE_BUCKET_QUOTA_BYTES) + expect(bucket.expiresAt).to.equal( + Math.floor(serviceExpiresAt / 1000) + DEFAULT_SERVICE_BUCKET_RETENTION_SECONDS + ) + const st = await fsp.stat(path.join(folder, 'buckets', bucket.bucketId)) + expect(st.isDirectory()).to.equal(true) + }) + + it('takes the quota and retention from config when set', async () => { + const node: any = { + getConfig: () => ({ + persistentStorage: { enabled: true, type: 'localfs', options: { folder } }, + serviceBucketQuotaBytes: 1024, + serviceBucketRetentionSeconds: 60 + }) + } + const configured = new PersistentStorageLocalFS(node) + const serviceExpiresAt = Date.now() + 3600_000 + const bucket = await configured.getOrCreateServiceBucket( + OWNER, + newServiceId(), + serviceExpiresAt + ) + expect(bucket.quotaBytes).to.equal(1024) + expect(bucket.expiresAt).to.equal(Math.floor(serviceExpiresAt / 1000) + 60) + }) + + it('is idempotent per serviceId and only ever extends the retention', async () => { + const serviceId = newServiceId() + const t = Date.now() + 3600_000 + const first = await storage.getOrCreateServiceBucket(OWNER, serviceId, t) + const later = await storage.getOrCreateServiceBucket(OWNER, serviceId, t + 7200_000) + expect(later.bucketId).to.equal(first.bucketId) + expect(later.expiresAt).to.equal(first.expiresAt + 7200) + // an earlier window never shortens it + await storage.extendBucketRetention(first.bucketId, t) + expect((await storage.getBucket(first.bucketId)).expiresAt).to.equal(later.expiresAt) + }) + + it('never gives an expiry to a bucket created without one', async () => { + const { bucketId } = await storage.createNewBucket([], OWNER) + await storage.extendBucketRetention(bucketId, Date.now()) + const row = await storage.getBucket(bucketId) + expect(row.expiresAt).to.equal(null) + expect(row.quotaBytes).to.equal(null) + expect(await storage.getBucketQuotaUsage(bucketId)).to.equal(null) + }) + + it('counts files in nested folders towards usage', async () => { + const bucket = await storage.getOrCreateServiceBucket( + OWNER, + newServiceId(), + Date.now() + ) + const dir = path.join(folder, 'buckets', bucket.bucketId) + await fsp.mkdir(path.join(dir, 'a', 'b'), { recursive: true }) + await fsp.writeFile(path.join(dir, 'top.txt'), Buffer.alloc(10)) + await fsp.writeFile(path.join(dir, 'a', 'b', 'deep.bin'), Buffer.alloc(25)) + expect(await storage.getBucketUsageBytes(bucket.bucketId)).to.equal(35) + }) + + describe('quota', () => { + let bucketId: string + + beforeEach(async () => { + ;({ bucketId } = await storage.createNewBucket([], OWNER, undefined, { + quotaBytes: 100 + })) + }) + + it('accepts an upload that fits', async () => { + const info = await storage.uploadFile( + bucketId, + 'ok.bin', + Readable.from([Buffer.alloc(100)]), + OWNER + ) + expect(info.size).to.equal(100) + // full now: the next upload is refused + await storage + .uploadFile(bucketId, 'more.bin', Readable.from([Buffer.alloc(1)]), OWNER) + .then( + () => expect.fail('a full bucket must refuse uploads'), + (e) => expect(e).to.be.instanceOf(PersistentStorageQuotaExceededError) + ) + }) + + it('accepts uploads again once files are deleted', async () => { + await storage.uploadFile( + bucketId, + 'a.bin', + Readable.from([Buffer.alloc(100)]), + OWNER + ) + // warm the cache with the full reading, as a status poll would + expect((await storage.getBucketQuotaUsage(bucketId, 60_000)).usedBytes).to.equal( + 100 + ) + await storage.deleteFile(bucketId, 'a.bin', OWNER) + expect((await storage.getBucketQuotaUsage(bucketId, 60_000)).usedBytes).to.equal(0) + const info = await storage.uploadFile( + bucketId, + 'b.bin', + Readable.from([Buffer.alloc(50)]), + OWNER + ) + expect(info.size).to.equal(50) + }) + + it('serves a cached reading within maxAgeMs and re-sizes after it', async () => { + const dir = path.join(folder, 'buckets', bucketId) + expect((await storage.getBucketQuotaUsage(bucketId, 60_000)).usedBytes).to.equal(0) + // a write the storage API never saw, as a service container makes + await fsp.writeFile(path.join(dir, 'from-container.bin'), Buffer.alloc(40)) + expect((await storage.getBucketQuotaUsage(bucketId, 60_000)).usedBytes).to.equal(0) + expect((await storage.getBucketQuotaUsage(bucketId)).usedBytes).to.equal(40) + }) + + it('rejects an upload that overflows and leaves no partial file behind', async () => { + await storage + .uploadFile(bucketId, 'big.bin', Readable.from([Buffer.alloc(101)]), OWNER) + .then( + () => expect.fail('upload should be rejected'), + (e) => expect(e).to.be.instanceOf(PersistentStorageQuotaExceededError) + ) + expect(await storage.getBucketUsageBytes(bucketId)).to.equal(0) + }) + + it('frees the size of the file an upload replaces, and keeps it when the upload fails', async () => { + await storage.uploadFile( + bucketId, + 'f.bin', + Readable.from([Buffer.alloc(80)]), + OWNER + ) + // 80 in use, but replacing f.bin frees those 80 bytes + await storage.uploadFile( + bucketId, + 'f.bin', + Readable.from([Buffer.alloc(90)]), + OWNER + ) + await storage + .uploadFile(bucketId, 'f.bin', Readable.from([Buffer.alloc(150)]), OWNER) + .catch(() => {}) + const files = await storage.listFiles(bucketId, OWNER) + expect(files.map((f) => [f.name, f.size])).to.deep.equal([['f.bin', 90]]) + }) + }) + + it('deletes expired buckets and keeps the rest', async () => { + const expired = await storage.getOrCreateServiceBucket( + OWNER, + newServiceId(), + Date.now() - (DEFAULT_SERVICE_BUCKET_RETENTION_SECONDS + 60) * 1000 + ) + const live = await storage.getOrCreateServiceBucket(OWNER, newServiceId(), Date.now()) + const permanent = await storage.createNewBucket([], OWNER) + + expect(await storage.deleteExpiredBuckets()).to.be.greaterThanOrEqual(1) + + expect(await storage.getBucket(expired.bucketId)).to.equal(null) + await fsp.stat(path.join(folder, 'buckets', expired.bucketId)).then( + () => expect.fail('expired bucket folder should be gone'), + (e) => expect(e.code).to.equal('ENOENT') + ) + expect(await storage.getBucket(live.bucketId)).to.not.equal(null) + expect(await storage.getBucket(permanent.bucketId)).to.not.equal(null) + }) +}) diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts index fe18f2efd..93090f7b9 100644 --- a/src/test/unit/service/serviceHandlers.test.ts +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -66,6 +66,8 @@ interface FakeOpts { cost?: number | null envId?: string streamableLogs?: Readable | null + // node persistent storage backend; omitted → persistent storage not configured + persistentStorageType?: 'localfs' | 's3' } function buildFakes(opts: FakeOpts = {}) { @@ -189,14 +191,20 @@ function buildFakes(opts: FakeOpts = {}) { // Valid by default; override validateBucket/assertConsumerAllowedForBucket to fail it. const persistentStorage: any = { validateBucket: sinon.stub(), - assertConsumerAllowedForBucket: sinon.stub().resolves(undefined) + assertConsumerAllowedForBucket: sinon.stub().resolves(undefined), + getBucketQuotaUsage: sinon.stub().resolves(null), + extendBucketRetention: sinon.stub().resolves(undefined), + getOrCreateServiceBucket: sinon.stub().resolves({ bucketId: 'auto-bucket' }) } const node: any = { getRequestMap: () => new Map(), getConfig: (): any => ({ rateLimit: undefined as number | undefined, - serviceTemplatesPath: undefined as string | undefined + serviceTemplatesPath: undefined as string | undefined, + persistentStorage: opts.persistentStorageType + ? { enabled: true, type: opts.persistentStorageType } + : undefined }), getC2DEngines: () => engines, getKeyManager: () => ({ @@ -293,6 +301,48 @@ describe('Service handlers', () => { expect(jobs[0]).to.not.have.property('userData') expect(jobs[0].serviceId).to.equal('svc-1') }) + + const statusTask = { + command: PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + consumerAddress: OWNER, + nonce: '1', + signature: '0xsig', + serviceId: 'svc-1' + } + + it('reports the output bucket fill level, flagging a full bucket', async () => { + const { node, persistentStorage } = buildFakes({ + serviceJobInDb: makeJob({ outputBucketId: 'bucket-42' }) + }) + persistentStorage.getBucketQuotaUsage.resolves({ quotaBytes: 100, usedBytes: 100 }) + const [job] = await body( + await new ServiceGetStatusHandler(node).handle({ ...statusTask } as any) + ) + expect(job.outputBucketUsage).to.deep.equal({ + quotaBytes: 100, + usedBytes: 100, + full: true + }) + const [bucketId, maxAgeMs] = persistentStorage.getBucketQuotaUsage.firstCall.args + expect(bucketId).to.equal('bucket-42') + expect(maxAgeMs).to.be.greaterThan(0) + }) + + it('omits the fill level when the bucket has no quota or cannot be sized', async () => { + for (const usage of [null, new Error('EACCES')]) { + const { node, persistentStorage } = buildFakes({ + serviceJobInDb: makeJob({ outputBucketId: 'bucket-42' }) + }) + if (usage instanceof Error) persistentStorage.getBucketQuotaUsage.rejects(usage) + else persistentStorage.getBucketQuotaUsage.resolves(usage) + const res = await new ServiceGetStatusHandler(node).handle({ + ...statusTask + } as any) + expect(res.status.httpStatus).to.equal(200) + const [job] = await body(res) + expect(job).to.not.have.property('outputBucketUsage') + } + }) }) describe('ServiceStopHandler', () => { @@ -711,6 +761,26 @@ describe('Service handlers', () => { expect(out[0]).to.not.have.property('userData') }) + it('extends the output bucket retention to the new expiresAt', async () => { + const job = makeJob({ outputBucketId: 'bucket-42' }) + const { node, persistentStorage } = buildFakes({ serviceJobInDb: job }) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + const out = await body(res) + expect(persistentStorage.extendBucketRetention.firstCall.args).to.deep.equal([ + 'bucket-42', + out[0].expiresAt + ]) + }) + + it('still extends the service when the bucket retention update fails', async () => { + const job = makeJob({ outputBucketId: 'bucket-42' }) + const { node, persistentStorage } = buildFakes({ serviceJobInDb: job }) + persistentStorage.extendBucketRetention.rejects(new Error('disk gone')) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + }) + it('auto-refunds an unresolved extension intent from a previous crash, then proceeds', async () => { const job = makeJob({ extendPayments: [ @@ -1146,6 +1216,64 @@ describe('Service handlers', () => { expect(engine.createServiceJob.lastCall.args.at(-1)).to.equal('bucket-42') }) + it('extends the retention of a supplied bucket and does not create one', async () => { + const { node, persistentStorage } = buildFakes({ persistentStorageType: 'localfs' }) + await new ServiceStartHandler(node).handle({ + ...baseTask, + outputBucketId: 'bucket-42' + } as any) + expect(persistentStorage.extendBucketRetention.firstCall.args[0]).to.equal( + 'bucket-42' + ) + expect(persistentStorage.getOrCreateServiceBucket.called).to.equal(false) + }) + + it('starts into a supplied bucket even when it is full (the quota is soft)', async () => { + const { node, engine, persistentStorage } = buildFakes() + persistentStorage.getBucketQuotaUsage.resolves({ quotaBytes: 100, usedBytes: 500 }) + const res = await new ServiceStartHandler(node).handle({ + ...baseTask, + outputBucketId: 'bucket-42' + } as any) + expect(res.status.httpStatus).to.equal(200) + expect(engine.createServiceJob.lastCall.args.at(-1)).to.equal('bucket-42') + }) + + it('creates a default bucket for the service on localfs and returns its id', async () => { + const { node, engine, persistentStorage } = buildFakes({ + persistentStorageType: 'localfs' + }) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + const [owner, serviceId, expiresAt] = + persistentStorage.getOrCreateServiceBucket.firstCall.args + expect(owner).to.equal(OWNER) + // keyed by the same serviceId the job is created with + expect(serviceId).to.equal(engine.createServiceJob.firstCall.args[13]) + expect(expiresAt).to.be.closeTo(Date.now() + baseTask.duration * 1000, 5000) + expect(engine.createServiceJob.firstCall.args.at(-1)).to.equal('auto-bucket') + }) + + it('runs without a bucket when persistent storage is not localfs', async () => { + for (const persistentStorageType of [undefined, 's3'] as const) { + const { node, engine, persistentStorage } = buildFakes({ persistentStorageType }) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + expect(persistentStorage.getOrCreateServiceBucket.called).to.equal(false) + expect(engine.createServiceJob.firstCall.args.at(-1)).to.equal(undefined) + } + }) + + it('does not create a bucket for a start that is refused', async () => { + const { node, engine, persistentStorage } = buildFakes({ + persistentStorageType: 'localfs' + }) + engine.escrow.getUserAvailableFunds.resolves(0n) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + expect(persistentStorage.getOrCreateServiceBucket.called).to.equal(false) + }) + it('forwards user metadata to createServiceJob (before outputBucketId)', async () => { const { node, engine } = buildFakes() const metadata = { run: 'experiment-7', attempt: 2, dryRun: false } diff --git a/src/utils/config/constants.ts b/src/utils/config/constants.ts index 924a6526a..c7d12162a 100644 --- a/src/utils/config/constants.ts +++ b/src/utils/config/constants.ts @@ -115,7 +115,11 @@ export const ENV_TO_CONFIG_MAPPING = { HTTP_CERT_PATH: 'httpCertPath', HTTP_KEY_PATH: 'httpKeyPath', ENABLE_BENCHMARK: 'enableBenchmark', - PERSISTENT_STORAGE: 'persistentStorage' + PERSISTENT_STORAGE: 'persistentStorage', + // Flat rather than under persistentStorage.*: that key is replaced wholesale when the + // PERSISTENT_STORAGE env var carries its JSON. + SERVICE_BUCKET_QUOTA_BYTES: 'serviceBucketQuotaBytes', + SERVICE_BUCKET_RETENTION_SECONDS: 'serviceBucketRetentionSeconds' } as const // Configuration defaults @@ -125,6 +129,9 @@ export const DEFAULT_RATE_LIMIT_PER_MINUTE = 30 export const DEFAULT_DB_INIT_MAX_ATTEMPTS = 10 export const DEFAULT_DB_INIT_RETRY_DELAY = 2000 export const DEFAULT_DB_INIT_MAX_RETRY_DELAY = 30000 +// Default output bucket SERVICE_START creates when the request carries no outputBucketId. +export const DEFAULT_SERVICE_BUCKET_QUOTA_BYTES = 5 * 1024 * 1024 * 1024 // 5 GB +export const DEFAULT_SERVICE_BUCKET_RETENTION_SECONDS = 7 * 24 * 60 * 60 // 1 week export const DEFAULT_MAX_CONNECTIONS_PER_MINUTE = 60 * 2 // 120 requests per minute export const SEPOLIA_CHAIN_ID = '11155111' export const BASE_CHAIN_ID = '8453' diff --git a/src/utils/config/schemas.ts b/src/utils/config/schemas.ts index ae64b6910..ef74f32dc 100644 --- a/src/utils/config/schemas.ts +++ b/src/utils/config/schemas.ts @@ -15,7 +15,9 @@ import { DEFAULT_FILTER_ANNOUNCED_ADDRESSES, DEFAULT_DB_INIT_MAX_ATTEMPTS, DEFAULT_DB_INIT_RETRY_DELAY, - DEFAULT_DB_INIT_MAX_RETRY_DELAY + DEFAULT_DB_INIT_MAX_RETRY_DELAY, + DEFAULT_SERVICE_BUCKET_QUOTA_BYTES, + DEFAULT_SERVICE_BUCKET_RETENTION_SECONDS } from './constants.js' import { P2P_TIMEOUT_DEFAULTS, @@ -1082,6 +1084,20 @@ export const OceanNodeConfigSchema = z return val }, PersistentStorageConfigSchema) .optional(), + // A 0 quota would stop every service on its first size check, so it must be >= 1. + // A 0 retention is allowed: the bucket then goes when the paid window ends. + serviceBucketQuotaBytes: z.coerce + .number() + .int() + .min(1) + .optional() + .default(DEFAULT_SERVICE_BUCKET_QUOTA_BYTES), + serviceBucketRetentionSeconds: z.coerce + .number() + .int() + .min(0) + .optional() + .default(DEFAULT_SERVICE_BUCKET_RETENTION_SECONDS), FEE_AMOUNT: z.string().optional(), FEE_TOKENS: z.string().optional(), diff --git a/src/utils/constants.ts b/src/utils/constants.ts index f62ed9ec4..5c0686a34 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -504,6 +504,16 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.DB_INIT_MAX_RETRY_DELAY, required: false }, + SERVICE_BUCKET_QUOTA_BYTES: { + name: 'SERVICE_BUCKET_QUOTA_BYTES', + value: process.env.SERVICE_BUCKET_QUOTA_BYTES, + required: false + }, + SERVICE_BUCKET_RETENTION_SECONDS: { + name: 'SERVICE_BUCKET_RETENTION_SECONDS', + value: process.env.SERVICE_BUCKET_RETENTION_SECONDS, + required: false + }, CRON_DELETE_DB_LOGS: { name: 'CRON_DELETE_DB_LOGS', value: process.env.CRON_DELETE_DB_LOGS, diff --git a/src/utils/cronjobs/scheduleCronJobs.ts b/src/utils/cronjobs/scheduleCronJobs.ts index f3aa8fd41..9757a5c54 100644 --- a/src/utils/cronjobs/scheduleCronJobs.ts +++ b/src/utils/cronjobs/scheduleCronJobs.ts @@ -26,6 +26,11 @@ export async function scheduleCronJobs(node: OceanNode) { } catch (e) { OCEAN_NODE_LOGGER.error(`Error when deleting expired c2d jobs: ${e.message}`) } + try { + scheduleDeleteExpiredBucketsJob(node) + } catch (e) { + OCEAN_NODE_LOGGER.error(`Error when deleting expired buckets: ${e.message}`) + } try { scheduleNodeMetricsJobs(node, await node.getDatabase()) } catch (e) { @@ -90,6 +95,21 @@ function scheduleDeleteLogsJob(dbconn: Database | null) { } } +// Service output buckets expire SERVICE_BUCKET_RETENTION_SECONDS (a week by default) after +// their service's paid window. Hourly is plenty at that granularity. +function scheduleDeleteExpiredBucketsJob(node: OceanNode) { + const storage = node.getPersistentStorage() + if (!storage) return + scheduleCron('0 * * * *', async () => { + try { + const deleted = await storage.deleteExpiredBuckets() + if (deleted > 0) OCEAN_NODE_LOGGER.info(`${deleted} expired buckets deleted.`) + } catch (err) { + OCEAN_NODE_LOGGER.error(`Error deleting expired buckets: ${err.message}`) + } + }) +} + function scheduleCleanExpiredC2DJobs(dbconn: Database | null) { // Schedule the cron job to run every 5 minutes or whatever specified