diff --git a/packages/mongodb-downloader/src/index.ts b/packages/mongodb-downloader/src/index.ts index e44e53c8..7ec00026 100644 --- a/packages/mongodb-downloader/src/index.ts +++ b/packages/mongodb-downloader/src/index.ts @@ -1,11 +1,12 @@ /* eslint-disable no-console */ import fetch from 'node-fetch'; import * as tar from 'tar'; +import { createHash } from 'crypto'; import { promisify } from 'util'; import { promises as fs, createWriteStream } from 'fs'; import path from 'path'; import decompress from 'decompress'; -import { pipeline } from 'stream'; +import { pipeline, Transform } from 'stream'; import getDownloadURL from 'mongodb-download-url'; import type { Options as DownloadOptions, @@ -31,6 +32,11 @@ export type MongoDBDownloaderOptions = { useLockfile: boolean; /** The options to pass to the download URL lookup. */ downloadOptions?: DownloadOptions; + /** + * A direct URL to a MongoDB tarball to download. If set, `version` and + * `downloadOptions` are ignored and no download URL lookup is performed. + */ + downloadUrl?: string; }; export class MongoDBDownloader { @@ -39,6 +45,7 @@ export class MongoDBDownloader { version = '*', directory, useLockfile, + downloadUrl, }: MongoDBDownloaderOptions): Promise { await fs.mkdir(directory, { recursive: true }); const isWindows = ['win32', 'windows'].includes( @@ -58,12 +65,20 @@ export class MongoDBDownloader { versionName = versionName + (isEnterprise ? '-enterprise' : '-community'); } - const downloadTarget = path.resolve( - directory, - `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ - process.arch - }-${versionName}`.replace(/[^a-zA-Z0-9_-]/g, ''), - ); + const downloadTarget = downloadUrl + ? path.resolve( + directory, + `mongodb-custom-${createHash('sha256') + .update(downloadUrl) + .digest('hex') + .slice(0, 16)}`, + ) + : path.resolve( + directory, + `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ + process.arch + }-${versionName}`.replace(/[^a-zA-Z0-9_-]/g, ''), + ); const bindir = path.resolve( downloadTarget, isCryptLibrary && !isWindows ? 'lib' : 'bin', @@ -99,11 +114,13 @@ export class MongoDBDownloader { } await fs.mkdir(downloadTarget, { recursive: true }); - const artifactInfo = await this.lookupDownloadUrl({ - targetVersion: version, - enterprise: isEnterprise, - options: downloadOptions, - }); + const artifactInfo = downloadUrl + ? ({ url: downloadUrl } as DownloadArtifactInfo) + : await this.lookupDownloadUrl({ + targetVersion: version, + enterprise: isEnterprise, + options: downloadOptions, + }); const { url } = artifactInfo; debug(`Downloading ${url} into ${downloadTarget}...`); @@ -139,11 +156,35 @@ export class MongoDBDownloader { const response = await fetch(url, { highWaterMark: MongoDBDownloader.HWM, } as Parameters[1]); + if (!response.ok) { + throw new Error( + `Failed to download ${url}: ${response.status} ${response.statusText}`, + ); + } + const totalBytes = +(response.headers.get('content-length') ?? ''); + const totalMB = totalBytes ? (totalBytes / 1048576).toFixed(1) : null; + debug(`Download started`, { url, totalMB }); + let downloadedBytes = 0; + let lastProgressLog = Date.now(); + const progress = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + downloadedBytes += chunk.length; + if (Date.now() - lastProgressLog >= 3000) { + lastProgressLog = Date.now(); + const downloadedMB = (downloadedBytes / 1048576).toFixed(1); + debug( + `Downloading: ${downloadedMB}MB${totalMB ? ` / ${totalMB}MB` : ''}`, + ); + } + callback(null, chunk); + }, + }); if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { // the server's tarballs can contain hard links, which the (unmaintained?) // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) await promisify(pipeline)( response.body, + progress, tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }), ); } else { @@ -153,6 +194,7 @@ export class MongoDBDownloader { ); await promisify(pipeline)( response.body, + progress, createWriteStream(filename, { highWaterMark: MongoDBDownloader.HWM }), ); debug(`Written file ${url} to ${filename}, extracting...`); @@ -230,32 +272,15 @@ async function withoutLock( const downloader = new MongoDBDownloader(); /** Download mongod + mongos with version info and return version info and the path to a directory containing them. */ -export async function downloadMongoDbWithVersionInfo({ - downloadOptions = {}, - version = '*', - directory, - useLockfile, -}: MongoDBDownloaderOptions): Promise { - return await downloader.downloadMongoDbWithVersionInfo({ - downloadOptions, - version, - directory, - useLockfile, - }); +export async function downloadMongoDbWithVersionInfo( + options: MongoDBDownloaderOptions, +): Promise { + return await downloader.downloadMongoDbWithVersionInfo(options); } /** Download mongod + mongos and return the path to a directory containing them. */ -export async function downloadMongoDb({ - downloadOptions = {}, - version = '*', - directory, - useLockfile, -}: MongoDBDownloaderOptions): Promise { - return ( - await downloader.downloadMongoDbWithVersionInfo({ - downloadOptions, - version, - directory, - useLockfile, - }) - ).downloadedBinDir; +export async function downloadMongoDb( + options: MongoDBDownloaderOptions, +): Promise { + return (await downloader.downloadMongoDbWithVersionInfo(options)) + .downloadedBinDir; } diff --git a/packages/mongodb-runner/README.md b/packages/mongodb-runner/README.md index ad0e553a..e1f9607d 100644 --- a/packages/mongodb-runner/README.md +++ b/packages/mongodb-runner/README.md @@ -99,6 +99,13 @@ You can specify a config file for the CLI using `--config`: $ npx mongodb-runner start --config ``` +## DSC clusters + +mongodb-runner can launch clusters that use DSC, backed by an SLS storage +layer, managing the storage backend's docker compose project alongside the +mongod processes. See +[docs/disaggregated-storage.md](./docs/disaggregated-storage.md). + ## License Apache 2.0 diff --git a/packages/mongodb-runner/docs/disaggregated-storage.md b/packages/mongodb-runner/docs/disaggregated-storage.md new file mode 100644 index 00000000..c4f4ce41 --- /dev/null +++ b/packages/mongodb-runner/docs/disaggregated-storage.md @@ -0,0 +1,273 @@ +# Launching a DSC cluster + +mongodb-runner can launch MongoDB clusters that use DSC, backed by an SLS +storage layer. In this mode, the runner: + +1. Starts a docker compose project providing the storage backend + (log servers, page servers, cell metadata services, schedulers, a localstack + S3/SQS emulation, etc.), +2. waits for the storage layer to become ready, +3. pre-allocates the ports of all `mongod` nodes (so the storage configuration + can reference the replica set members before they start), +4. runs a per-shard setup step against the storage layer, +5. starts every `mongod` with + `--setParameter disaggregatedStorageConfig=...` and + `--setParameter disaggregatedStorageEnabled=true`. + +The compose project is started **once per cluster** and shared by all shards; +each shard only runs its own setup step (e.g. starting an SLS log) against the +shared storage layer. When the cluster is stopped, the compose project is torn +down with it (`docker compose down --volumes`), including when cluster startup +fails partway through. + +For replica sets with DSC, the default node count is **2** +(1 primary + 1 secondary) instead of the usual 3. This applies to the shard +replica sets of a sharded cluster as well. Set `secondaries` explicitly to +override. + +## Prerequisites + +- Docker with `docker compose` v2. +- Access to the SLS container images. The default repository is a private ECR + registry, so log in first: + + ```bash + aws ecr get-login-password --region us-east-1 | \ + docker login --username AWS --password-stdin 664315256653.dkr.ecr.us-east-1.amazonaws.com + ``` + +- A `mongod` build that understands the `disaggregatedStorageConfig` + server parameter. Stock community/enterprise release binaries do **not** — + you need a build of the server with the atlas module. Provide it either as: + - `binDir`: a local directory containing the binaries, or + - `downloadUrl`: a URL to a tarball of such a build (cached by URL, standard + release-tarball layout with a top-level directory containing `bin/`). +- The SLS multi-cell compose file — mongodb-runner does not ship one; point + it at the file you want to use, typically + `buildscripts/modules/atlas/sls-multicell-docker-compose.yml` in a mongodb + server checkout. Files it references (`slsbackup.proto`, + `flags-state.json`) are resolved relative to it, and the services/ports are + parsed from it, so any version of the file works as-is. +- The SLS image tag to use, typically the `pinned_sls_commit` from + `buildscripts/modules/atlas/manifest.json` in the mongodb server repository. + +## Quick start + +Given the compose file and image tag, everything else (compose environment +variables, readiness polling, per-shard log creation, and the +`disaggregatedStorageConfig` server parameter) is generated automatically. +Full sequence, assuming a mongodb server checkout at `$MONGO_REPO`: + +```bash +# 1. Log in to the SLS image registry +aws ecr get-login-password --region us-east-1 | \ + docker login --username AWS --password-stdin 664315256653.dkr.ecr.us-east-1.amazonaws.com + +# 2. Look up the pinned SLS image tag +SLS_IMAGE_TAG=$(python3 -c "import json; print(json.load(open('$MONGO_REPO/buildscripts/modules/atlas/manifest.json'))['pinned_sls_commit'])") + +# 3. Start a 2-node replica set backed by SLS +mongodb-runner start -t replset \ + --slsCompose=$MONGO_REPO/buildscripts/modules/atlas/sls-multicell-docker-compose.yml \ + --slsImageTag=$SLS_IMAGE_TAG \ + --binDir=/path/to/dsc-mongod/bin \ + --debug +# or, instead of --binDir: +# --downloadUrl=https://.../dsc-mongod.tgz +``` + +This prints the connection string once the cluster is up. The first run pulls +all SLS images, which can take several minutes (`--debug` shows the +progress). `mongodb-runner stop --id=...` (printed by `start`) tears down the +mongod processes and the compose project together. + +The same works for `-t standalone` and `-t sharded`. A DSC-capable +`mongod` is required (see Prerequisites) — with a stock binary, startup fails +with `Unknown --setParameter 'disaggregatedStorageConfig'`. + +[`examples/sls-replset.js`](../examples/sls-replset.js) is the equivalent +using the programmatic API: + +```bash +cd packages/mongodb-runner && npm run compile +SLS_COMPOSE_FILE=$MONGO_REPO/buildscripts/modules/atlas/sls-multicell-docker-compose.yml \ +SLS_IMAGE_TAG=$SLS_IMAGE_TAG \ +MONGOD_BIN_DIR=/path/to/dsc-mongod/bin \ + node examples/sls-replset.js +``` + +## Programmatic use + +The `disaggregatedStorage` option is accepted for every topology +(standalone, replset, sharded). For an SLS project, one call creates +fully-populated options: + +```typescript +import { + MongoCluster, + createSLSDisaggregatedStorageOptions, +} from '@mongodb-js/mongodb-runner'; + +const disaggregatedStorage = await createSLSDisaggregatedStorageOptions({ + composeFile: '/path/to/sls-multicell-docker-compose.yml', + imageTag: 'abc123', // pinned_sls_commit + // optional: projectName, firstLogId, readyTimeoutSecs, imageRepo, + // testDataId, hostInternalIP +}); + +const cluster = await MongoCluster.start({ + topology: 'sharded', + shards: 2, + binDir: '/path/to/dsc-mongod/bin', // or downloadUrl: '...' + tmpDir: os.tmpdir(), + disaggregatedStorage, +}); + +console.log(cluster.connectionString); +// disaggregatedStorage.sls exposes the allocated ports and service URIs. +await cluster.close(); // also tears down the compose project +``` + +### Custom storage backends + +For a different storage backend — or to customize individual steps — provide +the `disaggregatedStorage` fields yourself. This is what +`createSLSDisaggregatedStorageOptions()` assembles under the hood: + +```typescript +import { + MongoCluster, + createSLSMultiCellEnvironment, + createSLSDisaggregatedStorageConfig, +} from '@mongodb-js/mongodb-runner'; + +// Parses the services out of the given compose file, allocates free host +// ports for them, and builds the env vars the file expects (image repo/tag, +// per-service ports, ...). +const sls = await createSLSMultiCellEnvironment({ + composeFile: '/path/to/sls-multicell-docker-compose.yml', + imageTag: 'abc123', // pinned_sls_commit +}); + +const cluster = await MongoCluster.start({ + topology: 'sharded', + shards: 2, + downloadUrl: 'https://.../dsc-mongod.tgz', // or binDir: '...' + tmpDir: os.tmpdir(), + disaggregatedStorage: { + // Any compose project providing the storage backend. + composeFile: sls.composeFile, + + // Environment variables for compose-file interpolation. Set + // COMPOSE_PROJECT_NAME for a predictable project/container naming scheme. + env: { ...sls.env, COMPOSE_PROJECT_NAME: 'my-sls-project' }, + + // Called once after `docker compose up`. Poll until the storage layer + // actually serves traffic; with the SLS compose file, waiting for + // the /ready file in the testdriver container is the canonical check. + waitForReady: async () => { + /* e.g. docker exec my-sls-project-testdriver-1 test -f /ready */ + }, + + // Called once per shard before that shard's mongod processes start. + // For sharded clusters, index 0 with isConfigServer: true is the config + // server replica set; for standalone/replset it is called once with + // { index: 0, isConfigServer: false }. The descriptor also carries the + // replica set name and the members with their pre-allocated ports. + setupShard: async ({ index, isConfigServer }) => { + /* e.g. StartLog via grpcurl in the testdriver container */ + }, + + // Value for `--setParameter disaggregatedStorageConfig=...`, injected + // into every mongod (config server included) together with + // `disaggregatedStorageEnabled=true`. With the SLS helper, this is + // built automatically from the shard descriptor (replica set name and + // pre-allocated member host:ports) and the SLS environment; the logId + // must match the log started in setupShard. + config: (shard) => + createSLSDisaggregatedStorageConfig(sls, shard, { + logId: 1 + shard.index, + }), + }, +}); + +console.log(cluster.connectionString); +// ... +await cluster.close(); // also tears down the compose project +``` + +### The SLS environment helper + +`createSLSMultiCellEnvironment(options)` mirrors the `SLSMultiCellFixture` +from the server codebase and returns: + +| Field | Description | +| ------------- | -------------------------------------------------------------------------------------------------------- | +| `composeFile` | Path to the compose file in use | +| `env` | All environment variables the compose file expects (`SLS_IMAGE_REPO`, `SLS_IMAGE_TAG`, `*_PORT` vars, …) | +| `ports` | The allocated host port per service, e.g. `ports['crs-cell1-0']` | +| `services` | Host `addr`/`uri` per service, e.g. `services['cms-cell1-0'].uri` | + +Required options: `composeFile`, `imageTag`. Optional: `imageRepo`, +`thirdPartyImageRepo`, `testDataId` (container label for test attribution), +`hostInternalIP`. The service list is parsed from the compose file's +`ports:` mappings (`parseSLSComposeServices`), so it adapts to whatever +version of the file is provided. + +Related exports: `createSLSDisaggregatedStorageConfig` (builds the mongod +server parameter for one shard; options: `logId`, `cellMetadataService`, +`zoneName`, `encryptionKeyFilePath`), `parseSLSComposeServices`, +`SLS_HOSTNAME`, `SLS_CELL1`/`SLS_CELL2`/`SLS_CELL3`, and +`DockerComposeProject` for managing compose projects directly. + +## CLI use + +For an SLS project, `--slsCompose` + `--slsImageTag` handle everything (see +Quick start): + +```bash +mongodb-runner start -t replset \ + --slsCompose=/path/to/sls-multicell-docker-compose.yml \ + --slsImageTag= --binDir=... +``` + +For a **custom** storage backend with its own compose file, use the low-level +flags — a static compose project and an explicit config value: + +```bash +mongodb-runner start -t replset \ + --downloadUrl https://.../dsc-mongod.tgz \ + --disaggregatedStorageCompose ./path/to/docker-compose.yml \ + --disaggregatedStorageConfig '{"...": "..."}' +``` + +`--disaggregatedStorageConfig` takes the JSON (or plain string) value for the +`disaggregatedStorageConfig` server parameter. Custom compose environment +variables, readiness polling, and per-shard setup steps require the +programmatic API. + +`mongodb-runner stop --id=...` restores the cluster metadata and tears down +the compose project along with the mongod processes, even from a different +process than the one that started it. + +## Troubleshooting + +- **`Unknown --setParameter 'disaggregatedStorageConfig'`** — the `mongod` + being launched is a stock build. Point `binDir`/`downloadUrl` at a + DSC-capable build. Also note that binaries must match the host platform; + a Linux build from a VM checkout cannot run directly on macOS. +- **`StartLog` fails with `log segment already exists`** — the requested log + ID is already in use. Log ID `9999` is reserved by the cell metadata + services in the SLS compose file; also, the storage layer's state + persists while the compose project is running, so re-running setup with the + same log ID conflicts. Either tolerate the error and reuse the log (see the + example script), or reset the storage layer. +- **Resetting the storage layer** — with a fixed `COMPOSE_PROJECT_NAME`: + + ```bash + docker compose -p down --volumes + ``` + +- **Seeing what's happening** — run with `DEBUG='mongodb-runner*,mongodb-downloader*'` + (or `--verbose` on the CLI) to get compose output, tarball download + progress, and per-server mongod log entries. diff --git a/packages/mongodb-runner/examples/sls-replset.js b/packages/mongodb-runner/examples/sls-replset.js new file mode 100644 index 00000000..2bcd4039 --- /dev/null +++ b/packages/mongodb-runner/examples/sls-replset.js @@ -0,0 +1,86 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ +'use strict'; + +// Starts a 2-node replica set backed by the SLS multi-cell disaggregated +// storage compose project. +// +// Usage: +// SLS_COMPOSE_FILE=/path/to/sls-multicell-docker-compose.yml \ +// SLS_IMAGE_TAG= \ +// MONGOD_BIN_DIR=/path/to/mongod/dir \ +// node examples/sls-replset.js +// +// This is equivalent to: +// mongodb-runner start -t replset \ +// --slsCompose= --slsImageTag= --binDir=... + +// Enable mongodb-runner debug output (compose progress, server startup, ...) +// unless the user already configured DEBUG themselves. +process.env.DEBUG = process.env.DEBUG || 'mongodb-runner,mongodb-downloader'; + +const os = require('os'); +const { + MongoCluster, + createSLSDisaggregatedStorageOptions, +} = require('../dist/index.js'); + +async function main() { + if (!process.env.MONGOD_BIN_DIR && !process.env.MONGOD_DOWNLOAD_URL) { + throw new Error( + 'Set MONGOD_BIN_DIR to a directory containing a mongod build that ' + + 'supports the disaggregatedStorageConfig setParameter, or ' + + 'MONGOD_DOWNLOAD_URL to a tarball URL of such a build. ' + + '(Otherwise mongodb-runner downloads a stock community binary, ' + + 'which will fail with "Unknown --setParameter".)', + ); + } + if (!process.env.SLS_COMPOSE_FILE || !process.env.SLS_IMAGE_TAG) { + throw new Error( + 'Set SLS_COMPOSE_FILE to the path of an SLS multi-cell ' + + 'docker-compose.yml and SLS_IMAGE_TAG to the SLS image tag to use ' + + '(e.g. the pinned_sls_commit from the server repo manifest).', + ); + } + + // Sets up everything the SLS compose project needs: env vars with + // allocated service ports, readiness polling, per-shard log creation, and + // the mongod disaggregatedStorageConfig server parameter. + const disaggregatedStorage = await createSLSDisaggregatedStorageOptions({ + composeFile: process.env.SLS_COMPOSE_FILE, + imageTag: process.env.SLS_IMAGE_TAG, + }); + + console.log('SLS service ports:', disaggregatedStorage.sls.ports); + console.log( + 'starting docker compose project (this pulls all SLS images on first ' + + 'run, which can take several minutes)...', + ); + + const cluster = await MongoCluster.start({ + topology: 'replset', + secondaries: 1, // 2 nodes total (this is also the DSC default) + tmpDir: os.tmpdir(), + binDir: process.env.MONGOD_BIN_DIR, + downloadUrl: process.env.MONGOD_DOWNLOAD_URL, + disaggregatedStorage, + }); + + console.log('Replica set running at:', cluster.connectionString); + console.log('Press Ctrl+C to shut down'); + + process.on('SIGINT', () => { + cluster.close().then( + () => process.exit(0), + (err) => { + console.error(err); + process.exit(1); + }, + ); + }); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/packages/mongodb-runner/src/cli.ts b/packages/mongodb-runner/src/cli.ts index b27cf054..a7e210f4 100644 --- a/packages/mongodb-runner/src/cli.ts +++ b/packages/mongodb-runner/src/cli.ts @@ -39,6 +39,11 @@ import type { MongoClientOptions } from 'mongodb'; type: 'string', describe: 'MongoDB server version to use', }) + .option('downloadUrl', { + type: 'string', + describe: + 'Direct URL to a MongoDB tarball to download (takes precedence over --version)', + }) .option('logDir', { type: 'string', describe: 'Directory to store server log files in', @@ -78,6 +83,25 @@ import type { MongoClientOptions } from 'mongodb'; type: 'string', describe: 'Configure OIDC authentication on the server', }) + .option('slsCompose', { + type: 'string', + describe: + 'Path to an SLS multi-cell docker-compose.yml; launches the SLS DSC project and configures mongod to use it (requires a DSC-capable mongod via --binDir or --downloadUrl)', + }) + .option('slsImageTag', { + type: 'string', + describe: + 'SLS docker image tag to use with --slsCompose (e.g. the pinned_sls_commit from the server repo manifest)', + }) + .option('disaggregatedStorageCompose', { + type: 'string', + describe: 'Path to docker-compose.yml for the DSC backend', + }) + .option('disaggregatedStorageConfig', { + type: 'string', + describe: + 'JSON value for the disaggregatedStorageConfig setParameter on each mongod', + }) .option('debug', { type: 'boolean', describe: 'Enable debug output' }) .option('verbose', { type: 'boolean', describe: 'Enable verbose output' }) .command('start', 'Start a MongoDB instance') @@ -96,10 +120,10 @@ import type { MongoClientOptions } from 'mongodb'; args.push(...argv.args.map(String)); } if (argv.debug || argv.verbose) { - createDebug.enable('mongodb-runner'); + createDebug.enable('mongodb-runner,mongodb-downloader'); } if (argv.verbose) { - createDebug.enable('mongodb-runner:*'); + createDebug.enable('mongodb-runner*,mongodb-downloader*'); } if (argv.oidc && process.platform !== 'linux') { @@ -114,7 +138,38 @@ import type { MongoClientOptions } from 'mongodb'; } async function start() { - const { cluster, id } = await utilities.start(argv, args); + if (argv.slsCompose && !argv.slsImageTag) { + throw new Error('--slsCompose requires --slsImageTag'); + } + const disaggregatedStorage = argv.slsCompose + ? await utilities.createSLSDisaggregatedStorageOptions({ + composeFile: argv.slsCompose, + imageTag: argv.slsImageTag!, + }) + : argv.disaggregatedStorageCompose + ? { + composeFile: argv.disaggregatedStorageCompose, + config: (() => { + try { + return JSON.parse(argv.disaggregatedStorageConfig ?? ''); + } catch { + return argv.disaggregatedStorageConfig ?? ''; + } + })(), + } + : undefined; + if (disaggregatedStorage && 'sls' in disaggregatedStorage) { + console.error('Allocated SLS service ports:'); + for (const [serviceName, { addr }] of Object.entries( + disaggregatedStorage.sls.services, + )) { + console.error(` ${serviceName}: ${addr}`); + } + } + const { cluster, id } = await utilities.start( + { ...argv, disaggregatedStorage }, + args, + ); const cs = new ConnectionString(cluster.connectionString); // Only the connection string should print to stdout so it can be captured // by a calling process. diff --git a/packages/mongodb-runner/src/docker-compose.ts b/packages/mongodb-runner/src/docker-compose.ts new file mode 100644 index 00000000..377dd589 --- /dev/null +++ b/packages/mongodb-runner/src/docker-compose.ts @@ -0,0 +1,117 @@ +import { spawn } from 'child_process'; +import { once } from 'events'; +import { debug, uuid } from './util'; + +export interface DockerComposeProjectOptions { + /** + * Environment variables used for variable interpolation in the compose file. + * Merged over the current process environment. + */ + env?: Record; + /** + * Compose project name (passed as `docker compose -p `). Defaults to a + * generated unique name so multiple projects from the same compose file can + * coexist. + */ + projectName?: string; +} + +async function runDockerCompose( + composeFile: string, + projectName: string, + env: Record | undefined, + args: string[], +): Promise<{ code: number | null; stderr: string }> { + const proc = spawn( + 'docker', + ['compose', '-f', composeFile, '-p', projectName, ...args], + { + stdio: ['inherit', 'pipe', 'pipe'], + env: { ...process.env, ...env }, + }, + ); + await once(proc, 'spawn'); + debug('docker compose: spawned', { pid: proc.pid, args }); + let stderr = ''; + proc.stderr.setEncoding('utf8'); + proc.stderr.on('data', (chunk: string) => { + stderr += chunk; + for (const line of chunk.split('\n')) { + if (line.trim()) debug('docker compose stderr:', line); + } + }); + proc.stdout.setEncoding('utf8'); + proc.stdout.on('data', (chunk: string) => { + for (const line of chunk.split('\n')) { + if (line.trim()) debug('docker compose stdout:', line); + } + }); + const [code] = await once(proc, 'exit'); + debug('docker compose: exited', { code }); + return { code: code as number | null, stderr }; +} + +export class DockerComposeProject { + private constructor( + private readonly composeFile: string, + private readonly projectName: string, + private readonly env?: Record, + ) {} + + static async start( + composeFile: string, + options: DockerComposeProjectOptions = {}, + ): Promise { + const projectName = + options.projectName ?? + options.env?.COMPOSE_PROJECT_NAME ?? + `mongodb-runner-${uuid()}`; + debug('starting docker compose project', { composeFile, projectName }); + const { code, stderr } = await runDockerCompose( + composeFile, + projectName, + options.env, + ['up', '-d'], + ); + if (code !== 0) { + throw new Error( + `docker compose up failed with exit code ${String(code)}: ${stderr}`, + ); + } + debug('docker compose project started'); + return new DockerComposeProject(composeFile, projectName, options.env); + } + + async close(): Promise { + debug('stopping docker compose project', { + composeFile: this.composeFile, + projectName: this.projectName, + }); + const { code, stderr } = await runDockerCompose( + this.composeFile, + this.projectName, + this.env, + ['down', '--volumes'], + ); + if (code !== 0) { + debug('docker compose down failed', { code: String(code), stderr }); + } + debug('docker compose project stopped'); + } + + serialize(): unknown { + return { + composeFile: this.composeFile, + projectName: this.projectName, + env: this.env, + }; + } + + static deserialize(serialized: any): DockerComposeProject { + return new DockerComposeProject( + serialized.composeFile, + serialized.projectName, + serialized.env, + ); + } +} diff --git a/packages/mongodb-runner/src/index.ts b/packages/mongodb-runner/src/index.ts index d59500c9..8ab97040 100644 --- a/packages/mongodb-runner/src/index.ts +++ b/packages/mongodb-runner/src/index.ts @@ -12,7 +12,29 @@ export { RSOptions as MongoClusterRSOptions, CommonOptions as MongoClusterCommonOptions, ShardedOptions as MongoClusterShardedOptions, + type ShardDescriptor, + type DisaggregatedStorageOptions, } from './mongocluster'; +export { + createSLSMultiCellEnvironment, + createSLSDisaggregatedStorageConfig, + createSLSDisaggregatedStorageOptions, + type SLSDisaggregatedStorageConfigOptions, + type SLSDisaggregatedStorageSetupOptions, + parseSLSComposeServices, + SLS_HOSTNAME, + SLS_CELL1, + SLS_CELL2, + SLS_CELL3, + type SLSCell, + type SLSServiceInfo, + type SLSMultiCellEnvironment, + type SLSMultiCellEnvironmentOptions, +} from './sls'; +export { + DockerComposeProject, + type DockerComposeProjectOptions, +} from './docker-compose'; export type { LogEntry } from './mongologreader'; export type { ConnectionString } from 'mongodb-connection-string-url'; export { prune, start, stop, exec, instances } from './runner-helpers'; diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 6f8d24a8..14b2b89e 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -20,8 +20,10 @@ import { makeConnectionString, safePromiseAll, hasPortArg, + allocatePort, } from './util'; import { OIDCMockProviderProcess } from './oidc'; +import { DockerComposeProject } from './docker-compose'; import { EventEmitter } from 'events'; import assert from 'assert'; import { handleTLSClientKeyOptions } from './tls-helpers'; @@ -99,6 +101,11 @@ export interface CommonOptions { * MongoDB server version to download and use (e.g. '6.0.3', '8.x-enterprise', 'latest-alpha', etc.) */ version?: string; + /** + * A direct URL to a MongoDB tarball to download and use. Takes precedence + * over `version`. + */ + downloadUrl?: string; /** * User accounts to create after starting the cluster. */ @@ -121,6 +128,13 @@ export interface CommonOptions { * Topology of the cluster. */ topology: 'standalone' | 'replset' | 'sharded'; + + /** + * When set, starts a docker compose project providing a DSC + * backend before spawning any mongod processes, and injects + * `--setParameter disaggregatedStorageConfig=...` into each one. + */ + disaggregatedStorage?: DisaggregatedStorageOptions; } /** @@ -142,6 +156,70 @@ export type ShardedOptions = { shards?: number | Partial[]; }; +/** A replica set member of a shard, with its pre-allocated port. */ +export interface ShardMemberDescriptor { + host: string; + port: number; + priority: number; + arbiterOnly: boolean; +} + +/** Identifies a shard within a cluster for DSC setup. */ +export interface ShardDescriptor { + /** Zero-based shard index. In a sharded cluster, 0 is the config server. */ + index: number; + /** True only for the dedicated config server replset in a sharded cluster. */ + isConfigServer: boolean; + /** The replica set name (absent for standalone topologies). */ + replSetName?: string; + /** + * The members of this shard with their pre-allocated ports. With + * DSC, ports are allocated before the servers start so + * that the storage configuration can reference them. + */ + members: ShardMemberDescriptor[]; +} + +/** + * Options for DSC support. When provided, mongodb-runner will + * launch a docker compose project providing the storage backend, then call the + * supplied hooks before starting mongod processes. + */ +export interface DisaggregatedStorageOptions { + /** Path to the docker-compose.yml file for the storage backend. */ + composeFile: string; + + /** + * Environment variables used for variable interpolation in the compose file + * (e.g. image repository/tag and host port mappings). Merged over the + * current process environment. + */ + env?: Record; + + /** + * Value for `--setParameter disaggregatedStorageConfig=...` injected into + * every mongod, including the config server. Accepts a per-shard function + * to produce different values per shard (e.g. different bucket names). + */ + config: + | string + | Record + | ((shard: ShardDescriptor) => string | Record); + + /** + * Called once after `docker compose up` returns, before any mongod starts. + * Should poll/retry until the storage layer is ready to accept connections. + */ + waitForReady?: () => Promise; + + /** + * Called once per shard before that shard's mongod processes start. + * Should create any resources (buckets, namespaces, etc.) needed by that shard. + * For standalone/replset topologies, called once with index=0, isConfigServer=false. + */ + setupShard?: (shard: ShardDescriptor) => Promise; +} + export type MongoClusterOptions = Pick< MongoServerOptions, | 'logDir' @@ -232,6 +310,36 @@ function processRSMembers(options: MongoClusterOptions): { }; } +function buildDisaggregatedArgs( + ds: DisaggregatedStorageOptions, + shard: ShardDescriptor, + baseArgs: string[], +): string[] { + const raw = typeof ds.config === 'function' ? ds.config(shard) : ds.config; + const value = typeof raw === 'string' ? raw : JSON.stringify(raw); + return [ + ...baseArgs, + '--setParameter', + `disaggregatedStorageConfig=${value}`, + '--setParameter', + 'disaggregatedStorageEnabled=true', + ]; +} + +/** Remove any `--port ` / `--port=` argument, flag included. */ +function stripPortArg(args: string[]): string[] { + return removePortArg(args).filter((arg) => arg !== '--port'); +} + +/** Return the value of a `--port ` / `--port=` argument, if any. */ +function getPortArg(args: string[] | undefined): number | undefined { + if (!args) return undefined; + const splitArgIndex = args.indexOf('--port'); + if (splitArgIndex !== -1) return +args[splitArgIndex + 1] || undefined; + const arg = args.find((a) => a.startsWith('--port=')); + return arg ? +arg.split('=')[1] || undefined : undefined; +} + function processShardOptions(options: MongoClusterOptions): { shards: Partial[]; mongosArgs: string[][]; @@ -271,6 +379,7 @@ export class MongoCluster extends EventEmitter { private servers: MongoServer[] = []; // mongod/mongos private shards: MongoCluster[] = []; // replsets private oidcMockProviderProcess?: OIDCMockProviderProcess; + private dockerComposeProject?: DockerComposeProject; private defaultConnectionOptions: Partial = {}; private users: MongoDBUserDoc[] = []; @@ -295,12 +404,14 @@ export class MongoCluster extends EventEmitter { tmpdir: string, targetVersionSemverSpecifier?: string | undefined, options?: DownloadOptions | undefined, + downloadUrl?: string | undefined, ): Promise { return downloadMongoDb({ directory: tmpdir, version: targetVersionSemverSpecifier, downloadOptions: options, useLockfile: true, + downloadUrl, }); } @@ -311,6 +422,7 @@ export class MongoCluster extends EventEmitter { servers: this.servers.map((srv) => srv.serialize()), shards: this.shards.map((shard) => shard.serialize()), oidcMockProviderProcess: this.oidcMockProviderProcess?.serialize(), + dockerComposeProject: this.dockerComposeProject?.serialize(), defaultConnectionOptions: jsonClone(this.defaultConnectionOptions ?? {}), users: jsonClone(this.users), }; @@ -319,7 +431,7 @@ export class MongoCluster extends EventEmitter { isClosed(): boolean { // Return true if and only if there are no running sub-clusters/servers // eslint-disable-next-line @typescript-eslint/no-unused-vars - for (const _ of this.children()) return true; + for (const _ of this.children()) return false; return true; } @@ -338,6 +450,9 @@ export class MongoCluster extends EventEmitter { cluster.oidcMockProviderProcess = serialized.oidcMockProviderProcess ? OIDCMockProviderProcess.deserialize(serialized.oidcMockProviderProcess) : undefined; + cluster.dockerComposeProject = serialized.dockerComposeProject + ? DockerComposeProject.deserialize(serialized.dockerComposeProject) + : undefined; return cluster; } @@ -369,9 +484,16 @@ export class MongoCluster extends EventEmitter { return this.servers[0].serverVariant; } - static async start({ - ...options - }: MongoClusterOptions): Promise { + static async start( + { ...options }: MongoClusterOptions, + // Internal parameter used when starting the shards of a sharded cluster: + // the parent cluster owns the docker compose project, and the shards + // inherit the DSC context from it. + _internal?: { + disaggregatedStorage: DisaggregatedStorageOptions; + shardContext: { index: number; isConfigServer: boolean }; + }, + ): Promise { options = { ...options, ...(await handleTLSClientKeyOptions(options)) }; const cluster = new MongoCluster(); @@ -383,6 +505,7 @@ export class MongoCluster extends EventEmitter { options.downloadDir ?? options.tmpDir, options.version, options.downloadOptions, + options.downloadUrl, ); } @@ -411,7 +534,79 @@ export class MongoCluster extends EventEmitter { ]; } + let disaggregatedStorage = _internal?.disaggregatedStorage; + if (!disaggregatedStorage && options.disaggregatedStorage) { + disaggregatedStorage = options.disaggregatedStorage; + delete options.disaggregatedStorage; + cluster.dockerComposeProject = await DockerComposeProject.start( + disaggregatedStorage.composeFile, + { env: disaggregatedStorage.env }, + ); + } + + try { + if (disaggregatedStorage) { + if (cluster.dockerComposeProject) { + // Only the compose-owning (top-level) cluster waits for readiness. + await disaggregatedStorage.waitForReady?.(); + } + if (options.secondaries === undefined) options.secondaries = 1; + } + return await this._startServers( + cluster, + options, + disaggregatedStorage, + _internal?.shardContext ?? { index: 0, isConfigServer: false }, + ); + } catch (err) { + // Don't leak the compose project if cluster setup fails partway through. + if (cluster.dockerComposeProject) { + try { + await cluster.dockerComposeProject.close(); + } catch { + /* ignore */ + } + } + throw err; + } + } + + private static async _startServers( + cluster: MongoCluster, + options: MongoClusterOptions, + disaggregatedStorage: DisaggregatedStorageOptions | undefined, + shardContext: { index: number; isConfigServer: boolean }, + ): Promise { if (options.topology === 'standalone') { + if (disaggregatedStorage) { + // Pre-allocate the port so the storage configuration can reference it. + let port = getPortArg(options.args); + if (!port) { + port = await allocatePort(); + options.args = [ + ...stripPortArg(options.args ?? []), + '--port', + String(port), + ]; + } + const descriptor: ShardDescriptor = { + ...shardContext, + members: [ + { + host: options.host ?? '127.0.0.1', + port, + priority: 1, + arbiterOnly: false, + }, + ], + }; + await disaggregatedStorage.setupShard?.(descriptor); + options.args = buildDisaggregatedArgs( + disaggregatedStorage, + descriptor, + options.args ?? [], + ); + } cluster.servers.push( await MongoServer.start({ ...options, @@ -420,6 +615,42 @@ export class MongoCluster extends EventEmitter { ); } else if (options.topology === 'replset') { const { rsMembers, replSetName } = processRSMembers(options); + if (disaggregatedStorage) { + // Pre-allocate member ports so the storage configuration (which + // embeds the replica set config, including member host:ports) can be + // computed before the servers start. + const members: ShardMemberDescriptor[] = []; + for (const member of rsMembers) { + let port = getPortArg(member.args); + if (!port) { + port = await allocatePort(); + member.args = [ + ...stripPortArg(member.args ?? []), + '--port', + String(port), + ]; + } + members.push({ + host: options.host ?? '127.0.0.1', + port, + priority: member.priority ?? 0, + arbiterOnly: member.arbiterOnly ?? false, + }); + } + const descriptor: ShardDescriptor = { + ...shardContext, + replSetName, + members, + }; + await disaggregatedStorage.setupShard?.(descriptor); + for (const member of rsMembers) { + member.args = buildDisaggregatedArgs( + disaggregatedStorage, + descriptor, + member.args ?? [], + ); + } + } debug('Starting replica set nodes', { replSetName, @@ -448,24 +679,29 @@ export class MongoCluster extends EventEmitter { const primary = cluster.servers[primaryIndex]; await primary.withClient(async (client) => { - debug('Running rs.initiate'); - const rsConf = { - _id: replSetName, - configsvr: rsMembers.some((m) => m.args?.includes('--configsvr')), - members: nodes.map(([srv, member], i) => { - return { - _id: i, - host: srv.hostport, - arbiterOnly: member.arbiterOnly ?? false, - priority: member.priority ?? (i === primaryIndex ? 1 : 0), - tags: member.tags || {}, - }; - }), - }; - debugVerbose('replSetInitiate:', rsConf); - await client.db('admin').command({ - replSetInitiate: rsConf, - }); + // With DSC, the replica set config is provided to + // the servers at startup (disaggregatedStorageConfig.replSetConfig) + // and replSetInitiate is not supported. + if (!disaggregatedStorage) { + debug('Running rs.initiate'); + const rsConf = { + _id: replSetName, + configsvr: rsMembers.some((m) => m.args?.includes('--configsvr')), + members: nodes.map(([srv, member], i) => { + return { + _id: i, + host: srv.hostport, + arbiterOnly: member.arbiterOnly ?? false, + priority: member.priority ?? (i === primaryIndex ? 1 : 0), + tags: member.tags || {}, + }; + }), + }; + debugVerbose('replSetInitiate:', rsConf); + await client.db('admin').command({ + replSetInitiate: rsConf, + }); + } for (let i = 0; i < 60; i++) { const status = await client.db('admin').command({ @@ -490,16 +726,27 @@ export class MongoCluster extends EventEmitter { const { shards, mongosArgs } = processShardOptions(options); debug('starting config server and shard servers', shards); const allShards = await safePromiseAll( - shards.map(async (s) => { - const isConfig = s.args?.includes('--configsvr'); - const cluster = await MongoCluster.start({ - ...options, - ...s, - topology: 'replset', - requireApiVersion: undefined, - users: isConfig ? undefined : options.users, // users go on the mongos/config server only for the config set - }); - return [cluster, isConfig] as const; + shards.map(async (s, i) => { + const isConfigServer = s.args?.includes('--configsvr') ?? false; + // The per-shard DSC setup (setupShard, config + // injection) happens inside the recursive replset start, where the + // member ports are known. + const shardCluster = await MongoCluster.start( + { + ...options, + ...s, + topology: 'replset', + requireApiVersion: undefined, + users: isConfigServer ? undefined : options.users, + }, + disaggregatedStorage + ? { + disaggregatedStorage, + shardContext: { index: i, isConfigServer }, + } + : undefined, + ); + return [shardCluster, isConfigServer] as const; }), ); const configsvr = allShards.find(([, isConfig]) => isConfig)![0]; @@ -639,6 +886,7 @@ export class MongoCluster extends EventEmitter { closable?.close(), ), ); + await this.dockerComposeProject?.close(); this.servers = []; this.shards = []; } diff --git a/packages/mongodb-runner/src/mongoserver.ts b/packages/mongodb-runner/src/mongoserver.ts index 4efc1d9e..0dc4ec26 100644 --- a/packages/mongodb-runner/src/mongoserver.ts +++ b/packages/mongodb-runner/src/mongoserver.ts @@ -81,6 +81,7 @@ interface SerializedServerProperties { isArbiter?: boolean; isMongos?: boolean; isConfigSvr?: boolean; + isDSC?: boolean; keyFileContents?: string; } @@ -112,6 +113,9 @@ export class MongoServer extends EventEmitter { public isArbiter = false; public isMongos = false; private isConfigSvr = false; + // Whether this server uses DSC. DSC servers do not + // allow writes to the `local` database, so metadata tracking is skipped. + private isDSC = false; private keyFileContents?: string; private defaultConnectionOptions?: Partial; @@ -137,6 +141,7 @@ export class MongoServer extends EventEmitter { isArbiter: this.isArbiter, isMongos: this.isMongos, isConfigSvr: this.isConfigSvr, + isDSC: this.isDSC, keyFileContents: this.keyFileContents, }; } @@ -151,6 +156,9 @@ export class MongoServer extends EventEmitter { srv.host = serialized.host; } srv.defaultConnectionOptions = serialized.defaultConnectionOptions; + // Set before _populateBuildInfo so the restore check skips the + // local-database metadata access for DSC servers. + srv.isDSC = !!serialized.isDSC; srv.closing = !!(await srv._populateBuildInfo('restore-check')); srv.isArbiter = !!serialized.isArbiter; srv.isMongos = !!serialized.isMongos; @@ -228,6 +236,7 @@ export class MongoServer extends EventEmitter { srv.isArbiter = !!options.isArbiter; srv.isMongos = options.binary === 'mongos'; srv.isConfigSvr = !!options.args?.includes('--configsvr'); + srv.isDSC = !!options.args?.includes('disaggregatedStorageEnabled=true'); if (options.host && !srv.isConfigSvr) { srv.host = options.host; } @@ -439,6 +448,12 @@ export class MongoServer extends EventEmitter { mode: 'insert-new' | 'restore-check', retryOptions?: { intervalMs?: number; timeoutMs?: number }, ): Promise { + if (this.isDSC) { + // DSC servers do not allow writes to the `local` + // database, so metadata tracking is not possible there. + debug('skipping metadata check for DSC server'); + return; + } let hello = await client.db('admin').command({ hello: 1 }); if (this.isArbiter) { // RS convergence: hello.arbiterOnly may lag behind the RS config while the @@ -614,7 +629,7 @@ export class MongoServer extends EventEmitter { if (!this.hasInsertedMetadataCollEntry) { debug('populating metadata collection entry after initial setup'); const err = await this._populateBuildInfo('insert-new'); - if (err && !this.isMongos && !this.isConfigSvr) throw err; + if (err && !this.isMongos && !this.isConfigSvr && !this.isDSC) throw err; } if (!this.buildInfo) { throw new Error( @@ -625,7 +640,8 @@ export class MongoServer extends EventEmitter { !this.hasInsertedMetadataCollEntry && !this.isArbiter && !this.isMongos && - !this.isConfigSvr + !this.isConfigSvr && + !this.isDSC ) { throw new Error( `Server has not inserted metadata collection entry ${JSON.stringify(this.serialize())}`, diff --git a/packages/mongodb-runner/src/oidc.ts b/packages/mongodb-runner/src/oidc.ts index ba5b4a13..a88e53a2 100644 --- a/packages/mongodb-runner/src/oidc.ts +++ b/packages/mongodb-runner/src/oidc.ts @@ -1,11 +1,12 @@ import { spawn } from 'child_process'; import { once } from 'events'; +import { randomUUID } from 'crypto'; import { parseCLIArgs, OIDCMockProvider } from '@mongodb-js/oidc-mock-provider'; import { debug } from './util'; if (process.env.RUN_OIDC_MOCK_PROVIDER !== undefined) { (async function main() { - const uuid = crypto.randomUUID(); + const uuid = randomUUID(); debug('starting OIDC mock provider with UUID', uuid); const config = parseCLIArgs(process.env.RUN_OIDC_MOCK_PROVIDER); const sampleTokenConfig = await config.getTokenPayload({ @@ -119,7 +120,7 @@ export class OIDCMockProviderProcess { } if (!this.issuer || !this.uuid) return; - await fetch(new URL(this.issuer, `/shutdown/${this.uuid}`)); + await fetch(new URL(`/shutdown/${this.uuid}`, this.issuer)); this.uuid = undefined; } } diff --git a/packages/mongodb-runner/src/sls.ts b/packages/mongodb-runner/src/sls.ts new file mode 100644 index 00000000..f355cf67 --- /dev/null +++ b/packages/mongodb-runner/src/sls.ts @@ -0,0 +1,381 @@ +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; +import { execFile as execFileCb } from 'child_process'; +import { promisify } from 'util'; +import { debug, allocatePort, sleep, uuid } from './util'; +import type { + DisaggregatedStorageOptions, + ShardDescriptor, +} from './mongocluster'; + +const execFile = promisify(execFileCb); + +/** + * Helpers for the SLS (DSC) multi-cell docker compose + * project shipped in the mongodb server repository + * (buildscripts/modules/atlas/sls-multicell-docker-compose.yml). + * Replicates the environment-variable setup performed by the + * SLSMultiCellFixture in that codebase. + */ + +/** Hostname through which SLS services are reachable from the host. */ +export const SLS_HOSTNAME = 'local.sls.mmscloudteam.com'; + +const DEFAULT_SLS_IMAGE_REPO = + '664315256653.dkr.ecr.us-east-1.amazonaws.com/disagg-storage/'; + +export interface SLSServiceInfo { + /** Environment variable through which the compose file receives the host port. */ + portVar: string; + /** Port the service listens on inside its container. */ + internalPort: number; +} + +/** + * Extract the host-exposed services and their port environment variables from + * an SLS docker compose file. Matches `ports:` entries of the form + * `"${SOME_PORT:-30001}:27998"`. + */ +export function parseSLSComposeServices( + composeFileContents: string, +): Record { + const services: Record = Object.create(null); + let topLevelKey: string | undefined; + let currentService: string | undefined; + for (const line of composeFileContents.split('\n')) { + let match; + if ((match = /^(\w[\w.-]*):/.exec(line))) { + topLevelKey = match[1]; + currentService = undefined; + } else if ( + topLevelKey === 'services' && + (match = /^ {2}([\w.-]+):\s*$/.exec(line)) + ) { + currentService = match[1]; + } else if ( + currentService && + (match = /\$\{([A-Z0-9_]+)(?::-\d+)?\}:(\d+)/.exec(line)) && + // Only 'ports:' list entries, not e.g. SERVER_URI environment values + /^\s+-\s/.exec(line) + ) { + (services[currentService] ??= []).push({ + portVar: match[1], + internalPort: +match[2], + }); + } + } + return services; +} + +export interface SLSCell { + cell: string; + zone: string; +} + +export const SLS_CELL1: SLSCell = Object.freeze({ + cell: 'cell1', + zone: 'zone1', +}); +export const SLS_CELL2: SLSCell = Object.freeze({ + cell: 'cell2', + zone: 'zone2', +}); +export const SLS_CELL3: SLSCell = Object.freeze({ + cell: 'cell3', + zone: 'zone3', +}); + +export interface SLSMultiCellEnvironmentOptions { + /** + * Path to the SLS multi-cell docker-compose.yml (typically + * buildscripts/modules/atlas/sls-multicell-docker-compose.yml in a mongodb + * server checkout). Files it references (slsbackup.proto, + * flags-state.json) are resolved relative to it. + */ + composeFile: string; + /** + * Image tag for the SLS images (typically the `pinned_sls_commit` from the + * server repository's buildscripts/modules/atlas/manifest.json). + */ + imageTag: string; + /** Docker image repository for SLS images. */ + imageRepo?: string; + /** Docker image repository for third-party images (default: imageRepo with 'disagg-storage' replaced by 'thirdparty'). */ + thirdPartyImageRepo?: string; + /** Test run identifier, applied as a `testdata_id` label on all containers. */ + testDataId?: string; + /** Host IP reachable from inside containers (compose default: 172.17.0.1). */ + hostInternalIP?: string; +} + +export interface SLSMultiCellEnvironment { + /** Path to the compose file in use. */ + composeFile: string; + /** Environment variables to pass to the docker compose project. */ + env: Record; + /** Allocated host port for each service, keyed by service name. */ + ports: Record; + /** Host address and URI for each service, keyed by service name. */ + services: Record; +} + +/** + * Allocate host ports and build the environment variables expected by an SLS + * multi-cell compose file. The services and their port variables are parsed + * from the compose file itself, so this stays compatible with whatever + * version of the file is provided. + */ +export async function createSLSMultiCellEnvironment( + options: SLSMultiCellEnvironmentOptions, +): Promise { + const { composeFile, imageTag } = options; + + const serviceInfo = parseSLSComposeServices( + await fs.readFile(composeFile, 'utf8'), + ); + if (!Object.keys(serviceInfo).length) { + throw new Error( + `Could not find any services with host port mappings in ${composeFile}`, + ); + } + + const imageRepo = options.imageRepo ?? DEFAULT_SLS_IMAGE_REPO; + const thirdPartyImageRepo = + options.thirdPartyImageRepo ?? + imageRepo.replace('disagg-storage', 'thirdparty'); + + const ports: Record = Object.create(null); + const env: Record = Object.assign(Object.create(null), { + SLS_IMAGE_REPO: imageRepo, + THIRD_PARTY_IMAGE_REPO: thirdPartyImageRepo, + SLS_IMAGE_TAG: imageTag, + TESTDATA_ID: options.testDataId ?? '', + }); + if (options.hostInternalIP) { + env.HOST_INTERNAL_IP = options.hostInternalIP; + } + + const services: Record = + Object.create(null); + for (const [serviceName, infos] of Object.entries(serviceInfo)) { + for (const [i, info] of infos.entries()) { + const port = await allocatePort(); + env[info.portVar] = String(port); + // The first host-exposed port is the service's main address. + if (i === 0) { + ports[serviceName] = port; + const addr = `${SLS_HOSTNAME}:${port}`; + services[serviceName] = { addr, uri: `http://${addr}` }; + } + } + } + + debug('created SLS multi-cell environment', { composeFile, ports }); + return { composeFile, env, ports, services }; +} + +export interface SLSDisaggregatedStorageConfigOptions { + /** SLS log ID for this shard (must match the log started via StartLog). */ + logId: number; + /** Cell metadata service to use (default: 'cms-cell1-0'). */ + cellMetadataService?: string; + /** Zone the mongod nodes report themselves in (default: cell1's zone). */ + zoneName?: string; + /** Path to an encryption key file, if encryption is used. */ + encryptionKeyFilePath?: string; +} + +/** + * Build the value for mongod's `disaggregatedStorageConfig` server parameter + * for one shard, based on the SLS environment and the shard's pre-allocated + * member ports. Pass this as the `config` callback of the cluster's + * `disaggregatedStorage` options: + * + * ``` + * config: (shard) => + * createSLSDisaggregatedStorageConfig(sls, shard, { logId: 1 + shard.index }), + * ``` + */ +export function createSLSDisaggregatedStorageConfig( + sls: SLSMultiCellEnvironment, + shard: ShardDescriptor, + options: SLSDisaggregatedStorageConfigOptions, +): Record { + const cellMetadataService = options.cellMetadataService ?? 'cms-cell1-0'; + const cmsService = sls.services[cellMetadataService]; + if (!cmsService) { + throw new Error(`Unknown cell metadata service: ${cellMetadataService}`); + } + return { + // The server expects a BSON long here; the extended JSON wrapper survives + // JSON.stringify, unlike a BSON.Long instance. + logID: { $numberLong: String(options.logId) }, + myZoneName: options.zoneName ?? SLS_CELL1.zone, + zones: [], + cellMetadataServer: cmsService.uri, + ...(options.encryptionKeyFilePath + ? { encryptionKeyFilePath: options.encryptionKeyFilePath } + : {}), + ...(shard.replSetName + ? { + replSetConfig: { + _id: shard.replSetName, + version: 1, + term: 1, + members: shard.members.map((member, i) => ({ + _id: i, + host: `${member.host}:${member.port}`, + priority: member.priority, + arbiterOnly: member.arbiterOnly, + })), + }, + } + : {}), + }; +} + +export interface SLSDisaggregatedStorageSetupOptions extends SLSMultiCellEnvironmentOptions { + /** Compose project name (default: a generated unique name). */ + projectName?: string; + /** How long to wait for the storage layer to become ready (default: 300s). */ + readyTimeoutSecs?: number; + /** + * SLS log ID assigned to the first shard; subsequent shards get consecutive + * IDs (default: 1). Log ID 9999 is reserved by the cell metadata services. + */ + firstLogId?: number; + /** + * Path to the encryption key file to use for DSC + * encryption. If not provided, a key file with a well-known test key is + * created in the OS temp directory (matching the server's jstest setup). + */ + encryptionKeyFilePath?: string; +} + +// Well-known test encryption key, matching createKeyFile() in the server +// repository's disagg_storage jstest library. +const TEST_ENCRYPTION_KEY = '3zKkqoh8BGyC5BnyMZOEXsuTCHTD286SeNXEXeMuMxM='; + +async function createTestEncryptionKeyFile(): Promise { + const fileName = path.join( + os.tmpdir(), + `mongodb-runner-sls-decrypt-key-${uuid()}`, + ); + await fs.writeFile(fileName, TEST_ENCRYPTION_KEY, { mode: 0o600 }); + return fileName; +} + +/** + * Create fully-populated `disaggregatedStorage` options for the SLS + * multi-cell compose project from a mongodb server repository checkout: + * environment variables, readiness polling (waiting for the testdriver + * container's /ready marker), per-shard log creation (StartLog via grpcurl + * in the testdriver container), and the mongod `disaggregatedStorageConfig` + * server parameter. + * + * ``` + * const disaggregatedStorage = await createSLSDisaggregatedStorageOptions({ + * composeFile: '/path/to/sls-multicell-docker-compose.yml', + * imageTag: '', + * }); + * const cluster = await MongoCluster.start({ + * topology: 'replset', + * disaggregatedStorage, + * // ... + * }); + * ``` + */ +export async function createSLSDisaggregatedStorageOptions( + options: SLSDisaggregatedStorageSetupOptions, +): Promise { + const sls = await createSLSMultiCellEnvironment(options); + const projectName = options.projectName ?? `mongodb-runner-sls-${uuid()}`; + const testdriverContainer = `${projectName}-testdriver-1`; + const firstLogId = options.firstLogId ?? 1; + const encryptionKeyFilePath = + options.encryptionKeyFilePath ?? (await createTestEncryptionKeyFile()); + + const grpcurl = async ( + service: string, + method: string, + payload: unknown, + ): Promise => { + const { stdout } = await execFile('docker', [ + 'exec', + testdriverContainer, + '/grpcurl', + '-plaintext', + '-d', + JSON.stringify(payload), + service, + method, + ]); + return stdout.trim() ? JSON.parse(stdout) : {}; + }; + + return { + sls, + composeFile: sls.composeFile, + env: { ...sls.env, COMPOSE_PROJECT_NAME: projectName }, + waitForReady: async () => { + const timeoutSecs = options.readyTimeoutSecs ?? 300; + const deadline = Date.now() + timeoutSecs * 1000; + debug('waiting for SLS storage layer', { testdriverContainer }); + while (Date.now() < deadline) { + try { + await execFile('docker', [ + 'exec', + testdriverContainer, + 'test', + '-f', + '/ready', + ]); + debug('SLS storage layer is ready'); + return; + } catch { + await sleep(2000); + } + } + throw new Error(`SLS storage layer not ready after ${timeoutSecs}s`); + }, + setupShard: async ({ index }) => { + const logId = firstLogId + index; + debug('starting SLS log for shard', { index, logId }); + const res = await grpcurl( + 'crs-cell1-0:27996', + 'schedulerservice.v1.SchedulerService/GetLogServers', + { cells: [SLS_CELL1, SLS_CELL2, SLS_CELL3] }, + ); + const serverIds = res.server_ids ?? res.serverIds ?? []; + if (!serverIds.length) { + throw new Error('SLS GetLogServers returned no servers'); + } + try { + await grpcurl( + 'crs-cell1-0:27996', + 'schedulerservice.v1.ControlPlaneService/StartLog', + { log_id: logId, server_ids: serverIds, ancestry: { ancestors: [] } }, + ); + } catch (err) { + // The storage layer's state persists while the compose project is + // running, so the log may already exist from a previous run. + const { stderr = '', stdout = '' } = (err ?? {}) as { + stderr?: string; + stdout?: string; + }; + const output = `${stderr}${stdout}`; + if (output.includes('already exists')) { + debug('SLS log already exists, reusing it', { logId }); + return; + } + throw err; + } + }, + config: (shard) => + createSLSDisaggregatedStorageConfig(sls, shard, { + logId: firstLogId + shard.index, + encryptionKeyFilePath, + }), + }; +} diff --git a/packages/mongodb-runner/src/util.ts b/packages/mongodb-runner/src/util.ts index 24f249c1..903fa10d 100644 --- a/packages/mongodb-runner/src/util.ts +++ b/packages/mongodb-runner/src/util.ts @@ -2,6 +2,9 @@ import type { MongoClientOptions } from 'mongodb'; import { BSON } from 'mongodb'; import createDebug from 'debug'; import { ConnectionString } from 'mongodb-connection-string-url'; +import { once } from 'events'; +import { createServer } from 'net'; +import type { AddressInfo } from 'net'; export const debug = createDebug('mongodb-runner'); export const debugVerbose = debug.extend('verbose'); @@ -55,13 +58,23 @@ export async function safePromiseAll( if (rejected.length) { if (rejected.length === 1) throw rejected[0].reason; throw new AggregateError( - [rejected.map((r) => r.reason)], + rejected.map((r) => r.reason), `${rejected.length} errors: ${rejected.map((r) => r.reason).join(', ')}`, ); } return results.map((r) => (r as PromiseFulfilledResult).value); } +/** Allocate a currently-free TCP port on 127.0.0.1. */ +export async function allocatePort(): Promise { + const server = createServer(); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const { port } = server.address() as AddressInfo; + await new Promise((resolve) => server.close(resolve)); + return port; +} + export function pick( obj: T, keys: K[], diff --git a/packages/saslprep/src/code-points-data.ts b/packages/saslprep/src/code-points-data.ts index ad121e4c..b75a0ff1 100644 --- a/packages/saslprep/src/code-points-data.ts +++ b/packages/saslprep/src/code-points-data.ts @@ -2,7 +2,7 @@ import { gunzipSync } from 'zlib'; export default gunzipSync( Buffer.from( - 'H4sIAAAAAAACA+3dTYgcaRkA4LemO9Mhxm0FITnE9Cwr4jHgwgZ22B6YywqCJ0HQg5CL4sGTuOjCtGSF4CkHEW856MlTQHD3EJnWkU0Owh5VxE3LHlYQdNxd2U6mU59UV/d09fw4M2EySSXPAzNdP1/9fX/99bzVNZEN4jisRDulVFnQmLxm1aXF9Id/2/xMxNJ4XZlg576yuYlGt9gupV6xoFf8jhu9YvulVrFlp5XSx+lfvYhORGPXvqIRWSxERKtIm8bKFd10WNfKDS5Fo9jJWrq2+M2IlW+8uHgl/+BsROfPF4v5L7148Ur68Sha6dqZpYiVVy8tvLCWXo80Sf/lS89dGX2wHGvpzoXVn75/YWH5wmqe8uika82ViJXTy83Ve2k5Urozm38wm4/ls6t5uT6yfsTSJ7J3T0VKt8c5ExEXI8aFkH729c3eT+7EC6ca8cVULZUiYacX0R5PNWNxlh9L1y90q5kyzrpyy+9WcvOV6URntqw7La9sNVstXyczWVaWYbaaTYqzOHpr7pyiNT3/YzKuT63Z/FqKZlFTiuXtFM2vVOtIq7jiyKJbWZaOWD0euz0yoV2Z7kY0xq2x0YhfzVpmM5px9nTEH7JZ0ot5u39p0ma75Z472/s/H+2yr2inYyuq7fMvJivH2rM72N/Z3lyL31F2b1ya1P0zn816k2KP6JU9UzseucdQH5YqVeH/lFajSN2udg+TLJ9rksNxlvV2lki19rXKI43TPLejFu4ov7k3nMbhyhfY3Xb37f8BAGCf0eMTOH5szf154KmnNgKcnLb+Fzi2AfXktbN7fJelwTAiO/W5uQ2KINXRYu+znqo/WTAdLadURHmy3qciazd3bra4T3w16/f7t7Ms9U5gfJu10955sx1r3vmhBAAAAAAAgId20J1iZbDowNvIjuH427Gr5l/eiC+8OplZON8sVjx/qr9y+Pj+YRItT+NqAM+kkZs3AAAAAID6yfx1FwCAI97/dCh1/ub6SA0AAAAAAAAAgNoT/wcAAAAAAACA+hP/BwAAAAAAAID6E/8HAAAAAAAAgPoT/wcAAAAAAACA+hP/BwAAAAAAAID6E/8HAAAAAAAAgPoT/wcAAAAAAACA+hP/BwAAAAAAAID6E/8HAAAAAAAAgPoT/wcAAAAAAACA+hutp5SiQpYAAAAAAAAAQO2MIpZiT804flnAE2fhwjOeAZXr76kOAAAAAAAA8FjNf4N/l0NE3U/vuVQskLpSd4/Yh2xu9xTu0tFeeNYsLI2f/VMdNxTzj6Je9E/+6pp6Nn3awW3A54goe4Bss6v+PGsjQGMAAAAAAOBp5XEgwH6e7J7rwEQHRb/XvAMAAAAAAAA8yzoDeQDwVGjIAgAAAAAAAACoPfF/AAAAAAAAAKg/8X8AAAAAAAAAqD/xfwAAAAAAAACoP/F/AAAAAAAAAKg/8X8AAAAAAAAAqD/xfwAAAAAAAACoP/F/AAAAAAAAAKg/8X8AAAAAAAAAqD/xfwAAAAAAAACoP/F/AAAAAAAAAKg/8X8AAAAAAAAAqL/GSkSkClkCAAAAAAAAALXTSAAAAAAAAABA3Y1kAQAAAAAAAADUX8RSXZ9dsHC9+M8Fg2Ex/em1lAZpEBGttcrVjZqLEa+k0XpKw9mG4zWx4ukPUMhkAQAAAAAAABzBqbSe3//rXOS9HxGdo4TqR2XkutCdBu+LaPZw/lBbO7cbHnh2C7N7AIo4evEznllqLqWUp/LnYOtpM2bnOH66wI1+9GO4sOuISwv/TOlumu56FDv3NZhc4mR9v7zYIrafr40j/Cccvj9Xns3t3mu99E7qxUv3bqS0/ouNH/08++RGemfQ+nsx/5uNXsQPGulynPvv3ZTW37zd+1ovrqaYpP/122X6Xpx779Z3zr/3YOPKW1lkaRDf31pPaf3j/msRsVGkL+d/f+/m4sJsPm1cfSsr16e8m9Ldj/KsnyIuR3nXw83Is3EhxLd/2V773ks3m/cj/THKUummdP9qKhIOImuOU0Xjwb3y+oqt735rpTetVbF9n8R4x9crRfO77TKqVOZpDclv5bfK18lMnk+q0K18UpxF/RrGXE0Zxtqx3tWSj+vxbL4XaasfKb0dRbtLW73JsfPGg177H+OmGKlfvS1msllt7JEJm9XOJqXR+Fkfo1H66uy5H1v3Xx5+uJmGLw9jro2u7Loj4PnuR6+f+e3d261+eazNhzrL7X83MohoHpS4PddV8ki1it61//pw1g7z6p1U/26Nm2llST57B5rUvuG0XqSU/rPd7jYrqWcbd+beJQ77BgPMDwn37/8BAGCf0eMTOH4cPlufv9VGgJOzqf8Fjm1APXkd7B7f5dF57GPMaWy/MTvjvNvtXj6h8W2+GXvnzXaseeeHEgAAAAAAAB7aQXeKlcGiadBoEOeLb2dtpGOL2MyOtf391a3P/zD96c3JzIP3t4oV797vrh8+vn+YRL5bBuj/AQAAAABqJvfHXQAAHkX82zfXAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACeAgkAAAAAAAAAqLuRLAAAAAAAAACA2hv9D1iu/VAYaAYA', + 'H4sIAAAAAAACE+3dTYgcWR0A8FfTnekQ47aCkBxiZpYV8RhwYQM7bA/ksoLgSRD0IOSiePAkLrowtWSF4CkHEW856MlTQHA9RKZ1ZJODsEcVcTOyhxUEHXdXtpPp1PNVV0939Xw4M2EySWV+v2G661W9+nrv1evX86/uCdl6OArLoRtjrM1ojZ+z+txy+sO/bXwmhMXRsirD9m1lMxOtXrlejHk5Iy8fw828XH+xU6650Inx4/ivNH8h5d2xrXQcWZhLz50ybxypFvTiQV2vVrgUWuVGVuL1+W+mE/7Gi/NXiw/Opq3++WKZ/tKLF6/GHw1DJ14/k45u+dVLcy+sxNfTsVf5v3zpuavDD5bS+ncvXPnJ+xfmli5cKWKRDup6ezktP73UvnI/LqX8d6fph9N0WDqb8o/SIeun0/9E9u6plLozKpnkYvotKyH+9Osb+Y/vhhdOtcIXY71WyowL6aE7mmqH+Wl5LN640KsXyqjoqjW/WyvNV7YmFqbzelv1lV3JrlTP40SWVXWYpsbVWe69M3NMKZmHozRqT51peiWGdtlSyvndNP2VehvplGecDqxXmxcP2TyeuF0KoVubTufWGl2N6fGX0yuznX7Ong7hD9k068Wi2780vmZ71ZYXJts/H7pVX9GNR1ZVk+MvJ2v72rU72NvZfOaK31Z3b14at/0zn83ycbWnGVXP1A2P3RNoD4u1pvB/aqtV5u7Wu4dxkc9ckoNRkeXba6Te+jrVnkZ5ntvWCrfV38wLTutg9QvsvHb37P8BAGCP0eNTOH7szPx54JmnNQIcn67+FziyAfX4eWHn+C6L64M0mj31uZkVyiDV+qF2Me2p+uMZW6PlGMsoT5Z/KmTd9vbV5vPdN5f1+/07WRbzYxjfZt24e9lMYs3b35QAAAAAAADwyPa7U6wKFu17G9kR7H8Su2r/5c3whVfHibnz7XLB86f6yweP7x8k09JWXA3gRBq6eQMAAAAAaJ7MX3cBADjk/U8H0uRPrg+1AAAAAAAAAABoPPF/AAAAAAAAAGg+8X8AAAAAAAAAaD7xfwAAAAAAAABoPvF/AAAAAAAAAGg+8X8AAAAAAAAAaD7xfwAAAAAAAABoPvF/AAAAAAAAAGg+8X8AAAAAAAAAaD7xfwAAAAAAAABoPvF/AAAAAAAAAGi+4Wp6CDWKBAAAAAAAAAAaZxjCYthVOxy9LMBTZ+7CCS+A2vnnmgMAAAAAAAA8UbOf4N/hAFH307vOzRUtDaXtHrIP2Zj0FO7Scb1w0swtjr77pz5uKNOPo130j//s2no2fdr+14D3EaHqAbKNnvZz0kaAxgAAAAAAwLPK14EAe3m6e659M+0X/V7xCgAAAAAAAHCSLawrA4BnQksRAAAAAAAAAEDjif8DAAAAAAAAQPOJ/wMAAAAAAABA84n/AwAAAAAAAEDzif8DAAAAAAAAQPOJ/wMAAAAAAABA84n/AwAAAAAAAEDzif8DAAAAAAAAQPOJ/wMAAAAAAABA84n/AwAAAAAAAEDzif8DAAAAAAAAQPOJ/wMAAAAAAABA87WW00OsUSQAAAAAAAAA0DitCAAAAAAAAAA03VARAAAAAAAAAEDzhbDY1O8umLtR/ueC9UE5/emVNBXX01RnpXZ2w/Z8CK/E4WqMg+mKoyVh2bc/QClTBAAAAAAAAHAIp+Jq8eCvM5H3fpq9cJhQ/bCKXJd6W8H7pB7ZLm1uX2+w79HNLU4myzh6+TtKLLYXYyxi9bu/1bgRpsdYHnG82Q/9MJjbscfFuX/GeC9ubXoYtm9rfXyK4+X96mTL2H6xMorwLx9vBZ6rjuZO/loe34l5eOn+zXTCP19742fZJ9fiO+udv5fpX6/lIfygFS+Hc/9NZ7f61p38a3m4lo69yv+rt6v8eTj33u3vnH/v4drV32QhS8X9/c3VtPzj/mtpX2tl/ir9+/u35uem6bh2LeUfpWORSujeR0XWT6nLobrr4VYoslElhG//orvyvZdutR+E+MdQ1UrK/+BaLDOuh6w9yhVaD+9X51eufe9by/lWq9qaN66ZG7Wq+d2kjmqNeauFFLeL29XzOFEU4yZ0uxhXZ9m+BmGmpQzCypHe1VKM2vE0nYe42U/7ezuU113czMf7LloP8+4/RpdiiP36bTHj1Rpjl0LYqHc2qSZH3/WRHr8aJ9/7sfng5cGHG3Hw8iDMXKPLO+4IeL730etnfnvvTqdf7WvjkY5y8u9GUv/S3i9zd6ar5LHqlL1r//XB9Dos6ndS/bszukxrc4rpK9C49Q222kV6+s/kutuo5Z6uvDDzKnHQFxhgdki4d/8PAAB7jB6fwvHj4GS9/9YaAY7Phv4XOLIB9fh5fef4rggLT3yMuTKO7bemR1z0er3LxzS+LTbC7mUziTVvf1MCAAAAAADAI9vvTrEqWLQVNFoP58tPZ63FI4vYTPc1+fzq5ud/GP/01jjx8P3NcsG7D3qrB4/vHyRTX90D+n8AAAAAgGYp/HEXAIDHEf/2yXUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgGdABAAAAAAAAACabqgIAAAAAAAAAKDxhv8DWK79UBhoBgA=', 'base64', ), );