diff --git a/packages/mongodb-runner/README.md b/packages/mongodb-runner/README.md index e614f222..c1626c15 100644 --- a/packages/mongodb-runner/README.md +++ b/packages/mongodb-runner/README.md @@ -40,6 +40,9 @@ Options: --arbiters number of arbiter nodes for each replica set [number] --secondaries number of secondaries for each replica set [number] --shards number of shards for sharded clusters [number] + --configShard for `start` with the sharded topology: turn the + config server into a config shard (requires MongoDB + 8.0+) [boolean] --version MongoDB server version to use [string] --logDir Directory to store server log files in [string] --binDir Directory containing mongod/mongos binaries [string] @@ -111,6 +114,24 @@ You can specify a config file for the CLI using `--config`: $ npx @mongodb-js/mongodb-runner start --config ``` +## Config shards + +By default the runner starts a sharded cluster with a dedicated config server. +Pass `--configShard` (or `configShard: true` in a config file / programmatic +options) to turn the config server replica set into a config shard, a config +server that also holds data, once the cluster is up. This requires MongoDB 8.0 +or newer (the underlying `transitionFromDedicatedConfigServer` command is new +in 8.0) and is only supported for the `sharded` topology. On clusters that +enforce authentication (keyfile or `--auth`), at least one user must be +configured, since the transition closes the localhost exception and dropping +the temporary user it uses would otherwise leave the cluster unusable. `start` +waits until the transition has completed, and `ls --json` (and `start --json`) +report `configShard: true` for such clusters: + +```sh +$ npx mongodb-runner start -t sharded --shards 1 --configShard +``` + ## Machine-readable output By default `start` prints diagnostics to stderr and the bare connection string diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index 2fcd489e..e7b3370f 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -306,6 +306,43 @@ describe('cli', function () { await runCli(['prune']); }); + it('starts a config shard cluster and reports it in start and ls --json', async function () { + this.timeout(300_000); + const startStdout = await runCli([ + 'start', + '--topology', + 'sharded', + '--shards', + '1', + '--configShard', + '--json', + ]); + const started = JSON.parse(startStdout) as { + id: string; + connectionString: string; + configShard?: boolean; + }; + + expect(started.configShard, 'start --json should report the config shard') + .to.be.true; + + const client = new MongoClient(started.connectionString); + const listShards = await client.db('admin').command({ listShards: 1 }); + await client.close(); + expect( + listShards.shards.map((s: any) => s._id), + 'the config server should be a shard of the cluster', + ).to.include('config'); + + const lsStdout = await runCli(['ls', '--json']); + const entry = JSON.parse(lsStdout).find((e: any) => e.id === started.id); + expect(entry.configShard, 'ls --json should report the config shard').to.be + .true; + + await runCli(['stop', '--all']); + await runCli(['prune']); + }); + it('emits an empty JSON array for ls --json with no instances', async function () { const emptyDir = path.join(tmpDir, 'empty-runner'); await fs.mkdir(emptyDir, { recursive: true }); diff --git a/packages/mongodb-runner/src/cli.ts b/packages/mongodb-runner/src/cli.ts index 27141420..3b309897 100644 --- a/packages/mongodb-runner/src/cli.ts +++ b/packages/mongodb-runner/src/cli.ts @@ -36,6 +36,11 @@ import type { MongoClientOptions } from 'mongodb'; }, // Can also be an array when coming from the config file describe: 'number of shards for sharded clusters', }) + .option('configShard', { + type: 'boolean', + describe: + 'for `start` with the sharded topology: turn the config server into a config shard (requires MongoDB 8.0+)', + }) .option('version', { type: 'string', describe: 'MongoDB server version to use', diff --git a/packages/mongodb-runner/src/mongocluster.spec.ts b/packages/mongodb-runner/src/mongocluster.spec.ts index 172ee381..9c6627f3 100644 --- a/packages/mongodb-runner/src/mongocluster.spec.ts +++ b/packages/mongodb-runner/src/mongocluster.spec.ts @@ -340,6 +340,111 @@ describe('MongoCluster', function () { expect(cluster.connectionString).to.include('localhost'); }); + it('can start a sharded cluster with a config shard', async function () { + cluster = await MongoCluster.start({ + version: '8.x', + topology: 'sharded', + tmpDir, + shards: 1, + secondaries: twoIfNotWindowsCI, + configShard: true, + }); + const listShards = await cluster.withClient((client) => + client.db('admin').command({ listShards: 1 }), + ); + const shardIds = listShards.shards.map((s: any) => s._id); + expect( + shardIds, + 'the config server should be listed as a shard', + ).to.include('config'); + expect(shardIds).to.have.lengthOf(2); + + const serialized = cluster.serialize(); + expect(serialized.configShard).to.be.true; + const roundTripped = await MongoCluster.deserialize(serialized); + expect( + roundTripped.serialize().configShard, + 'the config shard flag should survive a serialize round trip', + ).to.be.true; + }); + + it('can start a sharded cluster with a config shard and keyfile auth', async function () { + const keyFile = path.join(tmpDir, 'keyFileConfigShard'); + await fs.writeFile(keyFile, 'csauthsecret', { mode: 0o400 }); + cluster = await MongoCluster.start({ + version: '8.x', + topology: 'sharded', + tmpDir, + shards: 1, + secondaries: 0, + users: [ + { + username: 'testuser', + password: 'testpass', + roles: ['root'], + }, + ], + args: ['--keyFile', keyFile], + configShard: true, + }); + const listShards = await cluster.withClient((client) => + client.db('admin').command({ listShards: 1 }), + ); + expect( + listShards.shards.map((s: any) => s._id), + 'the config server should be listed as a shard', + ).to.include('config'); + + // The temporary user created for the transition must have been removed. + const usersInfo = await cluster.withClient((client) => + client.db('admin').command({ usersInfo: 1 }), + ); + expect( + usersInfo.users.map((u: any) => u.user), + 'the temporary transition user should be gone', + ).to.deep.equal(['testuser']); + }); + + it('rejects configShard with authentication but no configured users', async function () { + const keyFile = path.join(tmpDir, 'keyFileConfigShardNoUsers'); + await fs.writeFile(keyFile, 'csauthsecret', { mode: 0o400 }); + let err: unknown; + try { + await MongoCluster.start({ + version: '8.x', + topology: 'sharded', + tmpDir, + shards: 1, + secondaries: 0, + args: ['--keyFile', keyFile], + configShard: true, + }); + } catch (e) { + err = e; + } + expect(err, 'start should have failed').to.be.instanceOf(Error); + expect((err as Error).message).to.include( + 'configShard requires at least one user', + ); + }); + + it('rejects configShard for non-sharded topologies', async function () { + let err: unknown; + try { + await MongoCluster.start({ + topology: 'replset', + tmpDir, + configShard: true, + }); + } catch (e) { + err = e; + } + expect(err, 'start should have failed').to.be.instanceOf(Error); + expect((err as Error).message).to.include( + 'configShard requires the sharded topology', + ); + }); + it('can use localhost as the host for a sharded cluster', async function () { cluster = await MongoCluster.start({ host: 'localhost', diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 2c4526c6..46797c8c 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -160,6 +160,13 @@ export type ShardedOptions = { mongosArgs?: string[][]; /** Number of shards to create or explicit shard configurations. */ shards?: number | Partial[]; + /** + * Turn the config server replica set into a config shard (a config server + * that also holds data) once the cluster is up, by running + * `transitionFromDedicatedConfigServer` through mongos. Requires MongoDB + * 8.0 or newer. Only meaningful for the `sharded` topology. + */ + configShard?: boolean; }; /** A replica set member of a shard, with its pre-allocated port. */ @@ -264,6 +271,8 @@ export interface SerializedClusterProperties { dockerComposeProject?: ReturnType; defaultConnectionOptions: Partial; users: MongoDBUserDoc[]; + /** True when the config server replica set is also a data-bearing shard. */ + configShard?: boolean; options?: MongoClusterOptions; } @@ -493,6 +502,15 @@ export class MongoCluster extends EventEmitter { private defaultConnectionOptions: Partial = {}; private users: MongoDBUserDoc[] = []; private originalOptions?: MongoClusterOptions; + /** True when the config server replica set is also a data-bearing shard. */ + private configShard = false; + /** + * Credentials of the temporary user used to run the config shard + * transition on authenticated clusters, kept around so that the cluster + * users can be created while it exists (the localhost exception closes + * as soon as any user has been created and does not reopen). + */ + private transitionUser?: { username: string; password: string }; private constructor() { super(); @@ -536,6 +554,7 @@ export class MongoCluster extends EventEmitter { dockerComposeProject: this.dockerComposeProject?.serialize(), defaultConnectionOptions: jsonClone(this.defaultConnectionOptions ?? {}), users: jsonClone(this.users), + configShard: this.configShard || undefined, options: this.originalOptions ? jsonClone(this.originalOptions) : undefined, @@ -557,6 +576,7 @@ export class MongoCluster extends EventEmitter { cluster.replSetName = serialized.replSetName; cluster.defaultConnectionOptions = serialized.defaultConnectionOptions; cluster.users = serialized.users; + cluster.configShard = serialized.configShard ?? false; cluster.servers = await safePromiseAll( serialized.servers.map((srv) => MongoServer.deserialize(srv)), ); @@ -728,6 +748,9 @@ export class MongoCluster extends EventEmitter { shardContext: { index: number; isConfigServer: boolean }, ): Promise { if (options.topology === 'standalone') { + if (options.configShard) { + throw new Error('configShard requires the sharded topology'); + } if (disaggregatedStorage) { // Pre-allocate the port so the storage configuration can reference it. let port = getPortArg(options.args); @@ -764,6 +787,9 @@ export class MongoCluster extends EventEmitter { }), ); } else if (options.topology === 'replset') { + if (options.configShard) { + throw new Error('configShard requires the sharded topology'); + } const { rsMembers, replSetName } = processRSMembers(options); if (disaggregatedStorage) { // Pre-allocate member ports so the storage configuration (which @@ -888,6 +914,7 @@ export class MongoCluster extends EventEmitter { topology: 'replset', requireApiVersion: undefined, users: isConfigServer ? undefined : options.users, + configShard: undefined, }, { disaggregatedStorage, @@ -929,9 +956,104 @@ export class MongoCluster extends EventEmitter { } debug('added shards'); }); + if (options.configShard) { + debug('transitioning the config server to a config shard'); + // The localhost exception (which covers addShard above) does not + // permit this command, so when the cluster enforces authentication, + // create a temporary privileged user through the localhost exception + // and run the transition (and the readiness poll) as that user. + // `__system` cannot be used here because mongos only accepts it with + // enableTestCommands. + const runTransition = async (client: MongoClient): Promise => { + try { + await client.db('admin').command({ + transitionFromDedicatedConfigServer: 1, + }); + } catch (err) { + throw new Error( + `Failed to turn the config server into a config shard (requires MongoDB 8.0 or newer): ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } + // The command returns before the transition has finished; wait + // until the config server shows up as a shard so that the + // cluster is fully usable when start() returns. + const deadline = Date.now() + 60_000; + // eslint-disable-next-line no-constant-condition + while (true) { + const { shards: currentShards } = await client.db('admin').command({ + listShards: 1, + // Bound each poll by the remaining time budget so that a + // stalled mongos cannot hang start() indefinitely. + maxTimeMS: Math.max(1, deadline - Date.now()), + }); + if (currentShards.some((s: any) => s._id === 'config')) break; + if (Date.now() >= deadline) { + throw new Error( + 'Timed out waiting for transitionFromDedicatedConfigServer to complete', + ); + } + await sleep(1000); + } + }; + const needsCredentials = + mongos.keyFileContents !== undefined || mongos.isAuth; + if (!needsCredentials) { + await mongos.withClient(runTransition, { timeoutMS: 60_000 }); + } else { + // Creating the temporary user closes the localhost exception, + // and it does not reopen once the user is dropped again, so the + // user is kept until the cluster's own users have been created + // (see addAuthIfNeeded / dropTransitionUserIfNeeded). + if (!options.users?.length) { + throw new Error( + 'configShard requires at least one user to be configured on clusters that enforce authentication: ' + + 'the temporary user created for the transition closes the localhost exception, ' + + 'and dropping it again would leave the cluster with no users and no way to authenticate', + ); + } + cluster.transitionUser = { + username: `runner-config-shard-${uuid()}`, + password: uuid(), + }; + const { username, password } = cluster.transitionUser; + await mongos.withClient(async (client) => { + await client.db('admin').command({ + createUser: username, + pwd: password, + roles: ['root'], + // User management commands can only use w:1 or w:majority + writeConcern: { w: 'majority', j: true, wtimeout: 0 }, + }); + }); + try { + await mongos.withClient(runTransition, { + auth: { username, password }, + timeoutMS: 60_000, + }); + } catch (err) { + // Best effort: remove the temporary user again on failure so + // that it does not linger with root privileges. + await mongos + .withClient( + async (client) => { + await client.db('admin').command({ dropUser: username }); + }, + { auth: { username, password } }, + ) + .catch(() => undefined); + cluster.transitionUser = undefined; + throw err; + } + } + cluster.configShard = true; + debug('config server transitioned to a config shard'); + } } await cluster.addAuthIfNeeded(); + await cluster.dropTransitionUserIfNeeded(); await cluster.addRequireApiVersionIfNeeded(options); try { await cluster.assertAllServersHaveInsertedLocalMetadata(); @@ -994,6 +1116,12 @@ export class MongoCluster extends EventEmitter { if (!this.users?.length) return; // Sleep to give time for a possible replset election to settle. await sleep(1000); + // On authenticated config-shard clusters, the localhost exception has + // been closed by the temporary transition user, so create the real + // users as that user instead. + const clientOptions: MongoClientOptions = this.transitionUser + ? { auth: { ...this.transitionUser } } + : {}; await this.withClient(async (client) => { const admin = client.db('admin'); for (const user of this.users) { @@ -1008,12 +1136,37 @@ export class MongoCluster extends EventEmitter { ...rest, }); } - }); + }, clientOptions); await this.updateDefaultConnectionOptions({ auth: this.users[0], }); } + /** + * Remove the temporary user used for the config shard transition, if one + * was created. Must run after addAuthIfNeeded, since the temporary user + * is what allows creating the cluster's own users on authenticated + * clusters. + */ + async dropTransitionUserIfNeeded(): Promise { + if (!this.transitionUser) return; + const { username, password } = this.transitionUser; + const mongos = this.servers.find((srv) => srv.isMongos); + if (!mongos) { + throw new Error( + 'Cannot drop the temporary config shard transition user: no mongos found', + ); + } + await mongos.withClient( + async (client) => { + await client.db('admin').command({ dropUser: username }); + }, + { auth: { username, password } }, + ); + this.transitionUser = undefined; + debug('dropped the temporary config shard transition user', username); + } + async updateDefaultConnectionOptions( options: Partial, ): Promise { diff --git a/packages/mongodb-runner/src/mongoserver.ts b/packages/mongodb-runner/src/mongoserver.ts index 2fa9f3a8..fc7a6ea4 100644 --- a/packages/mongodb-runner/src/mongoserver.ts +++ b/packages/mongodb-runner/src/mongoserver.ts @@ -131,7 +131,7 @@ export class MongoServer extends EventEmitter { // Whether this server was started with --auth. Its metadata bookkeeping is // best-effort because the runner may not have credentials it can use. public isAuth = false; - private keyFileContents?: string; + private _keyFileContents?: string; private defaultConnectionOptions?: Partial; private commandline: string[] = []; @@ -139,6 +139,20 @@ export class MongoServer extends EventEmitter { return this.uuid; } + /** + * The contents of the keyfile this server was started with, if any. Can be + * used to authenticate as the internal `__system` user (whose password is + * the keyfile contents) for administrative commands that the localhost + * exception does not permit. + */ + get keyFileContents(): string | undefined { + return this._keyFileContents; + } + + set keyFileContents(contents: string | undefined) { + this._keyFileContents = contents; + } + private constructor() { super(); /* see .start() */