fix(runners): pace Parameter Store writes against the account-wide limit

docs/rate-limits-and-tuning.md

@@ -82,7 +82,7 @@ Rate limits are **disabled by default** on GitHub Enterprise Server and must be
 |---|---|
 | Combined throughput (Get + Put) | **40 TPS** (shared per-account per-region) |
 
-Each runner instance requires one `PutParameter` call for its JIT config. At 40 TPS shared across all operations in the account, a burst of 40+ concurrent writes will throttle.
+Each runner instance requires one `PutParameter` call for its JIT config. This is account-wide, not per-invocation or per-pool: several pools' scale-up/pool lambdas can each individually look fine while their combined writes exceed the account limit. The module paces its own writes to stay under a configured ceiling — `ssm_parameter_store_max_writes_per_second` (default `40`, matching the standard tier) — dividing the per-write delay across `ssm_parameter_store_max_concurrent_invocations` (default `1`), which should reflect how many scale-up/pool invocations across all pools can realistically run at the same time.
 
 **Higher throughput mode** raises the ceiling:
 
@@ -94,6 +94,8 @@ aws ssm update-service-setting \
 
 Cost: $0.05 per 10,000 API interactions beyond the standard tier.
 
+Enabling this alone does nothing for this module's own pacing — raise `ssm_parameter_store_max_writes_per_second` to match the new ceiling too, or the module keeps pacing writes at the old 40 TPS default regardless of what AWS now allows.
+
 ### EC2 CreateFleet
 
 The exact TPS limit for `CreateFleet` is not publicly documented. It uses a token-bucket algorithm per-account per-region. Empirically throttles at low single-digit TPS.
@@ -111,7 +113,7 @@ For deployments running more than a handful of concurrent runners:
 | Running On-Demand Standard (A,C,D,H,I,M,R,T,Z) instances | **5 vCPUs** | Service Quotas console |
 | All Standard Spot Instance Requests | **5 vCPUs** | Service Quotas console |
 | EC2 CreateFleet API rate | Undocumented | AWS Support ticket |
-| SSM Parameter Store throughput | 40 TPS | `update-service-setting` (see above) |
+| SSM Parameter Store throughput | 40 TPS | `update-service-setting` (see above), then raise `ssm_parameter_store_max_writes_per_second` to match |
 
 **vCPU quotas are measured in vCPUs, not instance count.** Running 50× `c5.large` (2 vCPU each) requires a quota of at least 100 vCPUs.
 
@@ -131,7 +133,7 @@ For deployments running more than a handful of concurrent runners:
 | `isJobQueued` calls | 100 | 100 (same total) |
 | JIT config generation calls | 100 | 100 (same total) |
 
-Larger `batch_size` reduces CreateFleet calls (the most constrained AWS API) and Lambda invocations. Per-runner work (SSM writes, JIT config, isJobQueued) stays the same total. SSM peak TPS is lower with larger batches because writes are serialized within each Lambda rather than concurrent across many.
+Larger `batch_size` reduces CreateFleet calls (the most constrained AWS API) and Lambda invocations. Per-runner work (SSM writes, JIT config, isJobQueued) stays the same total. SSM writes are paced within each Lambda regardless of batch size (see `ssm_parameter_store_max_writes_per_second`/`ssm_parameter_store_max_concurrent_invocations` above) — batch size does not change the SSM pacing behavior, only how many Lambda invocations are running concurrently in the first place.
 
 ### Tradeoffs
 
@@ -157,7 +159,7 @@ Higher windows improve batching efficiency but add latency to job pickup.
 |---|---|---|---|
 | Small (<50 concurrent jobs) | 1–5 | 90s | Defaults work |
 | Medium (50–200) | 5–10 | 180s | Monitor SSM throttling |
-| Large (200+) | 10 | 300s | Enable SSM higher throughput, raise vCPU quotas, request CreateFleet rate increase |
+| Large (200+) | 10 | 300s | Enable SSM higher throughput and raise `ssm_parameter_store_max_writes_per_second`/`ssm_parameter_store_max_concurrent_invocations` to match, raise vCPU quotas, request CreateFleet rate increase |
 
 ## Monitoring
 

lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts

@@ -1,4 +1,4 @@
-import { isJobQueued, createStartRunnerConfig } from './github-runner';
+import { addDelay, isJobQueued, createStartRunnerConfig } from './github-runner';
 import { metricGitHubAppRateLimit } from '../github/rate-limit';
 import type { ActionRequestMessage, CreateGitHubRunnerConfig } from './types';
 import type { RunnerConfigStore } from '@aws-github-runner/storage-providers';
@@ -11,8 +11,91 @@ vi.mock('../github/rate-limit', () => ({
 
 const mockedMetricGitHubAppRateLimit = vi.mocked(metricGitHubAppRateLimit);
 
+const cleanEnv = process.env;
+
 beforeEach(() => {
   vi.clearAllMocks();
+  process.env = { ...cleanEnv };
+  delete process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS;
+});
+
+describe('Test addDelay', () => {
+  it('does not delay when the store has no write limit', () => {
+    const store = { maxWritesPerSecond: undefined } as RunnerConfigStore;
+    const { isDelay, delayMilliseconds } = addDelay(['1', '2'], store);
+    expect(isDelay).toBe(false);
+    expect(delayMilliseconds).toBe(0);
+  });
+
+  it('does not delay for an empty batch', () => {
+    const store = { maxWritesPerSecond: 40 } as RunnerConfigStore;
+    const { isDelay } = addDelay([], store);
+    expect(isDelay).toBe(false);
+  });
+
+  it('paces a batch below the write limit, unlike the previous per-invocation-only guard', () => {
+    const store = { maxWritesPerSecond: 40 } as RunnerConfigStore;
+    // A batch of 2 is far below 40, but the account-wide limit does not care about batch size.
+    const { isDelay, delayMilliseconds } = addDelay(['1', '2'], store);
+    expect(isDelay).toBe(true);
+    expect(delayMilliseconds).toBe(25); // 1000 / 40
+  });
+
+  it('divides the per-write delay across the configured number of concurrent invocations', () => {
+    process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS = '5';
+    const store = { maxWritesPerSecond: 40 } as RunnerConfigStore;
+    const { delayMilliseconds } = addDelay(['1'], store);
+    expect(delayMilliseconds).toBe(125); // (1000 / 40) * 5
+  });
+
+  it.each(['0', '-1', 'not-a-number', ''])(
+    'treats an invalid concurrency value (%s) as a single invocation',
+    (value) => {
+      process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS = value;
+      const store = { maxWritesPerSecond: 40 } as RunnerConfigStore;
+      const { delayMilliseconds } = addDelay(['1'], store);
+      expect(delayMilliseconds).toBe(25);
+    },
+  );
+});
+
+describe('Test createStartRunnerConfig registration-token pacing', () => {
+  const mockOctokit = {
+    actions: {
+      createRegistrationTokenForOrg: vi.fn().mockResolvedValue({ data: { token: 'reg-token' } }),
+    },
+  } as unknown as Octokit;
+
+  const githubRunnerConfig: CreateGitHubRunnerConfig = {
+    ephemeral: false,
+    enableJitConfig: false,
+    runnerLabels: 'self-hosted',
+    runnerGroup: 'Default',
+    runnerNamePrefix: 'test-',
+    runnerOwner: 'my-org',
+    runnerType: 'Org',
+    disableAutoUpdate: false,
+  };
+
+  it('paces every write between concurrent pools, not just large single-invocation batches', async () => {
+    vi.useFakeTimers();
+    const create = vi.fn().mockResolvedValue(undefined);
+    const runnerConfigStore = { maxWritesPerSecond: 40, create } as unknown as RunnerConfigStore;
+
+    process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS = '4';
+    const runPromise = createStartRunnerConfig(githubRunnerConfig, ['a', 'b', 'c'], mockOctokit, {
+      runnerConfigStore,
+    });
+
+    // Each write is followed by a (1000 / 40) * 4 = 100ms pacing delay.
+    await vi.advanceTimersByTimeAsync(100);
+    await vi.advanceTimersByTimeAsync(100);
+    await vi.advanceTimersByTimeAsync(100);
+    await runPromise;
+
+    expect(create).toHaveBeenCalledTimes(3);
+    vi.useRealTimers();
+  });
 });
 
 describe('Test isJobQueued rate-limit metric on error', () => {

lambdas/functions/control-plane/src/scale-runners/github-runner.ts

@@ -227,11 +227,16 @@ export async function createStartRunnerConfig(
   }
 }
 
-function addDelay(runnerIds: string[], runnerConfigStore: RunnerConfigStore) {
+// maxWritesPerSecond is account-wide, so the per-write delay is spread across the configured
+// number of concurrent invocations rather than sized off this invocation's batch alone.
+export function addDelay(runnerIds: string[], runnerConfigStore: RunnerConfigStore) {
   const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
   const maxWritesPerSecond = runnerConfigStore.maxWritesPerSecond;
-  const isDelay = maxWritesPerSecond !== undefined && runnerIds.length >= maxWritesPerSecond;
-  const delayMilliseconds = maxWritesPerSecond === undefined ? 0 : 1000 / maxWritesPerSecond;
+  const isDelay = maxWritesPerSecond !== undefined && runnerIds.length > 0;
+  const configuredConcurrency = parseInt(process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS ?? '', 10);
+  const maxConcurrentInvocations = configuredConcurrency > 0 ? configuredConcurrency : 1;
+  const delayMilliseconds =
+    maxWritesPerSecond === undefined ? 0 : (1000 / maxWritesPerSecond) * maxConcurrentInvocations;
   return { isDelay, delay, delayMilliseconds };
 }
 

lambdas/libs/storage-providers/aws/ssm/runner-config-store.test.ts

@@ -1,7 +1,7 @@
 import { putParameter } from '@aws-github-runner/aws-ssm-util';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { createAwsSsmRunnerConfigStore } from './runner-config-store';
+import { createAwsSsmRunnerConfigStore, resolveMaxWritesPerSecond } from './runner-config-store';
 
 vi.mock('@aws-github-runner/aws-ssm-util', () => ({
   putParameter: vi.fn(),
@@ -91,6 +91,34 @@ describe('aws_ssm runner config store', () => {
     expect(putParameterMock).toHaveBeenCalledWith('/runner/tokens/runner-1', 'jit-config', true, { tags: [] });
   });
 
+  it('uses SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND when set, e.g. after enabling higher throughput', () => {
+    process.env.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND = '10000';
+    const store = createAwsSsmRunnerConfigStore();
+    expect(store.maxWritesPerSecond).toBe(10000);
+  });
+
+  it.each([undefined, '', '0', '-5', 'not-a-number'])(
+    'falls back to the standard-tier default of 40 for an invalid value (%j)',
+    (value) => {
+      if (value === undefined) {
+        delete process.env.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND;
+      } else {
+        process.env.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND = value;
+      }
+      const store = createAwsSsmRunnerConfigStore();
+      expect(store.maxWritesPerSecond).toBe(40);
+    },
+  );
+
+  it('honors an explicitly configured maxWritesPerSecond over the default', () => {
+    const store = createAwsSsmRunnerConfigStore({
+      tokenPath: '/runner/tokens',
+      parameterStoreTags: [],
+      maxWritesPerSecond: 1000,
+    });
+    expect(store.maxWritesPerSecond).toBe(1000);
+  });
+
   it('logs safe context when a runner configuration write fails', async () => {
     const error = Object.assign(new Error('encoded-jit-secret'), { name: 'ThrottlingException' });
     putParameterMock.mockRejectedValue(error);
@@ -108,6 +136,20 @@ describe('aws_ssm runner config store', () => {
   });
 });
 
+describe('resolveMaxWritesPerSecond', () => {
+  it.each([
+    ['10000', 10000],
+    ['1', 1],
+    [undefined, 40],
+    ['', 40],
+    ['0', 40],
+    ['-1', 40],
+    ['not-a-number', 40],
+  ])('resolves %j to %i', (rawValue, expected) => {
+    expect(resolveMaxWritesPerSecond(rawValue)).toBe(expected);
+  });
+});
+
 function setTokenPath(tokenPath: string | undefined): void {
   if (tokenPath === undefined) {
     delete process.env.SSM_TOKEN_PATH;

lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts

@@ -7,9 +7,19 @@ import { loadSsmParameterStoreTagsFromEnvironment } from './parameter-store-tags
 
 const logger = createAwsSsmStorageLogger('runner-config-store');
 
+// SSM Parameter Store's standard-tier default; accounts that enabled the higher-throughput tier
+// (up to several thousand TPS) should override this via SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND.
+const DEFAULT_MAX_WRITES_PER_SECOND = 40;
+
+export function resolveMaxWritesPerSecond(rawValue: string | undefined): number {
+  const parsed = parseInt(rawValue ?? '', 10);
+  return parsed > 0 ? parsed : DEFAULT_MAX_WRITES_PER_SECOND;
+}
+
 export interface AwsSsmRunnerConfigStoreConfig {
   tokenPath: string;
   parameterStoreTags: ReadonlyArray<Readonly<{ Key: string; Value: string }>>;
+  maxWritesPerSecond?: number;
 }
 
 export function createAwsSsmRunnerConfigStore(config?: AwsSsmRunnerConfigStoreConfig): RunnerConfigStore {
@@ -29,13 +39,16 @@ export function createAwsSsmRunnerConfigStore(config?: AwsSsmRunnerConfigStoreCo
   return new AwsSsmRunnerConfigStore({
     tokenPath,
     parameterStoreTags: loadSsmParameterStoreTagsFromEnvironment(),
+    maxWritesPerSecond: resolveMaxWritesPerSecond(process.env.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND),
   });
 }
 
 class AwsSsmRunnerConfigStore implements RunnerConfigStore {
-  readonly maxWritesPerSecond = 40;
+  readonly maxWritesPerSecond: number;
 
-  constructor(private readonly config: AwsSsmRunnerConfigStoreConfig) {}
+  constructor(private readonly config: AwsSsmRunnerConfigStoreConfig) {
+    this.maxWritesPerSecond = config.maxWritesPerSecond ?? DEFAULT_MAX_WRITES_PER_SECOND;
+  }
 
   async create(record: RunnerConfigRecord, options: { metadata?: RunnerConfigMetadata[] } = {}): Promise<void> {
     const parameterName = `${this.config.tokenPath}/${record.runnerId}`;

lambdas/libs/storage-providers/storage-providers.test.ts

@@ -1,9 +1,11 @@
 import { describe, expect, it, vi } from 'vitest';
 
 import { createStorageProviders } from './storage-providers';
+import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store';
 
 vi.mock('./aws/ssm/runner-config-store', () => ({
   createAwsSsmRunnerConfigStore: vi.fn(() => ({ create: vi.fn() })),
+  resolveMaxWritesPerSecond: vi.fn((rawValue: string | undefined) => (rawValue ? parseInt(rawValue, 10) : 40)),
 }));
 vi.mock('./aws/ssm/runner-group-cache-store', () => ({
   createAwsSsmRunnerGroupCacheStore: vi.fn(() => ({ get: vi.fn(), create: vi.fn() })),
@@ -39,4 +41,19 @@ describe('createStorageProviders', () => {
       githubAppCredentials: expect.any(Object),
     });
   });
+
+  it('forwards SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND to the runner config store', () => {
+    const environment = Object.freeze({
+      RUNNER_CONFIG_STORAGE_PROVIDER: 'AWS_SSM',
+      SSM_TOKEN_PATH: '/runners/tokens',
+      SSM_CONFIG_PATH: '/runners/config',
+      SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND: '10000',
+      PARAMETER_GITHUB_APP_ID_NAME: 'app-id',
+      PARAMETER_GITHUB_APP_KEY_BASE64_NAME: 'app-key',
+    });
+
+    createStorageProviders(environment);
+
+    expect(createAwsSsmRunnerConfigStore).toHaveBeenCalledWith(expect.objectContaining({ maxWritesPerSecond: 10000 }));
+  });
 });

lambdas/libs/storage-providers/storage-providers.ts

@@ -1,7 +1,7 @@
 import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
 import { createAwsSsmGitHubAppCredentialsStore } from './aws/ssm/github-app-credentials-store';
 import { createAwsSsmRunnerConfigConsumer } from './aws/ssm/runner-config-consumer';
-import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store';
+import { createAwsSsmRunnerConfigStore, resolveMaxWritesPerSecond } from './aws/ssm/runner-config-store';
 import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store';
 import type { CommonStorage, StorageProviders } from './core';
 import { loadRunnerConfigConsumerConfigFromEnvironment } from './runner-config-consumer';
@@ -29,7 +29,11 @@ export function createStorageProviders(environment: Environment = process.env):
   });
 
   return {
-    runnerConfig: createAwsSsmRunnerConfigStore({ tokenPath, parameterStoreTags }),
+    runnerConfig: createAwsSsmRunnerConfigStore({
+      tokenPath,
+      parameterStoreTags,
+      maxWritesPerSecond: resolveMaxWritesPerSecond(environment.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND),
+    }),
     runnerGroupCache: createAwsSsmRunnerGroupCacheStore({ configPath, parameterStoreTags }),
     consumer: createAwsSsmRunnerConfigConsumer({ SSM_TOKEN_PATH: tokenPath }, consumerConfig),
     ...createCommonStorage(environment),

main.tf

@@ -258,7 +258,9 @@ module "runners" {
   runner_name_prefix                                             = var.runner_name_prefix
   parameter_store_tags                                           = var.parameter_store_tags
 
-  scale_up_reserved_concurrent_executions = var.scale_up_reserved_concurrent_executions
+  scale_up_reserved_concurrent_executions        = var.scale_up_reserved_concurrent_executions
+  ssm_parameter_store_max_concurrent_invocations = var.ssm_parameter_store_max_concurrent_invocations
+  ssm_parameter_store_max_writes_per_second      = var.ssm_parameter_store_max_writes_per_second
 
   associate_public_ipv4_address = var.associate_public_ipv4_address
   network_interfaces            = var.runner_network_interfaces

modules/runners/pool.tf

@@ -57,14 +57,16 @@ module "pool" {
       role                                 = { arn = var.iam_overrides["override_runner_role"] ? var.iam_overrides["runner_role_arn"] : aws_iam_role.runner[0].arn }
       use_dedicated_host                   = var.use_dedicated_host
     }
-    subnet_ids                           = var.subnet_ids
-    ssm_token_path                       = "${var.ssm_paths.root}/${var.ssm_paths.tokens}"
-    ssm_config_path                      = "${var.ssm_paths.root}/${var.ssm_paths.config}"
-    ami_id_ssm_parameter_name            = local.ami_id_ssm_parameter_name
-    ami_id_ssm_parameter_read_policy_arn = local.ami_id_ssm_parameter_name != null ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null
-    tags                                 = local.tags
-    lambda_tags                          = var.lambda_tags
-    arn_ssm_parameters_path_config       = local.arn_ssm_parameters_path_config
+    subnet_ids                                     = var.subnet_ids
+    ssm_token_path                                 = "${var.ssm_paths.root}/${var.ssm_paths.tokens}"
+    ssm_config_path                                = "${var.ssm_paths.root}/${var.ssm_paths.config}"
+    ssm_parameter_store_max_concurrent_invocations = var.ssm_parameter_store_max_concurrent_invocations
+    ssm_parameter_store_max_writes_per_second      = var.ssm_parameter_store_max_writes_per_second
+    ami_id_ssm_parameter_name                      = local.ami_id_ssm_parameter_name
+    ami_id_ssm_parameter_read_policy_arn           = local.ami_id_ssm_parameter_name != null ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null
+    tags                                           = local.tags
+    lambda_tags                                    = var.lambda_tags
+    arn_ssm_parameters_path_config                 = local.arn_ssm_parameters_path_config
   }
 
   aws_partition  = var.aws_partition