From 4ec6bcbcd707dd0480f1221a6de045b287dd81cb Mon Sep 17 00:00:00 2001 From: Anatoli Tsikhamirau Date: Fri, 11 Sep 2026 07:57:40 +0200 Subject: [PATCH] fix(runners): pace Parameter Store writes against the account-wide limit The write-pacing guard only started delaying writes once a single invocation's own batch reached the store's maxWritesPerSecond threshold, and sized the delay off that same per-invocation number. Parameter Store's write-rate limit is account-wide: several pools' scale-up/pool lambdas can each stay under the threshold individually while their combined writes exceed the account limit, and a batch below the threshold got no pacing at all. The guard now always paces once a write limit is configured, and divides the per-write delay by a new ssm_parameter_store_max_concurrent_invocations variable so the configured account-wide write budget is shared across the expected number of concurrent invocations instead of assumed available to each one independently. The write limit itself was also hardcoded to SSM's standard-tier default of 40/sec. It's now configurable via a new ssm_parameter_store_max_writes_per_second variable, so accounts that enabled SSM's higher-throughput tier (up to several thousand writes/second) can raise the pacing ceiling to match instead of being paced against a limit that no longer applies to them. Both variables are wired through for the scale-up and pool lambdas; defaults (1 concurrent invocation, 40 writes/sec) match prior behavior. Documented in docs/rate-limits-and-tuning.md alongside the existing SSM/batch_size guidance. --- docs/rate-limits-and-tuning.md | 10 +- .../src/scale-runners/github-runner.test.ts | 92 +++++++++++++++++++ .../src/scale-runners/github-runner.ts | 11 ++- .../aws/ssm/runner-config-store.test.ts | 44 ++++++++- .../aws/ssm/runner-config-store.ts | 17 +++- .../storage-providers.test.ts | 17 ++++ .../storage-providers/storage-providers.ts | 8 +- main.tf | 4 +- modules/runners/pool.tf | 18 ++-- modules/runners/pool/main.tf | 76 +++++++-------- modules/runners/pool/variables.tf | 28 +++--- modules/runners/scale-up.tf | 84 ++++++++--------- modules/runners/variables.tf | 12 +++ variables.tf | 12 +++ 14 files changed, 321 insertions(+), 112 deletions(-) create mode 100644 lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts diff --git a/docs/rate-limits-and-tuning.md b/docs/rate-limits-and-tuning.md index 88c74bf6d5..0e933c1ffb 100644 --- a/docs/rate-limits-and-tuning.md +++ b/docs/rate-limits-and-tuning.md @@ -82,7 +82,7 @@ Rate limits are **disabled by default** on GitHub Enterprise Server and must be |---|---| | Combined throughput (Get + Put) | **40 TPS** (shared per-account per-region) | -Each runner instance requires one `PutParameter` call for its JIT config. At 40 TPS shared across all operations in the account, a burst of 40+ concurrent writes will throttle. +Each runner instance requires one `PutParameter` call for its JIT config. This is account-wide, not per-invocation or per-pool: several pools' scale-up/pool lambdas can each individually look fine while their combined writes exceed the account limit. The module paces its own writes to stay under a configured ceiling — `ssm_parameter_store_max_writes_per_second` (default `40`, matching the standard tier) — dividing the per-write delay across `ssm_parameter_store_max_concurrent_invocations` (default `1`), which should reflect how many scale-up/pool invocations across all pools can realistically run at the same time. **Higher throughput mode** raises the ceiling: @@ -94,6 +94,8 @@ aws ssm update-service-setting \ Cost: $0.05 per 10,000 API interactions beyond the standard tier. +Enabling this alone does nothing for this module's own pacing — raise `ssm_parameter_store_max_writes_per_second` to match the new ceiling too, or the module keeps pacing writes at the old 40 TPS default regardless of what AWS now allows. + ### EC2 CreateFleet The exact TPS limit for `CreateFleet` is not publicly documented. It uses a token-bucket algorithm per-account per-region. Empirically throttles at low single-digit TPS. @@ -111,7 +113,7 @@ For deployments running more than a handful of concurrent runners: | Running On-Demand Standard (A,C,D,H,I,M,R,T,Z) instances | **5 vCPUs** | Service Quotas console | | All Standard Spot Instance Requests | **5 vCPUs** | Service Quotas console | | EC2 CreateFleet API rate | Undocumented | AWS Support ticket | -| SSM Parameter Store throughput | 40 TPS | `update-service-setting` (see above) | +| SSM Parameter Store throughput | 40 TPS | `update-service-setting` (see above), then raise `ssm_parameter_store_max_writes_per_second` to match | **vCPU quotas are measured in vCPUs, not instance count.** Running 50× `c5.large` (2 vCPU each) requires a quota of at least 100 vCPUs. @@ -131,7 +133,7 @@ For deployments running more than a handful of concurrent runners: | `isJobQueued` calls | 100 | 100 (same total) | | JIT config generation calls | 100 | 100 (same total) | -Larger `batch_size` reduces CreateFleet calls (the most constrained AWS API) and Lambda invocations. Per-runner work (SSM writes, JIT config, isJobQueued) stays the same total. SSM peak TPS is lower with larger batches because writes are serialized within each Lambda rather than concurrent across many. +Larger `batch_size` reduces CreateFleet calls (the most constrained AWS API) and Lambda invocations. Per-runner work (SSM writes, JIT config, isJobQueued) stays the same total. SSM writes are paced within each Lambda regardless of batch size (see `ssm_parameter_store_max_writes_per_second`/`ssm_parameter_store_max_concurrent_invocations` above) — batch size does not change the SSM pacing behavior, only how many Lambda invocations are running concurrently in the first place. ### Tradeoffs @@ -157,7 +159,7 @@ Higher windows improve batching efficiency but add latency to job pickup. |---|---|---|---| | Small (<50 concurrent jobs) | 1–5 | 90s | Defaults work | | Medium (50–200) | 5–10 | 180s | Monitor SSM throttling | -| Large (200+) | 10 | 300s | Enable SSM higher throughput, raise vCPU quotas, request CreateFleet rate increase | +| Large (200+) | 10 | 300s | Enable SSM higher throughput and raise `ssm_parameter_store_max_writes_per_second`/`ssm_parameter_store_max_concurrent_invocations` to match, raise vCPU quotas, request CreateFleet rate increase | ## Monitoring diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts new file mode 100644 index 0000000000..12d799538c --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts @@ -0,0 +1,92 @@ +import { addDelay, createStartRunnerConfig } from './github-runner'; +import type { CreateGitHubRunnerConfig } from './types'; +import type { RunnerConfigStore } from '@aws-github-runner/storage-providers'; +import { Octokit } from '@octokit/rest'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; + +const cleanEnv = process.env; + +beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS; +}); + +describe('Test addDelay', () => { + it('does not delay when the store has no write limit', () => { + const store = { maxWritesPerSecond: undefined } as RunnerConfigStore; + const { isDelay, delayMilliseconds } = addDelay(['1', '2'], store); + expect(isDelay).toBe(false); + expect(delayMilliseconds).toBe(0); + }); + + it('does not delay for an empty batch', () => { + const store = { maxWritesPerSecond: 40 } as RunnerConfigStore; + const { isDelay } = addDelay([], store); + expect(isDelay).toBe(false); + }); + + it('paces a batch below the write limit, unlike the previous per-invocation-only guard', () => { + const store = { maxWritesPerSecond: 40 } as RunnerConfigStore; + // A batch of 2 is far below 40, but the account-wide limit does not care about batch size. + const { isDelay, delayMilliseconds } = addDelay(['1', '2'], store); + expect(isDelay).toBe(true); + expect(delayMilliseconds).toBe(25); // 1000 / 40 + }); + + it('divides the per-write delay across the configured number of concurrent invocations', () => { + process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS = '5'; + const store = { maxWritesPerSecond: 40 } as RunnerConfigStore; + const { delayMilliseconds } = addDelay(['1'], store); + expect(delayMilliseconds).toBe(125); // (1000 / 40) * 5 + }); + + it.each(['0', '-1', 'not-a-number', ''])( + 'treats an invalid concurrency value (%s) as a single invocation', + (value) => { + process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS = value; + const store = { maxWritesPerSecond: 40 } as RunnerConfigStore; + const { delayMilliseconds } = addDelay(['1'], store); + expect(delayMilliseconds).toBe(25); + }, + ); +}); + +describe('Test createStartRunnerConfig registration-token pacing', () => { + const mockOctokit = { + actions: { + createRegistrationTokenForOrg: vi.fn().mockResolvedValue({ data: { token: 'reg-token' } }), + }, + } as unknown as Octokit; + + const githubRunnerConfig: CreateGitHubRunnerConfig = { + ephemeral: false, + enableJitConfig: false, + runnerLabels: 'self-hosted', + runnerGroup: 'Default', + runnerNamePrefix: 'test-', + runnerOwner: 'my-org', + runnerType: 'Org', + disableAutoUpdate: false, + }; + + it('paces every write between concurrent pools, not just large single-invocation batches', async () => { + vi.useFakeTimers(); + const create = vi.fn().mockResolvedValue(undefined); + const runnerConfigStore = { maxWritesPerSecond: 40, create } as unknown as RunnerConfigStore; + + process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS = '4'; + const runPromise = createStartRunnerConfig(githubRunnerConfig, ['a', 'b', 'c'], mockOctokit, { + runnerConfigStore, + }); + + // Each write is followed by a (1000 / 40) * 4 = 100ms pacing delay. + await vi.advanceTimersByTimeAsync(100); + await vi.advanceTimersByTimeAsync(100); + await vi.advanceTimersByTimeAsync(100); + await runPromise; + + expect(create).toHaveBeenCalledTimes(3); + vi.useRealTimers(); + }); +}); diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index b344012149..c9e893ffec 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -215,11 +215,16 @@ export async function createStartRunnerConfig( } } -function addDelay(runnerIds: string[], runnerConfigStore: RunnerConfigStore) { +// maxWritesPerSecond is account-wide, so the per-write delay is spread across the configured +// number of concurrent invocations rather than sized off this invocation's batch alone. +export function addDelay(runnerIds: string[], runnerConfigStore: RunnerConfigStore) { const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const maxWritesPerSecond = runnerConfigStore.maxWritesPerSecond; - const isDelay = maxWritesPerSecond !== undefined && runnerIds.length >= maxWritesPerSecond; - const delayMilliseconds = maxWritesPerSecond === undefined ? 0 : 1000 / maxWritesPerSecond; + const isDelay = maxWritesPerSecond !== undefined && runnerIds.length > 0; + const configuredConcurrency = parseInt(process.env.SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS ?? '', 10); + const maxConcurrentInvocations = configuredConcurrency > 0 ? configuredConcurrency : 1; + const delayMilliseconds = + maxWritesPerSecond === undefined ? 0 : (1000 / maxWritesPerSecond) * maxConcurrentInvocations; return { isDelay, delay, delayMilliseconds }; } diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.test.ts index 9d3e948b9d..d70cc3de51 100644 --- a/lambdas/libs/storage-providers/aws/ssm/runner-config-store.test.ts +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.test.ts @@ -1,7 +1,7 @@ import { putParameter } from '@aws-github-runner/aws-ssm-util'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { createAwsSsmRunnerConfigStore } from './runner-config-store'; +import { createAwsSsmRunnerConfigStore, resolveMaxWritesPerSecond } from './runner-config-store'; vi.mock('@aws-github-runner/aws-ssm-util', () => ({ putParameter: vi.fn(), @@ -91,6 +91,34 @@ describe('aws_ssm runner config store', () => { expect(putParameterMock).toHaveBeenCalledWith('/runner/tokens/runner-1', 'jit-config', true, { tags: [] }); }); + it('uses SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND when set, e.g. after enabling higher throughput', () => { + process.env.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND = '10000'; + const store = createAwsSsmRunnerConfigStore(); + expect(store.maxWritesPerSecond).toBe(10000); + }); + + it.each([undefined, '', '0', '-5', 'not-a-number'])( + 'falls back to the standard-tier default of 40 for an invalid value (%j)', + (value) => { + if (value === undefined) { + delete process.env.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND; + } else { + process.env.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND = value; + } + const store = createAwsSsmRunnerConfigStore(); + expect(store.maxWritesPerSecond).toBe(40); + }, + ); + + it('honors an explicitly configured maxWritesPerSecond over the default', () => { + const store = createAwsSsmRunnerConfigStore({ + tokenPath: '/runner/tokens', + parameterStoreTags: [], + maxWritesPerSecond: 1000, + }); + expect(store.maxWritesPerSecond).toBe(1000); + }); + it('logs safe context when a runner configuration write fails', async () => { const error = Object.assign(new Error('encoded-jit-secret'), { name: 'ThrottlingException' }); putParameterMock.mockRejectedValue(error); @@ -108,6 +136,20 @@ describe('aws_ssm runner config store', () => { }); }); +describe('resolveMaxWritesPerSecond', () => { + it.each([ + ['10000', 10000], + ['1', 1], + [undefined, 40], + ['', 40], + ['0', 40], + ['-1', 40], + ['not-a-number', 40], + ])('resolves %j to %i', (rawValue, expected) => { + expect(resolveMaxWritesPerSecond(rawValue)).toBe(expected); + }); +}); + function setTokenPath(tokenPath: string | undefined): void { if (tokenPath === undefined) { delete process.env.SSM_TOKEN_PATH; diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts index 584b654ba9..22bf61c9df 100644 --- a/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts @@ -7,9 +7,19 @@ import { loadSsmParameterStoreTagsFromEnvironment } from './parameter-store-tags const logger = createAwsSsmStorageLogger('runner-config-store'); +// SSM Parameter Store's standard-tier default; accounts that enabled the higher-throughput tier +// (up to several thousand TPS) should override this via SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND. +const DEFAULT_MAX_WRITES_PER_SECOND = 40; + +export function resolveMaxWritesPerSecond(rawValue: string | undefined): number { + const parsed = parseInt(rawValue ?? '', 10); + return parsed > 0 ? parsed : DEFAULT_MAX_WRITES_PER_SECOND; +} + export interface AwsSsmRunnerConfigStoreConfig { tokenPath: string; parameterStoreTags: ReadonlyArray>; + maxWritesPerSecond?: number; } export function createAwsSsmRunnerConfigStore(config?: AwsSsmRunnerConfigStoreConfig): RunnerConfigStore { @@ -29,13 +39,16 @@ export function createAwsSsmRunnerConfigStore(config?: AwsSsmRunnerConfigStoreCo return new AwsSsmRunnerConfigStore({ tokenPath, parameterStoreTags: loadSsmParameterStoreTagsFromEnvironment(), + maxWritesPerSecond: resolveMaxWritesPerSecond(process.env.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND), }); } class AwsSsmRunnerConfigStore implements RunnerConfigStore { - readonly maxWritesPerSecond = 40; + readonly maxWritesPerSecond: number; - constructor(private readonly config: AwsSsmRunnerConfigStoreConfig) {} + constructor(private readonly config: AwsSsmRunnerConfigStoreConfig) { + this.maxWritesPerSecond = config.maxWritesPerSecond ?? DEFAULT_MAX_WRITES_PER_SECOND; + } async create(record: RunnerConfigRecord, options: { metadata?: RunnerConfigMetadata[] } = {}): Promise { const parameterName = `${this.config.tokenPath}/${record.runnerId}`; diff --git a/lambdas/libs/storage-providers/storage-providers.test.ts b/lambdas/libs/storage-providers/storage-providers.test.ts index f1b01ee66b..6721523d6b 100644 --- a/lambdas/libs/storage-providers/storage-providers.test.ts +++ b/lambdas/libs/storage-providers/storage-providers.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it, vi } from 'vitest'; import { createStorageProviders } from './storage-providers'; +import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; vi.mock('./aws/ssm/runner-config-store', () => ({ createAwsSsmRunnerConfigStore: vi.fn(() => ({ create: vi.fn() })), + resolveMaxWritesPerSecond: vi.fn((rawValue: string | undefined) => (rawValue ? parseInt(rawValue, 10) : 40)), })); vi.mock('./aws/ssm/runner-group-cache-store', () => ({ createAwsSsmRunnerGroupCacheStore: vi.fn(() => ({ get: vi.fn(), create: vi.fn() })), @@ -39,4 +41,19 @@ describe('createStorageProviders', () => { githubAppCredentials: expect.any(Object), }); }); + + it('forwards SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND to the runner config store', () => { + const environment = Object.freeze({ + RUNNER_CONFIG_STORAGE_PROVIDER: 'AWS_SSM', + SSM_TOKEN_PATH: '/runners/tokens', + SSM_CONFIG_PATH: '/runners/config', + SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND: '10000', + PARAMETER_GITHUB_APP_ID_NAME: 'app-id', + PARAMETER_GITHUB_APP_KEY_BASE64_NAME: 'app-key', + }); + + createStorageProviders(environment); + + expect(createAwsSsmRunnerConfigStore).toHaveBeenCalledWith(expect.objectContaining({ maxWritesPerSecond: 10000 })); + }); }); diff --git a/lambdas/libs/storage-providers/storage-providers.ts b/lambdas/libs/storage-providers/storage-providers.ts index 8390fbd2d3..238d365891 100644 --- a/lambdas/libs/storage-providers/storage-providers.ts +++ b/lambdas/libs/storage-providers/storage-providers.ts @@ -1,7 +1,7 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { createAwsSsmGitHubAppCredentialsStore } from './aws/ssm/github-app-credentials-store'; import { createAwsSsmRunnerConfigConsumer } from './aws/ssm/runner-config-consumer'; -import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; +import { createAwsSsmRunnerConfigStore, resolveMaxWritesPerSecond } from './aws/ssm/runner-config-store'; import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store'; import type { CommonStorage, StorageProviders } from './core'; import { loadRunnerConfigConsumerConfigFromEnvironment } from './runner-config-consumer'; @@ -29,7 +29,11 @@ export function createStorageProviders(environment: Environment = process.env): }); return { - runnerConfig: createAwsSsmRunnerConfigStore({ tokenPath, parameterStoreTags }), + runnerConfig: createAwsSsmRunnerConfigStore({ + tokenPath, + parameterStoreTags, + maxWritesPerSecond: resolveMaxWritesPerSecond(environment.SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND), + }), runnerGroupCache: createAwsSsmRunnerGroupCacheStore({ configPath, parameterStoreTags }), consumer: createAwsSsmRunnerConfigConsumer({ SSM_TOKEN_PATH: tokenPath }, consumerConfig), ...createCommonStorage(environment), diff --git a/main.tf b/main.tf index 26e7e472f9..2b9084e772 100644 --- a/main.tf +++ b/main.tf @@ -258,7 +258,9 @@ module "runners" { runner_name_prefix = var.runner_name_prefix parameter_store_tags = var.parameter_store_tags - scale_up_reserved_concurrent_executions = var.scale_up_reserved_concurrent_executions + scale_up_reserved_concurrent_executions = var.scale_up_reserved_concurrent_executions + ssm_parameter_store_max_concurrent_invocations = var.ssm_parameter_store_max_concurrent_invocations + ssm_parameter_store_max_writes_per_second = var.ssm_parameter_store_max_writes_per_second associate_public_ipv4_address = var.associate_public_ipv4_address diff --git a/modules/runners/pool.tf b/modules/runners/pool.tf index 11840a4638..747322871f 100644 --- a/modules/runners/pool.tf +++ b/modules/runners/pool.tf @@ -57,14 +57,16 @@ module "pool" { role = { arn = var.iam_overrides["override_runner_role"] ? var.iam_overrides["runner_role_arn"] : aws_iam_role.runner[0].arn } use_dedicated_host = var.use_dedicated_host } - subnet_ids = var.subnet_ids - ssm_token_path = "${var.ssm_paths.root}/${var.ssm_paths.tokens}" - ssm_config_path = "${var.ssm_paths.root}/${var.ssm_paths.config}" - ami_id_ssm_parameter_name = local.ami_id_ssm_parameter_name - ami_id_ssm_parameter_read_policy_arn = local.ami_id_ssm_parameter_name != null ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null - tags = local.tags - lambda_tags = var.lambda_tags - arn_ssm_parameters_path_config = local.arn_ssm_parameters_path_config + subnet_ids = var.subnet_ids + ssm_token_path = "${var.ssm_paths.root}/${var.ssm_paths.tokens}" + ssm_config_path = "${var.ssm_paths.root}/${var.ssm_paths.config}" + ssm_parameter_store_max_concurrent_invocations = var.ssm_parameter_store_max_concurrent_invocations + ssm_parameter_store_max_writes_per_second = var.ssm_parameter_store_max_writes_per_second + ami_id_ssm_parameter_name = local.ami_id_ssm_parameter_name + ami_id_ssm_parameter_read_policy_arn = local.ami_id_ssm_parameter_name != null ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null + tags = local.tags + lambda_tags = var.lambda_tags + arn_ssm_parameters_path_config = local.arn_ssm_parameters_path_config } aws_partition = var.aws_partition diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf index f9e140317d..5f4e45690d 100644 --- a/modules/runners/pool/main.tf +++ b/modules/runners/pool/main.tf @@ -27,43 +27,45 @@ resource "aws_lambda_function" "pool" { environment { variables = { - AMI_ID_SSM_PARAMETER_NAME = var.config.ami_id_ssm_parameter_name - DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate - ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral - ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config - ENVIRONMENT = var.config.prefix - GHES_URL = var.config.ghes.url - USER_AGENT = var.config.user_agent - INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy - INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price - INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type - INSTANCE_TYPE_PRIORITIES = var.config.instance_type_priorities != null ? jsonencode(var.config.instance_type_priorities) : "" - INSTANCE_TYPES = join(",", var.config.instance_types) - LAUNCH_TEMPLATE_NAME = var.config.runner.launch_template.name - LOG_LEVEL = upper(var.config.lambda.log_level) - NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1 - PARAMETER_GITHUB_APP_ID_NAME = var.config.github_app_parameters.id.name - PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name - PARAMETER_GITHUB_APPS_MANIFEST_NAME = var.config.github_app_parameters.additional_apps_manifest != null ? var.config.github_app_parameters.additional_apps_manifest.name : "" - POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false" - RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes - RUNNER_LABELS = lower(join(",", var.config.runner.labels)) - RUNNER_GROUP_NAME = var.config.runner.group_name - RUNNER_NAME_PREFIX = var.config.runner.name_prefix - RUNNER_OWNER = var.config.runner.pool_owner - RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count - SSM_TOKEN_PATH = var.config.ssm_token_path - SSM_CONFIG_PATH = var.config.ssm_config_path - SUBNET_IDS = join(",", var.config.subnet_ids) - POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool" - POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false - POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests - POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error - ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.runner.enable_on_demand_failover_for_errors) - SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags - SCALE_ERRORS = jsonencode(var.config.runner.scale_errors) - USE_DEDICATED_HOST = var.config.runner.use_dedicated_host - INCLUDE_BUSY_RUNNERS = var.config.include_busy_runners + AMI_ID_SSM_PARAMETER_NAME = var.config.ami_id_ssm_parameter_name + DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate + ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral + ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config + ENVIRONMENT = var.config.prefix + GHES_URL = var.config.ghes.url + USER_AGENT = var.config.user_agent + INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy + INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price + INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type + INSTANCE_TYPE_PRIORITIES = var.config.instance_type_priorities != null ? jsonencode(var.config.instance_type_priorities) : "" + INSTANCE_TYPES = join(",", var.config.instance_types) + LAUNCH_TEMPLATE_NAME = var.config.runner.launch_template.name + LOG_LEVEL = upper(var.config.lambda.log_level) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = var.config.github_app_parameters.id.name + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name + PARAMETER_GITHUB_APPS_MANIFEST_NAME = var.config.github_app_parameters.additional_apps_manifest != null ? var.config.github_app_parameters.additional_apps_manifest.name : "" + POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false" + RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes + RUNNER_LABELS = lower(join(",", var.config.runner.labels)) + RUNNER_GROUP_NAME = var.config.runner.group_name + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + RUNNER_OWNER = var.config.runner.pool_owner + RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count + SSM_TOKEN_PATH = var.config.ssm_token_path + SSM_CONFIG_PATH = var.config.ssm_config_path + SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS = var.config.ssm_parameter_store_max_concurrent_invocations + SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND = var.config.ssm_parameter_store_max_writes_per_second + SUBNET_IDS = join(",", var.config.subnet_ids) + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool" + POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error + ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.runner.enable_on_demand_failover_for_errors) + SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags + SCALE_ERRORS = jsonencode(var.config.runner.scale_errors) + USE_DEDICATED_HOST = var.config.runner.use_dedicated_host + INCLUDE_BUSY_RUNNERS = var.config.include_busy_runners } } diff --git a/modules/runners/pool/variables.tf b/modules/runners/pool/variables.tf index 4c3551c4c1..b9fc4a6185 100644 --- a/modules/runners/pool/variables.tf +++ b/modules/runners/pool/variables.tf @@ -65,19 +65,21 @@ variable "config" { schedule_expression_timezone = string size = number })) - include_busy_runners = bool - role_permissions_boundary = string - kms_key_arn = string - ami_kms_key_arn = string - ami_id_ssm_parameter_arn = string - role_path = string - ssm_token_path = string - ssm_config_path = string - ami_id_ssm_parameter_name = string - ami_id_ssm_parameter_read_policy_arn = string - arn_ssm_parameters_path_config = string - lambda_tags = map(string) - user_agent = string + include_busy_runners = bool + role_permissions_boundary = string + kms_key_arn = string + ami_kms_key_arn = string + ami_id_ssm_parameter_arn = string + role_path = string + ssm_token_path = string + ssm_config_path = string + ssm_parameter_store_max_concurrent_invocations = optional(number, 1) + ssm_parameter_store_max_writes_per_second = optional(number, 40) + ami_id_ssm_parameter_name = string + ami_id_ssm_parameter_read_policy_arn = string + arn_ssm_parameters_path_config = string + lambda_tags = map(string) + user_agent = string }) } diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index d78eb7bfa5..7031d50289 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -30,47 +30,49 @@ resource "aws_lambda_function" "scale_up" { depends_on = [aws_cloudwatch_log_group.scale_up] environment { variables = { - AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name - DISABLE_RUNNER_AUTOUPDATE = var.disable_runner_autoupdate - ENABLE_EPHEMERAL_RUNNERS = var.enable_ephemeral_runners - ENABLE_JIT_CONFIG = var.enable_jit_config - ENABLE_JOB_QUEUED_CHECK = local.enable_job_queued_check - ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit - ENABLE_ORGANIZATION_RUNNERS = var.enable_organization_runners - ENVIRONMENT = var.prefix - GHES_URL = var.ghes_url - USER_AGENT = var.user_agent - INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy - INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price - INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type - INSTANCE_TYPE_PRIORITIES = var.instance_type_priorities != null ? jsonencode(var.instance_type_priorities) : "" - INSTANCE_TYPES = join(",", var.instance_types) - LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name - LOG_LEVEL = upper(var.log_level) - MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.minimum_running_time_in_minutes, local.min_runtime_defaults[var.runner_os]) - NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1 - PARAMETER_GITHUB_APP_ID_NAME = var.github_app_parameters.id.name - PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.github_app_parameters.key_base64.name - PARAMETER_GITHUB_APPS_MANIFEST_NAME = var.github_app_parameters.additional_apps_manifest != null ? var.github_app_parameters.additional_apps_manifest.name : "" - POWERTOOLS_LOGGER_LOG_EVENT = var.log_level == "debug" ? "true" : "false" - POWERTOOLS_METRICS_NAMESPACE = var.metrics.namespace - POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false - POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests - POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error - RUNNER_LABELS = lower(join(",", var.runner_labels)) - RUNNER_GROUP_NAME = var.runner_group_name - RUNNER_NAME_PREFIX = var.runner_name_prefix - COMPUTE_PROVIDER_TYPE = "ec2" - RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count - POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-up" - SSM_TOKEN_PATH = local.token_path - SSM_CONFIG_PATH = "${var.ssm_paths.root}/${var.ssm_paths.config}" - SSM_PARAMETER_STORE_TAGS = local.parameter_store_tags - SUBNET_IDS = join(",", var.subnet_ids) - ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.enable_on_demand_failover_for_errors) - SCALE_ERRORS = jsonencode(var.scale_errors) - JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) - USE_DEDICATED_HOST = var.use_dedicated_host + AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name + DISABLE_RUNNER_AUTOUPDATE = var.disable_runner_autoupdate + ENABLE_EPHEMERAL_RUNNERS = var.enable_ephemeral_runners + ENABLE_JIT_CONFIG = var.enable_jit_config + ENABLE_JOB_QUEUED_CHECK = local.enable_job_queued_check + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit + ENABLE_ORGANIZATION_RUNNERS = var.enable_organization_runners + ENVIRONMENT = var.prefix + GHES_URL = var.ghes_url + USER_AGENT = var.user_agent + INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy + INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price + INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type + INSTANCE_TYPE_PRIORITIES = var.instance_type_priorities != null ? jsonencode(var.instance_type_priorities) : "" + INSTANCE_TYPES = join(",", var.instance_types) + LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name + LOG_LEVEL = upper(var.log_level) + MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.minimum_running_time_in_minutes, local.min_runtime_defaults[var.runner_os]) + NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = var.github_app_parameters.id.name + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.github_app_parameters.key_base64.name + PARAMETER_GITHUB_APPS_MANIFEST_NAME = var.github_app_parameters.additional_apps_manifest != null ? var.github_app_parameters.additional_apps_manifest.name : "" + POWERTOOLS_LOGGER_LOG_EVENT = var.log_level == "debug" ? "true" : "false" + POWERTOOLS_METRICS_NAMESPACE = var.metrics.namespace + POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error + RUNNER_LABELS = lower(join(",", var.runner_labels)) + RUNNER_GROUP_NAME = var.runner_group_name + RUNNER_NAME_PREFIX = var.runner_name_prefix + COMPUTE_PROVIDER_TYPE = "ec2" + RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count + POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-up" + SSM_TOKEN_PATH = local.token_path + SSM_CONFIG_PATH = "${var.ssm_paths.root}/${var.ssm_paths.config}" + SSM_PARAMETER_STORE_TAGS = local.parameter_store_tags + SSM_PARAMETER_STORE_MAX_CONCURRENT_INVOCATIONS = var.ssm_parameter_store_max_concurrent_invocations + SSM_PARAMETER_STORE_MAX_WRITES_PER_SECOND = var.ssm_parameter_store_max_writes_per_second + SUBNET_IDS = join(",", var.subnet_ids) + ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.enable_on_demand_failover_for_errors) + SCALE_ERRORS = jsonencode(var.scale_errors) + JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) + USE_DEDICATED_HOST = var.use_dedicated_host } } diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 1d2f9b35a9..88428d6379 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -320,6 +320,18 @@ variable "scale_up_reserved_concurrent_executions" { default = 1 } +variable "ssm_parameter_store_max_concurrent_invocations" { + description = "Expected number of concurrent scale-up/pool lambda invocations writing to Parameter Store, used to pace each invocation's writes to a share of the account-wide write-rate limit." + type = number + default = 1 +} + +variable "ssm_parameter_store_max_writes_per_second" { + description = "Parameter Store write-rate limit to pace against, in writes/second. Defaults to the standard-tier limit; raise this if SSM's higher-throughput tier is enabled (up to several thousand writes/second)." + type = number + default = 40 +} + variable "lambda_scale_up_memory_size" { description = "Memory size limit in MB for scale-up lambda." type = number diff --git a/variables.tf b/variables.tf index f384921c09..ded7ec4485 100644 --- a/variables.tf +++ b/variables.tf @@ -168,6 +168,18 @@ variable "scale_up_reserved_concurrent_executions" { default = 1 } +variable "ssm_parameter_store_max_concurrent_invocations" { + description = "Expected number of concurrent scale-up/pool lambda invocations writing to Parameter Store, used to pace each invocation's writes to a share of the account-wide write-rate limit." + type = number + default = 1 +} + +variable "ssm_parameter_store_max_writes_per_second" { + description = "Parameter Store write-rate limit to pace against, in writes/second. Defaults to the standard-tier limit; raise this if SSM's higher-throughput tier is enabled (up to several thousand writes/second)." + type = number + default = 40 +} + variable "webhook_lambda_zip" { description = "File location of the webhook lambda zip file." type = string