From e5da163e3d3847c456451f254878357aefe6acc1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 12 Nov 2026 20:36:25 -0800 Subject: [PATCH 1/3] fix(aws-marketplace): check the container license with CheckoutLicense so AWS's standard buyer policy works A buyer who follows AWS's generated launch steps gets the managed AWSLicenseManagerConsumptionPolicy, which does not allow ListReceivedLicenses, so the check failed and Sim stayed on the open-source core. Check out a provisional unit of the entitlement and return it at once, the integration AWS documents for contract-priced containers. License Manager answers a unit held by another replica, or one a crashed replica never returned, the same way as a missing entitlement. Startup retries that answer, and a running Enterprise deployment only restarts without Enterprise once it has held longer than an unreturned unit can last. --- .../aws-marketplace/container-license.test.ts | 149 +++++++++++++ .../aws-marketplace/container-license.ts | 198 ++++++++++++------ .../aws-marketplace-prerequisites.yaml | 9 +- helm/sim/examples/values-aws-marketplace.yaml | 5 +- 4 files changed, 287 insertions(+), 74 deletions(-) create mode 100644 apps/sim/lib/billing/aws-marketplace/container-license.test.ts diff --git a/apps/sim/lib/billing/aws-marketplace/container-license.test.ts b/apps/sim/lib/billing/aws-marketplace/container-license.test.ts new file mode 100644 index 00000000000..d53d4d8c458 --- /dev/null +++ b/apps/sim/lib/billing/aws-marketplace/container-license.test.ts @@ -0,0 +1,149 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSend } = vi.hoisted(() => ({ mockSend: vi.fn() })) + +vi.mock('@aws-sdk/client-license-manager', () => ({ + LicenseManagerClient: class { + send = mockSend + }, + CheckoutLicenseCommand: class { + constructor(public input: { ProductSKU: string }) {} + }, + CheckInLicenseCommand: class { + constructor(public input: { LicenseConsumptionToken: string }) {} + }, +})) +vi.mock('@/lib/core/network/proxy-agent.server', () => ({ awsRequestHandler: () => undefined })) +vi.mock('@sim/utils/random', () => ({ randomInt: (min: number) => min })) + +import { CheckInLicenseCommand } from '@aws-sdk/client-license-manager' +import { applyAwsMarketplaceContainerLicense } from '@/lib/billing/aws-marketplace/container-license' + +const MINUTE = 60 * 1000 +const TOKEN = 'consumption-token-placeholder' + +type Answer = 'entitled' | 'NoEntitlementsAllowedException' | 'AccessDeniedException' + +/** License Manager's checkout answers in order; the last one repeats once the list runs out. */ +let answers: Answer[] = [] +let checkInFails = false +const checkIns: string[] = [] + +function awsError(name: string): Error { + return Object.assign(new Error(name), { name }) +} + +function enterpriseEnabled(): boolean { + return process.env.ENTERPRISE_ENABLED === 'true' +} + +async function boot(): Promise { + const started = applyAwsMarketplaceContainerLicense() + await vi.advanceTimersByTimeAsync(30_000) + await started +} + +describe('applyAwsMarketplaceContainerLicense', () => { + let kill: ReturnType + + beforeEach(() => { + vi.useFakeTimers({ toFake: ['setTimeout', 'Date'] }) + answers = [] + checkInFails = false + checkIns.length = 0 + vi.stubEnv('AWS_MARKETPLACE_LICENSE_PRODUCT_SKU', 'product-placeholder') + kill = vi.spyOn(process, 'kill').mockImplementation(() => true) + mockSend.mockReset() + mockSend.mockImplementation(async (command: unknown) => { + if (command instanceof CheckInLicenseCommand) { + checkIns.push(command.input.LicenseConsumptionToken) + if (checkInFails) throw awsError('ServerInternalException') + return {} + } + const answer = answers.length > 1 ? answers.shift() : answers[0] + if (answer !== 'entitled') throw awsError(answer ?? 'ServerInternalException') + return { + LicenseConsumptionToken: TOKEN, + EntitlementsAllowed: [{ Name: 'enterprise_license', Unit: 'Count', Value: '1' }], + } + }) + }) + + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + kill.mockRestore() + vi.unstubAllEnvs() + Reflect.deleteProperty(process.env, 'ENTERPRISE_ENABLED') + Reflect.deleteProperty(process.env, 'NEXT_PUBLIC_ENTERPRISE_ENABLED') + }) + + it('does nothing outside a Marketplace install', async () => { + vi.stubEnv('AWS_MARKETPLACE_LICENSE_PRODUCT_SKU', undefined) + await boot() + expect(mockSend).not.toHaveBeenCalled() + }) + + it('turns Enterprise on and returns the unit it checked out', async () => { + answers = ['entitled'] + await boot() + expect(enterpriseEnabled()).toBe(true) + expect(checkIns).toEqual([TOKEN]) + }) + + it('keeps an active license when returning the unit fails', async () => { + answers = ['entitled'] + checkInFails = true + await boot() + expect(enterpriseEnabled()).toBe(true) + }) + + it('retries a unit another replica is holding at startup', async () => { + answers = ['NoEntitlementsAllowedException', 'entitled'] + await boot() + expect(enterpriseEnabled()).toBe(true) + }) + + it('runs without Enterprise when no unit is allowed at startup', async () => { + answers = ['NoEntitlementsAllowedException'] + await boot() + expect(enterpriseEnabled()).toBe(false) + }) + + it('runs without Enterprise when its IAM role cannot check out', async () => { + answers = ['AccessDeniedException'] + await boot() + expect(enterpriseEnabled()).toBe(false) + }) + + it('keeps Enterprise on through a unit held elsewhere for up to an hour', async () => { + answers = ['entitled', ...Array(5).fill('NoEntitlementsAllowedException'), 'entitled'] + await boot() + await vi.advanceTimersByTimeAsync(180 * MINUTE) + expect(kill).not.toHaveBeenCalled() + }) + + it('restarts without Enterprise once no entitlement outlasts any held unit', async () => { + answers = ['entitled', 'NoEntitlementsAllowedException'] + await boot() + await vi.advanceTimersByTimeAsync(85 * MINUTE) + expect(kill).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(10 * MINUTE) + expect(kill).toHaveBeenCalledWith(process.pid, 'SIGTERM') + }) + + it('never restarts a running Enterprise deployment over a permission failure', async () => { + answers = ['entitled', 'AccessDeniedException'] + await boot() + await vi.advanceTimersByTimeAsync(6 * 60 * MINUTE) + expect(kill).not.toHaveBeenCalled() + }) + + it('restarts into Enterprise once the license becomes active', async () => { + answers = [...Array(3).fill('NoEntitlementsAllowedException'), 'entitled'] + await boot() + expect(enterpriseEnabled()).toBe(false) + await vi.advanceTimersByTimeAsync(16 * MINUTE) + expect(kill).toHaveBeenCalledWith(process.pid, 'SIGTERM') + }) +}) diff --git a/apps/sim/lib/billing/aws-marketplace/container-license.ts b/apps/sim/lib/billing/aws-marketplace/container-license.ts index fa42f646256..2200f5633fa 100644 --- a/apps/sim/lib/billing/aws-marketplace/container-license.ts +++ b/apps/sim/lib/billing/aws-marketplace/container-license.ts @@ -1,10 +1,12 @@ import { - type GrantedLicense, + CheckInLicenseCommand, + CheckoutLicenseCommand, LicenseManagerClient, - ListReceivedLicensesCommand, } from '@aws-sdk/client-license-manager' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' import { randomInt } from '@sim/utils/random' import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' @@ -12,15 +14,19 @@ import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' * Enterprise entitlement for Sim installed from the AWS Marketplace container listing. * * The listing sells an annual contract whose one dimension is `enterprise_license`. AWS Marketplace - * grants the buyer a license in AWS License Manager, and the running product reads it. While the - * license is active, the Enterprise suite is on (`ENTERPRISE_ENABLED`). Without one, Sim runs as the - * open-source core, so a lapsed renewal or an unreachable License Manager never locks a buyer out - * of their deployment. + * grants the buyer a license in AWS License Manager, and the running product checks it the way AWS + * documents for contract-priced containers: a provisional `CheckoutLicense` of the entitlement, + * which needs only permissions in AWS's managed `AWSLicenseManagerConsumptionPolicy`. While the + * license is active, the Enterprise suite is on (`ENTERPRISE_ENABLED`). Without one, Sim runs as + * the open-source core, so a lapsed renewal or an unreachable License Manager never locks a buyer + * out of their deployment. * - * The check reads the license rather than checking a unit out of it. A contract usually covers the - * whole deployment with one unit, so checking it out from every replica would make them compete - * for it, and a unit left checked out (a failed check-in holds it for an hour) would read exactly - * like an expired license. + * A contract usually covers the deployment with one unit, which every replica checks out. Each + * check returns its unit at once (`CheckInLicense`), but two replicas checking together, or a unit + * a crashed replica never returned (License Manager takes it back after an hour), answer exactly + * like a license without the entitlement: AWS reports both as `NoEntitlementsAllowedException`. So + * that answer switches a running deployment's Enterprise off only after it has held for longer + * than a held unit can last. * * Opt-in: nothing here runs unless {@link PRODUCT_SKU_ENV} is set, which only the Marketplace * Helm profile does. Enterprise flags are read once at startup by both the server and the @@ -34,8 +40,8 @@ const logger = createLogger('AwsMarketplaceContainerLicense') const PRODUCT_SKU_ENV = 'AWS_MARKETPLACE_LICENSE_PRODUCT_SKU' /** - * Optional override for where the license is read. License Manager replicates a granted license to - * other Regions, so the default is the pod's own Region, the same chain AWS's EKS walkthrough uses. + * Optional override for where the license is checked. License Manager replicates a granted license + * to other Regions, so the default is the pod's own Region, the same chain AWS's EKS walkthrough uses. */ const LICENSE_REGION_ENV = 'AWS_MARKETPLACE_LICENSE_REGION' @@ -48,23 +54,45 @@ const ENTERPRISE_ENTITLEMENT = 'enterprise_license' /** AWS recommends re-checking every 15 minutes to pick up renewals, upgrades, and cancellations. */ const RECHECK_INTERVAL_MS = 15 * 60 * 1000 +/** Replicas start together; a random part of this added to each re-check keeps them apart. */ +const RECHECK_JITTER_MS = 2 * 60 * 1000 + +/** License Manager takes back a unit nobody checked in after an hour. */ +const UNRETURNED_UNIT_MS = 60 * 60 * 1000 + /** - * Replicas start together, so their re-checks line up; each waits a random part of this before - * restarting, so a renewal or lapse never takes every replica down at once. + * How long "no entitlement" must hold before a running Enterprise deployment believes it: longer + * than an unreturned unit lasts, plus one re-check, so a unit held elsewhere can never cause it. */ -const RESTART_SPREAD_MS = 10 * 60 * 1000 +const NO_ENTITLEMENT_CONFIRMATION_MS = UNRETURNED_UNIT_MS + RECHECK_INTERVAL_MS + +/** Replicas rolling out together can meet on the unit, so startup retries that answer. */ +const STARTUP_ATTEMPTS = 3 +const STARTUP_RETRY_MIN_MS = 1_000 +const STARTUP_RETRY_MAX_MS = 5_000 /** - * How the buyer's account holds the license. A delegate grant only shows licenses distributed to - * other accounts, and a grant still awaiting acceptance, or rejected, disabled, or deleted by the - * license administrator, entitles nothing. No status at all is a license the account bought itself. + * Replicas see a change together; each waits a random part of this before restarting, so a renewal + * or lapse never takes every replica down at once. */ -const USABLE_RECEIVED_STATUSES = new Set(['ACTIVE', 'WORKFLOW_COMPLETED']) +const RESTART_SPREAD_MS = 10 * 60 * 1000 const ENTERPRISE_FLAGS = ['ENTERPRISE_ENABLED', 'NEXT_PUBLIC_ENTERPRISE_ENABLED'] as const -/** `unknown` is a failed check (network, permissions, throttling), never a verdict on the license. */ -type LicenseState = 'entitled' | 'not-entitled' | 'unknown' +/** License Manager's answers when the account holds no unit of the entitlement it can use now. */ +const NO_ENTITLEMENT_ERRORS = new Set([ + 'NoEntitlementsAllowedException', + 'ResourceNotFoundException', +]) + +/** The ServiceAccount's IAM role does not allow the check: a setup problem, never a verdict. */ +const PERMISSION_ERRORS = new Set(['AccessDeniedException', 'AuthorizationException']) + +/** + * `no-entitlement` is AWS's answer for both an unlicensed account and a unit held elsewhere. + * `unknown` is a failed check (network, permissions, throttling), never a verdict on the license. + */ +type LicenseState = 'entitled' | 'no-entitlement' | 'unknown' function createClient(): LicenseManagerClient { const region = @@ -78,24 +106,34 @@ function createClient(): LicenseManagerClient { }) } -function isUsableGrant({ ReceivedMetadata: received }: GrantedLicense): boolean { - if (received?.GrantType === 'DelegateGrant') return false - return !received?.ReceivedStatus || USABLE_RECEIVED_STATUSES.has(received.ReceivedStatus) +/** A unit left out returns to the pool within the hour, so a failed check-in is only logged. */ +async function checkIn(client: LicenseManagerClient, licenseConsumptionToken: string) { + try { + await client.send( + new CheckInLicenseCommand({ LicenseConsumptionToken: licenseConsumptionToken }) + ) + } catch (error) { + logger.warn('Could not return the AWS Marketplace license unit; License Manager reclaims it', { + message: getErrorMessage(error, 'Unknown error'), + }) + } } -/** - * An AWS Marketplace license this product's buyer holds is current when License Manager reports it - * `AVAILABLE`, its issuer is AWS Marketplace, it carries the Enterprise entitlement, and the account - * holds it through a grant it may use. License Manager moves an ended, cancelled, or suspended - * license out of `AVAILABLE` itself. - */ -function isActiveEnterpriseLicense(license: GrantedLicense): boolean { - return ( - license.Status === 'AVAILABLE' && - isUsableGrant(license) && - license.Issuer?.KeyFingerprint === MARKETPLACE_KEY_FINGERPRINT && - (license.Entitlements ?? []).some((entitlement) => entitlement.Name === ENTERPRISE_ENTITLEMENT) - ) +function classifyFailure(error: unknown): LicenseState { + const name = error instanceof Error ? error.name : undefined + if (name && NO_ENTITLEMENT_ERRORS.has(name)) return 'no-entitlement' + if (name && PERMISSION_ERRORS.has(name)) { + logger.error( + 'The IAM role cannot check the AWS Marketplace license; allow license-manager:CheckoutLicense and license-manager:CheckInLicense', + { error: name } + ) + return 'unknown' + } + logger.warn('Could not check the AWS Marketplace Enterprise license', { + error: name, + message: getErrorMessage(error, 'Unknown error'), + }) + return 'unknown' } async function checkLicense( @@ -103,33 +141,50 @@ async function checkLicense( productSku: string ): Promise { try { - let nextToken: string | undefined - do { - const page = await client.send( - new ListReceivedLicensesCommand({ - Filters: [ - { Name: 'ProductSKU', Values: [productSku] }, - { Name: 'Fingerprint', Values: [MARKETPLACE_KEY_FINGERPRINT] }, - ], - NextToken: nextToken, - }) - ) - if ((page.Licenses ?? []).some(isActiveEnterpriseLicense)) return 'entitled' - nextToken = page.NextToken - } while (nextToken) - return 'not-entitled' + const checkout = await client.send( + new CheckoutLicenseCommand({ + ProductSKU: productSku, + CheckoutType: 'PROVISIONAL', + KeyFingerprint: MARKETPLACE_KEY_FINGERPRINT, + Entitlements: [{ Name: ENTERPRISE_ENTITLEMENT, Value: '1', Unit: 'Count' }], + ClientToken: generateId(), + }) + ) + if (checkout.LicenseConsumptionToken) await checkIn(client, checkout.LicenseConsumptionToken) + const allowed = (checkout.EntitlementsAllowed ?? []).some( + (entitlement) => entitlement.Name === ENTERPRISE_ENTITLEMENT + ) + return allowed ? 'entitled' : 'no-entitlement' } catch (error) { - logger.warn('Could not check the AWS Marketplace Enterprise license', { - message: getErrorMessage(error, 'Unknown error'), - }) - return 'unknown' + return classifyFailure(error) } } +async function checkAtStartup( + client: LicenseManagerClient, + productSku: string +): Promise { + let state = await checkLicense(client, productSku) + for (let attempt = 1; state === 'no-entitlement' && attempt < STARTUP_ATTEMPTS; attempt++) { + await sleep(randomInt(STARTUP_RETRY_MIN_MS, STARTUP_RETRY_MAX_MS)) + state = await checkLicense(client, productSku) + } + return state +} + function setEnterpriseFlags(enabled: boolean): void { for (const flag of ENTERPRISE_FLAGS) process.env[flag] = enabled ? 'true' : 'false' } +function restartToApply(state: LicenseState): void { + const delayMs = randomInt(0, RESTART_SPREAD_MS) + logger.info('AWS Marketplace Enterprise license changed; restarting to apply it', { + state, + delayMs, + }) + setTimeout(() => process.kill(process.pid, 'SIGTERM'), delayMs) +} + /** * Resolves the Enterprise license before the server loads, so every Enterprise gate reads the * answer through `ENTERPRISE_ENABLED`, then keeps re-checking. Returns without doing anything on a @@ -140,7 +195,7 @@ export async function applyAwsMarketplaceContainerLicense(): Promise { if (!productSku) return const client = createClient() - const bootState = await checkLicense(client, productSku) + const bootState = await checkAtStartup(client, productSku) const enterpriseAtBoot = bootState === 'entitled' setEnterpriseFlags(enterpriseAtBoot) logger.info( @@ -150,17 +205,22 @@ export async function applyAwsMarketplaceContainerLicense(): Promise { { state: bootState } ) - const recheck = setInterval(async () => { + let noEntitlementSince: number | null = null + const scheduleRecheck = () => { + setTimeout(recheck, RECHECK_INTERVAL_MS + randomInt(0, RECHECK_JITTER_MS)).unref() + } + const recheck = async () => { const state = await checkLicense(client, productSku) - const entitledNow = state === 'entitled' - if (state === 'unknown' || entitledNow === enterpriseAtBoot) return - clearInterval(recheck) - const delayMs = randomInt(0, RESTART_SPREAD_MS) - logger.info('AWS Marketplace Enterprise license changed; restarting to apply it', { - state, - delayMs, - }) - setTimeout(() => process.kill(process.pid, 'SIGTERM'), delayMs) - }, RECHECK_INTERVAL_MS) - recheck.unref() + if (state === 'entitled') { + noEntitlementSince = null + if (!enterpriseAtBoot) return restartToApply(state) + } else if (state === 'no-entitlement' && enterpriseAtBoot) { + noEntitlementSince ??= Date.now() + if (Date.now() - noEntitlementSince >= NO_ENTITLEMENT_CONFIRMATION_MS) { + return restartToApply(state) + } + } + scheduleRecheck() + } + scheduleRecheck() } diff --git a/helm/sim/examples/aws-marketplace-prerequisites.yaml b/helm/sim/examples/aws-marketplace-prerequisites.yaml index 2754ff4b49c..0635c6900a1 100644 --- a/helm/sim/examples/aws-marketplace-prerequisites.yaml +++ b/helm/sim/examples/aws-marketplace-prerequisites.yaml @@ -159,9 +159,12 @@ Resources: Version: '2012-10-17' Statement: - Effect: Allow - # Sim only reads the license. ListReceivedLicenses has no resource-level - # permissions, so it cannot be scoped below '*'. - Action: license-manager:ListReceivedLicenses + # Sim checks a unit of the license out and returns it at once, the two actions + # AWS's managed AWSLicenseManagerConsumptionPolicy also grants. Neither has + # resource-level permissions, so they cannot be scoped below '*'. + Action: + - license-manager:CheckoutLicense + - license-manager:CheckInLicense Resource: '*' Outputs: diff --git a/helm/sim/examples/values-aws-marketplace.yaml b/helm/sim/examples/values-aws-marketplace.yaml index 65b3ae6ae93..477124088c7 100644 --- a/helm/sim/examples/values-aws-marketplace.yaml +++ b/helm/sim/examples/values-aws-marketplace.yaml @@ -21,8 +21,9 @@ # provider (for IRSA). # 2. RDS for PostgreSQL 17 and ElastiCache reachable from the cluster. The listing's # CloudFormation template creates both in your VPC. -# 3. An IAM role for the ServiceAccount below that allows license-manager:ListReceivedLicenses -# (the CloudFormation template creates it). +# 3. An IAM role for the ServiceAccount below that allows license-manager:CheckoutLicense and +# license-manager:CheckInLicense (the CloudFormation template creates it; AWS's managed +# AWSLicenseManagerConsumptionPolicy grants both too). # 4. An ACM certificate for the two hostnames below. # 5. Two Secrets in the install namespace (no secret ever goes in this file). The template's # DatabasePasswordArn and CacheAuthTokenArn outputs are the ARNs of the Secrets Manager From a0b6763f376b39480ce1314a6cbe1e70a799d3e8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 12 Nov 2026 20:51:01 -0800 Subject: [PATCH 2/3] fix(aws-marketplace): count only an unbroken run of no-entitlement answers, assert the checkout request, bump the chart --- .../aws-marketplace/container-license.test.ts | 33 +++++++++++++++++-- .../aws-marketplace/container-license.ts | 13 ++++---- helm/sim/Chart.yaml | 2 +- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/apps/sim/lib/billing/aws-marketplace/container-license.test.ts b/apps/sim/lib/billing/aws-marketplace/container-license.test.ts index d53d4d8c458..6795d0e04ea 100644 --- a/apps/sim/lib/billing/aws-marketplace/container-license.test.ts +++ b/apps/sim/lib/billing/aws-marketplace/container-license.test.ts @@ -22,12 +22,17 @@ import { applyAwsMarketplaceContainerLicense } from '@/lib/billing/aws-marketpla const MINUTE = 60 * 1000 const TOKEN = 'consumption-token-placeholder' -type Answer = 'entitled' | 'NoEntitlementsAllowedException' | 'AccessDeniedException' +type Answer = + | 'entitled' + | 'NoEntitlementsAllowedException' + | 'AccessDeniedException' + | 'ServerInternalException' /** License Manager's checkout answers in order; the last one repeats once the list runs out. */ let answers: Answer[] = [] let checkInFails = false -const checkIns: string[] = [] +const checkoutRequests: unknown[] = [] +const checkIns: Array = [] function awsError(name: string): Error { return Object.assign(new Error(name), { name }) @@ -51,6 +56,7 @@ describe('applyAwsMarketplaceContainerLicense', () => { answers = [] checkInFails = false checkIns.length = 0 + checkoutRequests.length = 0 vi.stubEnv('AWS_MARKETPLACE_LICENSE_PRODUCT_SKU', 'product-placeholder') kill = vi.spyOn(process, 'kill').mockImplementation(() => true) mockSend.mockReset() @@ -60,6 +66,7 @@ describe('applyAwsMarketplaceContainerLicense', () => { if (checkInFails) throw awsError('ServerInternalException') return {} } + checkoutRequests.push((command as { input: unknown }).input) const answer = answers.length > 1 ? answers.shift() : answers[0] if (answer !== 'entitled') throw awsError(answer ?? 'ServerInternalException') return { @@ -73,7 +80,6 @@ describe('applyAwsMarketplaceContainerLicense', () => { vi.clearAllTimers() vi.useRealTimers() kill.mockRestore() - vi.unstubAllEnvs() Reflect.deleteProperty(process.env, 'ENTERPRISE_ENABLED') Reflect.deleteProperty(process.env, 'NEXT_PUBLIC_ENTERPRISE_ENABLED') }) @@ -88,6 +94,14 @@ describe('applyAwsMarketplaceContainerLicense', () => { answers = ['entitled'] await boot() expect(enterpriseEnabled()).toBe(true) + expect(checkoutRequests).toEqual([ + expect.objectContaining({ + ProductSKU: 'product-placeholder', + CheckoutType: 'PROVISIONAL', + KeyFingerprint: 'aws:294406891311:AWS/Marketplace:issuer-fingerprint', + Entitlements: [{ Name: 'enterprise_license', Value: '1', Unit: 'Count' }], + }), + ]) expect(checkIns).toEqual([TOKEN]) }) @@ -132,6 +146,19 @@ describe('applyAwsMarketplaceContainerLicense', () => { expect(kill).toHaveBeenCalledWith(process.pid, 'SIGTERM') }) + it('does not count failed checks toward switching Enterprise off', async () => { + answers = [ + 'entitled', + 'NoEntitlementsAllowedException', + ...Array(6).fill('ServerInternalException'), + 'NoEntitlementsAllowedException', + 'entitled', + ] + await boot() + await vi.advanceTimersByTimeAsync(130 * MINUTE) + expect(kill).not.toHaveBeenCalled() + }) + it('never restarts a running Enterprise deployment over a permission failure', async () => { answers = ['entitled', 'AccessDeniedException'] await boot() diff --git a/apps/sim/lib/billing/aws-marketplace/container-license.ts b/apps/sim/lib/billing/aws-marketplace/container-license.ts index 2200f5633fa..75744045c14 100644 --- a/apps/sim/lib/billing/aws-marketplace/container-license.ts +++ b/apps/sim/lib/billing/aws-marketplace/container-license.ts @@ -61,8 +61,9 @@ const RECHECK_JITTER_MS = 2 * 60 * 1000 const UNRETURNED_UNIT_MS = 60 * 60 * 1000 /** - * How long "no entitlement" must hold before a running Enterprise deployment believes it: longer - * than an unreturned unit lasts, plus one re-check, so a unit held elsewhere can never cause it. + * How long "no entitlement" must hold, with no other answer in between, before a running Enterprise + * deployment believes it: longer than an unreturned unit lasts, plus one re-check, so a unit held + * elsewhere can never cause it. */ const NO_ENTITLEMENT_CONFIRMATION_MS = UNRETURNED_UNIT_MS + RECHECK_INTERVAL_MS @@ -211,10 +212,10 @@ export async function applyAwsMarketplaceContainerLicense(): Promise { } const recheck = async () => { const state = await checkLicense(client, productSku) - if (state === 'entitled') { - noEntitlementSince = null - if (!enterpriseAtBoot) return restartToApply(state) - } else if (state === 'no-entitlement' && enterpriseAtBoot) { + // Only an unbroken run of "no entitlement" answers counts toward switching Enterprise off. + if (state !== 'no-entitlement') noEntitlementSince = null + if (state === 'entitled' && !enterpriseAtBoot) return restartToApply(state) + if (state === 'no-entitlement' && enterpriseAtBoot) { noEntitlementSince ??= Date.now() if (Date.now() - noEntitlementSince >= NO_ENTITLEMENT_CONFIRMATION_MS) { return restartToApply(state) diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index f98dd9f91f3..6b50f65e446 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.11.16 +version: 1.11.17 appVersion: "v0.8.26" kubeVersion: ">=1.25.0-0" home: https://sim.ai From e518dd0d42e29c35336c37451df58760372a76bc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 12 Nov 2026 21:11:58 -0800 Subject: [PATCH 3/3] fix(aws-marketplace): check the container license as an uncounted tier entitlement AWS documents a contract that unlocks a tier of features, rather than metering a quantity, as a provisional CheckoutLicense with unit None and no value. That checkout is not counted against the contract, so replicas never compete for a unit, and the check-in, startup retry, and one-hour confirmation window that guarded against a held unit go away. A definite answer that differs from the boot mode restarts; a failed check keeps the current mode. --- .../aws-marketplace/container-license.test.ts | 118 +++++------------- .../aws-marketplace/container-license.ts | 102 ++++----------- .../aws-marketplace-prerequisites.yaml | 10 +- helm/sim/examples/values-aws-marketplace.yaml | 6 +- 4 files changed, 62 insertions(+), 174 deletions(-) diff --git a/apps/sim/lib/billing/aws-marketplace/container-license.test.ts b/apps/sim/lib/billing/aws-marketplace/container-license.test.ts index 6795d0e04ea..bbb6d4a62b0 100644 --- a/apps/sim/lib/billing/aws-marketplace/container-license.test.ts +++ b/apps/sim/lib/billing/aws-marketplace/container-license.test.ts @@ -9,30 +9,24 @@ vi.mock('@aws-sdk/client-license-manager', () => ({ CheckoutLicenseCommand: class { constructor(public input: { ProductSKU: string }) {} }, - CheckInLicenseCommand: class { - constructor(public input: { LicenseConsumptionToken: string }) {} - }, })) vi.mock('@/lib/core/network/proxy-agent.server', () => ({ awsRequestHandler: () => undefined })) vi.mock('@sim/utils/random', () => ({ randomInt: (min: number) => min })) -import { CheckInLicenseCommand } from '@aws-sdk/client-license-manager' import { applyAwsMarketplaceContainerLicense } from '@/lib/billing/aws-marketplace/container-license' const MINUTE = 60 * 1000 -const TOKEN = 'consumption-token-placeholder' type Answer = | 'entitled' | 'NoEntitlementsAllowedException' + | 'ResourceNotFoundException' | 'AccessDeniedException' | 'ServerInternalException' /** License Manager's checkout answers in order; the last one repeats once the list runs out. */ let answers: Answer[] = [] -let checkInFails = false const checkoutRequests: unknown[] = [] -const checkIns: Array = [] function awsError(name: string): Error { return Object.assign(new Error(name), { name }) @@ -42,37 +36,21 @@ function enterpriseEnabled(): boolean { return process.env.ENTERPRISE_ENABLED === 'true' } -async function boot(): Promise { - const started = applyAwsMarketplaceContainerLicense() - await vi.advanceTimersByTimeAsync(30_000) - await started -} - describe('applyAwsMarketplaceContainerLicense', () => { let kill: ReturnType beforeEach(() => { - vi.useFakeTimers({ toFake: ['setTimeout', 'Date'] }) + vi.useFakeTimers({ toFake: ['setTimeout'] }) answers = [] - checkInFails = false - checkIns.length = 0 checkoutRequests.length = 0 vi.stubEnv('AWS_MARKETPLACE_LICENSE_PRODUCT_SKU', 'product-placeholder') kill = vi.spyOn(process, 'kill').mockImplementation(() => true) mockSend.mockReset() - mockSend.mockImplementation(async (command: unknown) => { - if (command instanceof CheckInLicenseCommand) { - checkIns.push(command.input.LicenseConsumptionToken) - if (checkInFails) throw awsError('ServerInternalException') - return {} - } - checkoutRequests.push((command as { input: unknown }).input) + mockSend.mockImplementation(async (command: { input: unknown }) => { + checkoutRequests.push(command.input) const answer = answers.length > 1 ? answers.shift() : answers[0] if (answer !== 'entitled') throw awsError(answer ?? 'ServerInternalException') - return { - LicenseConsumptionToken: TOKEN, - EntitlementsAllowed: [{ Name: 'enterprise_license', Unit: 'Count', Value: '1' }], - } + return { EntitlementsAllowed: [{ Name: 'enterprise_license', Unit: 'None' }] } }) }) @@ -86,91 +64,61 @@ describe('applyAwsMarketplaceContainerLicense', () => { it('does nothing outside a Marketplace install', async () => { vi.stubEnv('AWS_MARKETPLACE_LICENSE_PRODUCT_SKU', undefined) - await boot() + await applyAwsMarketplaceContainerLicense() expect(mockSend).not.toHaveBeenCalled() }) - it('turns Enterprise on and returns the unit it checked out', async () => { + it('turns Enterprise on with an uncounted checkout of the entitlement', async () => { answers = ['entitled'] - await boot() + await applyAwsMarketplaceContainerLicense() expect(enterpriseEnabled()).toBe(true) expect(checkoutRequests).toEqual([ expect.objectContaining({ ProductSKU: 'product-placeholder', CheckoutType: 'PROVISIONAL', KeyFingerprint: 'aws:294406891311:AWS/Marketplace:issuer-fingerprint', - Entitlements: [{ Name: 'enterprise_license', Value: '1', Unit: 'Count' }], + Entitlements: [{ Name: 'enterprise_license', Unit: 'None' }], }), ]) - expect(checkIns).toEqual([TOKEN]) - }) - - it('keeps an active license when returning the unit fails', async () => { - answers = ['entitled'] - checkInFails = true - await boot() - expect(enterpriseEnabled()).toBe(true) - }) - - it('retries a unit another replica is holding at startup', async () => { - answers = ['NoEntitlementsAllowedException', 'entitled'] - await boot() - expect(enterpriseEnabled()).toBe(true) }) - it('runs without Enterprise when no unit is allowed at startup', async () => { + it('runs without Enterprise when no license grants the entitlement', async () => { answers = ['NoEntitlementsAllowedException'] - await boot() + await applyAwsMarketplaceContainerLicense() expect(enterpriseEnabled()).toBe(false) }) it('runs without Enterprise when its IAM role cannot check out', async () => { answers = ['AccessDeniedException'] - await boot() + await applyAwsMarketplaceContainerLicense() expect(enterpriseEnabled()).toBe(false) }) - it('keeps Enterprise on through a unit held elsewhere for up to an hour', async () => { - answers = ['entitled', ...Array(5).fill('NoEntitlementsAllowedException'), 'entitled'] - await boot() - await vi.advanceTimersByTimeAsync(180 * MINUTE) - expect(kill).not.toHaveBeenCalled() - }) - - it('restarts without Enterprise once no entitlement outlasts any held unit', async () => { - answers = ['entitled', 'NoEntitlementsAllowedException'] - await boot() - await vi.advanceTimersByTimeAsync(85 * MINUTE) - expect(kill).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(10 * MINUTE) - expect(kill).toHaveBeenCalledWith(process.pid, 'SIGTERM') - }) - - it('does not count failed checks toward switching Enterprise off', async () => { - answers = [ - 'entitled', - 'NoEntitlementsAllowedException', - ...Array(6).fill('ServerInternalException'), - 'NoEntitlementsAllowedException', - 'entitled', - ] - await boot() - await vi.advanceTimersByTimeAsync(130 * MINUTE) - expect(kill).not.toHaveBeenCalled() - }) - - it('never restarts a running Enterprise deployment over a permission failure', async () => { - answers = ['entitled', 'AccessDeniedException'] - await boot() - await vi.advanceTimersByTimeAsync(6 * 60 * MINUTE) - expect(kill).not.toHaveBeenCalled() - }) + it.each(['NoEntitlementsAllowedException', 'ResourceNotFoundException'])( + 'restarts without Enterprise once the license ends (%s)', + async (answer) => { + answers = ['entitled', answer] + await applyAwsMarketplaceContainerLicense() + await vi.advanceTimersByTimeAsync(16 * MINUTE) + expect(kill).toHaveBeenCalledWith(process.pid, 'SIGTERM') + } + ) it('restarts into Enterprise once the license becomes active', async () => { - answers = [...Array(3).fill('NoEntitlementsAllowedException'), 'entitled'] - await boot() + answers = ['NoEntitlementsAllowedException', 'entitled'] + await applyAwsMarketplaceContainerLicense() expect(enterpriseEnabled()).toBe(false) await vi.advanceTimersByTimeAsync(16 * MINUTE) expect(kill).toHaveBeenCalledWith(process.pid, 'SIGTERM') }) + + it.each(['ServerInternalException', 'AccessDeniedException'])( + 'never restarts a running Enterprise deployment over a failed check (%s)', + async (answer) => { + answers = ['entitled', answer] + await applyAwsMarketplaceContainerLicense() + await vi.advanceTimersByTimeAsync(6 * 60 * MINUTE) + expect(kill).not.toHaveBeenCalled() + } + ) }) diff --git a/apps/sim/lib/billing/aws-marketplace/container-license.ts b/apps/sim/lib/billing/aws-marketplace/container-license.ts index 75744045c14..8d32361e432 100644 --- a/apps/sim/lib/billing/aws-marketplace/container-license.ts +++ b/apps/sim/lib/billing/aws-marketplace/container-license.ts @@ -1,11 +1,6 @@ -import { - CheckInLicenseCommand, - CheckoutLicenseCommand, - LicenseManagerClient, -} from '@aws-sdk/client-license-manager' +import { CheckoutLicenseCommand, LicenseManagerClient } from '@aws-sdk/client-license-manager' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { randomInt } from '@sim/utils/random' import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' @@ -15,18 +10,12 @@ import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' * * The listing sells an annual contract whose one dimension is `enterprise_license`. AWS Marketplace * grants the buyer a license in AWS License Manager, and the running product checks it the way AWS - * documents for contract-priced containers: a provisional `CheckoutLicense` of the entitlement, - * which needs only permissions in AWS's managed `AWSLicenseManagerConsumptionPolicy`. While the - * license is active, the Enterprise suite is on (`ENTERPRISE_ENABLED`). Without one, Sim runs as - * the open-source core, so a lapsed renewal or an unreachable License Manager never locks a buyer - * out of their deployment. - * - * A contract usually covers the deployment with one unit, which every replica checks out. Each - * check returns its unit at once (`CheckInLicense`), but two replicas checking together, or a unit - * a crashed replica never returned (License Manager takes it back after an hour), answer exactly - * like a license without the entitlement: AWS reports both as `NoEntitlementsAllowedException`. So - * that answer switches a running deployment's Enterprise off only after it has held for longer - * than a held unit can last. + * documents for a contract that unlocks a tier of features rather than metering a quantity: a + * provisional `CheckoutLicense` of the entitlement with unit `None`. Such a checkout is not counted + * against the contract, so every replica can check at once, and it needs only the permission AWS's + * managed `AWSLicenseManagerConsumptionPolicy` grants. While the license is active, the Enterprise + * suite is on (`ENTERPRISE_ENABLED`). Without one, Sim runs as the open-source core, so a lapsed + * renewal or an unreachable License Manager never locks a buyer out of their deployment. * * Opt-in: nothing here runs unless {@link PRODUCT_SKU_ENV} is set, which only the Marketplace * Helm profile does. Enterprise flags are read once at startup by both the server and the @@ -57,21 +46,6 @@ const RECHECK_INTERVAL_MS = 15 * 60 * 1000 /** Replicas start together; a random part of this added to each re-check keeps them apart. */ const RECHECK_JITTER_MS = 2 * 60 * 1000 -/** License Manager takes back a unit nobody checked in after an hour. */ -const UNRETURNED_UNIT_MS = 60 * 60 * 1000 - -/** - * How long "no entitlement" must hold, with no other answer in between, before a running Enterprise - * deployment believes it: longer than an unreturned unit lasts, plus one re-check, so a unit held - * elsewhere can never cause it. - */ -const NO_ENTITLEMENT_CONFIRMATION_MS = UNRETURNED_UNIT_MS + RECHECK_INTERVAL_MS - -/** Replicas rolling out together can meet on the unit, so startup retries that answer. */ -const STARTUP_ATTEMPTS = 3 -const STARTUP_RETRY_MIN_MS = 1_000 -const STARTUP_RETRY_MAX_MS = 5_000 - /** * Replicas see a change together; each waits a random part of this before restarting, so a renewal * or lapse never takes every replica down at once. @@ -80,7 +54,7 @@ const RESTART_SPREAD_MS = 10 * 60 * 1000 const ENTERPRISE_FLAGS = ['ENTERPRISE_ENABLED', 'NEXT_PUBLIC_ENTERPRISE_ENABLED'] as const -/** License Manager's answers when the account holds no unit of the entitlement it can use now. */ +/** License Manager's answers when the account holds no active license with the entitlement. */ const NO_ENTITLEMENT_ERRORS = new Set([ 'NoEntitlementsAllowedException', 'ResourceNotFoundException', @@ -89,11 +63,8 @@ const NO_ENTITLEMENT_ERRORS = new Set([ /** The ServiceAccount's IAM role does not allow the check: a setup problem, never a verdict. */ const PERMISSION_ERRORS = new Set(['AccessDeniedException', 'AuthorizationException']) -/** - * `no-entitlement` is AWS's answer for both an unlicensed account and a unit held elsewhere. - * `unknown` is a failed check (network, permissions, throttling), never a verdict on the license. - */ -type LicenseState = 'entitled' | 'no-entitlement' | 'unknown' +/** `unknown` is a failed check (network, permissions, throttling), never a verdict on the license. */ +type LicenseState = 'entitled' | 'not-entitled' | 'unknown' function createClient(): LicenseManagerClient { const region = @@ -107,25 +78,15 @@ function createClient(): LicenseManagerClient { }) } -/** A unit left out returns to the pool within the hour, so a failed check-in is only logged. */ -async function checkIn(client: LicenseManagerClient, licenseConsumptionToken: string) { - try { - await client.send( - new CheckInLicenseCommand({ LicenseConsumptionToken: licenseConsumptionToken }) - ) - } catch (error) { - logger.warn('Could not return the AWS Marketplace license unit; License Manager reclaims it', { - message: getErrorMessage(error, 'Unknown error'), - }) - } -} - function classifyFailure(error: unknown): LicenseState { const name = error instanceof Error ? error.name : undefined - if (name && NO_ENTITLEMENT_ERRORS.has(name)) return 'no-entitlement' + if (name && NO_ENTITLEMENT_ERRORS.has(name)) { + logger.info('AWS License Manager reports no active Enterprise entitlement', { error: name }) + return 'not-entitled' + } if (name && PERMISSION_ERRORS.has(name)) { logger.error( - 'The IAM role cannot check the AWS Marketplace license; allow license-manager:CheckoutLicense and license-manager:CheckInLicense', + 'The IAM role cannot check the AWS Marketplace license; allow license-manager:CheckoutLicense', { error: name } ) return 'unknown' @@ -147,32 +108,19 @@ async function checkLicense( ProductSKU: productSku, CheckoutType: 'PROVISIONAL', KeyFingerprint: MARKETPLACE_KEY_FINGERPRINT, - Entitlements: [{ Name: ENTERPRISE_ENTITLEMENT, Value: '1', Unit: 'Count' }], + Entitlements: [{ Name: ENTERPRISE_ENTITLEMENT, Unit: 'None' }], ClientToken: generateId(), }) ) - if (checkout.LicenseConsumptionToken) await checkIn(client, checkout.LicenseConsumptionToken) const allowed = (checkout.EntitlementsAllowed ?? []).some( (entitlement) => entitlement.Name === ENTERPRISE_ENTITLEMENT ) - return allowed ? 'entitled' : 'no-entitlement' + return allowed ? 'entitled' : 'not-entitled' } catch (error) { return classifyFailure(error) } } -async function checkAtStartup( - client: LicenseManagerClient, - productSku: string -): Promise { - let state = await checkLicense(client, productSku) - for (let attempt = 1; state === 'no-entitlement' && attempt < STARTUP_ATTEMPTS; attempt++) { - await sleep(randomInt(STARTUP_RETRY_MIN_MS, STARTUP_RETRY_MAX_MS)) - state = await checkLicense(client, productSku) - } - return state -} - function setEnterpriseFlags(enabled: boolean): void { for (const flag of ENTERPRISE_FLAGS) process.env[flag] = enabled ? 'true' : 'false' } @@ -196,7 +144,7 @@ export async function applyAwsMarketplaceContainerLicense(): Promise { if (!productSku) return const client = createClient() - const bootState = await checkAtStartup(client, productSku) + const bootState = await checkLicense(client, productSku) const enterpriseAtBoot = bootState === 'entitled' setEnterpriseFlags(enterpriseAtBoot) logger.info( @@ -206,21 +154,15 @@ export async function applyAwsMarketplaceContainerLicense(): Promise { { state: bootState } ) - let noEntitlementSince: number | null = null const scheduleRecheck = () => { setTimeout(recheck, RECHECK_INTERVAL_MS + randomInt(0, RECHECK_JITTER_MS)).unref() } const recheck = async () => { const state = await checkLicense(client, productSku) - // Only an unbroken run of "no entitlement" answers counts toward switching Enterprise off. - if (state !== 'no-entitlement') noEntitlementSince = null - if (state === 'entitled' && !enterpriseAtBoot) return restartToApply(state) - if (state === 'no-entitlement' && enterpriseAtBoot) { - noEntitlementSince ??= Date.now() - if (Date.now() - noEntitlementSince >= NO_ENTITLEMENT_CONFIRMATION_MS) { - return restartToApply(state) - } - } + // A failed check keeps the current mode; only a definite answer that differs restarts. + const changed = + (state === 'entitled' && !enterpriseAtBoot) || (state === 'not-entitled' && enterpriseAtBoot) + if (changed) return restartToApply(state) scheduleRecheck() } scheduleRecheck() diff --git a/helm/sim/examples/aws-marketplace-prerequisites.yaml b/helm/sim/examples/aws-marketplace-prerequisites.yaml index 0635c6900a1..12aa148a415 100644 --- a/helm/sim/examples/aws-marketplace-prerequisites.yaml +++ b/helm/sim/examples/aws-marketplace-prerequisites.yaml @@ -159,12 +159,10 @@ Resources: Version: '2012-10-17' Statement: - Effect: Allow - # Sim checks a unit of the license out and returns it at once, the two actions - # AWS's managed AWSLicenseManagerConsumptionPolicy also grants. Neither has - # resource-level permissions, so they cannot be scoped below '*'. - Action: - - license-manager:CheckoutLicense - - license-manager:CheckInLicense + # Sim checks the license with an uncounted checkout, which AWS's managed + # AWSLicenseManagerConsumptionPolicy also grants. CheckoutLicense has no + # resource-level permissions, so it cannot be scoped below '*'. + Action: license-manager:CheckoutLicense Resource: '*' Outputs: diff --git a/helm/sim/examples/values-aws-marketplace.yaml b/helm/sim/examples/values-aws-marketplace.yaml index 477124088c7..b4cee0b9083 100644 --- a/helm/sim/examples/values-aws-marketplace.yaml +++ b/helm/sim/examples/values-aws-marketplace.yaml @@ -21,9 +21,9 @@ # provider (for IRSA). # 2. RDS for PostgreSQL 17 and ElastiCache reachable from the cluster. The listing's # CloudFormation template creates both in your VPC. -# 3. An IAM role for the ServiceAccount below that allows license-manager:CheckoutLicense and -# license-manager:CheckInLicense (the CloudFormation template creates it; AWS's managed -# AWSLicenseManagerConsumptionPolicy grants both too). +# 3. An IAM role for the ServiceAccount below that allows license-manager:CheckoutLicense (the +# CloudFormation template creates it; AWS's managed AWSLicenseManagerConsumptionPolicy +# grants it too). # 4. An ACM certificate for the two hostnames below. # 5. Two Secrets in the install namespace (no secret ever goes in this file). The template's # DatabasePasswordArn and CacheAuthTokenArn outputs are the ARNs of the Secrets Manager