test(billing): assert observable verdicts and responses instead of mock calls

- The account block, continuation delegation, cache, billing-off and rollover tests assert the
  verdict or HTTP response. Where behaviour depends on an input, the fake answers by that input,
  as the real ledger and settlement do.
- Pins against real PostgreSQL that a reporting run's top-ups after its window ends are counted in
  that window, where its request was first charged, and a later run's charges in the next.

Author: waleedlatif1

apps/sim/app/api/billing/update-cost/route.test.ts

@@ -970,18 +970,18 @@ describe('POST /api/billing/update-cost — mid-run usage gate', () => {
   it('serves a cached admission to every step and re-reads a refusal', async () => {
     mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: false })
 
-    for (let step = 0; step < 5; step++) {
+    expect((await (await POST(attributedCallback())).json()).usageExceeded).toBe(false)
+    mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: true, scope: 'payer' })
+    for (let step = 0; step < 4; step++) {
       const body = await (await POST(attributedCallback())).json()
       expect(body.usageExceeded).toBe(false)
       expect(body).not.toHaveProperty('usageUpgrade')
     }
-    expect(mockCheckAttributedUsageLimits).toHaveBeenCalledTimes(1)
 
     resetUsageGateCache()
-    mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: true, scope: 'payer' })
-    await POST(attributedCallback())
-    await POST(attributedCallback())
-    expect(mockCheckAttributedUsageLimits).toHaveBeenCalledTimes(3)
+    expect((await (await POST(attributedCallback())).json()).usageExceeded).toBe(true)
+    mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: false })
+    expect((await (await POST(attributedCallback())).json()).usageExceeded).toBe(false)
   })
 
   it('answers a duplicate retry with the verdict its lost first answer carried', async () => {
@@ -1050,7 +1050,6 @@ describe('POST /api/billing/update-cost — mid-run usage gate', () => {
       const body = await (await POST(directCallback())).json()
 
       expect(body.usageExceeded).toBe(false)
-      expect(billingUsageMonitorMockFns.mockCheckUsageStatus).not.toHaveBeenCalled()
     })
   })
 
@@ -1061,56 +1060,70 @@ describe('POST /api/billing/update-cost — mid-run usage gate', () => {
       status: 'active',
       seats: 4,
     }
+    const ADMITTED_PERIOD = {
+      start: new Date('2026-07-01T00:00:00.000Z'),
+      end: new Date('2026-08-01T00:00:00.000Z'),
+    }
     const CURRENT_PERIOD = {
       start: new Date('2026-08-01T00:00:00.000Z'),
       end: new Date('2026-09-01T00:00:00.000Z'),
     }
 
+    function admittedWithSource(source: 'stripe' | 'reporting' | 'default') {
+      mockToBillingContext.mockReturnValue({
+        billingEntity: { type: 'organization', id: 'org-1' },
+        billingPeriod: { ...ADMITTED_PERIOD, source },
+      })
+    }
+
+    /** Threshold settlement for a payer whose charges belong to `period` refuses any other. */
+    function settlesOnlyAgainst(period: typeof ADMITTED_PERIOD) {
+      mockCheckAndBillPayerOverageThreshold.mockImplementation(
+        async (_payer: unknown, options: { expectedBillingPeriod: typeof ADMITTED_PERIOD }) => {
+          if (options.expectedBillingPeriod.start.getTime() !== period.start.getTime()) {
+            throw new Error('Settled against a period the charge did not land in')
+          }
+        }
+      )
+    }
+
     beforeEach(() => {
       mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: false })
       mockRequireBillingAttributionHeader.mockReturnValue({
         ...CURRENT_ATTRIBUTION,
         payerSubscription: PAYER_SUBSCRIPTION,
       })
-      mockToBillingContext.mockReturnValue({
-        billingEntity: { type: 'organization', id: 'org-1' },
-        billingPeriod: {
-          start: new Date('2026-07-01T00:00:00.000Z'),
-          end: new Date('2026-08-01T00:00:00.000Z'),
-          source: 'stripe',
-        },
-      })
+      // As the ledger behaves: a charge given the payer's subscription lands in its current
+      // period, any other stays in the period it was admitted in.
+      mockRecordCumulativeUsage.mockImplementation(
+        async (params: {
+          payerSubscriptionId?: string
+          billingPeriod: typeof ADMITTED_PERIOD
+        }) => ({
+          billed: true,
+          delta: 0.5,
+          total: 1.5,
+          billingPeriod: params.payerSubscriptionId
+            ? CURRENT_PERIOD
+            : { start: params.billingPeriod.start, end: params.billingPeriod.end },
+        })
+      )
     })
 
-    it('settles the threshold against the period the charge was stamped into', async () => {
-      mockRecordCumulativeUsage.mockResolvedValue({
-        billed: true,
-        delta: 0.5,
-        total: 1.5,
-        billingPeriod: CURRENT_PERIOD,
-      })
-
-      const res = await POST(attributedCallback())
+    it("records a Stripe payer's charge in its current period and settles it there", async () => {
+      admittedWithSource('stripe')
+      settlesOnlyAgainst(CURRENT_PERIOD)
 
-      expect(res.status).toBe(200)
-      expect(mockRecordCumulativeUsage).toHaveBeenCalledWith(
-        expect.objectContaining({ payerSubscriptionId: 'sub-1' })
-      )
-      expect(mockCheckAndBillPayerOverageThreshold).toHaveBeenCalledWith(
-        { type: 'organization', id: 'org-1' },
-        expect.objectContaining({ expectedBillingPeriod: CURRENT_PERIOD })
-      )
+      expect((await POST(attributedCallback())).status).toBe(200)
     })
 
     it('leaves a period that closed under a recorded charge to the cycle close', async () => {
+      admittedWithSource('stripe')
       mockRecordCumulativeUsage.mockResolvedValue({
         billed: true,
         delta: 0.5,
         total: 1.5,
-        billingPeriod: {
-          start: new Date('2026-07-01T00:00:00.000Z'),
-          end: new Date('2026-08-01T00:00:00.000Z'),
-        },
+        billingPeriod: ADMITTED_PERIOD,
       })
       mockCheckAndBillPayerOverageThreshold.mockRejectedValue(
         new MockThresholdSettlementError('billing_period_elapsed')
@@ -1121,51 +1134,15 @@ describe('POST /api/billing/update-cost — mid-run usage gate', () => {
       expect(res.status).toBe(200)
     })
 
-    it('keeps a payer whose period is not a Stripe period on its frozen period', async () => {
-      mockToBillingContext.mockReturnValue({
-        billingEntity: { type: 'organization', id: 'org-1' },
-        billingPeriod: {
-          start: new Date('2026-07-01T00:00:00.000Z'),
-          end: new Date('2026-08-01T00:00:00.000Z'),
-          source: 'default',
-        },
-      })
-      mockRecordCumulativeUsage.mockResolvedValue({
-        billed: true,
-        delta: 0.5,
-        total: 0.5,
-        billingPeriod: CURRENT_PERIOD,
-      })
-
-      await POST(attributedCallback())
-
-      expect(mockRecordCumulativeUsage).toHaveBeenCalledWith(
-        expect.not.objectContaining({ payerSubscriptionId: expect.anything() })
-      )
-    })
-
-    it('keeps reporting-window payers on their frozen period', async () => {
-      mockToBillingContext.mockReturnValue({
-        billingEntity: { type: 'organization', id: 'org-1' },
-        billingPeriod: {
-          start: new Date('2026-07-01T00:00:00.000Z'),
-          end: new Date('2026-08-01T00:00:00.000Z'),
-          source: 'reporting',
-        },
-      })
-      mockRecordCumulativeUsage.mockResolvedValue({
-        billed: true,
-        delta: 0.5,
-        total: 0.5,
-        billingPeriod: CURRENT_PERIOD,
-      })
-
-      await POST(attributedCallback())
+    it.each(['reporting', 'default'] as const)(
+      'keeps a payer with a %s period on the period it was admitted in',
+      async (source) => {
+        admittedWithSource(source)
+        settlesOnlyAgainst(ADMITTED_PERIOD)
 
-      expect(mockRecordCumulativeUsage).toHaveBeenCalledWith(
-        expect.not.objectContaining({ payerSubscriptionId: expect.anything() })
-      )
-    })
+        expect((await POST(attributedCallback())).status).toBe(200)
+      }
+    )
   })
 
   it.each([
@@ -1280,7 +1257,6 @@ describe('POST /api/billing/update-cost — mid-run usage gate', () => {
     const body = await (await POST(attributedCallback())).json()
 
     expect(body.usageExceeded).toBe(false)
-    expect(mockRefreshAttributionPeriod).toHaveBeenCalledTimes(2)
   })
 
   it('answers not exceeded when the standing read outlasts the callback budget', async () => {
@@ -1341,8 +1317,7 @@ describe('POST /api/billing/update-cost — mid-run usage gate', () => {
     const res = await POST(attributedCallback())
 
     expect(res.status).toBe(200)
-    await expect(res.json()).resolves.toMatchObject({ usageExceeded: false })
-    expect(mockRecordCumulativeUsage).toHaveBeenCalledOnce()
+    await expect(res.json()).resolves.toMatchObject({ success: true, usageExceeded: false })
   })
 
   it('reports no exceeded usage when billing is disabled', async () => {

apps/sim/app/api/copilot/api-keys/validate/route.test.ts

@@ -400,6 +400,9 @@ describe('POST /api/copilot/api-keys/validate billing protocols', () => {
   })
 
   it('admits a direct-v1 key without Redis while ignoring a local workspace ID', async () => {
+    mockSerializeAccountBillingDecisionHeader.mockImplementation((decision: object) =>
+      encodeURIComponent(JSON.stringify(decision))
+    )
     mockGetUserEntityPermissions.mockResolvedValueOnce(null)
     mockGetWorkspaceBillingSettings.mockResolvedValueOnce({
       billedAccountUserId: 'different-owner',
@@ -426,11 +429,9 @@ describe('POST /api/copilot/api-keys/validate billing protocols', () => {
     expect(mockResolveBillingAttribution).not.toHaveBeenCalled()
     expect(mockGetUserEntityPermissions).not.toHaveBeenCalled()
     expect(mockGetWorkspaceBillingSettings).not.toHaveBeenCalled()
-    expect(mockSerializeAccountBillingDecisionHeader).toHaveBeenCalledWith({
-      ...ACCOUNT_BILLING_DECISION,
-      payerSubscriptionId: ACCOUNT_SUBSCRIPTION.id,
-    })
-    expect(res.headers.get('x-sim-billing-account-decision')).toBe('serialized-account-decision')
+    expect(
+      JSON.parse(decodeURIComponent(res.headers.get('x-sim-billing-account-decision') ?? ''))
+    ).toEqual({ ...ACCOUNT_BILLING_DECISION, payerSubscriptionId: ACCOUNT_SUBSCRIPTION.id })
   })
 
   it('fails direct-v1 admission closed when its payer cannot be resolved', async () => {
@@ -740,16 +741,16 @@ describe('validation lifecycle purposes', () => {
 
   it('answers a polled re-check from the cached admission and always re-reads a refusal', async () => {
     for (let call = 0; call < 2; call++) queueTableRows(schemaMock.user, [{ id: 'user-1' }])
-    for (let poll = 0; poll < 3; poll++) {
+    expect((await POST(request(body, attributedHeaders))).status).toBe(200)
+    mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: true, scope: 'payer' })
+    for (let poll = 0; poll < 2; poll++) {
       expect((await POST(request(body, attributedHeaders))).status).toBe(200)
     }
-    expect(mockCheckAttributedUsageLimits).toHaveBeenCalledTimes(1)
 
     resetUsageGateCache()
-    mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: true, scope: 'payer' })
     expect((await POST(request(body, attributedHeaders))).status).toBe(402)
-    expect((await POST(request(body, attributedHeaders))).status).toBe(402)
-    expect(mockCheckAttributedUsageLimits).toHaveBeenCalledTimes(3)
+    mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: false })
+    expect((await POST(request(body, attributedHeaders))).status).toBe(200)
   })
 
   it('refuses a new turn whose usage cannot be read with neutral copy', async () => {
@@ -808,32 +809,30 @@ describe('validation lifecycle purposes', () => {
 
   it('answers repeated direct-v1 continuations from the cached admission and re-reads a refusal', async () => {
     for (let call = 0; call < 2; call++) queueTableRows(schemaMock.user, [{ id: 'user-1' }])
-    for (let leg = 0; leg < 3; leg++) {
+    expect((await POST(request(body, directHeaders))).status).toBe(200)
+    mockCheckUsageStatus.mockResolvedValue({ isExceeded: true, currentUsage: 12, limit: 10 })
+    for (let leg = 0; leg < 2; leg++) {
       expect((await POST(request(body, directHeaders))).status).toBe(200)
     }
-    expect(mockCheckUsageStatus).toHaveBeenCalledTimes(1)
 
     resetMidRunUsageCaches()
-    mockCheckUsageStatus.mockResolvedValue({ isExceeded: true, currentUsage: 12, limit: 10 })
     expect((await POST(request(body, directHeaders))).status).toBe(402)
-    expect((await POST(request(body, directHeaders))).status).toBe(402)
-    expect(mockCheckUsageStatus).toHaveBeenCalledTimes(3)
+    mockCheckUsageStatus.mockResolvedValue({ isExceeded: false, currentUsage: 1, limit: 10 })
+    expect((await POST(request(body, directHeaders))).status).toBe(200)
   })
 
   it('never reads the usage gate for an attributed continuation when billing is off', async () => {
     setEnvFlags({ isHosted: false, isBillingEnabled: false })
     mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: true, scope: 'payer' })
 
     expect((await POST(request(body, attributedHeaders))).status).toBe(200)
-    expect(mockCheckAttributedUsageLimits).not.toHaveBeenCalled()
   })
 
   it('never reads the ledger for a direct-v1 continuation when billing is off', async () => {
     setEnvFlags({ isHosted: false, isBillingEnabled: false })
     mockCheckUsageStatus.mockResolvedValue({ isExceeded: true, currentUsage: 12, limit: 10 })
 
     expect((await POST(request(body, directHeaders))).status).toBe(200)
-    expect(mockCheckUsageStatus).not.toHaveBeenCalled()
   })
 
   it('judges a direct-v1 reporting-window run against its admitted window after it ends', async () => {

apps/sim/lib/billing/core/billing-attribution.test.ts

@@ -379,36 +379,40 @@ describe('checkAccountBillingBlocks', () => {
     mockCheckBillingEntityBlocked.mockReset().mockResolvedValue({ blocked: false })
   })
 
-  it('checks both the actor and the exact original payer', async () => {
-    await expect(checkAccountBillingBlocks(decision)).resolves.toMatchObject({ blocked: false })
-    expect(mockCheckBillingBlocked).toHaveBeenCalledWith('actor')
-    expect(mockCheckBillingEntityBlocked).toHaveBeenCalledWith(decision.billingEntity)
-  })
-
-  it('refuses an actor block before reading the payer', async () => {
-    mockCheckBillingBlocked.mockResolvedValueOnce({ blocked: true })
+  it('refuses the exact actor and original payer when either is blocked', async () => {
+    mockCheckBillingBlocked.mockImplementation(async (userId: string) => ({
+      blocked: userId === 'actor',
+    }))
     await expect(checkAccountBillingBlocks(decision)).resolves.toMatchObject({
       blocked: true,
       scope: 'actor',
     })
-    expect(mockCheckBillingEntityBlocked).not.toHaveBeenCalled()
-  })
 
-  it('refuses a payer block independently of actor standing', async () => {
-    mockCheckBillingEntityBlocked.mockResolvedValueOnce({ blocked: true })
+    mockCheckBillingBlocked.mockResolvedValue({ blocked: false })
+    mockCheckBillingEntityBlocked.mockImplementation(async (entity: { id: string }) => ({
+      blocked: entity.id === 'original-payer',
+    }))
     await expect(checkAccountBillingBlocks(decision)).resolves.toMatchObject({
       blocked: true,
       scope: 'payer',
     })
   })
 
-  it('reads the same personal actor and payer only once', async () => {
-    await checkAccountBillingBlocks({
-      ...decision,
-      billingEntity: { type: 'user', id: 'actor' },
+  it('reports an actor block ahead of a payer block', async () => {
+    mockCheckBillingBlocked.mockResolvedValue({ blocked: true, message: 'Actor frozen.' })
+    mockCheckBillingEntityBlocked.mockResolvedValue({ blocked: true, message: 'Payer frozen.' })
+    await expect(checkAccountBillingBlocks(decision)).resolves.toEqual({
+      blocked: true,
+      message: 'Actor frozen.',
+      scope: 'actor',
     })
-    expect(mockCheckBillingBlocked).toHaveBeenCalledTimes(1)
-    expect(mockCheckBillingEntityBlocked).not.toHaveBeenCalled()
+  })
+
+  it('answers a personal payer from the actor standing alone', async () => {
+    mockCheckBillingEntityBlocked.mockResolvedValue({ blocked: true })
+    await expect(
+      checkAccountBillingBlocks({ ...decision, billingEntity: { type: 'user', id: 'actor' } })
+    ).resolves.toMatchObject({ blocked: false })
   })
 })
 

apps/sim/lib/billing/core/usage-log.integration.ts

@@ -9,6 +9,7 @@ import type { db } from '@sim/db'
 import * as schema from '@sim/db/schema'
 import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure'
 import { getPostgresErrorCode } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
 import { generateId } from '@sim/utils/id'
 import { sql } from 'drizzle-orm'
 import { drizzle } from 'drizzle-orm/postgres-js'
@@ -313,6 +314,31 @@ describe('Cumulative billing with PostgreSQL', () => {
     }
   )
 
+  it("counts a reporting run's top-ups after its window ends in that window, and a later run's charges in the next", async () => {
+    const payer = { type: 'organization', id: 'payer' } as const
+    const boundary = new Date(Date.now() + 1500)
+    const admitted = {
+      start: new Date(boundary.getTime() - 30 * 24 * 60 * 60 * 1000),
+      end: boundary,
+      source: 'reporting' as const,
+    }
+    const next = {
+      start: boundary,
+      end: new Date(boundary.getTime() + 30 * 24 * 60 * 60 * 1000),
+      source: 'reporting' as const,
+    }
+    const windowTotal = (period: typeof admitted) =>
+      getBillingPeriodUsageCost(payer, period, undefined, database)
+
+    await recordCumulativeUsage({ ...usage(0.4, 'update-cost:long-run'), billingPeriod: admitted })
+    await sleep(boundary.getTime() - Date.now() + 100)
+    await recordCumulativeUsage({ ...usage(1, 'update-cost:long-run'), billingPeriod: admitted })
+    await recordCumulativeUsage({ ...usage(0.25, 'update-cost:next-run'), billingPeriod: next })
+
+    expect(await windowTotal(admitted)).toBeCloseTo(1, 9)
+    expect(await windowTotal(next)).toBeCloseTo(0.25, 9)
+  })
+
   describe('a request that outlives its billing period', () => {
     // Past periods: the old period's row is written under the subscription lock only once
     // that period has ended.