From 6c16d8b1baf80dc211cd3dc8145631c9aa49c8d4 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 1 Sep 2026 14:05:27 +0100 Subject: [PATCH] feat(webapp): let the deployment S2 endpoint be overridden The S2 SDK honours only endpoints passed to its constructor. S2Environment.parse(), which reads the endpoint variables, is an opt-in helper the webapp never called, so deployment event logs always went to hosted S2 and could not be pointed at the local s2 container in docker compose. Realtime streams already had this knob. S2_DEPLOYMENT_ENDPOINT is a single value covering both the account and basin hosts. Two separate variables would let a half-set config send the access token to the hosted service while the operator believed the client was entirely local. With the variable unset the options object carries no endpoints key at all, so the call into the SDK is the one production already makes. An endpoints key with undefined members resolves to the same hosted URLs, so no assertion on the built client would catch a regression there; the options builder is asserted directly instead, from a module that reads no environment. The value is trimmed and validated. https is accepted anywhere, http only to a host that is not reachable from the public internet: loopback, a single-label container or service name as the self-hosted stack uses, or a private address. The access token is sent to whatever is configured as a bearer token, so cleartext to a routable host would leak it, and zod url() is too weak for this alone, accepting both "htp:/localhost" and any public http host. Cached S2 read tokens are scoped by endpoint. They are issued by whichever service the endpoint names and Redis outlives a restart, so a key scoped only by project would serve a token from the previous service for the rest of its hour. Hosted keeps its existing keys, so nothing is invalidated where the endpoint cannot change. --- apps/webapp/app/env.server.ts | 14 +++++ .../v3/DeploymentPresenter.server.ts | 7 +-- apps/webapp/app/utils/s2Endpoint.ts | 34 ++++++++++++ apps/webapp/app/v3/s2CacheScope.ts | 6 +++ apps/webapp/app/v3/s2Client.server.ts | 14 +++++ apps/webapp/app/v3/s2ClientConfig.test.ts | 53 +++++++++++++++++++ apps/webapp/app/v3/s2ClientConfig.ts | 30 +++++++++++ .../app/v3/services/deployment.server.ts | 8 +-- apps/webapp/test/s2CacheScope.test.ts | 20 +++++++ apps/webapp/test/s2Endpoint.test.ts | 41 ++++++++++++++ apps/webapp/vitest.config.ts | 1 + 11 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 apps/webapp/app/utils/s2Endpoint.ts create mode 100644 apps/webapp/app/v3/s2CacheScope.ts create mode 100644 apps/webapp/app/v3/s2Client.server.ts create mode 100644 apps/webapp/app/v3/s2ClientConfig.test.ts create mode 100644 apps/webapp/app/v3/s2ClientConfig.ts create mode 100644 apps/webapp/test/s2CacheScope.test.ts create mode 100644 apps/webapp/test/s2Endpoint.test.ts diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 2b1fba86980..10174e4f8ee 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -4,6 +4,7 @@ import { BoolEnv } from "./utils/boolEnv"; import { isValidDatabaseUrl } from "./utils/db"; import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; import { isValidRegex } from "./utils/regex"; +import { isValidS2Endpoint } from "./utils/s2Endpoint"; import { isValidDuration } from "./services/realtime/duration.server"; // `z.string()` constrained to a `parseDuration`-parseable string (e.g. @@ -79,6 +80,19 @@ const S2EnvSchema = z.preprocess( S2_ACCESS_TOKEN: z.string(), S2_DEPLOYMENT_LOGS_BASIN_NAME: z.string(), S2_DEPLOYMENT_STREAMS_LOCAL: z.string().default("0"), + // Points deployment event logs at an S2 service other than the hosted one, e.g. the + // local s2-lite in docker compose. One value covers both the account and basin + // endpoints: splitting them lets a half-set config send the access token to the + // hosted service while the operator believes they are entirely local. + S2_DEPLOYMENT_ENDPOINT: z + .string() + .trim() + .transform((value) => (value === "" ? undefined : value)) + .refine( + (value) => value === undefined || isValidS2Endpoint(value), + "must be an http(s) URL; http is only allowed for a loopback host, because the S2 access token is sent to it as a bearer token" + ) + .optional(), }), z.object({ S2_ENABLED: z.literal("0"), diff --git a/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts b/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts index 3e5df605257..1f382b2c4f5 100644 --- a/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts @@ -13,11 +13,12 @@ import { type User } from "~/models/user.server"; import { getUsername } from "~/utils/username"; import { processGitMetadata } from "./BranchesPresenter.server"; import { VercelProjectIntegrationDataSchema } from "~/v3/vercel/vercelProjectIntegrationSchema"; -import { S2 } from "@s2-dev/streamstore"; +import { createDeploymentS2Client } from "~/v3/s2Client.server"; import { env } from "~/env.server"; import { createRedisClient } from "~/redis.server"; import { tryCatch } from "@trigger.dev/core"; import { logger } from "~/services/logger.server"; +import { s2CacheScope } from "~/v3/s2CacheScope"; const S2_TOKEN_KEY_PREFIX = "s2-token:project:"; @@ -30,7 +31,7 @@ const s2TokenRedis = createRedisClient("s2-token-cache", { clusterMode: env.CACHE_REDIS_CLUSTER_MODE_ENABLED === "1", }); -const s2 = env.S2_ENABLED === "1" ? new S2({ accessToken: env.S2_ACCESS_TOKEN }) : undefined; +const s2 = createDeploymentS2Client(); export type ErrorData = { name: string; @@ -286,7 +287,7 @@ export class DeploymentPresenter { throw new Error("Failed getting S2 access token: S2 is not enabled"); } - const redisKey = `${S2_TOKEN_KEY_PREFIX}${projectRef}`; + const redisKey = `${S2_TOKEN_KEY_PREFIX}${s2CacheScope(env.S2_DEPLOYMENT_ENDPOINT)}${projectRef}`; const cachedToken = await s2TokenRedis.get(redisKey); if (cachedToken) { diff --git a/apps/webapp/app/utils/s2Endpoint.ts b/apps/webapp/app/utils/s2Endpoint.ts new file mode 100644 index 00000000000..fe7e7a47072 --- /dev/null +++ b/apps/webapp/app/utils/s2Endpoint.ts @@ -0,0 +1,34 @@ +const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]); + +function isPrivateIpv4(hostname: string): boolean { + const octets = hostname.split("."); + if (octets.length !== 4 || octets.some((o) => !/^\d{1,3}$/.test(o))) { + return false; + } + + const [a, b] = octets.map(Number) as [number, number, number, number]; + return a === 10 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168); +} + +// A single-label hostname (no dot) is a container or service name on a private network, which is +// how the self-hosted stack reaches S2, e.g. `http://s2/v1`. +function isPrivateHost(hostname: string): boolean { + return LOOPBACK_HOSTS.has(hostname) || !hostname.includes(".") || isPrivateIpv4(hostname); +} + +// The S2 access token is sent as a bearer header to whatever endpoint is configured, so cleartext +// is only acceptable to a host that is not reachable from the public internet. +export function isValidS2Endpoint(value: string): boolean { + let url: URL; + try { + url = new URL(value); + } catch { + return false; + } + + if (url.protocol === "https:") { + return true; + } + + return url.protocol === "http:" && url.hostname !== "" && isPrivateHost(url.hostname); +} diff --git a/apps/webapp/app/v3/s2CacheScope.ts b/apps/webapp/app/v3/s2CacheScope.ts new file mode 100644 index 00000000000..7d4040d3ccb --- /dev/null +++ b/apps/webapp/app/v3/s2CacheScope.ts @@ -0,0 +1,6 @@ +// Cached S2 read tokens are issued by whichever S2 service the endpoint names, and Redis outlives +// a restart, so a key scoped only by project would serve a token from the previous service after +// the endpoint changes. Hosted keeps its existing unscoped keys so nothing is invalidated. +export function s2CacheScope(endpoint: string | undefined): string { + return endpoint === undefined ? "" : `endpoint:${endpoint}:`; +} diff --git a/apps/webapp/app/v3/s2Client.server.ts b/apps/webapp/app/v3/s2Client.server.ts new file mode 100644 index 00000000000..184818f920e --- /dev/null +++ b/apps/webapp/app/v3/s2Client.server.ts @@ -0,0 +1,14 @@ +import type { S2 } from "@s2-dev/streamstore"; +import { env } from "~/env.server"; +import { buildDeploymentS2Client } from "~/v3/s2ClientConfig"; + +export function createDeploymentS2Client(): S2 | undefined { + if (env.S2_ENABLED !== "1") { + return undefined; + } + + return buildDeploymentS2Client({ + accessToken: env.S2_ACCESS_TOKEN, + endpoint: env.S2_DEPLOYMENT_ENDPOINT, + }); +} diff --git a/apps/webapp/app/v3/s2ClientConfig.test.ts b/apps/webapp/app/v3/s2ClientConfig.test.ts new file mode 100644 index 00000000000..21948066a47 --- /dev/null +++ b/apps/webapp/app/v3/s2ClientConfig.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { buildDeploymentS2Client, deploymentS2ClientOptions } from "./s2ClientConfig"; + +const BASIN = "trigger-local"; + +describe("buildDeploymentS2Client", () => { + // The SDK resolves an endpoints key with undefined members to the same hosted URLs, so nothing + // on the built client distinguishes the two calls. Assert on the options instead. + it("hands the SDK no endpoints key at all when no endpoint is configured", () => { + expect(deploymentS2ClientOptions({ accessToken: "token" })).toEqual({ accessToken: "token" }); + expect(deploymentS2ClientOptions({ accessToken: "token" })).not.toHaveProperty("endpoints"); + }); + + it("hands the SDK one endpoint for both hosts when configured", () => { + expect( + deploymentS2ClientOptions({ accessToken: "token", endpoint: "http://localhost:4566" }) + ).toEqual({ + accessToken: "token", + endpoints: { account: "http://localhost:4566", basin: "http://localhost:4566" }, + }); + }); + + it("still resolves the SDK's hosted defaults when no endpoint is configured", () => { + const client = buildDeploymentS2Client({ accessToken: "token" }); + + expect(client.endpoints.accountBaseUrl()).toBe("https://a.s2.dev/v1"); + expect(client.endpoints.basinBaseUrl(BASIN)).toBe(`https://${BASIN}.b.s2.dev/v1`); + expect(client.endpoints.includeBasinHeader).toBe(false); + }); + + it("points both the account and basin hosts at a configured endpoint", () => { + const client = buildDeploymentS2Client({ + accessToken: "token", + endpoint: "http://localhost:4566", + }); + + expect(client.endpoints.accountBaseUrl()).toBe("http://localhost:4566/v1"); + expect(client.endpoints.basinBaseUrl(BASIN)).toBe("http://localhost:4566/v1"); + expect(client.endpoints.includeBasinHeader).toBe(true); + }); + + // A split configuration would send the access token to the hosted service while the operator + // believed the client was entirely local, so one value has to drive both hosts. + it("never leaves one host hosted while the other is overridden", () => { + const client = buildDeploymentS2Client({ + accessToken: "token", + endpoint: "http://localhost:4566", + }); + + expect(client.endpoints.accountBaseUrl()).not.toContain("s2.dev"); + expect(client.endpoints.basinBaseUrl(BASIN)).not.toContain("s2.dev"); + }); +}); diff --git a/apps/webapp/app/v3/s2ClientConfig.ts b/apps/webapp/app/v3/s2ClientConfig.ts new file mode 100644 index 00000000000..5798a945b83 --- /dev/null +++ b/apps/webapp/app/v3/s2ClientConfig.ts @@ -0,0 +1,30 @@ +import { S2 } from "@s2-dev/streamstore"; + +export type DeploymentS2Config = { + accessToken: string; + endpoint?: string; +}; + +type DeploymentS2ClientOptions = { + accessToken: string; + endpoints?: { account: string; basin: string }; +}; + +// Exported so a test can pin the shape handed to the SDK: with no endpoint the options must carry +// no `endpoints` key, matching the call production already makes. +export function deploymentS2ClientOptions({ + accessToken, + endpoint, +}: DeploymentS2Config): DeploymentS2ClientOptions { + if (endpoint === undefined) { + return { accessToken }; + } + + // One value drives both hosts. Overriding just one would send the access token to the hosted + // service while the other half went elsewhere. + return { accessToken, endpoints: { account: endpoint, basin: endpoint } }; +} + +export function buildDeploymentS2Client(config: DeploymentS2Config): S2 { + return new S2(deploymentS2ClientOptions(config)); +} diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 5729053db79..9f7909120d2 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -23,8 +23,10 @@ import { import { FEATURE_FLAG, type FeatureFlagKey } from "../featureFlags"; import { flags } from "../featureFlags.server"; import { globalFlagsRegistry } from "../globalFlagsRegistry.server"; -import { AppendInput, AppendRecord, S2 } from "@s2-dev/streamstore"; +import { AppendInput, AppendRecord } from "@s2-dev/streamstore"; +import { createDeploymentS2Client } from "~/v3/s2Client.server"; import { createRedisClient } from "~/redis.server"; +import { s2CacheScope } from "~/v3/s2CacheScope"; const S2_TOKEN_KEY_PREFIX = "s2-token:read:deployment-event-stream:project:"; const s2TokenRedis = createRedisClient("s2-token-cache", { @@ -35,7 +37,7 @@ const s2TokenRedis = createRedisClient("s2-token-cache", { tlsDisabled: env.CACHE_REDIS_TLS_DISABLED === "true", clusterMode: env.CACHE_REDIS_CLUSTER_MODE_ENABLED === "1", }); -const s2 = env.S2_ENABLED === "1" ? new S2({ accessToken: env.S2_ACCESS_TOKEN }) : undefined; +const s2 = createDeploymentS2Client(); const DEPLOY_BUILD_PATH_ENV_FLAG: Partial> = { PREVIEW: FEATURE_FLAG.deployBuildPathPreview, @@ -522,7 +524,7 @@ export class DeploymentService extends BaseService { return errAsync({ type: "s2_is_disabled" as const }); } const basinName = env.S2_DEPLOYMENT_LOGS_BASIN_NAME; - const redisKey = `${S2_TOKEN_KEY_PREFIX}${project.externalRef}`; + const redisKey = `${S2_TOKEN_KEY_PREFIX}${s2CacheScope(env.S2_DEPLOYMENT_ENDPOINT)}${project.externalRef}`; const getTokenFromCache = () => fromPromise(s2TokenRedis.get(redisKey), (error) => ({ diff --git a/apps/webapp/test/s2CacheScope.test.ts b/apps/webapp/test/s2CacheScope.test.ts new file mode 100644 index 00000000000..1f21b19c39f --- /dev/null +++ b/apps/webapp/test/s2CacheScope.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { s2CacheScope } from "~/v3/s2CacheScope"; + +describe("s2CacheScope", () => { + // Hosted must keep the keys it already has in Redis, or every project takes a needless miss. + it("adds nothing when no endpoint is configured", () => { + expect(s2CacheScope(undefined)).toBe(""); + }); + + // Redis outlives a restart, so a token issued by the previous S2 service must not be served + // once the endpoint changes. + it("gives each endpoint its own namespace", () => { + const local = s2CacheScope("http://localhost:4566"); + const other = s2CacheScope("http://s2/v1"); + + expect(local).not.toBe(""); + expect(local).not.toBe(other); + expect(local).toBe(s2CacheScope("http://localhost:4566")); + }); +}); diff --git a/apps/webapp/test/s2Endpoint.test.ts b/apps/webapp/test/s2Endpoint.test.ts new file mode 100644 index 00000000000..1d78e9a0d2b --- /dev/null +++ b/apps/webapp/test/s2Endpoint.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { isValidS2Endpoint } from "~/utils/s2Endpoint"; + +describe("isValidS2Endpoint", () => { + it("accepts https anywhere", () => { + expect(isValidS2Endpoint("https://a.s2.dev")).toBe(true); + expect(isValidS2Endpoint("https://s2.internal:4566/v1")).toBe(true); + }); + + it("accepts http to a loopback host", () => { + expect(isValidS2Endpoint("http://localhost:4566")).toBe(true); + expect(isValidS2Endpoint("http://127.0.0.1:4566")).toBe(true); + expect(isValidS2Endpoint("http://[::1]:4566")).toBe(true); + }); + + // This is how the self-hosted stack reaches S2, so rejecting it would force TLS on a private + // container network. + it("accepts http to a container or service name and a private address", () => { + expect(isValidS2Endpoint("http://s2/v1")).toBe(true); + expect(isValidS2Endpoint("http://s2:80/v1")).toBe(true); + expect(isValidS2Endpoint("http://10.0.0.5:4566")).toBe(true); + expect(isValidS2Endpoint("http://172.20.0.3")).toBe(true); + expect(isValidS2Endpoint("http://192.168.1.9")).toBe(true); + }); + + // The access token is sent to whatever is configured, so cleartext to a routable host leaks it. + it("rejects http to a public host", () => { + expect(isValidS2Endpoint("http://s2.example.com")).toBe(false); + expect(isValidS2Endpoint("http://a.s2.dev")).toBe(false); + expect(isValidS2Endpoint("http://8.8.8.8")).toBe(false); + expect(isValidS2Endpoint("http://172.32.0.1")).toBe(false); + }); + + // All four pass zod's `.url()`, which is why the schema refines on this instead. + it("rejects malformed and non-http schemes", () => { + expect(isValidS2Endpoint("htp:/localhost:4566")).toBe(false); + expect(isValidS2Endpoint("ftp://localhost")).toBe(false); + expect(isValidS2Endpoint("not a url")).toBe(false); + expect(isValidS2Endpoint("")).toBe(false); + }); +}); diff --git a/apps/webapp/vitest.config.ts b/apps/webapp/vitest.config.ts index 3880e304573..a36c8102f2b 100644 --- a/apps/webapp/vitest.config.ts +++ b/apps/webapp/vitest.config.ts @@ -13,6 +13,7 @@ export default defineConfig({ "test/**/*.test.ts", "app/v3/runOpsMigration/**/*.test.ts", "app/v3/runStore.server.test.ts", + "app/v3/s2ClientConfig.test.ts", "app/v3/utils/**/*.test.ts", "app/v3/services/bulk/**/*.test.ts", "app/runEngine/concerns/**/*.test.ts",