diff --git a/apps/alerting/src/scheduled.test.ts b/apps/alerting/src/scheduled.test.ts index 1c8029d80..71615972c 100644 --- a/apps/alerting/src/scheduled.test.ts +++ b/apps/alerting/src/scheduled.test.ts @@ -5,7 +5,7 @@ import { buildLayer, catchTickFailure, selectScheduledProgram, type ScheduledTic const cronCases = [ ["*/5 * * * *", ["anomaly", "cloudflareAnalytics", "planetScale"]], - ["*/15 * * * *", ["digest"]], + ["*/15 * * * *", ["digest", "googleAnalytics"]], ["0 * * * *", ["serviceMapRollup"]], ["* * * * *", ["alert", "error", "escalation", "fixVerification"]], ] as const @@ -27,6 +27,7 @@ describe("alerting Effect root", () => { error: tick("error"), escalation: tick("escalation"), fixVerification: tick("fixVerification"), + googleAnalytics: tick("googleAnalytics"), planetScale: tick("planetScale"), serviceMapRollup: tick("serviceMapRollup"), } satisfies ScheduledTickPrograms @@ -56,6 +57,7 @@ describe("alerting Effect root", () => { error: errorGate.await.pipe(Effect.andThen(record("error"))), escalation: record("escalation"), fixVerification: record("fixVerification"), + googleAnalytics: record("googleAnalytics"), planetScale: record("planetScale"), serviceMapRollup: record("serviceMapRollup"), } satisfies ScheduledTickPrograms @@ -81,6 +83,7 @@ describe("alerting Effect root", () => { error: tick("error"), escalation: tick("escalation"), fixVerification: tick("fixVerification"), + googleAnalytics: tick("googleAnalytics"), planetScale: tick("planetScale"), serviceMapRollup: tick("serviceMapRollup"), } satisfies ScheduledTickPrograms diff --git a/apps/alerting/src/scheduled.ts b/apps/alerting/src/scheduled.ts index 0453696e8..3204bef34 100644 --- a/apps/alerting/src/scheduled.ts +++ b/apps/alerting/src/scheduled.ts @@ -27,6 +27,8 @@ import { ErrorsService, EscalationService, FixVerificationTickService, + GoogleAnalyticsOAuthService, + GoogleAnalyticsService, HazelOAuthService, layerPg, NotificationDispatcher, @@ -236,6 +238,12 @@ export const buildLayer = (env: AlertingWorkerEnv) => { ), ) + const GoogleAnalyticsOAuthServiceLive = GoogleAnalyticsOAuthService.layer.pipe(Layer.provide(BaseLive)) + + const GoogleAnalyticsServiceLive = GoogleAnalyticsService.layer.pipe( + Layer.provide(Layer.mergeAll(BaseLive, GoogleAnalyticsOAuthServiceLive, OrgIngestKeysServiceLive)), + ) + const PlanetScaleOAuthServiceLive = PlanetScaleOAuthService.layer.pipe(Layer.provide(BaseLive)) const PlanetScaleServiceLive = PlanetScaleService.layer.pipe( @@ -246,6 +254,7 @@ export const buildLayer = (env: AlertingWorkerEnv) => { AlertsServiceLive, AnomalyDetectionServiceLive, CloudflareAnalyticsServiceLive, + GoogleAnalyticsServiceLive, PlanetScaleServiceLive, DigestServiceLive, ErrorsServiceLive, @@ -420,6 +429,21 @@ const cloudflareAnalyticsTick = makeTick( }), ) +/** + * Runs on the 15-minute cron, not Cloudflare's 5-minute one. GA4 does not update fast enough to + * reward a tighter cadence, and every tick spends Data API quota tokens per property. + */ +const googleAnalyticsTick = makeTick( + GoogleAnalyticsService.use((analytics) => analytics.pollAllOrgs()), + "google_analytics", + (result) => ({ + properties: result.properties, + rowsIngested: result.rowsIngested, + skipped: result.skipped, + failures: result.failures, + }), +) + const planetScaleTick = makeTick( PlanetScaleService.use((planetscale) => planetscale.pollAllOrgs()), "planetscale", @@ -443,6 +467,7 @@ export interface ScheduledTickPrograms { readonly error: Effect.Effect readonly escalation: Effect.Effect readonly fixVerification: Effect.Effect + readonly googleAnalytics: Effect.Effect readonly planetScale: Effect.Effect readonly serviceMapRollup: Effect.Effect } @@ -463,7 +488,9 @@ export const selectScheduledProgram = ( discard: true, }), ), - Match.when("*/15 * * * *", () => ticks.digest), + Match.when("*/15 * * * *", () => + Effect.all([ticks.digest, ticks.googleAnalytics], { concurrency: 2, discard: true }), + ), Match.when("0 * * * *", () => ticks.serviceMapRollup), Match.when("* * * * *", () => // `fixVerification` is chained onto `error` rather than listed beside it: @@ -493,6 +520,7 @@ type ScheduledServices = | ErrorsService | EscalationService | FixVerificationTickService + | GoogleAnalyticsService | PlanetScaleService | ServiceMapRollupService @@ -504,6 +532,7 @@ export const scheduledTicks: ScheduledTickPrograms = { error: errorTick, escalation: escalationTick, fixVerification: fixVerificationTick, + googleAnalytics: googleAnalyticsTick, planetScale: planetScaleTick, serviceMapRollup: serviceMapRollupTick, } diff --git a/apps/alerting/src/worker.ts b/apps/alerting/src/worker.ts index f397658eb..785b69d4b 100644 --- a/apps/alerting/src/worker.ts +++ b/apps/alerting/src/worker.ts @@ -26,6 +26,7 @@ import { appUrlsEnv, authEnv, cloudflareOAuthEnv, + googleAnalyticsOAuthEnv, ingestKeyCryptoEnv, merge, optionalPlain, @@ -105,12 +106,13 @@ const configuredEnv = (stage: MapleStage) => optionalSecret("AUTUMN_SECRET_KEY"), optionalSecret("INTERNAL_SERVICE_TOKEN"), // The alerting worker is where incidents open and resolve, so it is the one - // that sends push (platform/Apns.ts) — and it runs the Cloudflare analytics - // and PlanetScale inventory pollers, each of which resolves and refreshes - // per-org OAuth tokens with the same config the api worker uses. + // that sends push (platform/Apns.ts) — and it runs the Cloudflare analytics, + // PlanetScale inventory and Google Analytics pollers, each of which resolves + // and refreshes per-org OAuth tokens with the same config the api worker uses. apnsEnv, cloudflareOAuthEnv, planetScaleOAuthEnv, + googleAnalyticsOAuthEnv, ) /** diff --git a/apps/api/src/alerting.ts b/apps/api/src/alerting.ts index 0627ac032..74e9fc583 100644 --- a/apps/api/src/alerting.ts +++ b/apps/api/src/alerting.ts @@ -9,6 +9,8 @@ export { BucketCacheService } from "@maple/query-engine/caching" export { CacheBackendLive } from "@/platform/CacheBackendLive" export { CloudflareAnalyticsService } from "./services/integrations/CloudflareAnalyticsService" export { CloudflareOAuthService } from "./services/auth/CloudflareOAuthService" +export { GoogleAnalyticsService } from "./services/integrations/GoogleAnalyticsService" +export { GoogleAnalyticsOAuthService } from "./services/auth/GoogleAnalyticsOAuthService" export { ErrorActorsService } from "./services/errors/ErrorActorsService" export { ErrorIssueReadModelsService } from "./services/errors/ErrorIssueReadModelsService" export { ErrorIssueWorkflowService } from "./services/errors/ErrorIssueWorkflowService" diff --git a/apps/api/src/dashboard-templates/index.ts b/apps/api/src/dashboard-templates/index.ts index b1512a1a4..79e18ce39 100644 --- a/apps/api/src/dashboard-templates/index.ts +++ b/apps/api/src/dashboard-templates/index.ts @@ -23,6 +23,7 @@ import { kubernetesPodTemplate } from "./infrastructure/kubernetes-pod" import { kafkaTemplate } from "./messaging/kafka" import { natsTemplate } from "./messaging/nats" import { rabbitmqTemplate } from "./messaging/rabbitmq" +import { googleAnalyticsTemplate } from "./product/google-analytics" import type { TemplateDefinition, TemplateMetadata, TemplatePreviewWidget } from "./types" export const DASHBOARD_TEMPLATES: ReadonlyArray = [ @@ -53,6 +54,8 @@ export const DASHBOARD_TEMPLATES: ReadonlyArray = [ kafkaTemplate, natsTemplate, rabbitmqTemplate, + // Product + googleAnalyticsTemplate, ] const TEMPLATE_BY_ID = new Map(DASHBOARD_TEMPLATES.map((t) => [t.id, t])) diff --git a/apps/api/src/dashboard-templates/product/google-analytics.ts b/apps/api/src/dashboard-templates/product/google-analytics.ts new file mode 100644 index 000000000..070b4f244 --- /dev/null +++ b/apps/api/src/dashboard-templates/product/google-analytics.ts @@ -0,0 +1,258 @@ +import { + CHART_DISPLAY_AREA, + CHART_DISPLAY_BAR, + CHART_DISPLAY_LINE, + buildPortableDashboard, + makeQueryDraft, + metricsTimeseries, + paramKey, + paramValue, + templateId, +} from "@/dashboard-templates/helpers" +import type { TemplateDefinition, WidgetDef } from "@/dashboard-templates/types" + +// GA4 metrics land under ServiceName `google-analytics/{propertyId}` (the +// GoogleAnalyticsService collector). +// +// EVERY widget here uses `sum`, and that is not a stylistic choice. These are +// DELTA-temporality sums — each row is one hour's increment, reconciled against the ledger so a +// revised hour arrives as a difference. `rate` and `increase` assume CUMULATIVE temporality and +// reconstruct increments with `lagInFrame`, so pointing either at these metrics +// double-differences the data (see packages/query-engine/src/query-builder/model.ts). +// +// The `.by_*` suffixes matter too: `channels`, `geo` and `device` each report the same session +// total sliced differently, so they are separate metrics rather than one metric with three +// attributes. Charting `google_analytics.sessions` never double-counts as a result. +/** + * GA4 property IDs are decimal, and this is interpolated into a filter expression — so anything + * else is rejected rather than embedded. A value carrying a quote would close the string early and + * leave an unparseable clause, and a clause the builder cannot parse is dropped, which would + * silently widen every widget from one property to ALL of them. Failing the parameter is the only + * safe response; an unfiltered dashboard is not a degraded result, it is the wrong one. + */ +const GA4_PROPERTY_ID = /^[0-9]+$/ + +function propertyWhere(propertyId?: string): string { + if (propertyId === undefined || propertyId === "") return "" + if (!GA4_PROPERTY_ID.test(propertyId)) { + throw new Error(`Google Analytics property must be a numeric GA4 property ID, got "${propertyId}"`) + } + return `service.name = "google-analytics/${propertyId}"` +} + +/** A single-metric stat: reduce one query-builder series to one number. */ +function metricStat(opts: { + id: string + name: string + metricName: string + where: string + unit: string + title: string + layout: WidgetDef["layout"] +}): WidgetDef { + return { + id: opts.id, + visualization: "stat", + dataSource: { + ...metricsTimeseries({ + id: opts.id, + name: opts.name, + metricName: opts.metricName, + metricType: "sum", + aggregation: "sum", + whereClause: opts.where, + }), + transform: { reduceToValue: { field: opts.name, aggregate: "sum" } }, + }, + display: { title: opts.title, unit: opts.unit }, + layout: opts.layout, + } +} + +/** A top-N breakdown: one series per attribute value, ranked by total over the window. */ +function breakdownChart(opts: { + id: string + name: string + metricName: string + attribute: string + where: string + title: string + layout: WidgetDef["layout"] + display: typeof CHART_DISPLAY_BAR | typeof CHART_DISPLAY_AREA +}): WidgetDef { + return { + id: opts.id, + visualization: "chart", + dataSource: metricsTimeseries({ + id: opts.id, + name: opts.name, + metricName: opts.metricName, + metricType: "sum", + aggregation: "sum", + whereClause: opts.where, + groupBy: [`attr.${opts.attribute}`], + }), + display: { title: opts.title, ...opts.display, unit: "number" }, + layout: opts.layout, + } +} + +function widgets(propertyId?: string): WidgetDef[] { + const where = propertyWhere(propertyId) + return [ + metricStat({ + id: "kpi-sessions", + name: "Sessions", + metricName: "google_analytics.sessions", + where, + unit: "number", + title: "Sessions", + layout: { x: 0, y: 0, w: 3, h: 2 }, + }), + metricStat({ + id: "kpi-active-users", + name: "Active users", + metricName: "google_analytics.active_users", + where, + unit: "number", + title: "Active Users", + layout: { x: 3, y: 0, w: 3, h: 2 }, + }), + metricStat({ + id: "kpi-new-users", + name: "New users", + metricName: "google_analytics.new_users", + where, + unit: "number", + title: "New Users", + layout: { x: 6, y: 0, w: 3, h: 2 }, + }), + metricStat({ + id: "kpi-page-views", + name: "Page views", + metricName: "google_analytics.page_views", + where, + unit: "number", + title: "Page Views", + layout: { x: 9, y: 0, w: 3, h: 2 }, + }), + + { + id: "traffic-over-time", + visualization: "chart", + dataSource: metricsTimeseries({ + id: "ga-sessions-over-time", + name: "Sessions", + metricName: "google_analytics.sessions", + metricType: "sum", + aggregation: "sum", + whereClause: where, + }), + display: { title: "Sessions Over Time", ...CHART_DISPLAY_AREA, unit: "number" }, + layout: { x: 0, y: 2, w: 6, h: 6 }, + }, + { + id: "engagement-over-time", + visualization: "chart", + dataSource: metricsTimeseries({ + id: "ga-engaged-sessions", + name: "Engaged sessions", + metricName: "google_analytics.engaged_sessions", + metricType: "sum", + aggregation: "sum", + whereClause: where, + }), + display: { title: "Engaged Sessions", ...CHART_DISPLAY_LINE, unit: "number" }, + layout: { x: 6, y: 2, w: 6, h: 6 }, + }, + + breakdownChart({ + id: "sessions-by-channel", + name: "Sessions", + metricName: "google_analytics.sessions.by_channel", + attribute: "google_analytics.channel_group", + where, + title: "Sessions by Channel", + display: CHART_DISPLAY_AREA, + layout: { x: 0, y: 8, w: 6, h: 6 }, + }), + breakdownChart({ + id: "page-views-by-page", + name: "Page views", + metricName: "google_analytics.page_views.by_page", + attribute: "url.path", + where, + title: "Page Views by Path", + display: CHART_DISPLAY_BAR, + layout: { x: 6, y: 8, w: 6, h: 6 }, + }), + breakdownChart({ + id: "sessions-by-country", + name: "Sessions", + metricName: "google_analytics.sessions.by_country", + attribute: "geo.country_iso_code", + where, + title: "Sessions by Country", + display: CHART_DISPLAY_BAR, + layout: { x: 0, y: 14, w: 4, h: 6 }, + }), + breakdownChart({ + id: "sessions-by-device", + name: "Sessions", + metricName: "google_analytics.sessions.by_device", + attribute: "google_analytics.device_category", + where, + title: "Sessions by Device", + display: CHART_DISPLAY_AREA, + layout: { x: 4, y: 14, w: 4, h: 6 }, + }), + breakdownChart({ + id: "key-events-by-name", + name: "Key events", + metricName: "google_analytics.key_events.by_event", + attribute: "google_analytics.event_name", + where, + title: "Key Events (Conversions)", + display: CHART_DISPLAY_BAR, + layout: { x: 8, y: 14, w: 4, h: 6 }, + }), + ] +} + +export const googleAnalyticsTemplate: TemplateDefinition = { + id: templateId("google-analytics"), + name: "Google Analytics", + description: + "Web analytics from the Google Analytics integration — sessions, users and page views, traffic and engagement over time, plus breakdowns by channel, page, country, device and key event.", + category: "product", + tags: ["google-analytics", "web", "marketing"], + requirement: { + kind: "integration", + label: "Google Analytics integration connected", + missing: "not connected", + collector: "the Google Analytics integration", + setupLabel: "the Google Analytics integration", + hint: "Connect a Google account with access to a GA4 property and every widget fills in on its own.", + }, + requiredMetricPrefixes: ["google_analytics."], + parameters: [ + { + key: paramKey("property_id"), + label: "Property", + description: "Optional — scope every widget to a single GA4 property.", + required: false, + placeholder: "123456789", + }, + ], + build: (params) => { + const propertyId = paramValue(params, "property_id") + return buildPortableDashboard({ + name: propertyId ? `${propertyId} — Google Analytics` : "Google Analytics", + description: + "Google Analytics 4 — sessions, users, page views, and breakdowns by channel, page, country and device.", + tags: ["google-analytics"], + timeRange: "24h", + widgets: widgets(propertyId), + }) + }, +} diff --git a/apps/api/src/platform/Env.ts b/apps/api/src/platform/Env.ts index 9a053c928..7fa3a23eb 100644 --- a/apps/api/src/platform/Env.ts +++ b/apps/api/src/platform/Env.ts @@ -145,6 +145,23 @@ export interface EnvConfig { * request may only name a subset of them. */ readonly PLANETSCALE_OAUTH_SCOPES: string + readonly GOOGLE_OAUTH_CLIENT_ID: Option.Option + /** Required alongside the client id — Google web-application clients are confidential. */ + readonly GOOGLE_OAUTH_CLIENT_SECRET: Option.Option> + readonly GOOGLE_OAUTH_AUTHORIZE_URL: string + readonly GOOGLE_OAUTH_TOKEN_URL: string + readonly GOOGLE_OAUTH_REVOKE_URL: string + /** OIDC userinfo — resolves the connecting Google identity for the integration card. */ + readonly GOOGLE_OAUTH_USERINFO_URL: string + /** + * Space-delimited OAuth scopes. `analytics.readonly` is a Google SENSITIVE scope: the + * client needs brand review + OAuth verification before serving more than 100 users. + */ + readonly GOOGLE_OAUTH_SCOPES: string + /** GA4 Data API base (`runReport`) — overridable for tests. */ + readonly MAPLE_GOOGLE_ANALYTICS_DATA_API_BASE_URL: string + /** GA4 Admin API base (`accountSummaries`) — overridable for tests. */ + readonly MAPLE_GOOGLE_ANALYTICS_ADMIN_API_BASE_URL: string } const portConfig = Config.number("PORT").pipe(Config.withDefault(3472)) @@ -296,6 +313,35 @@ const envConfig = Config.all({ "PLANETSCALE_OAUTH_SCOPES", "user:read_organizations organization:read_organization organization:read_databases organization:read_branches organization:read_backups organization:read_comments organization:read_deploy_requests branch:read_branch", ), + GOOGLE_OAUTH_CLIENT_ID: optionalString("GOOGLE_OAUTH_CLIENT_ID"), + GOOGLE_OAUTH_CLIENT_SECRET: optionalRedacted("GOOGLE_OAUTH_CLIENT_SECRET"), + GOOGLE_OAUTH_AUTHORIZE_URL: stringWithDefault( + "GOOGLE_OAUTH_AUTHORIZE_URL", + "https://accounts.google.com/o/oauth2/v2/auth", + ), + GOOGLE_OAUTH_TOKEN_URL: stringWithDefault("GOOGLE_OAUTH_TOKEN_URL", "https://oauth2.googleapis.com/token"), + GOOGLE_OAUTH_REVOKE_URL: stringWithDefault("GOOGLE_OAUTH_REVOKE_URL", "https://oauth2.googleapis.com/revoke"), + GOOGLE_OAUTH_USERINFO_URL: stringWithDefault( + "GOOGLE_OAUTH_USERINFO_URL", + "https://openidconnect.googleapis.com/v1/userinfo", + ), + // Read-only analytics data + the property list needed to discover what to poll: + // `analytics.readonly` alone covers both the Data API and Admin API reads we make. It is + // Google's only SENSITIVE scope here, and the one gating app verification. + // `openid email` is added so the integration card can say which Google account is connected + // and so the connection row has a stable external identity; both are non-sensitive. + GOOGLE_OAUTH_SCOPES: stringWithDefault( + "GOOGLE_OAUTH_SCOPES", + "https://www.googleapis.com/auth/analytics.readonly openid email", + ), + MAPLE_GOOGLE_ANALYTICS_DATA_API_BASE_URL: stringWithDefault( + "MAPLE_GOOGLE_ANALYTICS_DATA_API_BASE_URL", + "https://analyticsdata.googleapis.com/v1beta", + ), + MAPLE_GOOGLE_ANALYTICS_ADMIN_API_BASE_URL: stringWithDefault( + "MAPLE_GOOGLE_ANALYTICS_ADMIN_API_BASE_URL", + "https://analyticsadmin.googleapis.com/v1beta", + ), }) const makeEnv = Effect.gen(function* () { diff --git a/apps/api/src/resources/env.ts b/apps/api/src/resources/env.ts index a0cde766f..634f9d1a8 100644 --- a/apps/api/src/resources/env.ts +++ b/apps/api/src/resources/env.ts @@ -17,6 +17,7 @@ import { authEnv, cloudflareOAuthEnv, derived, + googleAnalyticsOAuthEnv, ingestKeyCryptoEnv, merge, optionalPlain, @@ -109,4 +110,5 @@ export const apiConfiguredEnv = (stage: MapleStage, domains: MapleDomains) => optionalPlain("GITHUB_API_BASE_URL"), cloudflareOAuthEnv, planetScaleOAuthEnv, + googleAnalyticsOAuthEnv, ) diff --git a/apps/api/src/routes/v1/integrations.http.ts b/apps/api/src/routes/v1/integrations.http.ts index e28b15c28..8e1b239c7 100644 --- a/apps/api/src/routes/v1/integrations.http.ts +++ b/apps/api/src/routes/v1/integrations.http.ts @@ -62,6 +62,11 @@ import { } from "@/services/integrations/cloudflare-analytics/queries" import { PlanetScaleConnectionService } from "@/services/integrations/PlanetScaleConnectionService" import { PlanetScaleService } from "@/services/integrations/PlanetScaleService" +import { + GOOGLE_ANALYTICS_CALLBACK_PATH, + GoogleAnalyticsOAuthService, +} from "@/services/auth/GoogleAnalyticsOAuthService" +import { GoogleAnalyticsService } from "@/services/integrations/GoogleAnalyticsService" import { PLANETSCALE_CALLBACK_PATH, PlanetScaleOAuthService } from "@/services/auth/PlanetScaleOAuthService" import { GithubConnectService } from "@/services/integrations/vcs/vendor/github/GithubConnectService" import { VcsCommitService } from "@/services/integrations/vcs/VcsCommitService" @@ -80,6 +85,7 @@ const HAZEL_MESSAGE_TYPE = "maple:integration:hazel" const GITHUB_MESSAGE_TYPE = "maple:integration:github" const CLOUDFLARE_MESSAGE_TYPE = "maple:integration:cloudflare" const PLANETSCALE_MESSAGE_TYPE = "maple:integration:planetscale" +const GOOGLE_ANALYTICS_MESSAGE_TYPE = "maple:integration:google-analytics" /** * How long `cloudflarePrime` spends on the post-connect poll. Long enough for zone discovery plus @@ -831,6 +837,8 @@ export const IntegrationsCallbackRouter = HttpRouter.use((router) => const cloudflareAnalytics = yield* CloudflareAnalyticsService const planetscaleOAuth = yield* PlanetScaleOAuthService const planetscaleConnection = yield* PlanetScaleConnectionService + const googleAnalyticsOAuth = yield* GoogleAnalyticsOAuthService + const googleAnalytics = yield* GoogleAnalyticsService const env = yield* Env const dashboardTargetOrigin = resolveDashboardTargetOrigin(env.MAPLE_APP_BASE_URL) @@ -855,6 +863,14 @@ export const IntegrationsCallbackRouter = HttpRouter.use((router) => messageType: CLOUDFLARE_MESSAGE_TYPE, label: "Cloudflare", }) + const googleAnalyticsCallbackPage = (params: Omit) => + renderCallbackPage({ + ...params, + targetOrigin: dashboardTargetOrigin, + messageType: GOOGLE_ANALYTICS_MESSAGE_TYPE, + label: "Google Analytics", + }) + const planetscaleCallbackPage = (params: Omit) => renderCallbackPage({ ...params, @@ -1276,5 +1292,81 @@ export const IntegrationsCallbackRouter = HttpRouter.use((router) => }) yield* router.add("GET", PLANETSCALE_CALLBACK_PATH, handlePlanetScale) + + const googleAnalyticsErrorPage = (message: string) => + htmlResponse(googleAnalyticsCallbackPage({ status: "error", message, returnTo: null }), 400) + + const handleGoogleAnalytics = Effect.fn("integrations.googleAnalyticsOAuthCallback")(function* ( + req: HttpServerRequest.HttpServerRequest, + ) { + const urlOption = Option.liftThrowable(() => new URL(req.url, "http://localhost"))() + if (Option.isNone(urlOption)) { + return googleAnalyticsErrorPage("Malformed callback URL") + } + const url = urlOption.value + const code = url.searchParams.get("code") + const state = url.searchParams.get("state") + const oauthError = url.searchParams.get("error") + + if (oauthError) { + // Google's own codes are terse; `access_denied` is the one users actually hit, + // by closing the consent screen. + return googleAnalyticsErrorPage( + oauthError === "access_denied" + ? "Google sign-in was cancelled — the connection wasn't authorized." + : `Google returned an error (${oauthError})`, + ) + } + + if (!code || !state) { + return googleAnalyticsErrorPage("Missing code or state in callback") + } + + return yield* googleAnalyticsOAuth.completeConnect(code, state).pipe( + // The first collection is NOT run here. It takes tens of seconds on a grant with + // several properties, and the popup would sit blank for all of it; the dashboard + // calls `prime` from the tab that stays open instead. + // Collector state is deliberately preserved across a reconnect — the ledger is what + // stops the restatement window's hours being emitted twice. See the note above + // `GoogleAnalyticsService`. + // The callback page reduces failures to short human copy — make sure the real + // cause still lands in the server log for diagnosis. + Effect.tapError((error) => + Effect.logError("Google Analytics OAuth completeConnect failed", { + tag: error._tag, + message: error.message, + }), + ), + Effect.map((result) => + htmlResponse( + googleAnalyticsCallbackPage({ + status: "success", + message: "Google Analytics connected. You can close this window and return to Maple.", + returnTo: result.returnTo, + }), + ), + ), + Effect.catchTags({ + // Validation/upstream messages are our own sanitized strings — and for this + // provider they carry the two refusals a user can actually act on: a grant + // with no refresh token, and one that reaches no GA4 property. + "@maple/http/errors/IntegrationsValidationError": (error) => + Effect.succeed(googleAnalyticsErrorPage(error.message)), + "@maple/http/errors/IntegrationsUpstreamError": (error) => + Effect.succeed(googleAnalyticsErrorPage(error.message)), + "@maple/http/errors/IntegrationsRevokedError": () => + Effect.succeed( + googleAnalyticsErrorPage( + "Google rejected the authorization — reconnect and try again", + ), + ), + "@maple/http/errors/IntegrationsPersistenceError": () => + Effect.succeed(googleAnalyticsErrorPage("Failed to complete Google Analytics connection")), + }), + ) + }) + + yield* router.add("GET", GOOGLE_ANALYTICS_CALLBACK_PATH, handleGoogleAnalytics) + }), ) diff --git a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts index ef5ce1e97..c5b220f21 100644 --- a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts +++ b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts @@ -51,6 +51,7 @@ import { ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, makeWarehouseServiceStub, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, TelemetryServiceStubsLayer, @@ -176,6 +177,7 @@ const makeHarness = () => { Layer.provide(V2TransportErrorBoundaryLive), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index 4e84f09ad..fcdee602f 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -37,6 +37,7 @@ import { ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, makeWarehouseServiceStub, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, TelemetryServiceStubsLayer, @@ -164,6 +165,7 @@ const makeHarness = ( Layer.provide(V2TransportErrorBoundaryLive), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), diff --git a/apps/api/src/routes/v2/api-keys.http.test.ts b/apps/api/src/routes/v2/api-keys.http.test.ts index 2b2500e84..bf50b30d5 100644 --- a/apps/api/src/routes/v2/api-keys.http.test.ts +++ b/apps/api/src/routes/v2/api-keys.http.test.ts @@ -19,6 +19,7 @@ import { AlertsServiceStubLayer, AllV2GroupLayersLive, ConfigResourceServiceStubsLayer, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, TelemetryServiceStubsLayer, @@ -64,6 +65,7 @@ const makeHarness = (checkRateLimit: RateLimiterApi["check"] = () => Effect.succ Layer.provide(V2TransportErrorBoundaryLive), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), diff --git a/apps/api/src/routes/v2/config-resources.http.test.ts b/apps/api/src/routes/v2/config-resources.http.test.ts index 1daca7a6c..9d4415c26 100644 --- a/apps/api/src/routes/v2/config-resources.http.test.ts +++ b/apps/api/src/routes/v2/config-resources.http.test.ts @@ -27,6 +27,7 @@ import { ApiV2RateLimiterAllowAllLayer, makeWarehouseServiceStub, Phase1ResourceStubsLayer, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, SetupAuditServiceStubLayer, @@ -109,6 +110,7 @@ const makeHarness = () => { Layer.provide(V2TransportErrorBoundaryLive), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(AlertsServiceStubLayer), Layer.provide(Phase1ResourceStubsLayer), Layer.provide(SetupAuditServiceStubLayer), diff --git a/apps/api/src/routes/v2/dashboards.http.test.ts b/apps/api/src/routes/v2/dashboards.http.test.ts index 2a414c511..8f098fed4 100644 --- a/apps/api/src/routes/v2/dashboards.http.test.ts +++ b/apps/api/src/routes/v2/dashboards.http.test.ts @@ -18,6 +18,7 @@ import { AllV2GroupLayersLive, ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, TelemetryServiceStubsLayer, @@ -62,6 +63,7 @@ const makeHarness = () => { Layer.provide(V2TransportErrorBoundaryLive), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), diff --git a/apps/api/src/routes/v2/integrations.http.test.ts b/apps/api/src/routes/v2/integrations.http.test.ts index e9d04f3a3..1a14f73cf 100644 --- a/apps/api/src/routes/v2/integrations.http.test.ts +++ b/apps/api/src/routes/v2/integrations.http.test.ts @@ -46,6 +46,7 @@ import { AllV2GroupLayersLive, ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, + GoogleAnalyticsServiceStubsLayer, TelemetryServiceStubsLayer, } from "./v2-test-support" @@ -167,6 +168,7 @@ const makeHarness = (slack: Partial = {}, planetscal Layer.provide(V2TransportErrorBoundaryLive), Layer.provide(slackServiceLayer(slack)), Layer.provide(planetscaleServiceLayer(planetscale)), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), diff --git a/apps/api/src/routes/v2/integrations.http.ts b/apps/api/src/routes/v2/integrations.http.ts index 4dfbc1c1f..f1a9ff5e1 100644 --- a/apps/api/src/routes/v2/integrations.http.ts +++ b/apps/api/src/routes/v2/integrations.http.ts @@ -8,6 +8,10 @@ import type { import { CurrentTenant } from "@maple/domain/http" import type { PlanetScaleDatabaseRow } from "@maple/db" import type { + V2GoogleAnalyticsConnectResponse, + V2GoogleAnalyticsDisconnectResponse, + V2GoogleAnalyticsIntegration, + V2GoogleAnalyticsPrimeResponse, V2PlanetScaleConnectResponse, V2PlanetScaleDatabase, V2PlanetScaleDatabaseList, @@ -35,12 +39,24 @@ import { recordHttpAudit } from "@/services/audit/AuditLogService" import { requireAdmin } from "@/services/auth/auth" import { Env } from "@/platform/Env" import { EdgeCacheService } from "@maple/cache" +import { + GOOGLE_ANALYTICS_CALLBACK_PATH, + GoogleAnalyticsOAuthService, +} from "@/services/auth/GoogleAnalyticsOAuthService" +import type { GoogleAnalyticsIntegrationStatus } from "@/services/integrations/GoogleAnalyticsService" +import { GoogleAnalyticsService } from "@/services/integrations/GoogleAnalyticsService" import { PLANETSCALE_CALLBACK_PATH, PlanetScaleOAuthService } from "@/services/auth/PlanetScaleOAuthService" import { PlanetScaleConnectionService } from "@/services/integrations/PlanetScaleConnectionService" import { PlanetScaleService } from "@/services/integrations/PlanetScaleService" import type { SlackChannelList, SlackInstallStatus } from "@/services/integrations/SlackIntegrationService" import { SLACK_CALLBACK_PATH, SlackIntegrationService } from "@/services/integrations/SlackIntegrationService" +/** + * How long `prime` spends on the post-connect poll. Long enough for property discovery plus a + * first window on an ordinary grant; a many-propertied one resumes on the next cron tick. + */ +const GOOGLE_ANALYTICS_PRIME_TIMEOUT = "20 seconds" + /** * Best-effort origin of the incoming request. `x-forwarded-*` is client-supplied * on any path that does not strip it, so the result is NOT trusted on its own — @@ -587,3 +603,152 @@ export const HttpV2PlanetScaleIntegrationsLive = HttpApiBuilder.group( ) }), ) + +/** + * Google Analytics 4. Reads are open to any org member; every mutation is admin-gated, matching + * the other integrations. The connect flow is the same trusted-origin dance as PlanetScale: the + * callback URL is persisted and replayed as `redirect_uri` at token exchange, and the origin is + * derived from a client-settable header, so an untrusted one would mint an authorize URL pointing + * at a host the caller controls. + */ +export const HttpV2GoogleAnalyticsIntegrationsLive = HttpApiBuilder.group( + MapleApiV2, + "googleAnalyticsIntegration", + (handlers) => + Effect.gen(function* () { + const analytics = yield* GoogleAnalyticsService + const googleOAuth = yield* GoogleAnalyticsOAuthService + const env = yield* Env + + const toStatus = (status: GoogleAnalyticsIntegrationStatus): V2GoogleAnalyticsIntegration => ({ + object: "google_analytics_integration" as const, + connected: status.connected, + connected_at: isoTimestampOrNull(status.connectedAt), + connected_email: status.externalUserEmail, + revoked: status.revoked, + properties: status.properties.map((property) => ({ + object: "google_analytics_property" as const, + property_id: property.propertyId, + property_name: property.propertyName, + account_name: property.accountName, + time_zone: property.timeZone, + enabled: property.enabled, + last_synced_at: isoTimestampOrNull(property.lastSyncedAt), + last_error: property.lastError, + watermark_at: isoTimestampOrNull(property.watermarkAt), + backfill_at: isoTimestampOrNull(property.backfillAt), + })), + }) + + return handlers + .handle("status", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const status = yield* analytics + .getIntegrationStatus(tenant.orgId) + .pipe(tapHttpErrors("Google Analytics status failed")) + return toStatus(status) + }), + ) + .handle("connect", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles, () => + V2InsufficientPermissions.make("Only org admins can connect Google Analytics"), + ) + const req = yield* HttpServerRequest.HttpServerRequest + const origin = resolveRequestOrigin(req) + if (!isTrustedCallbackOrigin(origin, env.MAPLE_APP_BASE_URL)) { + yield* Effect.logError( + "Rejected Google Analytics connect: untrusted callback origin", + { origin }, + ) + return yield* Effect.fail( + V2CallbackHostUnavailable.make( + "Google Analytics connections are not available from this host", + ), + ) + } + const result = yield* googleOAuth + .startConnect(tenant.orgId, tenant.userId, { + callbackUrl: `${origin}${GOOGLE_ANALYTICS_CALLBACK_PATH}`, + returnTo: payload.return_to, + }) + .pipe(tapHttpErrors("Google Analytics connect failed")) + yield* recordHttpAudit("google_analytics_integration.connect_started") + return { + object: "google_analytics_integration.connect" as const, + redirect_url: result.redirectUrl, + state: result.state, + } satisfies V2GoogleAnalyticsConnectResponse + }), + ) + .handle("disconnect", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles, () => + V2InsufficientPermissions.make("Only org admins can disconnect Google Analytics"), + ) + const result = yield* googleOAuth + .disconnect(tenant.orgId) + .pipe(tapHttpErrors("Google Analytics disconnect failed")) + // Collector state is deliberately NOT cleared here — see the note above + // `GoogleAnalyticsService`. Metrics already collected are retained, so the + // ledger has to outlive the grant or a reconnect inside the restatement + // window re-emits those hours on top of rows already in the warehouse. + yield* recordHttpAudit("google_analytics_integration.disconnected") + return { + object: "google_analytics_integration.disconnect" as const, + disconnected: result.disconnected, + } satisfies V2GoogleAnalyticsDisconnectResponse + }), + ) + .handle("prime", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles, () => + V2InsufficientPermissions.make("Only org admins can run a Google Analytics sync"), + ) + // Bounded: discovery plus a first window on an ordinary account fits well + // inside this, and whatever a many-propertied grant does not finish simply + // resumes on the next cron tick. + const result = yield* analytics + .pollOrg(tenant.orgId) + .pipe(Effect.timeoutOption(GOOGLE_ANALYTICS_PRIME_TIMEOUT)) + return { + object: "google_analytics_integration.prime" as const, + properties: Option.match(result, { + onNone: () => 0, + onSome: (value) => value.properties, + }), + rows_ingested: Option.match(result, { + onNone: () => 0, + onSome: (value) => value.rowsIngested, + }), + } satisfies V2GoogleAnalyticsPrimeResponse + }), + ) + .handle("updateProperty", ({ params, payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles, () => + V2InsufficientPermissions.make( + "Only org admins can change Google Analytics collection", + ), + ) + yield* analytics + .setPropertyEnabled(tenant.orgId, params.property_id, payload.enabled) + .pipe(tapHttpErrors("Google Analytics property update failed")) + yield* recordHttpAudit( + payload.enabled + ? "google_analytics_integration.property_enabled" + : "google_analytics_integration.property_disabled", + ) + const status = yield* analytics + .getIntegrationStatus(tenant.orgId) + .pipe(tapHttpErrors("Google Analytics status failed")) + return toStatus(status) + }), + ) + }), +) diff --git a/apps/api/src/routes/v2/mobile-devices.http.test.ts b/apps/api/src/routes/v2/mobile-devices.http.test.ts index 86b2c14a1..6bb9ea749 100644 --- a/apps/api/src/routes/v2/mobile-devices.http.test.ts +++ b/apps/api/src/routes/v2/mobile-devices.http.test.ts @@ -21,6 +21,7 @@ import { ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, Phase1ResourceStubsLayer, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, TelemetryServiceStubsLayer, @@ -78,6 +79,7 @@ const makeHarness = () => { Layer.provide(Phase1ResourceStubsLayer), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), Layer.provideMerge(AuditLogService.layerMemory), diff --git a/apps/api/src/routes/v2/phase1-resources.http.test.ts b/apps/api/src/routes/v2/phase1-resources.http.test.ts index 12771c1bb..be696dbdd 100644 --- a/apps/api/src/routes/v2/phase1-resources.http.test.ts +++ b/apps/api/src/routes/v2/phase1-resources.http.test.ts @@ -67,6 +67,7 @@ import { ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, makeWarehouseServiceStub, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, TelemetryServiceStubsLayer, @@ -559,6 +560,7 @@ const makeHarness = ( Layer.provide(V2TransportErrorBoundaryLive), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), diff --git a/apps/api/src/routes/v2/setup-audit.http.test.ts b/apps/api/src/routes/v2/setup-audit.http.test.ts index e3c6a7247..a44341f46 100644 --- a/apps/api/src/routes/v2/setup-audit.http.test.ts +++ b/apps/api/src/routes/v2/setup-audit.http.test.ts @@ -29,6 +29,7 @@ import { ApiV2RateLimiterAllowAllLayer, makeWarehouseServiceStub, Phase1ResourceStubsLayer, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, TelemetryServiceStubsLayer, @@ -140,6 +141,7 @@ const makeHarness = (warehouse: WarehouseQueryServiceApi = warehouseStub()) => { Layer.provide(Phase1ResourceStubsLayer), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provide(warehouseLive), Layer.provideMerge(ApiAuthorizationV2Layer), diff --git a/apps/api/src/routes/v2/telemetry.http.test.ts b/apps/api/src/routes/v2/telemetry.http.test.ts index b29bc9763..80bf560c0 100644 --- a/apps/api/src/routes/v2/telemetry.http.test.ts +++ b/apps/api/src/routes/v2/telemetry.http.test.ts @@ -27,6 +27,7 @@ import { ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, makeWarehouseServiceStub, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, } from "./v2-test-support" @@ -273,6 +274,7 @@ const makeHarness = ( Layer.provide(V2TransportErrorBoundaryLive), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), diff --git a/apps/api/src/routes/v2/v2-test-support.ts b/apps/api/src/routes/v2/v2-test-support.ts index 27c3fd826..823e7f35e 100644 --- a/apps/api/src/routes/v2/v2-test-support.ts +++ b/apps/api/src/routes/v2/v2-test-support.ts @@ -17,6 +17,8 @@ import { OrgIngestKeysService } from "@/services/org/OrgIngestKeysService" import { RecommendationIssueService } from "@/services/errors/RecommendationIssueService" import { PlanetScaleConnectionService } from "@/services/integrations/PlanetScaleConnectionService" import { PlanetScaleOAuthService } from "@/services/auth/PlanetScaleOAuthService" +import { GoogleAnalyticsOAuthService } from "@/services/auth/GoogleAnalyticsOAuthService" +import { GoogleAnalyticsService } from "@/services/integrations/GoogleAnalyticsService" import { PlanetScaleService } from "@/services/integrations/PlanetScaleService" import { ScrapeTargetsService } from "@/services/integrations/ScrapeTargetsService" import { SlackIntegrationService } from "@/services/integrations/SlackIntegrationService" @@ -35,7 +37,11 @@ import { HttpV2ApiKeysLive } from "./api-keys.http" import { HttpV2AttributeMappingsLive } from "./attribute-mappings.http" import { HttpV2DashboardsLive } from "./dashboards.http" import { HttpV2IngestKeysLive } from "./ingest-keys.http" -import { HttpV2PlanetScaleIntegrationsLive, HttpV2SlackIntegrationsLive } from "./integrations.http" +import { + HttpV2GoogleAnalyticsIntegrationsLive, + HttpV2PlanetScaleIntegrationsLive, + HttpV2SlackIntegrationsLive, +} from "./integrations.http" import { HttpV2ErrorIssuesLive } from "./error-issues.http" import { HttpV2AnomaliesLive } from "./anomalies.http" import { HttpV2InvestigationsLive } from "./investigations.http" @@ -81,6 +87,7 @@ export const AllV2GroupLayersLive = Layer.mergeAll( HttpV2ApiKeysLive, HttpV2SlackIntegrationsLive, HttpV2PlanetScaleIntegrationsLive, + HttpV2GoogleAnalyticsIntegrationsLive, HttpV2DashboardsLive, HttpV2AlertDeliveriesLive, HttpV2AlertRulesLive, @@ -323,6 +330,26 @@ export const PlanetScaleServiceStubsLayer = Layer.mergeAll( EdgeCacheService.layer.pipe(Layer.provide(MemoryCacheBackendLive)), ) +/** + * Inert Google Analytics services for harnesses that never touch that integration group. + */ +export const GoogleAnalyticsServiceStubsLayer = Layer.mergeAll( + Layer.succeed(GoogleAnalyticsService, { + pollAllOrgs: die, + pollOrg: die, + getIntegrationStatus: die, + setPropertyEnabled: die, + }), + Layer.succeed(GoogleAnalyticsOAuthService, { + startConnect: die, + completeConnect: die, + getStatus: die, + getValidAccessToken: die, + disconnect: die, + markConnectionRevoked: die, + }), +) + /** Inert SlackIntegrationService for harnesses that never touch the slack integration group. */ export const SlackIntegrationServiceStubLayer = Layer.succeed( SlackIntegrationService, diff --git a/apps/api/src/routes/v2/widget-credentials.http.test.ts b/apps/api/src/routes/v2/widget-credentials.http.test.ts index b785222b2..e07e92bac 100644 --- a/apps/api/src/routes/v2/widget-credentials.http.test.ts +++ b/apps/api/src/routes/v2/widget-credentials.http.test.ts @@ -21,6 +21,7 @@ import { ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, Phase1ResourceStubsLayer, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, TelemetryServiceStubsLayer, @@ -76,6 +77,7 @@ const makeHarness = () => { Layer.provide(Phase1ResourceStubsLayer), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), Layer.provideMerge(AuditLogService.layerMemory), diff --git a/apps/api/src/routes/v2/widget-summary.http.test.ts b/apps/api/src/routes/v2/widget-summary.http.test.ts index fc25fb99b..6e9b83d73 100644 --- a/apps/api/src/routes/v2/widget-summary.http.test.ts +++ b/apps/api/src/routes/v2/widget-summary.http.test.ts @@ -32,6 +32,7 @@ import { ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, makeWarehouseServiceStub, + GoogleAnalyticsServiceStubsLayer, PlanetScaleServiceStubsLayer, SlackIntegrationServiceStubLayer, } from "./v2-test-support" @@ -188,6 +189,7 @@ const makeHarness = (options: { Layer.provide(V2TransportErrorBoundaryLive), Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(GoogleAnalyticsServiceStubsLayer), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), diff --git a/apps/api/src/runtime/graph-boundaries.test.ts b/apps/api/src/runtime/graph-boundaries.test.ts index 6757e8c0e..d715901f2 100644 --- a/apps/api/src/runtime/graph-boundaries.test.ts +++ b/apps/api/src/runtime/graph-boundaries.test.ts @@ -92,6 +92,7 @@ describe("API runtime graph boundaries", () => { for (const routeOnlyService of [ "DailySpendService", "CloudflareAnalyticsService", + "GoogleAnalyticsService", "AnomalyDetectionService", "AiTriageService", "DigestService", diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index 7d57123b8..18891b8c8 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -44,7 +44,11 @@ import { HttpV2DashboardsLive } from "@/routes/v2/dashboards.http" import { V2TransportErrorBoundaryLive } from "@/routes/v2/error-envelope" import { HttpV2ErrorIssuesLive } from "@/routes/v2/error-issues.http" import { HttpV2IngestKeysLive } from "@/routes/v2/ingest-keys.http" -import { HttpV2PlanetScaleIntegrationsLive, HttpV2SlackIntegrationsLive } from "@/routes/v2/integrations.http" +import { + HttpV2GoogleAnalyticsIntegrationsLive, + HttpV2PlanetScaleIntegrationsLive, + HttpV2SlackIntegrationsLive, +} from "@/routes/v2/integrations.http" import { HttpV2InvestigationsLive } from "@/routes/v2/investigations.http" import { HttpV2MobileDevicesLive } from "@/routes/v2/mobile-devices.http" import { HttpV2OrganizationLive } from "@/routes/v2/organization.http" @@ -136,6 +140,7 @@ const ApiV2Routes = HttpApiBuilder.layer(MapleApiV2).pipe( HttpV2IngestKeysLive, HttpV2SlackIntegrationsLive, HttpV2PlanetScaleIntegrationsLive, + HttpV2GoogleAnalyticsIntegrationsLive, HttpV2ErrorIssuesLive, HttpV2AttributeMappingsLive, HttpV2AuditLogLive, diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 588144fae..d8647fea9 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -14,6 +14,7 @@ import { PlanetScaleOAuthService } from "@/services/auth/PlanetScaleOAuthService import { AuthService } from "@/services/auth/AuthService" import { CliDeviceAuthService } from "@/services/auth/CliDeviceAuthService" import { CloudflareOAuthService } from "@/services/auth/CloudflareOAuthService" +import { GoogleAnalyticsOAuthService } from "@/services/auth/GoogleAnalyticsOAuthService" import { HazelOAuthService } from "@/services/auth/HazelOAuthService" import { McpOAuthService } from "@/services/auth/McpOAuthService" import { OAuthStateRepository } from "@/services/auth/OAuthStateRepository" @@ -35,6 +36,7 @@ import { ErrorsService } from "@/services/errors/ErrorsService" import { InvestigationService } from "@/services/errors/InvestigationService" import { RecommendationIssueService } from "@/services/errors/RecommendationIssueService" import { CloudflareAnalyticsService } from "@/services/integrations/CloudflareAnalyticsService" +import { GoogleAnalyticsService } from "@/services/integrations/GoogleAnalyticsService" import { PlanetScaleConnectionService } from "@/services/integrations/PlanetScaleConnectionService" import { PlanetScaleDiscoveryService } from "@/services/integrations/PlanetScaleDiscoveryService" import { PlanetScaleService } from "@/services/integrations/PlanetScaleService" @@ -81,6 +83,7 @@ const CoreServicesLive = Layer.mergeAll( CliDeviceAuthService.layer, McpOAuthService.layer, CloudflareOAuthService.layer, + GoogleAnalyticsOAuthService.layer, DashboardPersistenceService.layer, SharedDashboardService.layer, HazelOAuthService.layer, @@ -117,6 +120,12 @@ const CloudflareAnalyticsServiceLive = CloudflareAnalyticsService.layer.pipe( Layer.provideMerge(Layer.mergeAll(CoreServicesLive, WarehouseQueryServiceLive)), ) +// Serves the integration card's per-property collection status; the poll loop +// itself runs in the alerting worker's cron, not here. +const GoogleAnalyticsServiceLive = GoogleAnalyticsService.layer.pipe( + Layer.provideMerge(CoreServicesLive), +) + const DemoServiceLive = DemoService.layer.pipe( Layer.provideMerge(Layer.mergeAll(CoreServicesLive, WarehouseQueryServiceLive)), ) @@ -293,6 +302,7 @@ const MainServicesLive = Layer.mergeAll( ProductEventsServiceLive, DailySpendServiceLive, CloudflareAnalyticsServiceLive, + GoogleAnalyticsServiceLive, AuditLogServiceLive, WarehouseQueryServiceLive, EdgeCacheServiceLive, diff --git a/apps/api/src/services/audit/audit-actions.ts b/apps/api/src/services/audit/audit-actions.ts index 2e9c1e1e5..6eb7f7b80 100644 --- a/apps/api/src/services/audit/audit-actions.ts +++ b/apps/api/src/services/audit/audit-actions.ts @@ -57,6 +57,9 @@ export const AuditResources = { planetscale_integration: { verbs: ["connect_started", "organization_selected", "metrics_token_set", "disconnected"], }, + google_analytics_integration: { + verbs: ["connect_started", "disconnected", "property_enabled", "property_disabled"], + }, slack_integration: { verbs: ["install_started", "uninstalled"] }, /** * Org membership, learned from Clerk's webhook — the web app changes members diff --git a/apps/api/src/services/auth/GoogleAnalyticsOAuthService.ts b/apps/api/src/services/auth/GoogleAnalyticsOAuthService.ts new file mode 100644 index 000000000..8da239cc4 --- /dev/null +++ b/apps/api/src/services/auth/GoogleAnalyticsOAuthService.ts @@ -0,0 +1,377 @@ +/** + * Google OAuth for the GA4 integration. + * + * Simpler than {@link CloudflareOAuthService} in one way and stricter in another. Simpler: a + * Google grant covers one identity, not a fan-out of accounts, so the connection row needs no + * `grantedAccountsJson`. Stricter: Google only issues a refresh token when the authorize request + * asks for `access_type=offline`, and only RE-issues one on a repeat consent when + * `prompt=consent` is also sent. Without both, a reconnect silently yields an access-token-only + * grant that dies within the hour — the same failure mode as the Cloudflare `offline_access` + * outage, which is why the refusal below is copied from it verbatim. + * + * All token storage, encryption, single-flight refresh and the `invalid_grant`-only revocation + * rule come from the shared `makeOAuthConnectionHelpers`; nothing about that is re-implemented. + */ +import { randomBytes } from "node:crypto" +import { + IntegrationsNotConnectedError, + IntegrationsPersistenceError, + IntegrationsRevokedError, + IntegrationsUpstreamError, + IntegrationsValidationError, + OrgId, + type UserId, +} from "@maple/domain/http" +import { oauthAuthStates } from "@maple/db" +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" +import { listProperties } from "@/services/integrations/GoogleAnalyticsApi" +import { Database } from "@/platform/DatabaseLive" +import { Env, type EnvConfig } from "@/platform/Env" +import { dateToMs, msToDate } from "@/platform/time" +import { makeOAuthConnectionHelpers, OAUTH_STATE_TTL_MS } from "./oauth/connection-helpers" + +const GOOGLE_ANALYTICS_PROVIDER = "google_analytics" + +/** + * Must match a redirect URI registered on the Google OAuth client exactly, path included — + * Google rejects the exchange otherwise. + */ +export const GOOGLE_ANALYTICS_CALLBACK_PATH = "/api/integrations/google-analytics/callback" + +const decodeOrgId = Schema.decodeUnknownSync(OrgId) + +const UserInfo = Schema.Struct({ + sub: Schema.optionalKey(Schema.String), + email: Schema.optionalKey(Schema.String), +}) +const decodeUserInfo = Schema.decodeUnknownEffect(UserInfo) + +interface ResolvedGoogleOAuthConfig { + readonly clientId: string + readonly clientSecret: Redacted.Redacted | null + readonly authorizeUrl: string + readonly tokenUrl: string + readonly revokeUrl: string + readonly userInfoUrl: string + readonly scopes: string +} + +const resolveConfig = Effect.fn("GoogleAnalyticsOAuthService.resolveConfig")(function* (env: EnvConfig) { + const clientId = yield* Option.match(env.GOOGLE_OAUTH_CLIENT_ID, { + onNone: () => + Effect.fail( + new IntegrationsValidationError({ + message: "GOOGLE_OAUTH_CLIENT_ID is required to use the Google Analytics integration", + }), + ), + onSome: (value) => Effect.succeed(value), + }) + // Unlike Cloudflare's public-client flow, Google web-application clients are confidential: + // the token exchange is rejected without the secret, so a missing one is a misconfiguration + // worth failing loudly at connect time rather than at the first refresh. + const clientSecret = yield* Option.match(env.GOOGLE_OAUTH_CLIENT_SECRET, { + onNone: () => + Effect.fail( + new IntegrationsValidationError({ + message: "GOOGLE_OAUTH_CLIENT_SECRET is required to use the Google Analytics integration", + }), + ), + onSome: (value) => Effect.succeed(value), + }) + + return { + clientId, + clientSecret, + authorizeUrl: env.GOOGLE_OAUTH_AUTHORIZE_URL, + tokenUrl: env.GOOGLE_OAUTH_TOKEN_URL, + revokeUrl: env.GOOGLE_OAUTH_REVOKE_URL, + userInfoUrl: env.GOOGLE_OAUTH_USERINFO_URL, + scopes: env.GOOGLE_OAUTH_SCOPES, + } satisfies ResolvedGoogleOAuthConfig +}) + +export interface GoogleAnalyticsConnectionStatus { + readonly connected: boolean + readonly connectedAt: number | null + readonly externalUserEmail: string | null + readonly connectedByUserId: string | null + readonly scope: string + readonly revoked: boolean +} + +interface GoogleAnalyticsOAuthServiceApi { + readonly startConnect: ( + orgId: OrgId, + userId: UserId, + options: { readonly callbackUrl: string; readonly returnTo?: string }, + ) => Effect.Effect< + { readonly redirectUrl: string; readonly state: string }, + IntegrationsValidationError | IntegrationsPersistenceError + > + readonly completeConnect: ( + code: string, + state: string, + ) => Effect.Effect< + { readonly orgId: OrgId; readonly returnTo: string | null }, + | IntegrationsValidationError + | IntegrationsUpstreamError + | IntegrationsRevokedError + | IntegrationsPersistenceError + > + readonly getStatus: ( + orgId: OrgId, + ) => Effect.Effect + readonly getValidAccessToken: ( + orgId: OrgId, + ) => Effect.Effect< + { readonly accessToken: string; readonly scope: string }, + | IntegrationsNotConnectedError + | IntegrationsRevokedError + | IntegrationsUpstreamError + | IntegrationsPersistenceError + | IntegrationsValidationError + > + readonly disconnect: ( + orgId: OrgId, + ) => Effect.Effect<{ readonly disconnected: boolean }, IntegrationsPersistenceError> + /** Stamp the grant revoked so pollers stop retrying it; cleared on reconnect. Best-effort. */ + readonly markConnectionRevoked: (orgId: OrgId) => Effect.Effect +} + +export class GoogleAnalyticsOAuthService extends Context.Service< + GoogleAnalyticsOAuthService, + GoogleAnalyticsOAuthServiceApi +>()("@maple/api/services/GoogleAnalyticsOAuthService", { + make: Effect.gen(function* () { + const database = yield* Database + const env = yield* Env + const httpClient = yield* HttpClient.HttpClient + const oauth = yield* makeOAuthConnectionHelpers({ + provider: GOOGLE_ANALYTICS_PROVIDER, + providerLabel: "Google Analytics", + database, + env, + }) + + /** Best-effort token revocation on disconnect — failures are logged, never surfaced. */ + const revokeToken = (config: ResolvedGoogleOAuthConfig, token: string) => + oauth.postForm(config.revokeUrl, { token }).pipe(Effect.ignore) + + /** The connecting Google identity, for the card's "connected as" line. Never fatal. */ + const fetchUserInfo = (config: ResolvedGoogleOAuthConfig, accessToken: string) => + httpClient + .execute( + HttpClientRequest.get(config.userInfoUrl).pipe( + HttpClientRequest.setHeaders({ + authorization: `Bearer ${accessToken}`, + accept: "application/json", + }), + ), + ) + .pipe( + Effect.flatMap((response) => response.json), + Effect.flatMap(decodeUserInfo), + Effect.option, + ) + + const startConnect = Effect.fn("GoogleAnalyticsOAuthService.startConnect")(function* ( + orgId: OrgId, + userId: UserId, + options: { readonly callbackUrl: string; readonly returnTo?: string }, + ) { + yield* Effect.annotateCurrentSpan({ orgId }) + const config = yield* resolveConfig(env) + const state = randomBytes(24).toString("base64url") + const currentTime = yield* Clock.currentTimeMillis + + yield* oauth.purgeExpiredStates(currentTime) + yield* oauth.dbExecute((db) => + db.insert(oauthAuthStates).values({ + state, + orgId, + provider: GOOGLE_ANALYTICS_PROVIDER, + initiatedByUserId: userId, + redirectUri: options.callbackUrl, + returnTo: options.returnTo ?? null, + // Google web-application clients are confidential and authenticate the exchange + // with the client secret; PKCE is accepted but adds nothing here. + codeVerifier: null, + createdAt: msToDate(currentTime), + expiresAt: msToDate(currentTime + OAUTH_STATE_TTL_MS), + }), + ) + + // `access_type=offline` asks for a refresh token; `prompt=consent` forces Google to + // RE-issue one on a repeat authorization. Without the second, a user reconnecting an + // already-authorized app gets an access token only, and the poller dies within the hour. + // `include_granted_scopes` keeps any scopes the user previously granted this client. + const params = new URLSearchParams({ + client_id: config.clientId, + redirect_uri: options.callbackUrl, + response_type: "code", + scope: config.scopes, + state, + access_type: "offline", + prompt: "consent", + include_granted_scopes: "true", + }) + return { redirectUrl: `${config.authorizeUrl}?${params.toString()}`, state } + }) + + const completeConnect = Effect.fn("GoogleAnalyticsOAuthService.completeConnect")(function* ( + code: string, + state: string, + ) { + const config = yield* resolveConfig(env) + const stateRow = yield* oauth.requireStateRow(state) + yield* oauth.deleteAuthState(state) + + const tokenResponse = yield* oauth.exchangeAuthorizationCode(config, code, stateRow.redirectUri) + const orgId = decodeOrgId(stateRow.orgId) + yield* Effect.annotateCurrentSpan({ orgId }) + + // A background poller must renew indefinitely. A grant with no refresh token silently + // stops working at the access token's ~1h expiry; refuse it loudly instead of storing a + // doomed connection. Best-effort revoke first — the token is never persisted, so this is + // the only moment we can invalidate it upstream. + if (!tokenResponse.refresh_token) { + yield* Effect.logWarning( + "Google OAuth token exchange returned no refresh token — refusing connection", + { orgId }, + ) + yield* revokeToken(config, tokenResponse.access_token) + return yield* Effect.fail( + new IntegrationsValidationError({ + message: + "Google returned no refresh token, so this connection would stop working within the hour. Remove Maple from your Google account's third-party access and connect again.", + }), + ) + } + + // A grant that reaches no GA4 property is useless, and finding out at the first poll + // means an integration card that looks connected and never fills in. Same refusal shape + // as Cloudflare's zero-accounts guard. + const properties = yield* listProperties({ + accessToken: tokenResponse.access_token, + adminBaseUrl: env.MAPLE_GOOGLE_ANALYTICS_ADMIN_API_BASE_URL, + }) + if (properties.length === 0) { + yield* revokeToken(config, tokenResponse.access_token) + return yield* Effect.fail( + new IntegrationsValidationError({ + message: + "That Google account can't see any Google Analytics 4 properties. Connect an account with at least Viewer access to a GA4 property.", + }), + ) + } + + // Userinfo is a nicety, not a requirement: it names the connected account in the UI. + // An empty identity still yields a valid connection row (see the fallbacks below). + const identity = Option.getOrElse( + yield* fetchUserInfo(config, tokenResponse.access_token), + (): typeof UserInfo.Type => ({}), + ) + + const accessEnc = yield* oauth.encryptValue(tokenResponse.access_token) + const refreshEnc = yield* oauth.encryptValue(tokenResponse.refresh_token) + const currentTime = yield* Clock.currentTimeMillis + const expiresAt = + tokenResponse.expires_in != null ? currentTime + tokenResponse.expires_in * 1000 : null + + yield* oauth.upsertConnection(orgId, currentTime, { + // `sub` is Google's stable per-client user id. Falling back to the email keeps the + // NOT NULL column honest when userinfo is unavailable; it is a label, not a key. + externalUserId: identity.sub ?? identity.email ?? "google-analytics", + externalUserEmail: identity.email ?? null, + externalAccountName: properties[0]?.accountName ?? null, + grantedAccountsJson: null, + connectedByUserId: stateRow.initiatedByUserId, + scope: tokenResponse.scope ?? config.scopes, + accessTokenCiphertext: accessEnc.ciphertext, + accessTokenIv: accessEnc.iv, + accessTokenTag: accessEnc.tag, + refreshTokenCiphertext: refreshEnc.ciphertext, + refreshTokenIv: refreshEnc.iv, + refreshTokenTag: refreshEnc.tag, + expiresAt: msToDate(expiresAt), + }) + + return { orgId, returnTo: stateRow.returnTo ?? null } + }) + + const getValidAccessToken = Effect.fn("GoogleAnalyticsOAuthService.getValidAccessToken")( + function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + const config = yield* resolveConfig(env) + const { accessToken, row } = yield* oauth.getValidConnectionToken(config, orgId) + return { accessToken, scope: row.scope } + }, + ) + + const getStatus = Effect.fn("GoogleAnalyticsOAuthService.getStatus")(function* (orgId: OrgId) { + const row = yield* oauth.loadConnection(orgId) + if (!row) { + return { + connected: false, + connectedAt: null, + externalUserEmail: null, + connectedByUserId: null, + scope: "", + revoked: false, + } satisfies GoogleAnalyticsConnectionStatus + } + return { + connected: true, + connectedAt: dateToMs(row.createdAt), + externalUserEmail: row.externalUserEmail, + connectedByUserId: row.connectedByUserId, + scope: row.scope, + revoked: row.revokedAt != null, + } satisfies GoogleAnalyticsConnectionStatus + }) + + const disconnect = Effect.fn("GoogleAnalyticsOAuthService.disconnect")(function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + // Best-effort upstream revocation before the row goes; a failure here must never block + // the disconnect, because the deleted row is the real backstop. + const row = yield* oauth.loadConnection(orgId) + if (row) { + const config = yield* resolveConfig(env).pipe(Effect.option) + // Revoking the REFRESH token invalidates the whole grant at Google, access token + // included — revoking the access token alone would leave the refresh grant live. + const refreshToken = + row.refreshTokenCiphertext && row.refreshTokenIv && row.refreshTokenTag + ? yield* oauth + .decryptValue({ + ciphertext: row.refreshTokenCiphertext, + iv: row.refreshTokenIv, + tag: row.refreshTokenTag, + }) + .pipe(Effect.option) + : Option.none() + if (Option.isSome(config) && Option.isSome(refreshToken)) { + yield* revokeToken(config.value, refreshToken.value) + } + } + return yield* oauth.deleteConnection(orgId) + }) + + const markConnectionRevoked = Effect.fn("GoogleAnalyticsOAuthService.markConnectionRevoked")( + function* (orgId: OrgId) { + yield* oauth.markConnectionRevoked(orgId) + }, + ) + + return { + startConnect, + completeConnect, + getStatus, + getValidAccessToken, + disconnect, + markConnectionRevoked, + } satisfies GoogleAnalyticsOAuthServiceApi + }), +}) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(FetchHttpClient.layer)) +} diff --git a/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts b/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts index cd6e7e916..87433f7d0 100644 --- a/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts +++ b/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts @@ -28,7 +28,7 @@ import { CloudflareOAuthService } from "@/services/auth/CloudflareOAuthService" import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" import { OrgIngestKeysService } from "@/services/org/OrgIngestKeysService" import type { MetricGaugeRow, MetricSumRow } from "./cloudflare-analytics/mapping" -import type { OtlpMetricsPayload } from "./cloudflare-analytics/otlp" +import type { OtlpMetricsPayload } from "./shared/otlp" import { compiledQueryOf } from "@maple/query-engine/execution" const trackedDbs: TestDb[] = [] diff --git a/apps/api/src/services/integrations/CloudflareAnalyticsService.ts b/apps/api/src/services/integrations/CloudflareAnalyticsService.ts index 1a33cc1d5..3aa6ca1d3 100644 --- a/apps/api/src/services/integrations/CloudflareAnalyticsService.ts +++ b/apps/api/src/services/integrations/CloudflareAnalyticsService.ts @@ -97,7 +97,7 @@ import { mapWorkersGroups, type CloudflareMetricRows, } from "./cloudflare-analytics/mapping" -import { metricRowsToOtlp } from "./cloudflare-analytics/otlp" +import { metricRowsToOtlp } from "./shared/otlp" import { accountAnalyticsDocument, DatasetSettings, diff --git a/apps/api/src/services/integrations/GoogleAnalyticsApi.ts b/apps/api/src/services/integrations/GoogleAnalyticsApi.ts new file mode 100644 index 000000000..4eeeea3de --- /dev/null +++ b/apps/api/src/services/integrations/GoogleAnalyticsApi.ts @@ -0,0 +1,355 @@ +// BOUNDARY: This module owns unparsed external values and narrows them before domain use. +/** + * Thin wrapper over the two Google Analytics 4 REST surfaces the collector needs: + * + * - **Admin API** `accountSummaries.list` — property discovery. One call enumerates every account + * the grant covers and the properties under each, so there is no per-account fan-out. + * - **Data API** `properties/{id}:runReport` — the actual numbers, one call per (property, dataset). + * + * Unlike {@link CloudflareApi} this is NOT a lazy dynamic-import facade. That indirection exists + * purely because the distilled Cloudflare SDK is a ~2.4MB module graph; GA4 is plain REST over the + * ambient `HttpClient`, so there is nothing heavy to defer and the extra hop would only obscure. + * + * Error mapping is the contract the poll loop reads: + * - 401 alone is {@link IntegrationsRevokedError} — the grant is gone and the connection is + * stamped revoked. 403 deliberately is NOT: Google overloads it for per-property permission + * loss and for a disabled API, neither of which a reconnect fixes. + * - Quota denials (429, or `RESOURCE_EXHAUSTED`) keep `status` on + * {@link IntegrationsUpstreamError} so the caller can hold its lease through a backoff instead + * of re-depleting the property's token budget. + * - Everything else is a plain upstream failure: the watermark simply does not advance. + */ +import { IntegrationsRevokedError, IntegrationsUpstreamError } from "@maple/domain/http" +import { Effect, Option, Schema } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" + +/** + * Each call provides its own client rather than demanding one from the caller's context — the + * same shape `CloudflareApiImpl` uses, so these stay callable from a service closure without + * leaking `HttpClient` into its requirement channel. `FetchHttpClient.Fetch` is a + * `Context.Reference` with a default, so a test that overrides it further out still wins. + */ +const withHttpClient = (effect: Effect.Effect) => + effect.pipe( + // Not hoistable into the static service graph: these are leaf calls made from a service + // closure, and the layer closes over nothing per-invocation, so providing it here is what + // keeps `HttpClient` out of every caller's requirement channel. + // oxlint-disable-next-line effecttsgo/strict-effect-provide + Effect.provide(FetchHttpClient.layer), + ) + +/** GA4 quota denial — the caller backs off rather than retrying within the tick. */ +export const GA_QUOTA_STATUS = 429 + +const PropertySummary = Schema.Struct({ + // "properties/123456789" + property: Schema.String, + displayName: Schema.optionalKey(Schema.String), + propertyType: Schema.optionalKey(Schema.String), +}) + +const AccountSummary = Schema.Struct({ + displayName: Schema.optionalKey(Schema.String), + propertySummaries: Schema.optionalKey(Schema.Array(PropertySummary)), +}) + +const AccountSummariesResponse = Schema.Struct({ + accountSummaries: Schema.optionalKey(Schema.Array(AccountSummary)), + nextPageToken: Schema.optionalKey(Schema.String), +}) + +const MetricHeader = Schema.Struct({ + name: Schema.optionalKey(Schema.String), + type: Schema.optionalKey(Schema.String), +}) + +const ReportValue = Schema.Struct({ value: Schema.optionalKey(Schema.String) }) + +const ReportRow = Schema.Struct({ + dimensionValues: Schema.optionalKey(Schema.Array(ReportValue)), + metricValues: Schema.optionalKey(Schema.Array(ReportValue)), +}) + +/** + * `rows` is absent, not empty, when a report matches nothing — hence `optionalKey` throughout. + * `dimensionHeaders`/`metricHeaders` echo the request order, which is what lets the mapper pair a + * row's positional values back to names without trusting our own request-building twice. + */ +const RunReportResponse = Schema.Struct({ + dimensionHeaders: Schema.optionalKey(Schema.Array(Schema.Struct({ name: Schema.optionalKey(Schema.String) }))), + metricHeaders: Schema.optionalKey(Schema.Array(MetricHeader)), + rows: Schema.optionalKey(Schema.Array(ReportRow)), + rowCount: Schema.optionalKey(Schema.Number), +}) + +export type GoogleAnalyticsRunReportResponse = typeof RunReportResponse.Type + +const decodeAccountSummaries = Schema.decodeUnknownEffect(AccountSummariesResponse) +const decodeRunReport = Schema.decodeUnknownEffect(RunReportResponse) + +export interface GoogleAnalyticsProperty { + /** Bare id ("123456789"), with the API's "properties/" resource prefix stripped. */ + readonly propertyId: string + readonly propertyName: string | null + readonly accountName: string | null +} + +/** The Data API's own `runReport` request body — optional members are OMITTED, never `undefined`. */ +interface RunReportBody { + dimensions: Array<{ name: string }> + metrics: Array<{ name: string }> + dateRanges: Array<{ startDate: string; endDate: string }> + keepEmptyRows: boolean + limit?: string + offset?: string + orderBys?: Array<{ metric: { metricName: string }; desc: boolean }> + dimensionFilter?: unknown +} + +/** One `runReport` request, in the Data API's own vocabulary. */ +export interface RunReportRequest { + readonly dimensions: ReadonlyArray + readonly metrics: ReadonlyArray + /** Inclusive `YYYY-MM-DD` bounds, in the property's configured reporting timezone. */ + readonly startDate: string + readonly endDate: string + readonly limit?: number + /** Row offset, for walking a report whose match count exceeds `limit`. */ + readonly offset?: number + /** Descending order-by on this metric — how a breakdown dataset takes its top N. */ + readonly orderByMetric?: string + /** Sent verbatim as the Data API's `dimensionFilter`. */ + readonly dimensionFilter?: unknown +} + +/** Constructor payload for {@link IntegrationsUpstreamError}, whose extras are `optionalKey`. */ +interface UpstreamErrorFields { + message: string + status?: number + cause?: unknown +} + +const upstream = (message: string, status?: number, cause?: unknown) => { + // `status` and `cause` are `optionalKey` on the error, so an explicit `undefined` is not the + // same as omission — assigned only when present. + const fields: UpstreamErrorFields = { message } + if (status !== undefined) fields.status = status + if (cause !== undefined) fields.cause = cause + return new IntegrationsUpstreamError(fields) +} + +/** + * Google's error envelope: `{ error: { code, status, message } }`. `status` is the symbolic + * enum ("PERMISSION_DENIED", "RESOURCE_EXHAUSTED"), which is what distinguishes a dead grant + * from a quota denial — both arrive as HTTP 403. + */ +const GoogleErrorEnvelope = Schema.Struct({ + error: Schema.optionalKey(Schema.Struct({ status: Schema.optionalKey(Schema.String) })), +}) +const decodeErrorEnvelope = Schema.decodeUnknownOption(Schema.fromJsonString(GoogleErrorEnvelope)) + +const errorStatusOf = (text: string): string | null => + Option.match(decodeErrorEnvelope(text), { + // Google does not promise this envelope on every failure path (a gateway 502 is plain + // HTML), so an undecodable body just means "no symbolic status", not a bug. + onNone: () => null, + onSome: (envelope) => envelope.error?.status ?? null, + }) + +const classifyFailure = (httpStatus: number, text: string, label: string) => { + const symbolic = errorStatusOf(text) + const snippet = text.slice(0, 300) + // Only a 401 means the GRANT is dead, and revoking is drastic: it stops collection for every + // property the org has and puts "Reconnect needed" on the card. + // + // 403 must NOT be treated that way, even though it is the shape a dead grant can also take. + // Google overloads it for conditions that say nothing about the credential: + // `PERMISSION_DENIED` when the account lost access to ONE property, `SERVICE_DISABLED` when + // the Data API is not enabled on the Cloud project, `RESOURCE_EXHAUSTED` for quota. Revoking + // on any of those would disconnect a whole org because one property was reshared, or because + // of a project setting no reconnect can fix. They stay per-window failures: the window is + // recorded and retried, and the other properties keep collecting. + if (httpStatus === 401) { + return new IntegrationsRevokedError({ + message: `Google Analytics ${label} rejected the stored grant (401${symbolic ? ` ${symbolic}` : ""}) — reconnect required`, + }) + } + if (httpStatus === 429 || symbolic === "RESOURCE_EXHAUSTED") { + return upstream( + `Google Analytics ${label} quota exhausted (${httpStatus}${symbolic ? ` ${symbolic}` : ""})`, + GA_QUOTA_STATUS, + ) + } + return upstream(`Google Analytics ${label} returned ${httpStatus}: ${snippet}`, httpStatus) +} + +const authorized = (request: HttpClientRequest.HttpClientRequest, accessToken: string) => + request.pipe( + HttpClientRequest.setHeaders({ + authorization: `Bearer ${accessToken}`, + accept: "application/json", + }), + ) + +/** + * Every GA4 property the grant can see, across every account it covers. Paginated: the Admin API + * caps `pageSize` at 200 and a single agency grant can exceed that, so the loop is not optional. + */ +export const listProperties = Effect.fn("GoogleAnalyticsApi.listProperties")(function* (options: { + readonly accessToken: string + readonly adminBaseUrl: string + /** Safety stop so a pathological `nextPageToken` cycle cannot spin the tick. */ + readonly maxPages?: number +}) { + const httpClient = yield* HttpClient.HttpClient + const properties: Array = [] + let pageToken: string | undefined + const maxPages = options.maxPages ?? 10 + + for (let page = 0; page < maxPages; page++) { + const url = new URL(`${options.adminBaseUrl.replace(/\/+$/, "")}/accountSummaries`) + url.searchParams.set("pageSize", "200") + if (pageToken !== undefined) url.searchParams.set("pageToken", pageToken) + + const response = yield* httpClient + .execute(authorized(HttpClientRequest.get(url.toString()), options.accessToken)) + .pipe( + Effect.annotateSpans("peer.service", "google-analytics-admin"), + Effect.catchTag("HttpClientError", (error) => + Effect.fail(upstream(`Google Analytics admin request failed: ${error.message}`, undefined, error)), + ), + ) + + if (response.status >= 300) { + const text = yield* response.text.pipe(Effect.orElseSucceed(() => "")) + return yield* Effect.fail(classifyFailure(response.status, text, "Admin API")) + } + + const json = yield* response.json.pipe( + Effect.mapError(() => upstream("Google Analytics Admin API returned a non-JSON response")), + ) + const decoded = yield* decodeAccountSummaries(json).pipe( + Effect.mapError(() => upstream("Google Analytics Admin API returned an unexpected payload")), + ) + + for (const account of decoded.accountSummaries ?? []) { + for (const summary of account.propertySummaries ?? []) { + // "properties/123456789" → "123456789". A name that does not carry the prefix is + // not a property resource we understand, so it is skipped rather than guessed at. + const propertyId = summary.property.startsWith("properties/") + ? summary.property.slice("properties/".length) + : null + if (propertyId === null || propertyId === "") continue + properties.push({ + propertyId, + propertyName: summary.displayName ?? null, + accountName: account.displayName ?? null, + }) + } + } + + pageToken = decoded.nextPageToken + if (pageToken === undefined || pageToken === "") break + } + + return properties +}, withHttpClient) + +const PropertyDetail = Schema.Struct({ + displayName: Schema.optionalKey(Schema.String), + timeZone: Schema.optionalKey(Schema.String), +}) +const decodePropertyDetail = Schema.decodeUnknownEffect(PropertyDetail) + +/** + * A property's IANA reporting timezone. Not carried by `accountSummaries`, and required before a + * property can be polled at all — `dateHour` is expressed in it — so this is fetched once per + * property and cached on the state row rather than per discovery pass. + */ +export const getPropertyTimeZone = Effect.fn("GoogleAnalyticsApi.getPropertyTimeZone")( + function* (options: { + readonly accessToken: string + readonly adminBaseUrl: string + readonly propertyId: string + }) { + const httpClient = yield* HttpClient.HttpClient + const url = `${options.adminBaseUrl.replace(/\/+$/, "")}/properties/${options.propertyId}` + const response = yield* httpClient + .execute(authorized(HttpClientRequest.get(url), options.accessToken)) + .pipe( + Effect.annotateSpans("peer.service", "google-analytics-admin"), + Effect.catchTag("HttpClientError", (error) => + Effect.fail(upstream(`Google Analytics admin request failed: ${error.message}`, undefined, error)), + ), + ) + + if (response.status >= 300) { + const text = yield* response.text.pipe(Effect.orElseSucceed(() => "")) + return yield* Effect.fail(classifyFailure(response.status, text, "Admin API")) + } + + const json = yield* response.json.pipe( + Effect.mapError(() => upstream("Google Analytics Admin API returned a non-JSON response")), + ) + const detail = yield* decodePropertyDetail(json).pipe( + Effect.mapError(() => upstream("Google Analytics Admin API returned an unexpected payload")), + ) + return { timeZone: detail.timeZone ?? null, displayName: detail.displayName ?? null } + }, + withHttpClient, +) + +/** One Data API `runReport` against a single property. */ +export const runReport = Effect.fn("GoogleAnalyticsApi.runReport")(function* (options: { + readonly accessToken: string + readonly dataBaseUrl: string + readonly propertyId: string + readonly request: RunReportRequest +}) { + const httpClient = yield* HttpClient.HttpClient + const { request } = options + // Google's own "(other)" bucket silently replaces the tail once a report exceeds its + // cardinality limit. Asking for the totals row would not tell us it happened, so instead the + // caller caps with `limit` and folds its own explicit remainder — see the mapper. + const body: RunReportBody = { + dimensions: request.dimensions.map((name) => ({ name })), + metrics: request.metrics.map((name) => ({ name })), + dateRanges: [{ startDate: request.startDate, endDate: request.endDate }], + keepEmptyRows: false, + } + // Omitted rather than sent as `undefined`: the Data API rejects a null `limit`. + if (request.limit !== undefined) body.limit = String(request.limit) + if (request.offset !== undefined) body.offset = String(request.offset) + if (request.orderByMetric !== undefined) { + body.orderBys = [{ metric: { metricName: request.orderByMetric }, desc: true }] + } + if (request.dimensionFilter !== undefined) body.dimensionFilter = request.dimensionFilter + + const url = `${options.dataBaseUrl.replace(/\/+$/, "")}/properties/${options.propertyId}:runReport` + const response = yield* httpClient + .execute( + authorized(HttpClientRequest.post(url), options.accessToken).pipe( + HttpClientRequest.bodyJsonUnsafe(body), + ), + ) + .pipe( + Effect.annotateSpans("peer.service", "google-analytics-data"), + Effect.catchTag("HttpClientError", (error) => + Effect.fail(upstream(`Google Analytics data request failed: ${error.message}`, undefined, error)), + ), + ) + + if (response.status >= 300) { + const text = yield* response.text.pipe(Effect.orElseSucceed(() => "")) + return yield* Effect.fail(classifyFailure(response.status, text, "Data API")) + } + + const json = yield* response.json.pipe( + Effect.mapError(() => upstream("Google Analytics Data API returned a non-JSON response")), + ) + return yield* decodeRunReport(json).pipe( + Effect.mapError(() => upstream("Google Analytics Data API returned an unexpected payload")), + ) +}, withHttpClient) + +export type GoogleAnalyticsApiError = IntegrationsUpstreamError | IntegrationsRevokedError diff --git a/apps/api/src/services/integrations/GoogleAnalyticsService.test.ts b/apps/api/src/services/integrations/GoogleAnalyticsService.test.ts new file mode 100644 index 000000000..ea49bbc1d --- /dev/null +++ b/apps/api/src/services/integrations/GoogleAnalyticsService.test.ts @@ -0,0 +1,482 @@ +// SAFETY-FILE: JSON in this test is emitted by the fixture or unit under test before its fields are asserted. +import { afterEach, assert, describe, it } from "@effect/vitest" +import { OrgId } from "@maple/domain/http" +import { googleAnalyticsLedger, googleAnalyticsState, oauthConnections } from "@maple/db" +import { ConfigProvider, Effect, Layer, Schema } from "effect" +import { TestClock } from "effect/testing" +import { FetchHttpClient } from "effect/unstable/http" +import { eq } from "drizzle-orm" +import { encryptAes256Gcm, parseBase64Aes256GcmKey } from "@/platform/Crypto" +import { Database } from "@/platform/DatabaseLive" +import { Env } from "@/platform/Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { GoogleAnalyticsOAuthService } from "@/services/auth/GoogleAnalyticsOAuthService" +import { OrgIngestKeysService } from "@/services/org/OrgIngestKeysService" +import { GoogleAnalyticsService } from "./GoogleAnalyticsService" +import type { OtlpMetricsPayload } from "./shared/otlp" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const asOrgId = Schema.decodeUnknownSync(OrgId) +const ORG = asOrgId("org_ga") +const PROPERTY_ID = "123456789" + +const ENCRYPTION_KEY_B64 = Buffer.alloc(32, 7).toString("base64") + +const baseConfig = { + PORT: "3472", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: ENCRYPTION_KEY_B64, + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + MAPLE_INGEST_PUBLIC_URL: "https://ingest.example.com", + GOOGLE_OAUTH_CLIENT_ID: "google-client-id", + GOOGLE_OAUTH_CLIENT_SECRET: "google-client-secret", +} + +/** Sessions GA4 reports for a given `dateHour`, per call. Successive polls read successive entries. */ +interface FetchOptions { + /** One entry per `runReport` call: dateHour → sessions. */ + readonly reports: ReadonlyArray> + readonly timeZone?: string + /** Force every Data API call to fail with this HTTP status. */ + readonly dataApiStatus?: number + readonly dataApiBody?: string + otlpCalls: Array + reportCalls: Array<{ propertyId: string; body: unknown }> + /** How many times the Admin API was asked for a property's timezone. */ + propertyLookups: number +} + +/** T0's property-local date — see {@link T0}. Declared here because the fetch mock needs it. */ +const T0_DATE = "2026-09-09" + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + +/** + * `HttpClientRequest.bodyJsonUnsafe` does not hand `fetch` a string, so `String(init.body)` yields + * "[object Object]" and every JSON.parse in the mock throws — which surfaces as an upstream + * failure on the real code path rather than as an obviously broken test. + */ +const readBody = async (init: RequestInit | undefined): Promise => { + if (init?.body == null) return "" + return typeof init.body === "string" ? init.body : await new Response(init.body as BodyInit).text() +} + +/** + * Stands in for Google's two APIs plus the ingest gateway. Only the `traffic` dataset returns + * rows; the other five report nothing, which keeps assertions about emitted values unambiguous. + */ +const mockGoogleFetch = (options: FetchOptions): typeof globalThis.fetch => { + let reportIndex = 0 + return (async (input: string | URL | Request, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url + + if (url.includes("/v1/metrics")) { + options.otlpCalls.push(JSON.parse(await readBody(init)) as OtlpMetricsPayload) + return json({ ok: true }) + } + + if (url.includes("/accountSummaries")) { + return json({ + accountSummaries: [ + { + displayName: "Acme", + propertySummaries: [{ property: `properties/${PROPERTY_ID}`, displayName: "example.com" }], + }, + ], + }) + } + + if (url.includes(":runReport")) { + if (options.dataApiStatus !== undefined) { + return new Response(options.dataApiBody ?? "{}", { status: options.dataApiStatus }) + } + const body = JSON.parse(await readBody(init)) as { + dimensions: Array<{ name: string }> + metrics: Array<{ name: string }> + dateRanges: Array<{ startDate: string; endDate: string }> + } + const propertyId = url.split("/properties/")[1]?.split(":")[0] ?? "" + options.reportCalls.push({ propertyId, body }) + + // Breakdown datasets report nothing — only `traffic` (dateHour alone) carries values. + if (body.dimensions.length !== 1) return json({ rows: [] }) + // Each poll asks twice for `traffic`: the live head window and one backfill round. + // Only the head window covers T0's hour, so the backfill legitimately reports nothing — + // and, more to the point, must not consume a fixture entry, or "one entry per poll" + // would silently become "one per call" and the revision assertions would drift. + if (body.dateRanges[0]?.endDate !== T0_DATE) return json({ rows: [] }) + + const report = options.reports[Math.min(reportIndex, options.reports.length - 1)] + reportIndex += 1 + const metricNames = body.metrics.map((metric) => metric.name) + return json({ + dimensionHeaders: [{ name: "dateHour" }], + metricHeaders: metricNames.map((name) => ({ name })), + rows: [...(report ?? new Map())].map(([dateHour, sessions]) => ({ + dimensionValues: [{ value: dateHour }], + // Only `sessions` carries a value; the rest report zero. + metricValues: metricNames.map((name) => ({ value: name === "sessions" ? String(sessions) : "0" })), + })), + }) + } + + // properties/{id} — the timezone lookup. + if (url.includes("/properties/")) { + options.propertyLookups += 1 + return json({ displayName: "example.com", timeZone: options.timeZone ?? "UTC" }) + } + + return json({ error: { message: `unexpected url ${url}` } }, 500) + }) as typeof globalThis.fetch +} + +const makeLayer = (testDb: TestDb, fetchOptions: FetchOptions) => + GoogleAnalyticsService.layer.pipe( + Layer.provideMerge(GoogleAnalyticsOAuthService.layer), + Layer.provideMerge(OrgIngestKeysService.layer), + Layer.provideMerge(testDb.layer), + Layer.provideMerge(Env.layer), + Layer.provideMerge(ConfigProvider.layer(ConfigProvider.fromUnknown(baseConfig))), + Layer.provideMerge(Layer.succeed(FetchHttpClient.Fetch, mockGoogleFetch(fetchOptions))), + ) + +/** A live grant with a real encrypted, non-expiring access token. */ +const seedConnection = Effect.gen(function* () { + const database = yield* Database + const key = yield* parseBase64Aes256GcmKey(ENCRYPTION_KEY_B64, (message) => new Error(message)) + const accessEnc = yield* encryptAes256Gcm("ga-access-token", key, (message) => new Error(message)) + const refreshEnc = yield* encryptAes256Gcm("ga-refresh-token", key, (message) => new Error(message)) + yield* database.execute((db) => + db.insert(oauthConnections).values({ + id: "conn-ga", + orgId: ORG, + provider: "google_analytics", + externalUserId: "google-sub", + externalUserEmail: "owner@example.com", + externalAccountName: "Acme", + connectedByUserId: "user_1", + scope: "https://www.googleapis.com/auth/analytics.readonly", + accessTokenCiphertext: accessEnc.ciphertext, + accessTokenIv: accessEnc.iv, + accessTokenTag: accessEnc.tag, + refreshTokenCiphertext: refreshEnc.ciphertext, + refreshTokenIv: refreshEnc.iv, + refreshTokenTag: refreshEnc.tag, + // Far future: the refresh path is exercised by the shared OAuth helper's own tests. + // Derived from T0, not the real clock: the service reads token expiry through the + // Effect clock, which is pinned to T0, so a wall-clock value would read as expired + // once real time moved a day past it and silently divert into the refresh path. + expiresAt: new Date(T0 + 86_400_000), + createdAt: new Date(T0), + updatedAt: new Date(T0), + }), + ) +}) + +/** Every `google_analytics.sessions` data point across all OTLP batches, in order. */ +const sessionPoints = (calls: ReadonlyArray) => + calls.flatMap((payload) => + payload.resourceMetrics.flatMap((resource) => + resource.scopeMetrics.flatMap((scope) => + scope.metrics + .filter((metric) => metric.name === "google_analytics.sessions") + .flatMap((metric) => metric.sum?.dataPoints ?? []), + ), + ), + ) + +/** + * Fixed test wall-clock, and the `dateHour` that matches it. + * + * `it.effect` runs on a TestClock that starts at epoch 0, so every window the collector computes + * would sit in 1970 and no fixture hour would ever fall inside it — the reconciliation assertions + * would pass vacuously against an empty result. Each test sets the clock to T0 first, and the + * fixture reports for T0's own hour. + */ +const T0 = Date.parse("2026-09-09T14:00:00Z") +const HOUR = "2026090914" + +const options = (reports: ReadonlyArray>, extra: Partial = {}) => + ({ reports, otlpCalls: [], reportCalls: [], propertyLookups: 0, ...extra }) satisfies FetchOptions + +describe("GoogleAnalyticsService", () => { + it.effect("discovers properties and creates a state row per dataset", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map([[HOUR, 10]])]) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + + const database = yield* Database + const rows = yield* database.execute((db) => + db.select().from(googleAnalyticsState).where(eq(googleAnalyticsState.orgId, ORG)), + ) + // Six datasets plus the discovery anchor. + assert.strictEqual(rows.length, 7) + const anchor = rows.find((row) => row.dataset === "__discovery__") + assert.isNotNull(anchor?.discoveredAt ?? null) + const traffic = rows.find((row) => row.dataset === "traffic") + assert.strictEqual(traffic?.propertyName, "example.com") + assert.strictEqual(traffic?.accountName, "Acme") + assert.strictEqual(traffic?.timeZone, "UTC") + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("emits the raw value first, then only the delta on a revision", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + // Second poll revises the same hour upward; third revises it back down. + const fetchOptions = options([ + new Map([[HOUR, 100]]), + new Map([[HOUR, 140]]), + new Map([[HOUR, 60]]), + ]) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + yield* service.pollOrg(ORG) + yield* service.pollOrg(ORG) + + const points = sessionPoints(fetchOptions.otlpCalls) + assert.deepStrictEqual( + points.map((point) => point.asDouble), + [100, 40, -80], + ) + // The whole point: the deltas sum to GA4's latest answer for the bucket. + assert.strictEqual( + points.reduce((total, point) => total + point.asDouble, 0), + 60, + ) + // ...and every one of them is a DELTA sum at the same instant. + const timestamps = new Set(points.map((point) => point.timeUnixNano)) + assert.strictEqual(timestamps.size, 1) + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("writes nothing when a re-poll finds no change", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map([[HOUR, 100]]), new Map([[HOUR, 100]])]) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + const afterFirst = fetchOptions.otlpCalls.length + yield* service.pollOrg(ORG) + + assert.isAbove(afterFirst, 0) + assert.strictEqual(sessionPoints(fetchOptions.otlpCalls).length, 1) + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("records what it emitted in the ledger, scoped to the property and dataset", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map([[HOUR, 100]])]) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + + const database = yield* Database + const ledger = yield* database.execute((db) => + db.select().from(googleAnalyticsLedger).where(eq(googleAnalyticsLedger.orgId, ORG)), + ) + // Only `traffic` returned rows, so only it has a ledger entry: the five breakdown + // datasets emitted nothing and must not accrue empty rows. + assert.strictEqual(ledger.length, 1) + assert.strictEqual(ledger[0]?.dataset, "traffic") + assert.strictEqual(ledger[0]?.propertyId, PROPERTY_ID) + const emitted = JSON.parse(ledger[0]?.emittedJson ?? "{}") as Record + assert.strictEqual(emitted["google_analytics.sessions"], 100) + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("records the failure and keeps the frontier when the Data API errors", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map()], { + dataApiStatus: 500, + dataApiBody: '{"error":{"status":"INTERNAL"}}', + }) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + const result = yield* service.pollOrg(ORG) + + assert.isAbove(result.failures, 0) + const database = yield* Database + const rows = yield* database.execute((db) => + db.select().from(googleAnalyticsState).where(eq(googleAnalyticsState.orgId, ORG)), + ) + const traffic = rows.find((row) => row.dataset === "traffic") + // Nothing landed, so the head frontier must not have moved. + assert.isNull(traffic?.watermarkAt ?? null) + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("stamps the connection revoked when Google rejects the grant", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map()], { + dataApiStatus: 401, + dataApiBody: '{"error":{"status":"UNAUTHENTICATED"}}', + }) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + + const database = yield* Database + const rows = yield* database.execute((db) => + db.select().from(oauthConnections).where(eq(oauthConnections.orgId, ORG)), + ) + assert.isNotNull(rows[0]?.revokedAt ?? null) + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("holds the lease through a quota backoff instead of clearing it", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map()], { + dataApiStatus: 429, + dataApiBody: '{"error":{"status":"RESOURCE_EXHAUSTED"}}', + }) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + + const database = yield* Database + const rows = yield* database.execute((db) => + db.select().from(googleAnalyticsState).where(eq(googleAnalyticsState.orgId, ORG)), + ) + // Compared against T0, not `new Date()`: `leaseUntil` is derived from the TestClock, + // so a real-clock comparison would start failing on a date rather than on a change. + const leased = rows.filter((row) => row.leaseUntil != null && row.leaseUntil.getTime() > T0) + // The lease must still be in the future — clearing it would let the next tick spend + // the rest of the org's GA4 budget immediately. + assert.isAbove(leased.length, 0) + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("skips a property the user disabled", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map([[HOUR, 10]])]) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + yield* service.setPropertyEnabled(ORG, PROPERTY_ID, false) + + const before = fetchOptions.reportCalls.length + yield* service.pollOrg(ORG) + assert.strictEqual(fetchOptions.reportCalls.length, before) + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("reports per-property status for the integration card", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map([[HOUR, 10]])]) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + + const status = yield* service.getIntegrationStatus(ORG) + assert.isTrue(status.connected) + assert.strictEqual(status.externalUserEmail, "owner@example.com") + // One entry per PROPERTY, not per dataset row. + assert.strictEqual(status.properties.length, 1) + assert.strictEqual(status.properties[0]?.propertyId, PROPERTY_ID) + assert.strictEqual(status.properties[0]?.timeZone, "UTC") + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("resolves a property's timezone once per tick, not once per dataset", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map([[HOUR, 10]])]) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + + // `rows` is snapshotted before the loop and `patchRow` writes the database, not the + // snapshot — without the per-tick memo each of the property's six dataset rows + // would miss and re-resolve. + assert.strictEqual(fetchOptions.propertyLookups, 1) + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) + + it.effect("does not re-emit a bucket after a disconnect and reconnect", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const fetchOptions = options([new Map([[HOUR, 100]]), new Map([[HOUR, 100]])]) + + yield* Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection + const service = yield* GoogleAnalyticsService + yield* service.pollOrg(ORG) + + // Disconnect drops the grant but deliberately leaves the ledger: metrics already + // collected are retained, so re-emitting this hour on reconnect would double it. + const database = yield* Database + yield* database.execute((db) => + db.delete(oauthConnections).where(eq(oauthConnections.orgId, ORG)), + ) + yield* seedConnection + yield* service.pollOrg(ORG) + + const points = sessionPoints(fetchOptions.otlpCalls) + assert.deepStrictEqual( + points.map((point) => point.asDouble), + [100], + ) + }).pipe(Effect.provide(makeLayer(testDb, fetchOptions))) + }), + ) +}) diff --git a/apps/api/src/services/integrations/GoogleAnalyticsService.ts b/apps/api/src/services/integrations/GoogleAnalyticsService.ts new file mode 100644 index 000000000..6b3190907 --- /dev/null +++ b/apps/api/src/services/integrations/GoogleAnalyticsService.ts @@ -0,0 +1,1039 @@ +// BOUNDARY: This module owns unparsed external values and narrows them before domain use. +/** + * Google Analytics 4 collector. + * + * Polls the GA4 Data API for every org with a connected Google account and writes the results + * into the regular OTel metrics pipeline (`metrics_sum`), so the metric explorer, dashboard + * builder and alerting all work on web-analytics data with zero new query paths — the same trade + * that makes {@link CloudflareAnalyticsService} cheap. + * + * The datasets are described by the {@link DATASETS} registry; one generic poll pipeline drives + * them all, one Data API call per (property, dataset, window). + * + * State lives in `google_analytics_state` (one row per org × property × dataset, plus a discovery + * anchor row per org) and `google_analytics_ledger`. Three frontiers, not Cloudflare's two: + * + * - `watermarkAt` — HEAD: end of the newest hour ingested. + * - `backfillAt` — history, walking down toward {@link BACKFILL_FLOOR_MS}. + * - `frozenThroughAt` — the one GA4 forces on us. GA4 revises `dateHour` for ~48h, so an hour is + * only final once it falls behind this line. Everything between here and HEAD is re-polled and + * emitted as a DELTA against the ledger (see `reconcile.ts`); everything behind it is frozen and + * its ledger rows are pruned. + * + * Delivery is at-least-once in the same narrow sense as the Cloudflare poller: a crash between the + * gateway accepting a batch and the ledger write landing re-emits that window's deltas next tick. + * Unlike Cloudflare's replay, this one self-heals — the next successful reconcile diffs against + * the last PERSISTED ledger, so the bucket converges on GA4's answer rather than drifting. + */ +import { + IntegrationsPersistenceError, + IntegrationsRevokedError, + IntegrationsUpstreamError, + UserId as UserIdSchema, + type OrgId, +} from "@maple/domain/http" +import { + googleAnalyticsLedger, + googleAnalyticsState, + oauthConnections, + type GoogleAnalyticsStateRow, +} from "@maple/db" +import { and, eq, gte, inArray, isNull, lt, or, sql } from "drizzle-orm" +import { Clock, Context, Effect, Layer, Schema } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" +import { Database } from "@/platform/DatabaseLive" +import { makeDbExecute, makePersistenceErrorMapper } from "@/platform/db-execute" +import { Env } from "@/platform/Env" +import { msToDate } from "@/platform/time" +import type { MetricSumRow } from "@/services/warehouse/metric-rows" +import { GoogleAnalyticsOAuthService } from "@/services/auth/GoogleAnalyticsOAuthService" +import { OrgIngestKeysService } from "@/services/org/OrgIngestKeysService" +import { + getPropertyTimeZone, + listProperties, + runReport, + GA_QUOTA_STATUS, + type GoogleAnalyticsProperty, +} from "./GoogleAnalyticsApi" +import { + DATASETS, + DISCOVERY_DATASET, + DISCOVERY_PROPERTY_ID, + type GaDatasetDef, +} from "./google-analytics/datasets" +import { mapReport } from "./google-analytics/mapping" +import { type LedgerBucket, parseLedger, reconcile, serializeLedger } from "./google-analytics/reconcile" +import { utcMsToZonedDate } from "./google-analytics/timezone" +import { metricRowsToOtlp } from "./shared/otlp" + +const HOUR_MS = 3_600_000 + +/** + * How long GA4 may keep revising an hour. Google documents "up to 48 hours" for full processing + * of a standard property; everything inside this window is re-polled and reconciled. + */ +const RESTATEMENT_WINDOW_MS = 48 * HOUR_MS + +/** + * Re-polled on EVERY tick. Fresh data changes fastest, so the cheap frequent pass covers only the + * recent tail; the expensive full-window sweep runs on {@link FULL_RECONCILE_INTERVAL_MS}. + */ +const HEAD_WINDOW_MS = 3 * HOUR_MS + +/** How often the whole 48h restatement window is swept. Hourly is well inside GA4's revision pace. */ +const FULL_RECONCILE_INTERVAL_MS = HOUR_MS + +/** How far back a newly connected property backfills. */ +const BACKFILL_FLOOR_MS = 30 * 24 * HOUR_MS + +/** One backfill round per dataset per tick, so history fills in behind live data. */ +const BACKFILL_ROUND_MS = 24 * HOUR_MS + +/** + * Lease held while a tick polls an org. Longer than the 15-minute cron interval would stall + * recovery after a crash; shorter than a slow tick would let two ticks overlap. + */ +const LEASE_MS = 10 * 60_000 + +/** + * Held (not cleared) after a quota rejection so the next tick skips the org instead of + * re-depleting the property's GA4 token budget — the lesson the Cloudflare poller learned the + * expensive way. + */ +const QUOTA_BACKOFF_MS = 30 * 60_000 + +/** + * Ceiling on Data API calls per org per tick. GA4 meters tokens per property per day and per + * hour, and a grant may cover many properties; the budget is what stops one agency-sized org from + * spending its whole hourly allowance in a single tick. + */ +const MAX_CALLS_PER_ORG_TICK = 30 + +const ORG_CONCURRENCY = 3 + +/** Property discovery TTL. Properties are created rarely; hourly is generous. */ +const DISCOVERY_TTL_MS = HOUR_MS + +/** GA4 caps a report at 250k rows; 10k is ample per page and keeps responses small. */ +const REPORT_ROW_LIMIT = 10_000 + +/** + * Page ceiling per (property, dataset, window). A report needing more than this is not one we can + * reconcile honestly, so the window fails rather than half-landing — see `pollWindow`. + */ +const MAX_REPORT_PAGES = 5 + +/** Attribution for the ingest key this collector mints on an org's behalf. */ +const SYSTEM_USER_ID = Schema.decodeUnknownSync(UserIdSchema)("system-google-analytics") + +const floorToHour = (ms: number) => Math.floor(ms / HOUR_MS) * HOUR_MS + +/** + * The tick's remaining Data API allowance, charged per REQUEST rather than per window. + * + * Mutable and threaded into `pollWindow` on purpose. A window is no longer one request now that + * reports paginate, so charging it once would let 30 "charged" windows issue up to 150 requests + * and quietly overrun the ceiling. Returning a page count from `pollWindow` would not fix it + * either: `recoverWindow` turns a non-fatal failure into `null` after its requests have already + * been spent. + */ +interface CallBudget { + calls: number +} + +export interface GoogleAnalyticsPropertyStatus { + readonly propertyId: string + readonly propertyName: string | null + readonly accountName: string | null + readonly timeZone: string | null + readonly enabled: boolean + readonly lastSyncedAt: number | null + readonly lastError: string | null + readonly watermarkAt: number | null + readonly backfillAt: number | null +} + +export interface GoogleAnalyticsIntegrationStatus { + readonly connected: boolean + readonly connectedAt: number | null + readonly externalUserEmail: string | null + readonly revoked: boolean + readonly properties: ReadonlyArray +} + +export interface GoogleAnalyticsPollResult { + readonly properties: number + readonly rowsIngested: number + readonly skipped: number + readonly failures: number +} + +interface GoogleAnalyticsServiceApi { + /** Cron entry point: every org with a live grant, bounded concurrency. */ + readonly pollAllOrgs: () => Effect.Effect + /** One org, used by the cron fan-out and by the post-connect prime. */ + readonly pollOrg: (orgId: OrgId) => Effect.Effect + readonly getIntegrationStatus: ( + orgId: OrgId, + ) => Effect.Effect + readonly setPropertyEnabled: ( + orgId: OrgId, + propertyId: string, + enabled: boolean, + ) => Effect.Effect +} + +/** + * There is deliberately no `resetOrgState`, and a disconnect leaves both tables alone. + * + * Metrics already collected are RETAINED when an org disconnects, so the ledger is the only record + * of what has been emitted for the hours still inside the restatement window. Dropping it and then + * reconnecting within 48h would re-emit each of those hours in full on top of rows already in the + * warehouse — the exact double-count the ledger exists to prevent. Dropping the state rows is worse + * again: it resets `backfillAt`, so the 30-day backfill re-runs over hours whose ledger entries have + * already been pruned, and every one of them double-counts with nothing left to reconcile against. + * + * Reconnecting with a different Google account needs no cleanup either — discovery soft-disables + * properties the new grant cannot see, and their ledger rows age out of the restatement window. + * Org DELETION is the case where these rows genuinely should go, and that is handled deliberately + * by the registry in `OrganizationService` rather than here. + */ + +export class GoogleAnalyticsService extends Context.Service< + GoogleAnalyticsService, + GoogleAnalyticsServiceApi +>()("@maple/api/services/GoogleAnalyticsService", { + make: Effect.gen(function* () { + const database = yield* Database + const env = yield* Env + const oauth = yield* GoogleAnalyticsOAuthService + const ingestKeys = yield* OrgIngestKeysService + const httpClient = yield* HttpClient.HttpClient + + const toPersistenceError = makePersistenceErrorMapper( + IntegrationsPersistenceError, + "Google Analytics integration storage is unavailable", + ) + const dbExecute = makeDbExecute(database, "GoogleAnalyticsService", toPersistenceError) + + const adminBaseUrl = env.MAPLE_GOOGLE_ANALYTICS_ADMIN_API_BASE_URL + const dataBaseUrl = env.MAPLE_GOOGLE_ANALYTICS_DATA_API_BASE_URL + const ingestMetricsUrl = `${env.MAPLE_INGEST_PUBLIC_URL.replace(/\/+$/, "")}/v1/metrics` + + // ── State access ────────────────────────────────────────────────────── + + const loadRows = (orgId: OrgId) => + dbExecute((db) => + db.select().from(googleAnalyticsState).where(eq(googleAnalyticsState.orgId, orgId)), + ) + + const rowId = (orgId: OrgId, propertyId: string, dataset: string) => + `${orgId}:${propertyId}:${dataset}` + + const upsertRow = (row: { + readonly orgId: OrgId + readonly propertyId: string + readonly dataset: string + readonly propertyName: string | null + readonly accountName: string | null + readonly now: number + }) => + dbExecute((db) => + db + .insert(googleAnalyticsState) + .values({ + id: rowId(row.orgId, row.propertyId, row.dataset), + orgId: row.orgId, + propertyId: row.propertyId, + dataset: row.dataset, + propertyName: row.propertyName, + accountName: row.accountName, + createdAt: msToDate(row.now), + updatedAt: msToDate(row.now), + }) + .onConflictDoUpdate({ + target: [ + googleAnalyticsState.orgId, + googleAnalyticsState.propertyId, + googleAnalyticsState.dataset, + ], + // Names refresh on every discovery, but `enabled` is deliberately NOT reset: + // a property the user switched off must stay off across discovery passes. + set: { + propertyName: row.propertyName, + accountName: row.accountName, + updatedAt: msToDate(row.now), + }, + }), + ) + + const patchRow = (id: string, patch: Partial) => + dbExecute((db) => db.update(googleAnalyticsState).set(patch).where(eq(googleAnalyticsState.id, id))) + + /** + * Claim the org for this tick. One conditional UPDATE across the org's rows: whoever moves + * `leaseUntil` past now owns the tick, and a competing tick reads zero updated rows and + * skips. Returns false when the org is already claimed. + */ + const claimLease = Effect.fn("GoogleAnalyticsService.claimLease")(function* ( + orgId: OrgId, + now: number, + ) { + const claimed = yield* dbExecute((db) => + db + .update(googleAnalyticsState) + .set({ leaseUntil: msToDate(now + LEASE_MS), updatedAt: msToDate(now) }) + .where( + and( + eq(googleAnalyticsState.orgId, orgId), + or(isNull(googleAnalyticsState.leaseUntil), lt(googleAnalyticsState.leaseUntil, msToDate(now))), + ), + ) + .returning({ id: googleAnalyticsState.id }), + ) + return claimed.length > 0 + }) + + const releaseLease = (orgId: OrgId, until: number | null, now: number) => + dbExecute((db) => + db + .update(googleAnalyticsState) + .set({ leaseUntil: until === null ? null : msToDate(until), updatedAt: msToDate(now) }) + .where(eq(googleAnalyticsState.orgId, orgId)), + ) + + const recordOrgError = (orgId: OrgId, message: string, now: number) => + dbExecute((db) => + db + .update(googleAnalyticsState) + .set({ + lastError: message.slice(0, 500), + lastErrorAt: msToDate(now), + updatedAt: msToDate(now), + }) + .where(eq(googleAnalyticsState.orgId, orgId)), + ) + + // ── Ledger ──────────────────────────────────────────────────────────── + + const loadLedger = (orgId: OrgId, propertyId: string, dataset: string, fromMs: number) => + dbExecute((db) => + db + .select() + .from(googleAnalyticsLedger) + .where( + and( + eq(googleAnalyticsLedger.orgId, orgId), + eq(googleAnalyticsLedger.propertyId, propertyId), + eq(googleAnalyticsLedger.dataset, dataset), + gte(googleAnalyticsLedger.bucketAt, msToDate(fromMs)), + ), + ), + ).pipe( + Effect.map((rows): ReadonlyArray => + rows.map((row) => ({ + bucketMs: row.bucketAt.getTime(), + emitted: parseLedger(row.emittedJson), + })), + ), + ) + + const saveLedger = Effect.fn("GoogleAnalyticsService.saveLedger")(function* (options: { + readonly orgId: OrgId + readonly propertyId: string + readonly dataset: string + readonly buckets: ReadonlyArray + readonly now: number + }) { + // Buckets that ended up with nothing emitted carry no information and are deleted + // rather than stored as `{}` — otherwise a quiet property accretes a row per hour. + const empty = options.buckets.filter((bucket) => Object.keys(bucket.emitted).length === 0) + const populated = options.buckets.filter((bucket) => Object.keys(bucket.emitted).length > 0) + + if (populated.length > 0) { + yield* dbExecute((db) => + db + .insert(googleAnalyticsLedger) + .values( + populated.map((bucket) => ({ + id: `${options.orgId}:${options.propertyId}:${options.dataset}:${bucket.bucketMs}`, + orgId: options.orgId, + propertyId: options.propertyId, + dataset: options.dataset, + bucketAt: msToDate(bucket.bucketMs), + emittedJson: serializeLedger(bucket.emitted), + createdAt: msToDate(options.now), + updatedAt: msToDate(options.now), + })), + ) + .onConflictDoUpdate({ + target: [ + googleAnalyticsLedger.orgId, + googleAnalyticsLedger.propertyId, + googleAnalyticsLedger.dataset, + googleAnalyticsLedger.bucketAt, + ], + // `excluded` is the row this statement tried to insert — the multi-row upsert + // needs each conflict to take ITS OWN new blob, not one value for all. + set: { + emittedJson: sql`excluded.emitted_json`, + updatedAt: msToDate(options.now), + }, + }), + ) + } + + if (empty.length > 0) { + yield* dbExecute((db) => + db.delete(googleAnalyticsLedger).where( + and( + eq(googleAnalyticsLedger.orgId, options.orgId), + eq(googleAnalyticsLedger.propertyId, options.propertyId), + eq(googleAnalyticsLedger.dataset, options.dataset), + inArray( + googleAnalyticsLedger.bucketAt, + empty.map((bucket) => msToDate(bucket.bucketMs)), + ), + ), + ), + ) + } + }) + + /** Drop ledger rows for hours GA4 can no longer revise — they are settled by definition. */ + const pruneLedger = (orgId: OrgId, propertyId: string, dataset: string, frozenThroughMs: number) => + dbExecute((db) => + db + .delete(googleAnalyticsLedger) + .where( + and( + eq(googleAnalyticsLedger.orgId, orgId), + eq(googleAnalyticsLedger.propertyId, propertyId), + eq(googleAnalyticsLedger.dataset, dataset), + lt(googleAnalyticsLedger.bucketAt, msToDate(frozenThroughMs)), + ), + ), + ) + + // ── Ingest ──────────────────────────────────────────────────────────── + + const getOrgIngestKey = (orgId: OrgId) => + ingestKeys.getOrCreate(orgId, SYSTEM_USER_ID).pipe(Effect.map((keys) => keys.publicKey)) + + /** + * Ship reconciled deltas to the ingest gateway as one OTLP/JSON request, so per-org routing + * (managed Tinybird vs BYO ClickHouse), schema-version gating, WAL durability and Autumn + * metering all apply exactly as they do for the org's own telemetry. + */ + const emitMetrics = Effect.fn("GoogleAnalyticsService.emitMetrics")( + function* (ingestKey: string, rows: ReadonlyArray) { + if (rows.length === 0) return 0 + const request = HttpClientRequest.post(ingestMetricsUrl, { + headers: { authorization: `Bearer ${ingestKey}`, "content-type": "application/json" }, + }).pipe(HttpClientRequest.bodyJsonUnsafe(metricRowsToOtlp(rows, []))) + const response = yield* httpClient + .execute(request) + .pipe(Effect.annotateSpans("peer.service", "ingest")) + if (response.status >= 300) { + const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")) + return yield* Effect.fail( + new IntegrationsUpstreamError({ + message: `Google Analytics metrics ingest returned ${response.status}: ${body.slice(0, 300)}`, + status: response.status, + }), + ) + } + return rows.length + }, + (effect) => + effect.pipe( + Effect.mapError((error) => + error instanceof IntegrationsUpstreamError + ? error + : new IntegrationsUpstreamError({ + message: `Google Analytics metrics ingest request failed: ${error instanceof Error ? error.message : String(error)}`, + cause: error, + }), + ), + ), + ) + + // ── Poll ────────────────────────────────────────────────────────────── + + /** + * One (property, dataset, window): fetch, map, reconcile against the ledger, ingest, then + * persist. The ledger is written only AFTER the gateway accepts the batch — the other order + * would record deltas as emitted that never landed, and those are unrecoverable. + */ + const pollWindow = Effect.fn("GoogleAnalyticsService.pollWindow")(function* (context: { + readonly orgId: OrgId + readonly row: GoogleAnalyticsStateRow + readonly dataset: GaDatasetDef + readonly accessToken: string + readonly ingestKey: string + readonly timeZone: string + readonly fromMs: number + readonly toMs: number + readonly now: number + readonly budget: CallBudget + }) { + const { dataset, fromMs, toMs, timeZone } = context + const startDate = utcMsToZonedDate(fromMs, timeZone) + // GA4 date ranges are inclusive on both ends and expressed in property-local dates, so + // the end date is the local date of the last covered instant, not of the exclusive bound. + const endDate = utcMsToZonedDate(toMs - 1, timeZone) + if (startDate === null || endDate === null) return 0 + + // Paged to COMPLETION, and that is a correctness requirement rather than a nicety. + // Reconciliation reads a series that is in the ledger but absent from the response as + // "revised to zero" and retracts it. A half-fetched report would therefore retract real + // data — and flap, re-emitting it next tick as the truncation point moved. `dateHour` × + // `pagePath` over a 48h window exceeds one page on any busy site, so this is reachable, + // not theoretical. + context.budget.calls += 1 + let response = yield* runReport({ + accessToken: context.accessToken, + dataBaseUrl, + propertyId: context.row.propertyId, + request: { + dimensions: dataset.breakdown + ? ["dateHour", dataset.breakdown.dimension] + : ["dateHour"], + metrics: dataset.metrics.map((metric) => metric.ga), + startDate, + endDate, + limit: REPORT_ROW_LIMIT, + // Only a breakdown has a tail to rank; the totals dataset returns one row per hour. + orderByMetric: dataset.breakdown?.rankBy, + }, + }) + const merged = [...(response.rows ?? [])] + const total = response.rowCount ?? merged.length + for ( + let page = 1; + merged.length < total && + page < MAX_REPORT_PAGES && + context.budget.calls < MAX_CALLS_PER_ORG_TICK; + page++ + ) { + context.budget.calls += 1 + const next = yield* runReport({ + accessToken: context.accessToken, + dataBaseUrl, + propertyId: context.row.propertyId, + request: { + dimensions: dataset.breakdown + ? ["dateHour", dataset.breakdown.dimension] + : ["dateHour"], + metrics: dataset.metrics.map((metric) => metric.ga), + startDate, + endDate, + limit: REPORT_ROW_LIMIT, + offset: merged.length, + orderByMetric: dataset.breakdown?.rankBy, + }, + }) + const nextRows = next.rows ?? [] + // A page that returns nothing while `rowCount` still claims more would spin the + // loop; stop and let the incompleteness check below decide. + if (nextRows.length === 0) break + merged.push(...nextRows) + } + + // Still short — because the page ceiling was hit, or because the tick ran out of + // budget mid-report. Either way emit nothing rather than retract series the report + // simply did not reach. The frontier does not advance, so the window is retried next + // tick with a fresh budget. + if (merged.length < total) { + return yield* Effect.fail( + new IntegrationsUpstreamError({ + message: `Google Analytics report for ${dataset.id} returned ${merged.length} of ${total} rows — refusing to reconcile a partial window`, + }), + ) + } + response = { ...response, rows: merged } + + // The date range is whole property-local DAYS, so it necessarily overreaches the + // requested hour window at both ends. Reconciliation must only judge the hours actually + // asked about, hence the explicit covered bounds rather than the response's own extent. + const points = mapReport({ dataset, response, timeZone }) + const ledger = yield* loadLedger(context.orgId, context.row.propertyId, dataset.id, fromMs) + const result = reconcile({ + orgId: context.orgId, + propertyId: context.row.propertyId, + propertyName: context.row.propertyName, + accountName: context.row.accountName, + dataset, + points, + ledger, + coveredFromMs: fromMs, + coveredToMs: toMs, + }) + + const ingested = yield* emitMetrics(context.ingestKey, result.rows) + yield* saveLedger({ + orgId: context.orgId, + propertyId: context.row.propertyId, + dataset: dataset.id, + buckets: result.ledger.filter( + (bucket) => bucket.bucketMs >= fromMs && bucket.bucketMs < toMs, + ), + now: context.now, + }) + return ingested + }) + + /** + * Property discovery, on an hourly TTL. Rows for properties that vanished are soft-disabled + * rather than deleted, so a property that comes back resumes from its old watermark instead + * of re-backfilling a month of history. + */ + const discoverProperties = Effect.fn("GoogleAnalyticsService.discoverProperties")(function* ( + orgId: OrgId, + accessToken: string, + existing: ReadonlyArray, + now: number, + ) { + const properties = yield* listProperties({ accessToken, adminBaseUrl }) + const live = new Set(properties.map((property) => property.propertyId)) + + for (const property of properties) { + for (const dataset of DATASETS) { + yield* upsertRow({ + orgId, + propertyId: property.propertyId, + dataset: dataset.id, + propertyName: property.propertyName, + accountName: property.accountName, + now, + }) + } + } + + const vanished = existing.filter( + (row) => + row.dataset !== DISCOVERY_DATASET && row.enabled && !live.has(row.propertyId), + ) + for (const row of vanished) { + yield* patchRow(row.id, { enabled: false, updatedAt: msToDate(now) }) + } + + yield* dbExecute((db) => + db + .insert(googleAnalyticsState) + .values({ + id: rowId(orgId, DISCOVERY_PROPERTY_ID, DISCOVERY_DATASET), + orgId, + propertyId: DISCOVERY_PROPERTY_ID, + dataset: DISCOVERY_DATASET, + discoveredAt: msToDate(now), + createdAt: msToDate(now), + updatedAt: msToDate(now), + }) + .onConflictDoUpdate({ + target: [ + googleAnalyticsState.orgId, + googleAnalyticsState.propertyId, + googleAnalyticsState.dataset, + ], + set: { discoveredAt: msToDate(now), updatedAt: msToDate(now) }, + }), + ) + + return properties + }) + + /** + * Resolve and cache a property's reporting timezone. Without it nothing can be polled. + * + * `resolved` is the per-tick memo, and it is what makes this once per PROPERTY rather than + * once per property × dataset. `rows` is a snapshot taken before the loop and `patchRow` + * writes the database, not the snapshot — so without the memo every one of a property's six + * dataset rows would miss, and a newly connected property would spend six Admin API calls + * and thirty-six row writes resolving one timezone. The memo also holds a null, so a + * property whose zone cannot be resolved is asked about once per tick, not six times. + */ + const ensureTimeZone = Effect.fn("GoogleAnalyticsService.ensureTimeZone")(function* ( + rows: ReadonlyArray, + resolved: Map, + accessToken: string, + propertyId: string, + now: number, + ) { + const memoized = resolved.get(propertyId) + if (memoized !== undefined) return memoized + const known = rows.find((row) => row.propertyId === propertyId && row.timeZone != null) + if (known?.timeZone != null) { + resolved.set(propertyId, known.timeZone) + return known.timeZone + } + const detail = yield* getPropertyTimeZone({ accessToken, adminBaseUrl, propertyId }) + resolved.set(propertyId, detail.timeZone) + if (detail.timeZone === null) return null + for (const row of rows.filter((candidate) => candidate.propertyId === propertyId)) { + yield* patchRow(row.id, { timeZone: detail.timeZone, updatedAt: msToDate(now) }) + } + return detail.timeZone + }) + + /** + * Which failures end the whole tick rather than just this window. + * + * A dead grant and an exhausted quota are properties of the CONNECTION, not of one report: + * grinding through the remaining (property, dataset) pairs would produce the identical + * failure each time, and in the quota case would spend the org's remaining GA4 budget doing + * it. Everything else is local — a malformed report for one dataset says nothing about the + * next — so it is recorded and the loop continues with that window's frontier untouched. + */ + const isConnectionFatal = (error: unknown) => + error instanceof IntegrationsRevokedError || + (error instanceof IntegrationsUpstreamError && error.status === GA_QUOTA_STATUS) + + /** + * Per-window recovery: re-raise the connection-fatal failures so `pollOrgSafely` can stamp + * the grant or start a quota backoff, and turn everything else into `null` after recording + * it on the row. Swallowing all of them — the obvious `Effect.option` — is what would let a + * dead grant retry forever, silently, on every tick. + */ + const recoverWindow = + (stateRowId: string, now: number) => + ( + effect: Effect.Effect, + ) => + effect.pipe( + Effect.catch((error) => + isConnectionFatal(error) + ? Effect.fail(error) + : patchRow(stateRowId, { + lastError: String(error instanceof Error ? error.message : error).slice(0, 500), + lastErrorAt: msToDate(now), + updatedAt: msToDate(now), + }).pipe( + Effect.ignore, + Effect.as(null), + ), + ), + ) + + const pollOrg = Effect.fn("GoogleAnalyticsService.pollOrg")(function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + const now = yield* Clock.currentTimeMillis + let rowsIngested = 0 + let failures = 0 + let skipped = 0 + const seenProperties = new Set() + + const claimed = yield* claimLease(orgId, now) + // A brand-new connection has no rows yet, so nothing to claim — discovery below creates + // them. Only an org that HAS rows and failed to claim is genuinely busy. + const existingBefore = yield* loadRows(orgId) + if (!claimed && existingBefore.length > 0) { + return { properties: 0, rowsIngested: 0, skipped: 1, failures: 0 } + } + + const { accessToken } = yield* oauth.getValidAccessToken(orgId) + const ingestKey = yield* getOrgIngestKey(orgId) + + const anchor = existingBefore.find((row) => row.dataset === DISCOVERY_DATASET) + const discoveryDue = + anchor?.discoveredAt == null || now - anchor.discoveredAt.getTime() >= DISCOVERY_TTL_MS + let properties: ReadonlyArray = [] + if (discoveryDue) { + properties = yield* discoverProperties(orgId, accessToken, existingBefore, now) + } + + const rows = yield* loadRows(orgId) + const pollable = rows.filter((row) => row.dataset !== DISCOVERY_DATASET && row.enabled) + + /** Per-tick timezone memo — see `ensureTimeZone`. */ + const timeZones = new Map() + + // Budget is shared across the org's properties: GA4 meters per property, but a grant + // covering 200 of them would still blow through a tick's wall-clock and the gateway's + // patience without a ceiling here. Charged per REQUEST — see `CallBudget`. + const budget: CallBudget = { calls: 0 } + + for (const row of pollable) { + if (budget.calls >= MAX_CALLS_PER_ORG_TICK) { + skipped += 1 + continue + } + const dataset = DATASETS.find((candidate) => candidate.id === row.dataset) + if (dataset === undefined) continue + + // Recovered per property, not per tick. A property the account lost access to answers + // this lookup with a 403, which is a non-fatal upstream error — but it is raised + // outside `recoverWindow`, so left unhandled it would abort `pollOrg` and skip every + // property after it. One reshared property must not stop the rest from collecting. + const timeZone = yield* ensureTimeZone( + rows, + timeZones, + accessToken, + row.propertyId, + now, + ).pipe( + Effect.catch((error) => { + if (isConnectionFatal(error)) return Effect.fail(error) + timeZones.set(row.propertyId, null) + return patchRow(row.id, { + lastError: String(error instanceof Error ? error.message : error).slice(0, 500), + lastErrorAt: msToDate(now), + updatedAt: msToDate(now), + }).pipe(Effect.ignore, Effect.as(null)) + }), + ) + if (timeZone === null) { + skipped += 1 + continue + } + seenProperties.add(row.propertyId) + + const frozenThrough = floorToHour(now - RESTATEMENT_WINDOW_MS) + const head = floorToHour(now) + HOUR_MS + const fullSweepDue = + row.lastSuccessAt == null || now - row.lastSuccessAt.getTime() >= FULL_RECONCILE_INTERVAL_MS + const from = Math.max(frozenThrough, head - (fullSweepDue ? RESTATEMENT_WINDOW_MS : HEAD_WINDOW_MS)) + + const ingested = yield* pollWindow({ + orgId, + row, + dataset, + accessToken, + ingestKey, + timeZone, + fromMs: from, + toMs: head, + now, + budget, + }).pipe(recoverWindow(row.id, now)) + + if (ingested === null) { + failures += 1 + continue + } + rowsIngested += ingested + + yield* patchRow(row.id, { + watermarkAt: msToDate(head), + frozenThroughAt: msToDate(frozenThrough), + backfillAt: row.backfillAt ?? msToDate(from), + lastSuccessAt: msToDate(now), + lastError: null, + lastErrorAt: null, + updatedAt: msToDate(now), + }) + yield* pruneLedger(orgId, row.propertyId, dataset.id, frozenThrough) + + // History fills in behind live data, one round per dataset per tick, and only from + // hours GA4 can no longer revise — a backfilled hour needs no ledger entry. + const backfillTo = row.backfillAt?.getTime() ?? from + const floor = floorToHour(now - BACKFILL_FLOOR_MS) + if (backfillTo > floor && budget.calls < MAX_CALLS_PER_ORG_TICK) { + const backfillFrom = Math.max(floor, backfillTo - BACKFILL_ROUND_MS) + const backfilled = yield* pollWindow({ + orgId, + row, + dataset, + accessToken, + ingestKey, + timeZone, + fromMs: backfillFrom, + toMs: backfillTo, + now, + budget, + }).pipe(recoverWindow(row.id, now)) + if (backfilled !== null) { + rowsIngested += backfilled + yield* patchRow(row.id, { + backfillAt: msToDate(backfillFrom), + updatedAt: msToDate(now), + }) + } + } + } + + return { + properties: seenProperties.size || properties.length, + rowsIngested, + skipped, + failures, + } + }) + + /** + * Wrap one org's poll with the outcomes the cron cares about. A revoked grant stamps the + * connection and stops; a quota rejection holds the lease through a backoff; anything else + * is recorded and retried next tick with the frontiers untouched. + */ + const FAILED_TICK: GoogleAnalyticsPollResult = { + properties: 0, + rowsIngested: 0, + skipped: 0, + failures: 1, + } + + const pollOrgSafely = (orgId: OrgId): Effect.Effect => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + // Branch on the error VALUE, not on `catchTags`: these failures carry namespaced tags + // ("@maple/http/errors/IntegrationsRevokedError"), so a tag-keyed catch silently + // matches nothing and every case falls through to the generic handler. + return yield* pollOrg(orgId).pipe( + Effect.catch((error) => + Effect.gen(function* () { + if (error instanceof IntegrationsRevokedError) { + yield* oauth.markConnectionRevoked(orgId) + yield* recordOrgError(orgId, error.message, now).pipe(Effect.ignore) + return FAILED_TICK + } + if (error instanceof IntegrationsUpstreamError) { + yield* recordOrgError(orgId, error.message, now).pipe(Effect.ignore) + if (error.status === GA_QUOTA_STATUS) { + // Hold the lease rather than clearing it, so the next tick skips this + // org instead of spending the rest of its GA4 budget. + yield* releaseLease(orgId, now + QUOTA_BACKOFF_MS, now).pipe(Effect.ignore) + } + return FAILED_TICK + } + yield* recordOrgError(orgId, String(error), now).pipe(Effect.ignore) + return FAILED_TICK + }), + ), + Effect.catchCause((cause) => + Effect.logError("Google Analytics poll failed", cause).pipe(Effect.as(FAILED_TICK)), + ), + ) + }).pipe( + // The lease is only cleared on a clean finish. A quota backoff has already set its + // own future lease above, and `ensuring` must not undo it. + Effect.tap((result) => + result.failures === 0 + ? Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => releaseLease(orgId, null, now)), + Effect.ignore, + ) + : Effect.void, + ), + ) + + const pollAllOrgs = Effect.fn("GoogleAnalyticsService.pollAllOrgs")(function* () { + const connections = yield* dbExecute((db) => + db + .select({ orgId: oauthConnections.orgId }) + .from(oauthConnections) + .where( + and( + eq(oauthConnections.provider, "google_analytics"), + isNull(oauthConnections.revokedAt), + ), + ), + ).pipe(Effect.orElseSucceed(() => [] as ReadonlyArray<{ orgId: OrgId }>)) + + const results = yield* Effect.forEach(connections, (connection) => pollOrgSafely(connection.orgId), { + concurrency: ORG_CONCURRENCY, + }) + + return results.reduce( + (total, result) => ({ + properties: total.properties + result.properties, + rowsIngested: total.rowsIngested + result.rowsIngested, + skipped: total.skipped + result.skipped, + failures: total.failures + result.failures, + }), + { properties: 0, rowsIngested: 0, skipped: 0, failures: 0 }, + ) + }) + + // ── Read surface ────────────────────────────────────────────────────── + + const getIntegrationStatus = Effect.fn("GoogleAnalyticsService.getIntegrationStatus")( + function* (orgId: OrgId) { + // A status read must never 500 because the connection row is unreadable: the card + // then shows "not connected", which is the honest answer from the caller's side. + const connection = yield* oauth.getStatus(orgId).pipe( + Effect.orElseSucceed(() => ({ + connected: false, + connectedAt: null, + externalUserEmail: null, + connectedByUserId: null, + scope: "", + revoked: false, + })), + ) + const rows = yield* loadRows(orgId) + + // One status entry per PROPERTY, not per row: the datasets are an implementation + // detail, so the property's health is the worst of its datasets and its progress the + // least advanced of them. + const byProperty = new Map>() + for (const row of rows) { + if (row.dataset === DISCOVERY_DATASET) continue + const group = byProperty.get(row.propertyId) + if (group === undefined) byProperty.set(row.propertyId, [row]) + else group.push(row) + } + + const properties = [...byProperty.entries()] + .map(([propertyId, group]): GoogleAnalyticsPropertyStatus => { + const withError = group.find((row) => row.lastError != null) + const oldest = (pick: (row: GoogleAnalyticsStateRow) => Date | null) => + group + .map(pick) + .filter((value): value is Date => value != null) + .reduce( + (min, value) => (min === null ? value.getTime() : Math.min(min, value.getTime())), + null, + ) + return { + propertyId, + propertyName: group[0]?.propertyName ?? null, + accountName: group[0]?.accountName ?? null, + timeZone: group[0]?.timeZone ?? null, + enabled: group.some((row) => row.enabled), + lastSyncedAt: oldest((row) => row.lastSuccessAt), + lastError: withError?.lastError ?? null, + watermarkAt: oldest((row) => row.watermarkAt), + backfillAt: oldest((row) => row.backfillAt), + } + }) + .sort((a, b) => a.propertyId.localeCompare(b.propertyId)) + + return { + connected: connection.connected, + connectedAt: connection.connectedAt, + externalUserEmail: connection.externalUserEmail, + revoked: connection.revoked, + properties, + } satisfies GoogleAnalyticsIntegrationStatus + }, + ) + + const setPropertyEnabled = Effect.fn("GoogleAnalyticsService.setPropertyEnabled")(function* ( + orgId: OrgId, + propertyId: string, + enabled: boolean, + ) { + const now = yield* Clock.currentTimeMillis + yield* dbExecute((db) => + db + .update(googleAnalyticsState) + .set({ enabled, updatedAt: msToDate(now) }) + .where( + and( + eq(googleAnalyticsState.orgId, orgId), + eq(googleAnalyticsState.propertyId, propertyId), + ), + ), + ) + }) + + return { + pollAllOrgs, + pollOrg: (orgId: OrgId) => pollOrgSafely(orgId), + getIntegrationStatus, + setPropertyEnabled, + } satisfies GoogleAnalyticsServiceApi + }), +}) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(FetchHttpClient.layer)) +} diff --git a/apps/api/src/services/integrations/cloudflare-analytics/mapping.ts b/apps/api/src/services/integrations/cloudflare-analytics/mapping.ts index 25ed264ce..8c7666408 100644 --- a/apps/api/src/services/integrations/cloudflare-analytics/mapping.ts +++ b/apps/api/src/services/integrations/cloudflare-analytics/mapping.ts @@ -14,6 +14,7 @@ * sampling-adjusted by Cloudflare, so they pass through untouched. */ import { fmtMetricTs, type MetricGaugeRow, type MetricSumRow } from "@/services/warehouse/metric-rows" +import { foldTail, OTHER_BUCKET, topNKeys } from "../shared/cardinality" import type { DnsGroupDefinition, DurableObjectsGroupDefinition, @@ -139,16 +140,6 @@ export const MAX_DNS_QUERY_NAMES = 20 export const MAX_HTTP_PATHS = 50 /** Countries are naturally bounded (~250); the cap is a safety net, not a design constraint. */ export const MAX_COUNTRIES = 50 -export const OTHER_BUCKET = "other" - -/** Top-N keys by weight; ties break lexicographically so folding is deterministic across runs. */ -export const topNKeys = (weights: ReadonlyMap, n: number): ReadonlySet => - new Set( - [...weights.entries()] - .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) - .slice(0, n) - .map(([key]) => key), - ) const httpResourceAttrs = (orgId: string, zoneId: string, zoneName: string): Attrs => ({ maple_org_id: orgId, @@ -265,12 +256,6 @@ export const mapHttpGroups = (input: MapHttpGroupsInput): CloudflareMetricRows = return { sumRows, gaugeRows } } -/** Weight-rank an unbounded dimension across a window and fold the tail into {@link OTHER_BUCKET}. */ -const foldTail = (weights: ReadonlyMap, n: number): ((key: string) => string) => { - const top = topNKeys(weights, n) - return (key) => (top.has(key) ? key : OTHER_BUCKET) -} - /** Cap stored path length so one pathological URL can't bloat the attribute map. */ const MAX_PATH_LEN = 200 diff --git a/apps/api/src/services/integrations/google-analytics/datasets.ts b/apps/api/src/services/integrations/google-analytics/datasets.ts new file mode 100644 index 000000000..e38071094 --- /dev/null +++ b/apps/api/src/services/integrations/google-analytics/datasets.ts @@ -0,0 +1,223 @@ +/** + * The GA4 report registry. One entry per Data API `runReport` call the collector makes per + * property per tick; one generic poll pipeline drives them all. + * + * Metric naming follows the Cloudflare collector's convention, and the `.by_*` suffix on + * breakdowns is load-bearing rather than cosmetic. `channels`, `geo` and `device` each report the + * SAME underlying total sliced a different way, so if they all wrote `google_analytics.sessions` + * a chart with no group-by would silently show four times the real session count. The suffix + * keeps each slice its own metric, exactly as `cloudflare.http.requests.by_country` does. + * + * Every value is emitted as a DELTA-temporality sum against the reconciliation ledger — see + * `reconcile.ts` for why, and chart them with `sum`, never `rate`/`increase`. + */ + +/** Instrumentation scope for every row this collector writes. */ +export const SCOPE_NAME = "@maple/google-analytics" + +/** Shared metric-name prefix. Dashboard-template readiness gates on exactly this string. */ +export const METRIC_PREFIX = "google_analytics." + +export interface GaMetricDef { + /** The Data API metric name, sent verbatim in the request. */ + readonly ga: string + /** The Maple metric name written to `metrics_sum`. */ + readonly metric: string + readonly unit: string + readonly description: string +} + +export interface GaBreakdown { + /** The Data API dimension name, sent alongside `dateHour`. */ + readonly dimension: string + /** Attribute key on the emitted metric row. */ + readonly attributeKey: string + /** + * Cardinality cap. `Attributes` sits in the metrics tables' sorting key, so an unbounded + * dimension degrades every read of the metric, not just this breakdown. The tail folds into + * one `other` series so the parts still sum to the unbroken total. + */ + readonly maxValues: number + /** Metric the tail-fold ranks by — the "heaviest" values are the ones worth keeping. */ + readonly rankBy: string + /** Truncation for a single pathological value (a 4KB URL). */ + readonly maxValueLength?: number +} + +export interface GaDatasetDef { + readonly id: string + /** Null for the unbroken totals dataset. */ + readonly breakdown: GaBreakdown | null + readonly metrics: ReadonlyArray +} + +const SESSIONS: GaMetricDef = { + ga: "sessions", + metric: "google_analytics.sessions", + unit: "{sessions}", + description: "Sessions", +} + +/** + * Unbroken totals. The headline numbers, and the only dataset whose metrics carry no breakdown + * attribute — which is what makes them safe to chart without a group-by. + */ +const trafficDataset: GaDatasetDef = { + id: "traffic", + breakdown: null, + metrics: [ + SESSIONS, + { + ga: "activeUsers", + metric: "google_analytics.active_users", + unit: "{users}", + description: "Active users", + }, + { + ga: "newUsers", + metric: "google_analytics.new_users", + unit: "{users}", + description: "First-time users", + }, + { + ga: "screenPageViews", + metric: "google_analytics.page_views", + unit: "{page_views}", + description: "Page and screen views", + }, + { + ga: "engagedSessions", + metric: "google_analytics.engaged_sessions", + unit: "{sessions}", + description: "Sessions that lasted over 10s, had a key event, or had 2+ page views", + }, + { + ga: "userEngagementDuration", + metric: "google_analytics.engagement_duration", + unit: "s", + description: "Total time the site was in the foreground", + }, + ], +} + +/** Where traffic came from — GA4's own channel grouping, not a re-derivation of source/medium. */ +const channelsDataset: GaDatasetDef = { + id: "channels", + breakdown: { + dimension: "sessionDefaultChannelGroup", + attributeKey: "google_analytics.channel_group", + // GA4's default channel grouping is a closed set of ~17 values; the cap is a safety net. + maxValues: 30, + rankBy: "sessions", + }, + metrics: [ + { ...SESSIONS, metric: "google_analytics.sessions.by_channel" }, + { + ga: "activeUsers", + metric: "google_analytics.active_users.by_channel", + unit: "{users}", + description: "Active users by acquisition channel", + }, + ], +} + +const geoDataset: GaDatasetDef = { + id: "geo", + breakdown: { + // `countryId`, not `country`: the former is ISO 3166-1 alpha-2, which matches the + // `geo.country_iso_code` semconv key and the Cloudflare collector's spelling. `country` + // returns localized display names, which would split "United States" per viewer locale. + dimension: "countryId", + attributeKey: "geo.country_iso_code", + maxValues: 50, + rankBy: "sessions", + }, + metrics: [{ ...SESSIONS, metric: "google_analytics.sessions.by_country" }], +} + +const deviceDataset: GaDatasetDef = { + id: "device", + breakdown: { + dimension: "deviceCategory", + attributeKey: "google_analytics.device_category", + // desktop / mobile / tablet / smart tv — bounded by GA4 itself. + maxValues: 10, + rankBy: "sessions", + }, + metrics: [{ ...SESSIONS, metric: "google_analytics.sessions.by_device" }], +} + +const pagesDataset: GaDatasetDef = { + id: "pages", + breakdown: { + dimension: "pagePath", + attributeKey: "url.path", + // The highest-cardinality dimension here by a wide margin, and the most valuable — hence a + // higher cap than the others rather than the same one. + maxValues: 100, + rankBy: "screenPageViews", + maxValueLength: 200, + }, + metrics: [ + { + ga: "screenPageViews", + metric: "google_analytics.page_views.by_page", + unit: "{page_views}", + description: "Page views by path", + }, + { + ga: "activeUsers", + metric: "google_analytics.active_users.by_page", + unit: "{users}", + description: "Active users by path", + }, + ], +} + +const eventsDataset: GaDatasetDef = { + id: "events", + breakdown: { + dimension: "eventName", + attributeKey: "google_analytics.event_name", + maxValues: 100, + rankBy: "eventCount", + maxValueLength: 100, + }, + metrics: [ + { + ga: "eventCount", + metric: "google_analytics.event_count.by_event", + unit: "{events}", + description: "Events by name", + }, + { + ga: "keyEvents", + metric: "google_analytics.key_events.by_event", + unit: "{events}", + description: "Key events (conversions) by name", + }, + ], +} + +export const DATASETS: ReadonlyArray = [ + trafficDataset, + channelsDataset, + geoDataset, + deviceDataset, + pagesDataset, + eventsDataset, +] + +export const DATASET_BY_ID: ReadonlyMap = new Map( + DATASETS.map((dataset) => [dataset.id, dataset]), +) + +/** + * Reserved state rows, which are not datasets and must never be polled as one. + * The discovery anchor holds grant-wide `discoveredAt`; see the schema's DISCOVERY ANCHOR note. + */ +export const DISCOVERY_DATASET = "__discovery__" +export const DISCOVERY_PROPERTY_ID = "" + +/** Service name for a property's rows — mirrors Cloudflare's `cloudflare/{zoneName}`. */ +export const serviceNameFor = (propertyId: string): string => `google-analytics/${propertyId}` diff --git a/apps/api/src/services/integrations/google-analytics/mapping.ts b/apps/api/src/services/integrations/google-analytics/mapping.ts new file mode 100644 index 000000000..b53440d6e --- /dev/null +++ b/apps/api/src/services/integrations/google-analytics/mapping.ts @@ -0,0 +1,135 @@ +/** + * Pure mapping from a decoded GA4 `runReport` response to flat series points. + * + * Deliberately does NOT produce metric rows: what gets written depends on the reconciliation + * ledger (see `reconcile.ts`), so this stage stops at "here is the value GA4 currently reports + * for this series in this hour" and leaves the delta arithmetic downstream. + * + * Two things are resolved here that the rest of the pipeline then never has to think about: + * - `dateHour` is in the property's reporting timezone, so it is converted to a UTC instant. + * - The breakdown dimension is capped and its tail folded, so `Attributes` — which sits in the + * metrics tables' sorting key — stays bounded. + */ +import type { GoogleAnalyticsRunReportResponse } from "../GoogleAnalyticsApi" +import { foldTail, OTHER_BUCKET } from "../shared/cardinality" +import type { GaDatasetDef, GaMetricDef } from "./datasets" +import { dateHourToUtcMs } from "./timezone" + +/** GA4's own overflow bucket, folded into ours so the two do not appear as separate series. */ +const GA_OTHER_ROW = "(other)" + +export interface GaSeriesPoint { + /** UTC epoch ms of the hour bucket's start. */ + readonly bucketMs: number + readonly metric: GaMetricDef + readonly attributes: Record + readonly value: number +} + +/** + * Stable identity for a series within a (property, dataset, bucket), used as the ledger key. + * Attribute order is normalized so a key never depends on object insertion order. + */ +export const seriesKey = (metricName: string, attributes: Record): string => { + const entries = Object.entries(attributes).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + return entries.length === 0 + ? metricName + : `${metricName} ${entries.map(([key, value]) => `${key}=${value}`).join(" ")}` +} + +const numberOf = (raw: string | undefined): number => { + if (raw === undefined || raw === "") return 0 + const parsed = Number(raw) + // GA4 returns metric values as strings; a non-numeric one is a contract break, and counting + // it as zero is safer than letting NaN reach the warehouse and poison a bucket's sum. + return Number.isFinite(parsed) ? parsed : 0 +} + +const truncate = (value: string, max: number | undefined): string => + max !== undefined && value.length > max ? value.slice(0, max) : value + +const indexOfHeader = ( + headers: ReadonlyArray<{ readonly name?: string }> | undefined, + name: string, +): number => (headers ?? []).findIndex((header) => header.name === name) + +/** + * Map one report. Rows whose `dateHour` cannot be placed on the timeline (malformed, or an + * unknown property timezone) are dropped rather than guessed at — a missing point is recoverable + * on the next reconcile pass, a misplaced one is not. + */ +export const mapReport = (options: { + readonly dataset: GaDatasetDef + readonly response: GoogleAnalyticsRunReportResponse + readonly timeZone: string +}): ReadonlyArray => { + const { dataset, response, timeZone } = options + const rows = response.rows ?? [] + if (rows.length === 0) return [] + + const hourIndex = indexOfHeader(response.dimensionHeaders, "dateHour") + if (hourIndex < 0) return [] + + const breakdown = dataset.breakdown + const breakdownIndex = breakdown === null ? -1 : indexOfHeader(response.dimensionHeaders, breakdown.dimension) + // The report echoes back the dimensions we asked for; a response missing the breakdown column + // is not one we can attribute, so emitting it unbroken would double-count against `traffic`. + if (breakdown !== null && breakdownIndex < 0) return [] + + const metricIndexes = dataset.metrics.map((metric) => ({ + metric, + index: indexOfHeader(response.metricHeaders, metric.ga), + })) + + const dimensionValue = (row: (typeof rows)[number], index: number): string => + row.dimensionValues?.[index]?.value ?? "" + + // Pass 1: rank the breakdown values across the whole window, so the surviving top-N is stable + // for every bucket in it. Ranking per bucket instead would churn the series set hour to hour. + let fold: (key: string) => string = (key) => key + if (breakdown !== null) { + const rankIndex = indexOfHeader(response.metricHeaders, breakdown.rankBy) + const weights = new Map() + for (const row of rows) { + const raw = dimensionValue(row, breakdownIndex) + if (raw === "" || raw === GA_OTHER_ROW) continue + const value = truncate(raw, breakdown.maxValueLength) + const weight = rankIndex < 0 ? 1 : numberOf(row.metricValues?.[rankIndex]?.value) + weights.set(value, (weights.get(value) ?? 0) + weight) + } + fold = foldTail(weights, breakdown.maxValues) + } + + /** A row's metric attributes: the folded breakdown value, or none for the totals dataset. */ + const attributesFor = (row: (typeof rows)[number]): GaSeriesPoint["attributes"] => { + if (breakdown === null) return {} + const raw = dimensionValue(row, breakdownIndex) + const folded = + raw === "" || raw === GA_OTHER_ROW ? OTHER_BUCKET : fold(truncate(raw, breakdown.maxValueLength)) + return { [breakdown.attributeKey]: folded } + } + + // Pass 2: accumulate, because folding merges many raw values into one `other` series and the + // same (bucket, series) must arrive at the ledger exactly once. + const accumulated = new Map() + for (const row of rows) { + const bucketMs = dateHourToUtcMs(dimensionValue(row, hourIndex), timeZone) + if (bucketMs === null) continue + + const attributes = attributesFor(row) + + for (const { metric, index } of metricIndexes) { + if (index < 0) continue + const value = numberOf(row.metricValues?.[index]?.value) + const key = `${bucketMs} ${seriesKey(metric.metric, attributes)}` + const existing = accumulated.get(key) + if (existing === undefined) { + accumulated.set(key, { bucketMs, metric, attributes, value }) + } else { + accumulated.set(key, { ...existing, value: existing.value + value }) + } + } + } + + return [...accumulated.values()] +} diff --git a/apps/api/src/services/integrations/google-analytics/reconcile.test.ts b/apps/api/src/services/integrations/google-analytics/reconcile.test.ts new file mode 100644 index 000000000..ca1fbc3f5 --- /dev/null +++ b/apps/api/src/services/integrations/google-analytics/reconcile.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, it } from "vitest" +import { DATASET_BY_ID } from "./datasets" +import { mapReport } from "./mapping" +import { type LedgerBucket, parseLedger, reconcile, serializeLedger } from "./reconcile" + +const traffic = DATASET_BY_ID.get("traffic")! +const channels = DATASET_BY_ID.get("channels")! + +const HOUR = 3_600_000 +const BUCKET = Date.parse("2026-09-09T14:00:00.000Z") + +/** A `runReport` response for the traffic dataset with a single hour of sessions. */ +const trafficResponse = (sessions: number) => ({ + dimensionHeaders: [{ name: "dateHour" }], + metricHeaders: [{ name: "sessions" }], + rows: [{ dimensionValues: [{ value: "2026090914" }], metricValues: [{ value: String(sessions) }] }], +}) + +/** A `runReport` response for the channels dataset: one row per (hour, channel). */ +const channelsResponse = (perChannel: ReadonlyArray) => ({ + dimensionHeaders: [{ name: "dateHour" }, { name: "sessionDefaultChannelGroup" }], + metricHeaders: [{ name: "sessions" }, { name: "activeUsers" }], + rows: perChannel.map(([channel, sessions]) => ({ + dimensionValues: [{ value: "2026090914" }, { value: channel }], + metricValues: [{ value: String(sessions) }, { value: "0" }], + })), +}) + +const run = (options: { + readonly dataset: typeof traffic + readonly response: ReturnType | ReturnType + readonly ledger: ReadonlyArray +}) => + reconcile({ + orgId: "org_test", + propertyId: "123456789", + propertyName: "Example", + accountName: "Acme", + dataset: options.dataset, + points: mapReport({ dataset: options.dataset, response: options.response, timeZone: "UTC" }), + ledger: options.ledger, + coveredFromMs: BUCKET, + coveredToMs: BUCKET + HOUR, + }) + +/** What the warehouse would report for a bucket: every delta ever emitted for it, summed. */ +const bucketSum = (...results: ReadonlyArray>) => + results.flatMap((result) => result.rows).reduce((total, row) => total + row.value, 0) + +describe("reconcile", () => { + it("emits the raw value the first time a bucket is seen", () => { + const result = run({ dataset: traffic, response: trafficResponse(100), ledger: [] }) + const sessions = result.rows.filter((row) => row.metric_name === "google_analytics.sessions") + expect(sessions).toHaveLength(1) + expect(sessions[0]!.value).toBe(100) + }) + + it("emits DELTA temporality, non-monotonic, at the bucket's own timestamp", () => { + const [row] = run({ dataset: traffic, response: trafficResponse(100), ledger: [] }).rows + expect(row!.aggregation_temporality).toBe(1) + expect(row!.is_monotonic).toBe(false) + expect(row!.timestamp).toBe("2026-09-09 14:00:00.000") + expect(row!.service_name).toBe("google-analytics/123456789") + }) + + // The core contract: however many times GA4 revises an hour, the bucket's summed deltas + // equal GA4's latest answer — which is exactly what `sum(Value)` per bucket reads back. + it("sums to the revised value after an upward revision", () => { + const first = run({ dataset: traffic, response: trafficResponse(100), ledger: [] }) + const second = run({ dataset: traffic, response: trafficResponse(140), ledger: first.ledger }) + + const delta = second.rows.filter((row) => row.metric_name === "google_analytics.sessions") + expect(delta).toHaveLength(1) + expect(delta[0]!.value).toBe(40) + expect(bucketSum(first, second)).toBe(140) + }) + + it("sums to the revised value after a DOWNWARD revision", () => { + const first = run({ dataset: traffic, response: trafficResponse(100), ledger: [] }) + const second = run({ dataset: traffic, response: trafficResponse(60), ledger: first.ledger }) + + const delta = second.rows.filter((row) => row.metric_name === "google_analytics.sessions") + expect(delta[0]!.value).toBe(-40) + expect(bucketSum(first, second)).toBe(60) + }) + + it("stays correct across a long chain of revisions", () => { + let ledger: ReadonlyArray = [] + const results = [] + for (const value of [10, 25, 25, 24, 90, 3, 3, 117]) { + const result = run({ dataset: traffic, response: trafficResponse(value), ledger }) + ledger = result.ledger + results.push(result) + } + expect(bucketSum(...results)).toBe(117) + }) + + it("writes nothing at all when a re-poll is unchanged", () => { + const first = run({ dataset: traffic, response: trafficResponse(100), ledger: [] }) + const second = run({ dataset: traffic, response: trafficResponse(100), ledger: first.ledger }) + expect(second.rows).toHaveLength(0) + expect(bucketSum(first, second)).toBe(100) + }) + + it("retracts a series that disappears from the report", () => { + // "Paid Social" is reported, then revised away entirely. Without a retraction its old + // value would linger in the bucket's sum forever. + const first = run({ + dataset: channels, + response: channelsResponse([ + ["Organic Search", 80], + ["Paid Social", 20], + ]), + ledger: [], + }) + const second = run({ + dataset: channels, + response: channelsResponse([["Organic Search", 80]]), + ledger: first.ledger, + }) + + const retraction = second.rows.find( + (row) => row.metric_attributes["google_analytics.channel_group"] === "Paid Social", + ) + expect(retraction?.value).toBe(-20) + expect(bucketSum(first, second)).toBe(80) + }) + + it("round-trips a breakdown value containing spaces", () => { + // The retraction path has to rebuild attributes from the ledger key, and GA4's channel + // groups are multi-word ("Organic Search", "Paid Social", "Cross-network"). + const first = run({ + dataset: channels, + response: channelsResponse([["Organic Search", 55]]), + ledger: [], + }) + const second = run({ dataset: channels, response: channelsResponse([]), ledger: first.ledger }) + + const retraction = second.rows.find((row) => row.metric_name === "google_analytics.sessions.by_channel") + expect(retraction?.metric_attributes["google_analytics.channel_group"]).toBe("Organic Search") + expect(retraction?.value).toBe(-55) + expect(bucketSum(first, second)).toBe(0) + }) + + it("zeroes a covered bucket when the report comes back empty", () => { + // GA4 was asked about this hour and answered "nothing" — a revision to zero, not a gap. + const first = run({ dataset: traffic, response: trafficResponse(100), ledger: [] }) + const second = run({ + dataset: traffic, + response: { dimensionHeaders: [{ name: "dateHour" }], metricHeaders: [{ name: "sessions" }], rows: [] }, + ledger: first.ledger, + }) + expect(second.rows.map((row) => row.value)).toEqual([-100]) + expect(bucketSum(first, second)).toBe(0) + }) + + it("leaves buckets outside the covered window untouched", () => { + const stale: LedgerBucket = { bucketMs: BUCKET - 10 * HOUR, emitted: { "google_analytics.sessions": 7 } } + const result = run({ dataset: traffic, response: trafficResponse(100), ledger: [stale] }) + + // No retraction for the old bucket — the report said nothing about it, which is not the + // same as saying it is zero. + expect(result.rows.every((row) => row.timestamp === "2026-09-09 14:00:00.000")).toBe(true) + expect(result.ledger).toContainEqual(stale) + }) + + it("drops zero-valued series from the ledger instead of accumulating them", () => { + const result = run({ + dataset: channels, + response: channelsResponse([ + ["Organic Search", 80], + ["Paid Social", 0], + ]), + ledger: [], + }) + const bucket = result.ledger.find((entry) => entry.bucketMs === BUCKET) + expect(Object.keys(bucket?.emitted ?? {})).toEqual([ + "google_analytics.sessions.by_channel google_analytics.channel_group=Organic Search", + ]) + }) +}) + +describe("ledger serialization", () => { + it("round-trips", () => { + const emitted = { "google_analytics.sessions": 12, "google_analytics.sessions.by_channel x=y": 3.5 } + expect(parseLedger(serializeLedger(emitted))).toEqual(emitted) + }) + + it("treats an absent or corrupt blob as nothing-emitted", () => { + expect(parseLedger(null)).toEqual({}) + expect(parseLedger("")).toEqual({}) + expect(parseLedger("{ not json")).toEqual({}) + expect(parseLedger("[1,2,3]")).toEqual({}) + }) + + it("discards non-finite values rather than letting NaN reach a delta", () => { + expect(parseLedger('{"a": 1, "b": "x", "c": null}')).toEqual({ a: 1 }) + }) +}) diff --git a/apps/api/src/services/integrations/google-analytics/reconcile.ts b/apps/api/src/services/integrations/google-analytics/reconcile.ts new file mode 100644 index 000000000..34b7a1905 --- /dev/null +++ b/apps/api/src/services/integrations/google-analytics/reconcile.ts @@ -0,0 +1,230 @@ +/** + * Delta reconciliation — the piece that makes a revisable source safe to write into an + * append-only warehouse. + * + * `metrics_sum` is a plain MergeTree with no dedupe, and GA4 keeps revising `dateHour` rows for + * ~48h after the fact. Re-polling an hour and writing the new value again would leave two rows at + * the same timestamp, and every reducer would then read wrong: `sum` double-counts, `avg` blends + * stale with fresh, `max` breaks on a downward revision. + * + * So nothing is ever written as an absolute value. Each series remembers what it has already + * emitted for a bucket (the ledger), and a re-poll emits only the DIFFERENCE. `sum(Value)` per + * bucket is then exactly GA4's current answer, by construction, no matter how many times the hour + * is revisited — and a downward revision is simply a negative delta. This is what + * `AggregationTemporality = 1` (DELTA) means, and it is why these rows are emitted with + * `is_monotonic: false`: the corrections are genuinely allowed to be negative. + * + * Chart these metrics with `sum`. Never `rate` or `increase` — those assume cumulative + * temporality and would double-difference the data (see + * `packages/query-engine/src/query-builder/model.ts`). + */ +import { Option, Schema } from "effect" +import { fmtMetricTs, type MetricAttrs, type MetricSumRow } from "@/services/warehouse/metric-rows" +import type { GaDatasetDef } from "./datasets" +import { SCOPE_NAME, serviceNameFor } from "./datasets" +import { type GaSeriesPoint, seriesKey } from "./mapping" + +/** What has already been emitted for one (property, dataset, bucket): seriesKey → value. */ +export interface LedgerBucket { + readonly bucketMs: number + readonly emitted: Readonly> +} + +export interface ReconcileResult { + /** DELTA rows to ship. Series whose value is unchanged produce nothing. */ + readonly rows: ReadonlyArray + /** Ledger buckets to persist. A bucket that ends up empty is flagged for deletion. */ + readonly ledger: ReadonlyArray +} + +/** + * Parse a stored ledger blob. A corrupt or non-object blob decodes to "nothing emitted yet", + * which re-emits the bucket's full current value — a visible double-count in one hour, versus + * silently freezing that hour forever if we treated the failure as "already up to date". + */ +const LedgerBlob = Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)) +const decodeLedgerBlob = Schema.decodeUnknownOption(LedgerBlob) +const decodeEmittedValue = Schema.decodeUnknownOption(Schema.Finite) + +export const parseLedger = (json: string | null | undefined): Readonly> => { + if (json == null || json === "") return {} + const blob = Option.getOrNull(decodeLedgerBlob(json)) + if (blob === null) return {} + // Filtered per key rather than validated as a whole: the blob is a map of INDEPENDENT series, + // so one unreadable entry should cost that series a re-emission, not force every other series + // in the hour to be re-emitted alongside it. + const emitted: Record = {} + for (const [key, value] of Object.entries(blob)) { + const parsed = decodeEmittedValue(value) + if (Option.isSome(parsed)) emitted[key] = parsed.value + } + return emitted +} + +export const serializeLedger = (emitted: Readonly>): string => JSON.stringify(emitted) + +const resourceAttributes = (options: { + readonly orgId: string + readonly propertyId: string + readonly propertyName: string | null + readonly accountName: string | null +}): MetricAttrs => { + const attributes: MetricAttrs = { + maple_org_id: options.orgId, + "service.name": serviceNameFor(options.propertyId), + "google_analytics.property.id": options.propertyId, + } + // Display names are absent until discovery has named the property; an empty-string attribute + // would read as "named, blank" everywhere downstream. + if (options.propertyName != null) attributes["google_analytics.property.name"] = options.propertyName + if (options.accountName != null) attributes["google_analytics.account.name"] = options.accountName + return attributes +} + +/** + * Reconcile one dataset's freshly-polled points against the ledger. + * + * `coveredFromMs`/`coveredToMs` bound the half-open range the report actually asked GA4 about. + * They are what make the "series disappeared" case safe: inside the window, a series present in + * the ledger but absent from the response has genuinely gone to zero (revised away, or folded + * into `other` as top-N membership shifted) and must be zeroed out with a negative delta, or its + * old value would linger in the bucket's sum forever. Outside the window we know nothing, so + * those buckets are passed through untouched. + */ +export const reconcile = (options: { + readonly orgId: string + readonly propertyId: string + readonly propertyName: string | null + readonly accountName: string | null + readonly dataset: GaDatasetDef + readonly points: ReadonlyArray + readonly ledger: ReadonlyArray + readonly coveredFromMs: number + readonly coveredToMs: number +}): ReconcileResult => { + const { points, coveredFromMs, coveredToMs } = options + const resource = resourceAttributes(options) + const serviceName = serviceNameFor(options.propertyId) + + const ledgerByBucket = new Map(options.ledger.map((bucket) => [bucket.bucketMs, bucket.emitted])) + + // Group the freshly-polled points by bucket so each bucket is reconciled as a unit — the + // disappeared-series check below is only sound against a bucket's complete new picture. + const freshByBucket = new Map>() + for (const point of points) { + if (point.bucketMs < coveredFromMs || point.bucketMs >= coveredToMs) continue + let bucket = freshByBucket.get(point.bucketMs) + if (bucket === undefined) { + bucket = new Map() + freshByBucket.set(point.bucketMs, bucket) + } + bucket.set(seriesKey(point.metric.metric, point.attributes), point) + } + + // A covered bucket the report said nothing about is still covered: GA4 was asked and answered + // "nothing here". Seeding it empty is what lets the retraction pass below zero it out — without + // this, an hour revised away entirely (or a breakdown that stopped receiving traffic) would + // keep its last value in the warehouse forever, since no fresh point would ever name it again. + for (const bucket of options.ledger) { + if (bucket.bucketMs < coveredFromMs || bucket.bucketMs >= coveredToMs) continue + if (!freshByBucket.has(bucket.bucketMs)) freshByBucket.set(bucket.bucketMs, new Map()) + } + + const rows: Array = [] + const nextLedger: Array = [] + + const emit = (bucketMs: number, point: GaSeriesPoint, delta: number) => { + const ts = fmtMetricTs(bucketMs) + rows.push({ + timestamp: ts, + start_timestamp: ts, + metric_name: point.metric.metric, + metric_description: point.metric.description, + metric_unit: point.metric.unit, + metric_attributes: point.attributes, + service_name: serviceName, + resource_schema_url: "", + resource_attributes: resource, + scope_schema_url: "", + scope_name: SCOPE_NAME, + scope_version: "", + scope_attributes: {}, + value: delta, + flags: 0, + exemplars_trace_id: [], + exemplars_span_id: [], + exemplars_timestamp: [], + exemplars_value: [], + exemplars_filtered_attributes: [], + // DELTA. Each row is an increment for its hour, not a running total. + aggregation_temporality: 1, + // A correction may be negative, so this sum is explicitly non-monotonic. It also keeps + // these rows out of the cumulative rate path, which selects on `IsMonotonic = 1`. + is_monotonic: false, + }) + } + + // Buckets the poll covered: emit the difference for every series, in either direction. + for (const [bucketMs, fresh] of freshByBucket) { + const previous = ledgerByBucket.get(bucketMs) ?? {} + const emitted: Record = {} + + for (const [key, point] of fresh) { + const delta = point.value - (previous[key] ?? 0) + if (delta !== 0) emit(bucketMs, point, delta) + // Zero-valued series are dropped from the ledger rather than recorded as 0: keeping + // them would grow the blob with every path that ever appeared once. + if (point.value !== 0) emitted[key] = point.value + } + + for (const [key, previousValue] of Object.entries(previous)) { + if (fresh.has(key) || previousValue === 0) continue + // Gone from the report, so it is now zero. Reconstruct just enough of the series to + // emit the retraction. + const series = parseSeriesKey(key, options.dataset) + if (series === null) continue + emit(bucketMs, { bucketMs, metric: series.metric, attributes: series.attributes, value: 0 }, -previousValue) + } + + nextLedger.push({ bucketMs, emitted }) + } + + // Buckets outside the covered window are left exactly as they were. + for (const bucket of options.ledger) { + if (!freshByBucket.has(bucket.bucketMs)) nextLedger.push(bucket) + } + + return { rows, ledger: nextLedger } +} + +/** + * Inverse of {@link seriesKey} for the retraction path. + * + * Not a general parser, and it must not become one: it works because a dataset has at most ONE + * breakdown, so the key is exactly `" ="` and the value is + * everything after the first `=` following a known attribute key. Splitting on whitespace instead + * would corrupt every value that contains a space — `sessionDefaultChannelGroup` alone yields + * "Organic Search", "Paid Social", "Cross-network". + * + * Returns null when the key does not belong to this dataset (a stale blob written by an older + * registry), which drops the retraction rather than emitting it against a guessed series. + */ +const parseSeriesKey = ( + key: string, + dataset: GaDatasetDef, +): { readonly metric: GaDatasetDef["metrics"][number]; readonly attributes: Record } | null => { + // Longest first: within a dataset one metric name can prefix another + // (`…page_views` vs `…page_views.by_page`), and the shorter would match wrongly. + const candidates = [...dataset.metrics].sort((a, b) => b.metric.length - a.metric.length) + for (const metric of candidates) { + if (!key.startsWith(metric.metric)) continue + const rest = key.slice(metric.metric.length) + if (rest === "") return { metric, attributes: {} } + const breakdown = dataset.breakdown + if (breakdown === null) continue + const prefix = ` ${breakdown.attributeKey}=` + if (!rest.startsWith(prefix)) continue + return { metric, attributes: { [breakdown.attributeKey]: rest.slice(prefix.length) } } + } + return null +} diff --git a/apps/api/src/services/integrations/google-analytics/timezone.test.ts b/apps/api/src/services/integrations/google-analytics/timezone.test.ts new file mode 100644 index 000000000..c7b648077 --- /dev/null +++ b/apps/api/src/services/integrations/google-analytics/timezone.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest" +import { dateHourToUtcMs, utcMsToZonedDate } from "./timezone" + +const iso = (ms: number) => new Date(ms).toISOString() + +describe("dateHourToUtcMs", () => { + it("is the identity for a UTC property", () => { + expect(iso(dateHourToUtcMs("2026090914", "UTC")!)).toBe("2026-09-09T14:00:00.000Z") + }) + + it("shifts a fixed-offset zone by its offset", () => { + // Asia/Tokyo is UTC+9 year-round: 14:00 local is 05:00 UTC. + expect(iso(dateHourToUtcMs("2026090914", "Asia/Tokyo")!)).toBe("2026-09-09T05:00:00.000Z") + }) + + it("applies the summer offset for a DST zone in summer", () => { + // America/Los_Angeles is UTC-7 in September (PDT): 14:00 local is 21:00 UTC. + expect(iso(dateHourToUtcMs("2026090914", "America/Los_Angeles")!)).toBe("2026-09-09T21:00:00.000Z") + }) + + it("applies the winter offset for the same zone in winter", () => { + // ...and UTC-8 in January (PST): the same wall-clock hour is 22:00 UTC. A naive fixed + // offset would put this bucket an hour out for half the year. + expect(iso(dateHourToUtcMs("2026011514", "America/Los_Angeles")!)).toBe("2026-01-15T22:00:00.000Z") + }) + + it("resolves the hour either side of a fall-back transition", () => { + // 2026-11-01, US DST ends at 02:00 local. 01:00 local occurs twice; the hour before and + // the hour after must still land on distinct, correctly-ordered instants. + const before = dateHourToUtcMs("2026110100", "America/Los_Angeles")! + const after = dateHourToUtcMs("2026110103", "America/Los_Angeles")! + expect(iso(before)).toBe("2026-11-01T07:00:00.000Z") + expect(iso(after)).toBe("2026-11-01T11:00:00.000Z") + expect(after).toBeGreaterThan(before) + }) + + it("handles a half-hour offset zone", () => { + // Asia/Kolkata is UTC+5:30 — the case that catches an implementation assuming whole hours. + expect(iso(dateHourToUtcMs("2026090914", "Asia/Kolkata")!)).toBe("2026-09-09T08:30:00.000Z") + }) + + it("handles midnight without rolling the day", () => { + expect(iso(dateHourToUtcMs("2026090900", "UTC")!)).toBe("2026-09-09T00:00:00.000Z") + expect(iso(dateHourToUtcMs("2026090900", "Asia/Tokyo")!)).toBe("2026-09-08T15:00:00.000Z") + }) + + it("returns null rather than guessing at malformed input", () => { + expect(dateHourToUtcMs("", "UTC")).toBeNull() + expect(dateHourToUtcMs("20260909", "UTC")).toBeNull() + expect(dateHourToUtcMs("2026090924", "UTC")).toBeNull() + expect(dateHourToUtcMs("2026139914", "UTC")).toBeNull() + expect(dateHourToUtcMs("(other)", "UTC")).toBeNull() + }) + + it("returns null for a timezone the runtime does not know", () => { + expect(dateHourToUtcMs("2026090914", "Mars/Olympus_Mons")).toBeNull() + }) +}) + +describe("utcMsToZonedDate", () => { + it("renders the property-local date, not the UTC one", () => { + const instant = Date.parse("2026-09-09T02:00:00.000Z") + expect(utcMsToZonedDate(instant, "UTC")).toBe("2026-09-09") + // 02:00 UTC is still the previous evening in Los Angeles — asking GA4 for the UTC date + // would miss a whole local day at the window edge. + expect(utcMsToZonedDate(instant, "America/Los_Angeles")).toBe("2026-09-08") + expect(utcMsToZonedDate(instant, "Asia/Tokyo")).toBe("2026-09-09") + }) + + it("returns null for an unknown timezone", () => { + expect(utcMsToZonedDate(Date.now(), "Mars/Olympus_Mons")).toBeNull() + }) +}) diff --git a/apps/api/src/services/integrations/google-analytics/timezone.ts b/apps/api/src/services/integrations/google-analytics/timezone.ts new file mode 100644 index 000000000..c7381a395 --- /dev/null +++ b/apps/api/src/services/integrations/google-analytics/timezone.ts @@ -0,0 +1,101 @@ +/** + * GA4 `dateHour` → UTC epoch milliseconds. + * + * The Data API expresses `dateHour` ("YYYYMMDDHH") in the PROPERTY'S configured reporting + * timezone, not UTC, and says so nowhere in the response. A property set to + * `America/Los_Angeles` reporting hour `2026090914` means 14:00 Pacific — 21:00 or 22:00 UTC + * depending on the date. Treating the string as UTC would shift every bucket by a whole number + * of hours, consistently and invisibly: the chart would look plausible and be wrong, and the + * reconciliation ledger would happily keep it consistent with itself. + * + * The conversion is done against the platform's own tz database via `Intl` rather than a date + * library, so it tracks DST rule changes without a dependency to keep current. + */ + +/** Cached per zone — constructing an `Intl.DateTimeFormat` is expensive and this runs per row. */ +const formatters = new Map() + +const formatterFor = (timeZone: string): Intl.DateTimeFormat | null => { + const cached = formatters.get(timeZone) + if (cached !== undefined) return cached + try { + const formatter = new Intl.DateTimeFormat("en-US", { + timeZone, + // h23, not hour12:false — the latter renders midnight as "24" on some engines, which + // would push every midnight bucket a day forward. + hourCycle: "h23", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }) + formatters.set(timeZone, formatter) + return formatter + } catch { + // An unknown IANA zone (a GA4 property configured with something this runtime's tz + // database has never heard of). Reported as null so the caller skips the property rather + // than silently filing its data under the wrong hour. + return null + } +} + +/** Offset in ms of `timeZone` at a given UTC instant: (wall-clock time there) − (UTC time). */ +const offsetAt = (formatter: Intl.DateTimeFormat, utcMs: number): number => { + const parts = formatter.formatToParts(new Date(utcMs)) + const lookup = (type: Intl.DateTimeFormatPartTypes): number => { + const part = parts.find((candidate) => candidate.type === type) + return part === undefined ? 0 : Number(part.value) + } + const asIfUtc = Date.UTC( + lookup("year"), + lookup("month") - 1, + lookup("day"), + lookup("hour"), + lookup("minute"), + lookup("second"), + ) + return asIfUtc - utcMs +} + +/** + * Convert a GA4 `dateHour` in `timeZone` to the UTC epoch ms of that hour's start. + * + * Returns null for a malformed `dateHour` or an unknown timezone — both mean "we cannot place + * this row on the timeline", and a skipped row is strictly better than a misplaced one. + */ +export const dateHourToUtcMs = (dateHour: string, timeZone: string): number | null => { + if (!/^\d{10}$/.test(dateHour)) return null + const year = Number(dateHour.slice(0, 4)) + const month = Number(dateHour.slice(4, 6)) + const day = Number(dateHour.slice(6, 8)) + const hour = Number(dateHour.slice(8, 10)) + if (month < 1 || month > 12 || day < 1 || day > 31 || hour > 23) return null + + const formatter = formatterFor(timeZone) + if (formatter === null) return null + + // Solve `localWallClock(utc) == target` by fixed point. The first guess uses the offset at the + // naive instant; one refinement settles it, because a second application lands on the correct + // side of any DST transition (offsets change by at most a couple of hours, far less than the + // day-wide window the first guess is already inside). + const target = Date.UTC(year, month - 1, day, hour) + const firstGuess = target - offsetAt(formatter, target) + const refined = target - offsetAt(formatter, firstGuess) + return refined +} + +/** UTC epoch ms → the `YYYY-MM-DD` the Data API wants for a date range, in `timeZone`. */ +export const utcMsToZonedDate = (utcMs: number, timeZone: string): string | null => { + const formatter = formatterFor(timeZone) + if (formatter === null) return null + const parts = formatter.formatToParts(new Date(utcMs)) + const value = (type: Intl.DateTimeFormatPartTypes): string => + parts.find((candidate) => candidate.type === type)?.value ?? "" + const year = value("year") + const month = value("month") + const day = value("day") + if (year === "" || month === "" || day === "") return null + return `${year}-${month}-${day}` +} diff --git a/apps/api/src/services/integrations/shared/cardinality.ts b/apps/api/src/services/integrations/shared/cardinality.ts new file mode 100644 index 000000000..edfb76762 --- /dev/null +++ b/apps/api/src/services/integrations/shared/cardinality.ts @@ -0,0 +1,29 @@ +/** + * Attribute-cardinality folding for integration pollers. + * + * Every provider hands back at least one unbounded dimension — hostnames, WAF rule ids, DNS + * query names, GA4 page paths — and `Attributes` sits in the metrics tables' sorting key, so an + * uncapped dimension bloats it and degrades every read of that metric, not just the breakdown. + * The fix is the same everywhere: keep the N heaviest values across the window and fold the tail + * into one explicit {@link OTHER_BUCKET} series, so the total still reconciles. + * + * Shared by the Cloudflare edge-analytics mapper and the Google Analytics collector. + */ + +/** The folded tail. One series, so a breakdown's parts still sum to the unbroken total. */ +export const OTHER_BUCKET = "other" + +/** Top-N keys by weight; ties break lexicographically so folding is deterministic across runs. */ +export const topNKeys = (weights: ReadonlyMap, n: number): ReadonlySet => + new Set( + [...weights.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, n) + .map(([key]) => key), + ) + +/** Weight-rank an unbounded dimension across a window and fold the tail into {@link OTHER_BUCKET}. */ +export const foldTail = (weights: ReadonlyMap, n: number): ((key: string) => string) => { + const top = topNKeys(weights, n) + return (key) => (top.has(key) ? key : OTHER_BUCKET) +} diff --git a/apps/api/src/services/integrations/cloudflare-analytics/otlp.test.ts b/apps/api/src/services/integrations/shared/otlp.test.ts similarity index 100% rename from apps/api/src/services/integrations/cloudflare-analytics/otlp.test.ts rename to apps/api/src/services/integrations/shared/otlp.test.ts diff --git a/apps/api/src/services/integrations/cloudflare-analytics/otlp.ts b/apps/api/src/services/integrations/shared/otlp.ts similarity index 89% rename from apps/api/src/services/integrations/cloudflare-analytics/otlp.ts rename to apps/api/src/services/integrations/shared/otlp.ts index 63f4f4c02..a1ceba927 100644 --- a/apps/api/src/services/integrations/cloudflare-analytics/otlp.ts +++ b/apps/api/src/services/integrations/shared/otlp.ts @@ -1,12 +1,15 @@ /** * Encode collector-shaped metric rows (`MetricSumRow` / `MetricGaugeRow`) into an OTLP/HTTP JSON - * `ExportMetricsServiceRequest`, so the Cloudflare poller can ship its synthetic edge metrics - * through the ingest gateway (`POST /v1/metrics`) exactly like every other telemetry source — - * which is what gives per-org routing (managed Tinybird vs BYO ClickHouse), schema-version gating, - * WAL durability, and Autumn metering for free. The rows are already OTel-metric-shaped, so this is + * `ExportMetricsServiceRequest`, so an integration poller can ship its synthetic metrics through + * the ingest gateway (`POST /v1/metrics`) exactly like every other telemetry source — which is + * what gives per-org routing (managed Tinybird vs BYO ClickHouse), schema-version gating, WAL + * durability, and Autumn metering for free. The rows are already OTel-metric-shaped, so this is * a mechanical re-expression: no query paths change (the downstream collector still lands them in * `metrics_sum` / `metrics_gauge`). * + * Provider-agnostic on purpose — the Cloudflare edge-analytics poller and the Google Analytics + * collector both emit through it, and any future polling integration should too. + * * Rows are grouped resource → scope → metric to match the OTLP envelope: sum rows become `sum` * metrics (carrying the row's `aggregation_temporality` — 1 = DELTA — and `is_monotonic`), gauge * rows become bare `gauge` metrics. Timestamps are the row's ClickHouse DateTime64 literal @@ -106,7 +109,7 @@ const scopeKey = (row: MetricGaugeRow): string => `${row.scope_schema_url}\x00${row.scope_name}\x00${row.scope_version}\x00${stableJson(row.scope_attributes)}` /** - * Convert Cloudflare metric rows into a single OTLP/JSON metrics request. Sum and gauge metrics + * Convert integration metric rows into a single OTLP/JSON metrics request. Sum and gauge metrics * coexist in the same envelope; the downstream collector fans them back out to * `metrics_sum` / `metrics_gauge`. */ diff --git a/apps/api/src/services/org/OrganizationService.ts b/apps/api/src/services/org/OrganizationService.ts index f6affe57d..485c31cc6 100644 --- a/apps/api/src/services/org/OrganizationService.ts +++ b/apps/api/src/services/org/OrganizationService.ts @@ -26,6 +26,8 @@ import { errorIssueStates, errorIssues, errorNotificationPolicies, + googleAnalyticsLedger, + googleAnalyticsState, liveActivities, mcpOAuthAuthorizations, mcpOAuthRefreshTokens, @@ -108,6 +110,13 @@ const ORG_SCOPED_TABLES = [ // APNs update tokens for running Live Activities. `mobile_devices` is purged // here already; leaving these behind keeps a live push channel open. liveActivities, + // GA4 collector state and its reconciliation ledger. Neither holds a secret, so the + // unpurged list's criterion would have allowed them — but deletion is terminal, which + // removes the one reason to keep them: the ledger exists so a RECONNECT does not re-emit + // hours already in the warehouse, and a deleted org never reconnects. What is left is + // third-party property and account names belonging to an org that is gone, so they go. + googleAnalyticsState, + googleAnalyticsLedger, ] as const /** diff --git a/apps/clickhouse-builder-docs/src/sidebar-icons.tsx b/apps/clickhouse-builder-docs/src/sidebar-icons.tsx index 8e86fb6b2..36f692900 100644 --- a/apps/clickhouse-builder-docs/src/sidebar-icons.tsx +++ b/apps/clickhouse-builder-docs/src/sidebar-icons.tsx @@ -301,8 +301,13 @@ const icons = { type SidebarIconName = keyof typeof icons +/** + * Sidebar names come from doc frontmatter, so an unknown one is expected, not a bug — and since + * the name is arbitrary text, `in` is the wrong test: it also matches inherited members, so + * `constructor` or `toString` would pass the guard and render a function as an icon. + */ function isSidebarIconName(name: string): name is SidebarIconName { - return name in icons + return Object.hasOwn(icons, name) } export function sidebarIcon(name: string | undefined) { diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index c68b2076c..6f32b4dd4 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -7327,6 +7327,10 @@ "description": "Connect PlanetScale to your organization and manage what Maple collects from it: connection status, organization binding, the metrics service token that enables branch-metrics scraping, the database inventory, webhook setup, query insights, and the lifecycle event timeline.", "name": "PlanetScale Integration" }, + { + "description": "Connect Google Analytics 4 to your organization and manage what Maple collects from it: connection status, the properties discovered under the grant, and which of them are collected. Collected data lands as regular metrics, so it charts and alerts alongside your traces.", + "name": "Google Analytics Integration" + }, { "description": "Deduplicated errors and alert-backed issues tracked through Maple's triage workflow.", "name": "Error Issues" diff --git a/apps/web/perf/check-bundle-budget.ts b/apps/web/perf/check-bundle-budget.ts index f507d3ddc..774080c53 100644 --- a/apps/web/perf/check-bundle-budget.ts +++ b/apps/web/perf/check-bundle-budget.ts @@ -65,7 +65,18 @@ const gzipBytes = chunks.reduce((total, chunk) => total + chunk.gzipBytes, 0) // the role/intent id literals and legacy-save maps in the quick-start atom the // root gate reads, and one lab registry entry. The cards' own copy is split // off so the route chunk carries it. main was at 683.4 KB. -const maxGzipBytes = 685 * 1024 +// 687 KB from the Google Analytics integration (2026-09-09): 1.6 KB of startup, +// all of it the v2 domain contract every page's API client carries. Measured by +// registering and unregistering that one group against the same build — 686.2 +// with `V2GoogleAnalyticsIntegrationsApiGroup` on `MapleApiV2`, 684.6 without — +// so the card, the catalog entry, the icon and the template-icon entry cost +// nothing measurable between them. It is the same category as the Releases and +// AI-detect contracts above, just larger: five endpoints and six schemas rather +// than one. The weight is the OpenAPI descriptions, and those are the public API +// documentation — cutting them to buy back a kilobyte of startup is the wrong +// trade. Splitting the group out of the client is not available either: every +// page's client is built from the whole `MapleApiV2` surface. +const maxGzipBytes = 687 * 1024 const budgetLabel = `${(maxGzipBytes / 1024).toFixed(1)} KB` // Anything lazy-only: chat, replay, and every dev-only lab surface. The diff --git a/apps/web/src/components/dashboard-builder/templates/template-icons.ts b/apps/web/src/components/dashboard-builder/templates/template-icons.ts index 6cd11a7a0..97e138c36 100644 --- a/apps/web/src/components/dashboard-builder/templates/template-icons.ts +++ b/apps/web/src/components/dashboard-builder/templates/template-icons.ts @@ -5,6 +5,7 @@ import { CloudflareIcon, DatabaseIcon, GlobeIcon, + GoogleAnalyticsIcon, GridSquareCirclePlusIcon, type IconComponent, KafkaIcon, @@ -42,6 +43,7 @@ const TEMPLATE_ICONS: Record = { "mongodb-overview": MongodbIcon, cloudflare: CloudflareIcon, planetscale: PlanetScaleIcon, + "google-analytics": GoogleAnalyticsIcon, "host-metrics": ServerIcon, "kubernetes-cluster": KubernetesIcon, "kubernetes-pod": KubernetesIcon, @@ -56,6 +58,7 @@ const CATEGORY_ICONS: Record = { database: DatabaseIcon, infrastructure: ServerIcon, messaging: PaperPlaneIcon, + product: ChartLineIcon, } satisfies Record export function templateIcon(templateId: string, category: string): IconComponent { diff --git a/apps/web/src/components/icons/google-analytics.tsx b/apps/web/src/components/icons/google-analytics.tsx new file mode 100644 index 000000000..a196f69dc --- /dev/null +++ b/apps/web/src/components/icons/google-analytics.tsx @@ -0,0 +1,21 @@ +import type { IconProps } from "./icon" + +// Source: simple-icons (MIT) — https://simpleicons.org/icons/googleanalytics +function GoogleAnalyticsIcon({ size = 24, className, ...props }: IconProps) { + return ( + + ) +} + +export { GoogleAnalyticsIcon } diff --git a/apps/web/src/components/icons/index.ts b/apps/web/src/components/icons/index.ts index ad71d1018..b88b78ebb 100644 --- a/apps/web/src/components/icons/index.ts +++ b/apps/web/src/components/icons/index.ts @@ -113,6 +113,7 @@ export { FolderIcon } from "./folder" export { GearIcon } from "./gear" export { GeminiIcon } from "./gemini" export { GithubIcon } from "./github" +export { GoogleAnalyticsIcon } from "./google-analytics" export { GoogleIcon } from "./google" export { GrokIcon } from "./grok" export { HaystackIcon } from "./haystack" diff --git a/apps/web/src/components/integrations/google-analytics-integration-card.tsx b/apps/web/src/components/integrations/google-analytics-integration-card.tsx new file mode 100644 index 000000000..501f7248b --- /dev/null +++ b/apps/web/src/components/integrations/google-analytics-integration-card.tsx @@ -0,0 +1,247 @@ +import { useState } from "react" +import { Exit } from "effect" +import { Badge } from "@maple/ui/components/ui/badge" +import { Button } from "@maple/ui/components/ui/button" +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { Switch } from "@maple/ui/components/ui/switch" +import { toastManager } from "@maple/ui/components/ui/toast" + +import { GoogleAnalyticsIcon, LoaderIcon } from "@/components/icons" +import { formatRelativeTime } from "@maple/ui/lib/time-format" +import { Result, useAtomRefresh, useAtomSet, useAtomValue } from "@/lib/effect-atom" +import { MapleApiV2AtomClient, retainedQueryV2 } from "@/lib/services/common/v2-atom-client" +import { showErrorToast } from "@/lib/error-toast" +import { IntegrationIconPlate, catalogEntry } from "./integration-catalog" +import { useIntegrationConnect } from "./integration-connect" +import { + IntegrationEmpty, + IntegrationEmptyCard, + IntegrationEmptyFeature, + IntegrationEmptyFeatures, + IntegrationEmptyHint, + IntegrationEmptyMedia, +} from "./integration-empty-state" + +const GA_ENTRY = catalogEntry("google-analytics") + +/** + * Google Analytics 4 connection card: authorize a Google account in a popup, and every GA4 + * property that account can see is discovered and collected automatically. Per-property toggles + * are the one knob — an agency account reaching hundreds of properties should not have to collect + * all of them. + * + * Disconnect lives in the card body rather than the route header, matching PlanetScale and Slack; + * Cloudflare's header actions are the outlier. + */ +export function GoogleAnalyticsIntegrationCard() { + const statusQuery = retainedQueryV2("googleAnalyticsIntegration", "status", { + reactivityKeys: ["googleAnalyticsIntegration"], + }) + const statusResult = useAtomValue(statusQuery) + const refreshStatus = useAtomRefresh(statusQuery) + + const connectFlow = useIntegrationConnect() + if (connectFlow === null) { + throw new Error("GoogleAnalyticsIntegrationCard must render inside IntegrationConnectProvider") + } + + const disconnect = useAtomSet( + MapleApiV2AtomClient.mutation("googleAnalyticsIntegration", "disconnect"), + { mode: "promiseExit" }, + ) + const updateProperty = useAtomSet( + MapleApiV2AtomClient.mutation("googleAnalyticsIntegration", "updateProperty"), + { mode: "promiseExit" }, + ) + + const handleDisconnect = async () => { + const result = await disconnect({ reactivityKeys: ["googleAnalyticsIntegration"] }) + if (Exit.isSuccess(result)) { + toastManager.add({ title: "Google Analytics disconnected", type: "success" }) + refreshStatus() + } else { + showErrorToast(result, { fallbackTitle: "Failed to disconnect Google Analytics" }) + } + } + + // Which property's toggle is in flight — and every row is disabled while one is, not just that + // row. The mutation atom is shared and non-concurrent, so a second toggle interrupts the first + // one's client-side operation without cancelling the PATCH that already reached the API, and + // `setPropertyEnabled` is an unconditional write with no ordering check: the older request can + // land last and undo the user's final choice. Per-row disabling was not enough on its own — + // whichever operation settled first cleared this back to null and re-enabled every row while + // another update was still in flight. + const [pendingProperty, setPendingProperty] = useState(null) + + const handleToggle = async (propertyId: string, enabled: boolean) => { + setPendingProperty(propertyId) + const result = await updateProperty({ + params: { property_id: propertyId }, + payload: { enabled }, + reactivityKeys: ["googleAnalyticsIntegration"], + }) + setPendingProperty(null) + if (Exit.isSuccess(result)) { + refreshStatus() + } else { + showErrorToast(result, { fallbackTitle: "Failed to update the property" }) + } + } + + if (Result.isInitial(statusResult)) { + return + } + + // A failed status fetch is not "not connected" — don't offer the connect CTA over an + // account that may already be authorized. + if (Result.isFailure(statusResult)) { + return ( +
+ +
+

Google Analytics

+

+ Couldn't load the Google Analytics connection status — refresh the page to try + again. +

+
+
+ ) + } + + const status = statusResult.value + + if (!status.connected) { + return ( + + + + + + + + + + Every GA4 property the Google account can see appears here after connecting. + + + + + ) + } + + return ( +
+
+
+ +
+
+

Google Analytics

+ {status.revoked ? ( + Reconnect needed + ) : ( + Connected + )} +
+

+ {status.connected_email ?? "Connected Google account"} +

+
+
+
+ {status.revoked && ( + + )} + +
+
+ + {status.revoked && ( +

+ Google rejected the stored authorization, so collection has stopped. Everything already + collected is still here — reconnect to resume. +

+ )} + +
+ {status.properties.length === 0 ? ( +

+ No GA4 properties discovered yet. Discovery runs hourly after connecting. +

+ ) : ( +
    + {status.properties.map((property) => ( +
  • +
    +
    + + {property.property_name ?? property.property_id} + + + {property.property_id} + +
    +

    + {propertyDetail(property)} +

    +
    + void handleToggle(property.property_id, checked)} + aria-label={`Collect ${property.property_name ?? property.property_id}`} + /> +
    • + ))} +
+ )} +
+
+ ) +} + +type PropertyStatus = { + readonly enabled: boolean + readonly account_name: string | null + readonly time_zone: string | null + readonly last_error: string | null + readonly last_synced_at: string | null +} + +/** The one line under a property's name: whatever most needs saying about it. */ +function propertyDetail(property: PropertyStatus): string { + if (!property.enabled) return "Not collected" + if (property.last_error !== null) return property.last_error + // No timezone means the property has never been collected — GA4 reports hourly data in the + // property's own zone, so nothing can be placed on the timeline until it resolves. + if (property.time_zone === null) return "Waiting for the first collection" + const account = property.account_name ?? "Google Analytics" + return property.last_synced_at === null + ? account + : `${account} · synced ${formatRelativeTime(property.last_synced_at)}` +} diff --git a/apps/web/src/components/integrations/integration-catalog.tsx b/apps/web/src/components/integrations/integration-catalog.tsx index d48d1adbb..5e05e9aba 100644 --- a/apps/web/src/components/integrations/integration-catalog.tsx +++ b/apps/web/src/components/integrations/integration-catalog.tsx @@ -8,6 +8,7 @@ import { ChevronRightIcon, CloudflareIcon, GithubIcon, + GoogleAnalyticsIcon, HazelIcon, PlanetScaleIcon, PrometheusIcon, @@ -29,6 +30,7 @@ export type IntegrationId = | "hazel" | "github" | "slack" + | "google-analytics" /** * Third-party brand accents for the icon-plate wash — no app token applies. @@ -37,6 +39,8 @@ export type IntegrationId = export const GITHUB_ACCENT = "#181717" export const HAZEL_ACCENT = "#F46F0F" export const CLOUDFLARE_ACCENT = "#F38020" +/** Google Analytics 4 brand orange. */ +export const GOOGLE_ANALYTICS_ACCENT = "#E37400" /** * Slack's deep aubergine — the brand's identity color, and the light-theme value. @@ -148,6 +152,15 @@ const CATALOG: ReadonlyArray = [ accent: SLACK_ACCENT, docsUrl: "https://maple.dev/docs/integrations/slack", }, + { + id: "google-analytics", + name: "Google Analytics", + description: + "Connect a Google account to chart GA4 sessions, users and page views next to your traces and errors.", + icon: GoogleAnalyticsIcon, + accent: GOOGLE_ANALYTICS_ACCENT, + docsUrl: "https://maple.dev/docs/integrations/google-analytics", + }, ] /** @@ -210,6 +223,11 @@ export function useIntegrationStatuses(): Partial null) .orElse(() => STATUS_UNAVAILABLE) + const googleAnalytics: CardStatus | null = Result.builder(googleAnalyticsResult) + .onSuccess((status): CardStatus => { + if (!status.connected) return NOT_CONNECTED + // A revoked grant still has a connection row, so "Not connected" would be wrong and + // "Connected" would be a lie — collection has stopped until someone reconnects. + if (status.revoked) return { label: "Reconnect needed", variant: "warning" } + const collecting = status.properties.filter((property) => property.enabled).length + return { + label: collecting > 0 ? `${collecting} propert${collecting === 1 ? "y" : "ies"}` : "Connected", + variant: "success", + } + }) + .onInitial(() => null) + .orElse(() => STATUS_UNAVAILABLE) + return { cloudflare, prometheus: scrapeStatus("prometheus"), @@ -294,6 +327,7 @@ export function useIntegrationStatuses(): Partial { @@ -614,6 +653,40 @@ export function useIntegrationOverviews(): Record null) .orElse(() => UNAVAILABLE) + const googleAnalytics: IntegrationOverview = Result.builder(googleAnalyticsResult) + .onSuccess((status): IntegrationOverview => { + if (!status.connected) return CONNECT + const collecting = status.properties.filter((property) => property.enabled) + const erroring = collecting.filter((property) => property.last_error !== null) + // A property with no timezone yet has never been collected — Google reports hourly + // data in the property's own zone, so nothing can be placed until it resolves. + const unresolved = collecting.filter((property) => property.time_zone === null) + const issue = status.revoked + ? "authorization revoked" + : erroring.length > 0 + ? `${plural(erroring.length, "property")} erroring` + : unresolved.length > 0 + ? `${plural(unresolved.length, "property")} not started` + : null + return { + kind: "connected", + health: issue ? "attention" : "healthy", + stateLabel: status.revoked ? "Reconnect needed" : issue ? "Needs attention" : "Healthy", + context: status.connected_email, + stat: collecting.length > 0 ? `${plural(collecting.length, "property")} collected` : null, + lastSyncLabel: syncedLabel( + maxMs( + collecting.map((property) => + property.last_synced_at ? Date.parse(property.last_synced_at) : null, + ), + ), + ), + issue, + } + }) + .onInitial(() => null) + .orElse(() => UNAVAILABLE) + return { cloudflare, prometheus: scrapeOverview("prometheus"), @@ -623,6 +696,7 @@ export function useIntegrationOverviews(): Record{children} case "planetscale": return {children} + case "google-analytics": + return {children} default: return children } @@ -389,3 +391,65 @@ function PlanetscaleConnectBoundary({ children }: { children: React.ReactNode }) return {children} } + +function GoogleAnalyticsConnectBoundary({ children }: { children: React.ReactNode }) { + const refreshStatus = useAtomRefresh( + retainedQueryV2("googleAnalyticsIntegration", "status", { + reactivityKeys: ["googleAnalyticsIntegration"], + }), + ) + const startConnect = useAtomSet( + MapleApiV2AtomClient.mutation("googleAnalyticsIntegration", "connect"), + { mode: "promiseExit" }, + ) + const prime = useAtomSet(MapleApiV2AtomClient.mutation("googleAnalyticsIntegration", "prime"), { + mode: "promiseExit", + }) + + // The callback deliberately does NOT run the first collection — it takes tens of seconds on a + // grant with several properties and the popup would sit blank for all of it. This tab is still + // open, so it runs it here. + // + // Two paths can reach it: the success message, and the popup simply closing (`postMessage` is + // lost under COOP, which `useOAuthPopupFlow` documents). Both are needed — without the close + // path, a COOP-blocked browser connects successfully and then waits up to fifteen minutes for + // the cron before showing a single number. The ref is what stops them running it twice. + const primed = useRef(false) + const primeOnce = useEffectEvent(() => { + if (primed.current) return + primed.current = true + void prime({ reactivityKeys: ["googleAnalyticsIntegration"] }).finally(refreshStatus) + }) + + useIntegrationMessage("maple:integration:google-analytics", (data) => { + if (data.status === "success") { + primeOnce() + refreshStatus() + } else if (data.status === "error") { + toastManager.add({ title: data.message ?? "Google Analytics connection failed", type: "error" }) + } + }) + + const value = useOAuthPopupFlow({ + windowName: "maple-google-analytics-connect", + label: "Google Analytics", + windowFeatures: "popup,width=520,height=680", + start: () => { + // Per ATTEMPT, not per mount: the card offers Reconnect on a revoked grant without + // unmounting this boundary, so a latched `primed` would skip the first collection on + // every attempt after the first and leave the reconnect waiting on cron. + primed.current = false + return startConnect({ + payload: { return_to: currentReturnPath() }, + reactivityKeys: ["googleAnalyticsIntegration"], + }).then(Exit.map(({ redirect_url }) => ({ redirectUrl: redirect_url }))) + }, + startErrorTitle: "Failed to start Google Analytics connect flow", + onClosed: () => { + refreshStatus() + primeOnce() + }, + }) + + return {children} +} diff --git a/apps/web/src/components/layout/dashboard-layout.tsx b/apps/web/src/components/layout/dashboard-layout.tsx index 69f629fe0..05b49a86b 100644 --- a/apps/web/src/components/layout/dashboard-layout.tsx +++ b/apps/web/src/components/layout/dashboard-layout.tsx @@ -85,12 +85,22 @@ function Breadcrumbs({ items, children }: { items: BreadcrumbEntry[]; children?:
- - + {/* The header is a fixed `h-16`, so a wrapping trail does not grow it — it spills out + and gets clipped by the border. Kept to one line instead: see the per-crumb rules + below for what gives way when a trail like "Settings › Integrations › Google + Analytics" meets a 375px viewport. */} + + {items.map((item, index) => ( - {index > 0 && } - + {index > 0 && } + {/* Below `sm` only the leaf survives: the full trail cannot share a + 375px row with the action cluster, and letting the ancestors shrink + instead collapses every crumb at once so their un-truncated link + text overlaps. Above `sm` the whole trail is back. */} + {item.href ? ( (() => { const { pathname, search } = parseSearchFromHref(item.href) @@ -110,7 +120,7 @@ function Breadcrumbs({ items, children }: { items: BreadcrumbEntry[]; children?: ) })() ) : ( - {item.label} + {item.label} )} diff --git a/apps/web/src/routes/integrations.tsx b/apps/web/src/routes/integrations.tsx index c40f8c674..42604033b 100644 --- a/apps/web/src/routes/integrations.tsx +++ b/apps/web/src/routes/integrations.tsx @@ -11,6 +11,7 @@ import { import { GithubIntegrationCard } from "@/components/integrations/github-integration-card" import { HazelIntegrationCard } from "@/components/integrations/hazel-integration-card" import { PlanetScaleIntegrationCard } from "@/components/integrations/planetscale-integration-card" +import { GoogleAnalyticsIntegrationCard } from "@/components/integrations/google-analytics-integration-card" import { SlackIntegrationCard } from "@/components/integrations/slack-integration-card" import { IntegrationCatalog, @@ -226,6 +227,8 @@ function IntegrationsPage() { ) : integration === "slack" ? ( + ) : integration === "google-analytics" ? ( + ) : ( // prometheus + warpstream share the generic scrape-target flow @@ -260,7 +263,11 @@ function IntegrationHeader({ integration }: { integration: IntegrationId }) { : null return ( -
+ // Wraps rather than overflowing: the action group is `shrink-0` (a Connect button that + // shrinks becomes an ellipsis), so on a narrow viewport it has to move to its own row or + // it runs off the edge. `entry.name` is the other half — the longest one in the catalog is + // "Google Analytics", which is what made the collision obvious. +