feat(observability): send metrics and logs to Grafana Cloud

AGENTS.md

@@ -18,6 +18,7 @@ BRANCH is a non-profit accounting platform (projects, donors, donations, expendi
 | `shared/rbac/` | `@branch/rbac` — **the** authorization policy, shared by lambdas + frontend | `shared/rbac/README.md` |
 | `shared/lambda-auth/` | `@branch/lambda-auth` — runtime Cognito auth/authz pkg | see backend doc |
 | `shared/lambda-http/` | `@branch/lambda-http` — route table, dispatch, permission enforcement | see backend doc |
+| `shared/lambda-telemetry/` | `@branch/lambda-telemetry` — OTLP metrics + structured logs to Grafana Cloud | `shared/lambda-telemetry/README.md` |
 | `infrastructure/` | Terraform: `aws/`, `github/`, `preview/`, `preview-shared/`, `test/` | `infrastructure/AGENTS.md` |
 | `.github/workflows/` | CI/CD + PR review bot | `.github/AGENTS.md` |
 
@@ -30,12 +31,13 @@ BRANCH is a non-profit accounting platform (projects, donors, donations, expendi
 
 ## Shared packages (critical)
 
-Four `file:`-linked packages dedupe code across the repo:
+Five `file:`-linked packages dedupe code across the repo:
 
 - **`@branch/types`** (`shared/types/`) — types only, no runtime. Exports DB row types (`DB`, `BranchUsers`, ...) + auth DTOs (`AuthContext`, `AuthenticatedUser`, `AccessLevel`, `AuthorizationCheck`). It is the **single declaration** of those DTOs: `@branch/lambda-auth` depends on this package and re-exports them, so never add a second copy anywhere. `db-types.d.ts` is **generated** from `apps/backend/db/migrations/**` by the `Schema Change Checks` workflow (or locally by `make types`) — never hand-edit it.
 - **`@branch/rbac`** (`shared/rbac/`) — the authorization policy, as one table of rules plus a pure `can`/`authorize`. **The lambdas and the frontend both evaluate this module**, so a disabled button and the 403 behind it cannot disagree. It is the only place a permission is defined; do not re-derive one from `isAdmin` in a component or a controller. Read `shared/rbac/README.md` — it carries the role matrix — before changing who may do what.
 - **`@branch/lambda-auth`** (`shared/lambda-auth/`) — runtime auth: `authenticateRequest(db, event)`, `extractToken(event)`, `loadRbacSubject(db, ctx)` (one query for the caller's memberships, which is also what `GET /auth/me` ships to the browser). Lambdas wrap it in their local `auth.ts`.
-- **`@branch/lambda-http`** (`shared/lambda-http/`) — runtime routing: `dispatch(event, { prefix, routes, resolveAuth })` replaces the old per-lambda if-chain handler with a declarative `Route[]` table (`routes.ts`). Every route declares `access: 'public' | 'authenticated'` or a `permission`, and dispatch enforces it before the controller runs; the union has no default arm, so omitting the gate is a type error. Also exports `json`, `parseBody`, `requirePermission`, `createAuthResolver`. Depends on `@branch/rbac`'s and `@branch/lambda-auth`'s `dist/`, so build those first (`.github/actions/build-shared-packages` discovers `shared/*/` and builds in `@branch`-dep order). See `apps/backend/lambdas/AGENTS.md`.
+- **`@branch/lambda-http`** (`shared/lambda-http/`) — runtime routing: `dispatch(event, { prefix, routes, resolveAuth })` replaces the old per-lambda if-chain handler with a declarative `Route[]` table (`routes.ts`). Every route declares `access: 'public' | 'authenticated'` or a `permission`, and dispatch enforces it before the controller runs; the union has no default arm, so omitting the gate is a type error. Also exports `json`, `parseBody`, `requirePermission`, `createAuthResolver`. Depends on `@branch/rbac`'s, `@branch/lambda-auth`'s and `@branch/lambda-telemetry`'s `dist/`, so build those first (`.github/actions/build-shared-packages` discovers `shared/*/` and builds in `@branch`-dep order). See `apps/backend/lambdas/AGENTS.md`.
+- **`@branch/lambda-telemetry`** (`shared/lambda-telemetry/`) — dependency, runtime observability: OTLP metrics and structured logs to Grafana Cloud. `dispatch()` calls it, so **every route in every service already reports** latency, status, cold starts, auth refusals and query duration, and emits one access log per request; controllers only add domain metrics (`recordEvent(METRICS.LOGIN, …)`). It is the only consumer of the `OTEL_EXPORTER_OTLP_*` env vars Terraform sets. With no endpoint configured the OTel SDK is never loaded, so local dev and tests are unaffected. Read `shared/lambda-telemetry/README.md` before adding a metric — the label-cardinality rules there are not optional.
 
 ## Root commands
 

apps/backend/AGENTS.md

@@ -52,10 +52,11 @@ http://localhost:3000/<service>/health
 
 ## Shared packages
 
-All three linked via `file:` deps in each lambda's `package.json`:
+All four linked via `file:` deps in each lambda's `package.json`:
 - `@branch/types` (`../../../../shared/types`) — devDependency, types only. Must stay a dependency-free leaf: `@branch/lambda-auth` depends on it.
 - `@branch/lambda-auth` (`../../../../shared/lambda-auth`) — dependency, runtime auth. Build it (`npm run build` in `shared/lambda-auth`) when its source changes; lambdas consume `dist/`. It depends on `@branch/types` and re-exports the auth DTOs from there, so those types have exactly one declaration; changing `shared/lambda-auth/package.json` deps invalidates every lambda's `package-lock.json`, so regenerate all six.
-- `@branch/lambda-http` (`../../../../shared/lambda-http`) — dependency, runtime routing: `dispatch(event, { prefix, routes, resolveAuth })` plus `json`/`parseBody`/`requirePermission`/`createAuthResolver`. Every lambda's `handler.ts` is a 5-line call into it; each lambda's own `routes.ts` supplies the `Route[]` table, and every route in it declares a `permission` or an `access` level that dispatch enforces. Depends on `@branch/rbac`'s and `@branch/lambda-auth`'s `dist/`, so build those first. See `lambdas/AGENTS.md`.
+- `@branch/lambda-http` (`../../../../shared/lambda-http`) — dependency, runtime routing: `dispatch(event, { prefix, routes, resolveAuth })` plus `json`/`parseBody`/`requirePermission`/`createAuthResolver`. Every lambda's `handler.ts` is a 5-line call into it; each lambda's own `routes.ts` supplies the `Route[]` table, and every route in it declares a `permission` or an `access` level that dispatch enforces. Depends on `@branch/rbac`'s, `@branch/lambda-auth`'s and `@branch/lambda-telemetry`'s `dist/`, so build those first. See `lambdas/AGENTS.md`.
+- `@branch/lambda-telemetry` (`../../../../shared/lambda-telemetry`) — dependency, runtime metrics + structured logs over OTLP to Grafana Cloud. Wired into `dispatch()` (RED metrics, cold starts, auth refusals, access log) and into each `db.ts` via `log: kyselyTelemetryLog` (query duration, slow/failed query lines). Controllers add domain counters with `recordEvent`/`recordValue`. Inert unless `OTEL_EXPORTER_OTLP_ENDPOINT` is set, which is why local dev and tests need no collector.
 
 ## Deploy
 
@@ -70,6 +71,8 @@ Automatic on push to `main` touching `apps/backend/lambdas/**` or `shared/types/
 
 `DB_HOST DB_PORT DB_USER DB_PASSWORD DB_NAME`, `COGNITO_USER_POOL_ID`, `COGNITO_CLIENT_ID` (or `COGNITO_APP_CLIENT_ID`), `REPORTS_BUCKET_NAME` (reports, expenditures and projects — one bucket holds both the `reports/` and `receipts/` prefixes, and all three delete out of it), `AWS_REGION` (default `us-east-2`, Lambda-reserved — never set it in Terraform).
 
+Observability adds `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_EXPORTER_OTLP_HEADERS` (read by `@branch/lambda-telemetry`; absent means no export at all, which is the local and test default) and the optional `LOG_LEVEL`.
+
 **Anything a lambda reads from `process.env` must be declared in the `environment` block of `infrastructure/aws/lambda.tf`.** That block is authoritative and is deliberately not in `lifecycle.ignore_changes`, so a value set by hand in the console is deleted on the next apply. Locally the Cognito values must be the real shared dev-pool IDs (`apps/backend/.env`) — auth talks to the real pool for JWKS and `InitiateAuth` — but no AWS credentials are needed, because every Cognito API on the sign-in path is unsigned.
 
 ## Auth

apps/backend/lambdas/AGENTS.md

@@ -51,6 +51,7 @@ export const getDonor: RouteHandler = async ({ params }) => {
 };
 ```
 - `dispatch()` handles OPTIONS preflight, `GET /<prefix>/health`, 404 and 500 centrally — routes.ts only lists real endpoints.
+- **`dispatch()` is also where the backend is instrumented.** Latency, status, cold starts and auth refusals are recorded there for every route, and one `Request served` log line carries the request id, route, status, duration and caller. Controllers add only *domain* metrics — `recordEvent(METRICS.EXPENDITURE_CHANGED, { action: 'reviewed', status })` — never a per-request counter of their own, which would double-count. Route *patterns* are the label; a concrete path or a row id never is. See `shared/lambda-telemetry/README.md`.
 - **A caught error is an error Sentry cannot see.** The Sentry layer wraps the handler and records uncaught throws only, and we always catch so API Gateway returns a JSON 500 instead of a 502. `dispatch()` reports what reaches its catch; a controller that returns its own 500 must use `serverError(err, 'Failed to …')` from `@branch/lambda-http` rather than `console.error` + `json(500, …)`. Same status to the caller, one less invisible failure.
 - **NEVER remove or modify the `ROUTES-START` / `ROUTES-END` markers** — the CLI inserts new table entries between them.
 - Patterns are always full-prefixed (`/donors/:id`, never `/:id`) so one table works whether the event arrives via API Gateway's `{proxy+}` (full path) or the shared dev-server (prefix stripped) — `dispatch()` canonicalizes both to the prefixed form.
@@ -116,7 +117,7 @@ the same predicate** or the total leaks what the page does not.
 
 ## DB access
 
-`db.ts` exports a `Kysely<DB>` (`DB` from `@branch/types`) over a `pg.Pool`. Always qualify the schema:
+`db.ts` exports a `Kysely<DB>` (`DB` from `@branch/types`) over a `pg.Pool`, with `log: kyselyTelemetryLog` so every statement's duration is measured and anything slow or failed gets a log line. A new lambda's `db.ts` must keep that hook. Always qualify the schema:
 ```ts
 await db.selectFrom('branch.users').where('cognito_sub', '=', sub).selectAll().executeTakeFirst();
 await db.selectFrom('branch.users').select(db.fn.count('user_id').as('count')).executeTakeFirst();

apps/backend/lambdas/auth/controllers/auth.ts

@@ -9,6 +9,7 @@ import {
 } from '@aws-sdk/client-cognito-identity-provider';
 import { json, parseBody, reportError, serverError } from '@branch/lambda-http';
 import type { RouteHandler } from '@branch/lambda-http';
+import { METRICS, recordEvent } from '@branch/lambda-telemetry';
 import { resolveProfileImage } from '../photos';
 import {
   cognitoClient,
@@ -58,10 +59,14 @@ export async function handleLogin(event: any): Promise<APIGatewayProxyResult> {
     const response = await cognitoClient.send(new InitiateAuthCommand(params));
 
     if (response.AuthenticationResult) {
+      recordEvent(METRICS.LOGIN, { outcome: 'success' });
       return authResultResponse(response.AuthenticationResult);
     }
 
     if (response.ChallengeName) {
+      // A small fixed set, so safe as a label.
+      recordEvent(METRICS.LOGIN, { outcome: 'challenge', challenge: response.ChallengeName });
+
       // MFA_SETUP cannot be answered by RespondToAuthChallenge alone -- it needs
       // AssociateSoftwareToken/VerifySoftwareToken enrollment, which is not
       // built yet. Return the Session anyway so a future enrollment endpoint can
@@ -81,6 +86,8 @@ export async function handleLogin(event: any): Promise<APIGatewayProxyResult> {
       'Unexpected response from authentication service',
     );
   } catch (error: any) {
+    // The address is not a label: PII, and unbounded cardinality.
+    recordEvent(METRICS.LOGIN, { outcome: 'failure', 'error.type': error?.name ?? 'Unknown' });
     return mapCognitoAuthError(error, 'login');
   }
 }

apps/backend/lambdas/auth/controllers/register.ts

@@ -9,6 +9,7 @@ import {
   ResendConfirmationCodeCommand,
 } from '@aws-sdk/client-cognito-identity-provider';
 import { json, reportError, serverError } from '@branch/lambda-http';
+import { METRICS, recordEvent } from '@branch/lambda-telemetry';
 import db from '../db';
 import { cognitoClient, USER_POOL_CLIENT_ID, USER_POOL_ID, validatePassword } from '../services/cognito';
 
@@ -71,6 +72,7 @@ export async function handleRegister(event: any): Promise<APIGatewayProxyResult>
     // email addresses have been invited, so the response is deliberately the
     // same whether or not the address is known.
     if (!existingUser) {
+      recordEvent(METRICS.REGISTRATION, { outcome: 'invitation_required' });
       return json(403, {
         message:
           'Registration is by invitation only. Ask an administrator to create your account.',
@@ -214,6 +216,7 @@ export async function handleRegister(event: any): Promise<APIGatewayProxyResult>
       return json(500, { message: 'Failed to create user account' });
     }
 
+    recordEvent(METRICS.REGISTRATION, { outcome: 'claimed' });
     return json(201, {
       message: 'User registered successfully',
       userId: cognitoUserSub,

apps/backend/lambdas/auth/db.ts

@@ -2,8 +2,10 @@
 import { Kysely, PostgresDialect } from 'kysely'
 import { Pool } from 'pg'
 import type { DB } from '@branch/types'
+import { kyselyTelemetryLog } from '@branch/lambda-telemetry'
 
 const db = new Kysely<DB>({
+  log: kyselyTelemetryLog,
   dialect: new PostgresDialect({
     pool: new Pool({
       host: process.env.DB_HOST ?? 'localhost',

apps/backend/lambdas/auth/package.json

@@ -30,6 +30,7 @@
     "@aws-sdk/s3-request-presigner": "^3.995.0",
     "@branch/lambda-auth": "file:../../../../shared/lambda-auth",
     "@branch/lambda-http": "file:../../../../shared/lambda-http",
+    "@branch/lambda-telemetry": "file:../../../../shared/lambda-telemetry",
     "@branch/rbac": "file:../../../../shared/rbac",
     "aws-jwt-verify": "^5.1.1",
     "dotenv": "^17.2.3",

apps/backend/lambdas/donors/controllers/donations.ts

@@ -1,6 +1,7 @@
 import type { RouteCtx } from '@branch/lambda-http';
 import { json, requirePermission } from '@branch/lambda-http';
 import { projectScopeIds } from '@branch/rbac';
+import { METRICS, recordEvent, recordValue } from '@branch/lambda-telemetry';
 import { sql, type SqlBool } from 'kysely';
 import db from '../db';
 
@@ -143,6 +144,10 @@ export async function createDonation({ event }: RouteCtx) {
       .returningAll()
       .executeTakeFirstOrThrow();
 
+    // Ids stay out of the labels; the rollups answer per-project questions.
+    recordEvent(METRICS.DONATION_RECORDED, { backdated: donatedAt !== undefined });
+    recordValue(METRICS.DONATION_AMOUNT, donationAmount);
+
     return json(201, { data: donation });
   } catch (err: any) {
     if (err?.code === '23505') {

apps/backend/lambdas/donors/db.ts

@@ -1,9 +1,11 @@
 import { Kysely, PostgresDialect } from 'kysely'
 import { Pool } from 'pg'
 import type { DB } from '@branch/types'
+import { kyselyTelemetryLog } from '@branch/lambda-telemetry'
 
 
 const db = new Kysely<DB>({
+  log: kyselyTelemetryLog,
   dialect: new PostgresDialect({
     pool: new Pool({
       host: process.env.DB_HOST ?? 'localhost',

apps/backend/lambdas/donors/package.json

@@ -25,6 +25,7 @@
   "dependencies": {
     "@branch/lambda-auth": "file:../../../../shared/lambda-auth",
     "@branch/lambda-http": "file:../../../../shared/lambda-http",
+    "@branch/lambda-telemetry": "file:../../../../shared/lambda-telemetry",
     "@branch/rbac": "file:../../../../shared/rbac",
     "aws-jwt-verify": "^5.1.1",
     "kysely": "^0.28.8",