feat(rules): let rules act on who the User-Agent says it is

packages/webdecoy/scripts/gen-bot-registry.mjs

@@ -0,0 +1,131 @@
+#!/usr/bin/env node
+/**
+ * Generate src/bots/registry.generated.ts from the Go agent registry.
+ *
+ * The Go table in WebDecoy/app `pkg/agents` is the single source of truth. This
+ * script turns its JSON export into a TypeScript module so the SDK can classify a
+ * User-Agent locally, with no network call, in the request path.
+ *
+ * Usage, from the WebDecoy/app checkout:
+ *
+ *   cd pkg && go run ./cmd/export-agent-registry \
+ *     | node ../../webdecoy-node/packages/webdecoy/scripts/gen-bot-registry.mjs
+ *
+ * or with an explicit file:
+ *
+ *   node scripts/gen-bot-registry.mjs agents.json
+ *
+ * The exporter emits agents already in `agents.MatchUserAgent` order, and this
+ * script preserves that order. Do not sort the output: the matcher takes the
+ * first hit, and several agents share User-Agent substrings, so reordering
+ * silently reclassifies real traffic.
+ */
+
+import { writeFileSync, readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+
+const OUT = join(dirname(fileURLToPath(import.meta.url)), '..', 'src', 'bots', 'registry.generated.ts');
+
+function readInput() {
+  const arg = process.argv[2];
+  if (arg) return readFileSync(arg, 'utf8');
+  return readFileSync(0, 'utf8');
+}
+
+const raw = readInput().trim();
+if (!raw) {
+  console.error('gen-bot-registry: no input. Pipe the exporter output or pass a JSON path.');
+  process.exit(1);
+}
+
+const { categories, agents } = JSON.parse(raw);
+
+if (!Array.isArray(agents) || agents.length === 0) {
+  console.error('gen-bot-registry: input has no agents.');
+  process.exit(1);
+}
+if (!Array.isArray(categories) || categories.length === 0) {
+  console.error('gen-bot-registry: input has no categories.');
+  process.exit(1);
+}
+
+// A pattern that is not already lowercase would never match, because the matcher
+// lowercases the User-Agent and compares against the pattern verbatim — same as
+// Go's strings.Contains(uaLower, pattern). Catch it here rather than shipping a
+// rule that silently never fires.
+for (const a of agents) {
+  for (const p of a.uaPatterns ?? []) {
+    if (p !== p.toLowerCase()) {
+      console.error(`gen-bot-registry: agent "${a.id}" has a non-lowercase UA pattern ${JSON.stringify(p)}, which can never match.`);
+      process.exit(1);
+    }
+  }
+}
+
+const q = (s) => JSON.stringify(s);
+
+const lines = agents.map((a) => {
+  const patterns = (a.uaPatterns ?? []).map(q).join(', ');
+  return (
+    `  { id: ${q(a.id)}, name: ${q(a.name)}, category: ${q(a.category)}, ` +
+    `organization: ${q(a.organization)}, baseScore: ${a.baseScore}, ` +
+    `respectsRobots: ${a.respectsRobots === true}, uaPatterns: [${patterns}] },`
+  );
+});
+
+const out = `/**
+ * GENERATED FILE — DO NOT EDIT.
+ *
+ * Source of truth: WebDecoy/app \`pkg/agents\` (Go).
+ * Regenerate with \`packages/webdecoy/scripts/gen-bot-registry.mjs\`; see that
+ * script for the command.
+ *
+ * ORDER IS SIGNIFICANT. Entries are in \`agents.MatchUserAgent\` order — AI data
+ * scrapers first, then AI search crawlers, AI agents, search engines, then the
+ * remaining categories. \`matchUserAgent()\` returns the first hit, and several
+ * agents share User-Agent substrings, so sorting this array changes how real
+ * traffic is classified.
+ *
+ * ${agents.length} agents across ${categories.length} categories.
+ */
+
+/**
+ * Classification categories, mirroring \`agents.Category\` in Go.
+ *
+ * Declared here rather than hand-written next door so a category added in Go
+ * cannot go missing in TypeScript.
+ */
+export type BotCategory =
+${categories.map((c) => `  | ${q(c)}`).join('\n')};
+
+/** One entry in the registry. */
+export interface BotAgent {
+  /** Stable lowercase slug, e.g. \`gptbot\`. */
+  id: string;
+  /** Display name, e.g. \`GPTBot\`. */
+  name: string;
+  category: BotCategory;
+  /** Operator, e.g. \`OpenAI\`. */
+  organization: string;
+  /** The registry's default threat score for this agent (0-100). */
+  baseScore: number;
+  /** Whether the operator documents honouring robots.txt. */
+  respectsRobots: boolean;
+  /** Lowercase User-Agent substrings that identify it. */
+  uaPatterns: readonly string[];
+}
+
+/** Every declared category, including any with no agents yet. */
+export const BOT_CATEGORIES: readonly BotCategory[] = [
+${categories.map((c) => `  ${q(c)},`).join('\n')}
+];
+
+/** The agent table, in match order. */
+export const BOT_REGISTRY: readonly BotAgent[] = [
+${lines.join('\n')}
+];
+`;
+
+writeFileSync(OUT, out);
+console.error(`gen-bot-registry: wrote ${agents.length} agents, ${categories.length} categories -> ${OUT}`);

packages/webdecoy/src/bots/bots.test.ts

@@ -0,0 +1,126 @@
+import {
+  matchUserAgent,
+  classifyUserAgent,
+  clearBotCache,
+  BOT_REGISTRY,
+  BOT_CATEGORIES,
+} from './index';
+
+beforeEach(() => clearBotCache());
+
+describe('matchUserAgent', () => {
+  it('identifies AI training crawlers from a real User-Agent', () => {
+    const gpt = matchUserAgent('Mozilla/5.0 (compatible; GPTBot/1.1; +https://openai.com/gptbot)');
+    expect(gpt?.id).toBe('gptbot');
+    expect(gpt?.category).toBe('training_crawler');
+    expect(gpt?.organization).toBe('OpenAI');
+
+    const claude = matchUserAgent('Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)');
+    expect(claude?.id).toBe('claudebot');
+    expect(claude?.category).toBe('training_crawler');
+  });
+
+  it('is case-insensitive, because User-Agent casing is not a contract', () => {
+    expect(matchUserAgent('GPTBOT/1.1')?.id).toBe('gptbot');
+    expect(matchUserAgent('gptbot/1.1')?.id).toBe('gptbot');
+  });
+
+  it('returns undefined for an ordinary browser', () => {
+    expect(
+      matchUserAgent(
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
+      ),
+    ).toBeUndefined();
+  });
+
+  it('returns undefined for empty and missing input rather than throwing', () => {
+    expect(matchUserAgent(undefined)).toBeUndefined();
+    expect(matchUserAgent(null)).toBeUndefined();
+    expect(matchUserAgent('')).toBeUndefined();
+  });
+
+  it('caches both hits and misses', () => {
+    // A miss caches `null`, which must not be confused with "not cached" —
+    // otherwise every browser request rescans all 168 agents forever.
+    const ua = 'Mozilla/5.0 (X11; Linux x86_64) Chrome/120.0.0.0';
+    expect(matchUserAgent(ua)).toBeUndefined();
+    expect(matchUserAgent(ua)).toBeUndefined();
+    expect(matchUserAgent('GPTBot/1.1')?.id).toBe('gptbot');
+    expect(matchUserAgent('GPTBot/1.1')?.id).toBe('gptbot');
+  });
+
+  it('bounds the cache against attacker-supplied User-Agents', () => {
+    // The key is client-controlled. Unbounded growth here would be a memory
+    // exhaustion vector, so the map must not keep growing past its cap.
+    for (let i = 0; i < 2500; i++) matchUserAgent(`UniqueAgent/${i}`);
+    // Still correct after whatever eviction happened.
+    expect(matchUserAgent('GPTBot/1.1')?.id).toBe('gptbot');
+  });
+});
+
+describe('registry integrity', () => {
+  it('carries the full generated table', () => {
+    expect(BOT_REGISTRY.length).toBe(168);
+    expect(BOT_CATEGORIES).toContain('training_crawler');
+    expect(BOT_CATEGORIES).toContain('search_crawler');
+  });
+
+  it('has only lowercase patterns, which the matcher relies on', () => {
+    // The matcher lowercases the UA and compares verbatim, mirroring Go's
+    // strings.Contains(uaLower, pattern). An uppercase pattern would be dead.
+    const offenders = BOT_REGISTRY.flatMap((a) =>
+      a.uaPatterns.filter((p) => p !== p.toLowerCase()).map((p) => `${a.id}:${p}`),
+    );
+    expect(offenders).toEqual([]);
+  });
+
+  it('has no empty pattern, which would match every request', () => {
+    const offenders = BOT_REGISTRY.filter((a) => a.uaPatterns.some((p) => p === ''));
+    expect(offenders).toEqual([]);
+  });
+
+  it('puts AI data scrapers before other categories, as Go does', () => {
+    // Match precedence is positional. If the generator ever sorts the table,
+    // this is what notices.
+    const firstTraining = BOT_REGISTRY.findIndex((a) => a.category === 'training_crawler');
+    const firstGeneric = BOT_REGISTRY.findIndex((a) => a.category === 'generic_scraper');
+    expect(firstTraining).toBe(0);
+    expect(firstGeneric).toBeGreaterThan(firstTraining);
+  });
+});
+
+describe('classifyUserAgent', () => {
+  it('describes a known agent completely', () => {
+    const v = classifyUserAgent('Mozilla/5.0 (compatible; GPTBot/1.1; +https://openai.com/gptbot)');
+    expect(v).toMatchObject({
+      known: true,
+      id: 'gptbot',
+      name: 'GPTBot',
+      category: 'training_crawler',
+      organization: 'OpenAI',
+      isAI: true,
+    });
+    expect(v.score).toBeGreaterThan(0);
+  });
+
+  it('reports an unknown agent as known:false / category:none, never undefined', () => {
+    const v = classifyUserAgent('Mozilla/5.0 Chrome/137.0.0.0');
+    expect(v.known).toBe(false);
+    expect(v.category).toBe('none');
+    expect(v.score).toBe(0);
+    expect(v.isAI).toBe(false);
+  });
+
+  it('flags every AI category as AI and ordinary crawlers as not', () => {
+    expect(classifyUserAgent('GPTBot/1.1').isAI).toBe(true);
+    expect(classifyUserAgent('PerplexityBot/1.0').isAI).toBe(true);
+    // Googlebot is a search engine, not an AI client. Getting this wrong means a
+    // customer's `ai: true` rule deindexes their site.
+    const google = classifyUserAgent(
+      'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+    );
+    expect(google.known).toBe(true);
+    expect(google.category).toBe('search_crawler');
+    expect(google.isAI).toBe(false);
+  });
+});