diff --git a/packages/webdecoy/scripts/gen-bot-registry.mjs b/packages/webdecoy/scripts/gen-bot-registry.mjs new file mode 100644 index 0000000..284db34 --- /dev/null +++ b/packages/webdecoy/scripts/gen-bot-registry.mjs @@ -0,0 +1,131 @@ +#!/usr/bin/env node +/** + * Generate src/bots/registry.generated.ts from the Go agent registry. + * + * The Go table in WebDecoy/app `pkg/agents` is the single source of truth. This + * script turns its JSON export into a TypeScript module so the SDK can classify a + * User-Agent locally, with no network call, in the request path. + * + * Usage, from the WebDecoy/app checkout: + * + * cd pkg && go run ./cmd/export-agent-registry \ + * | node ../../webdecoy-node/packages/webdecoy/scripts/gen-bot-registry.mjs + * + * or with an explicit file: + * + * node scripts/gen-bot-registry.mjs agents.json + * + * The exporter emits agents already in `agents.MatchUserAgent` order, and this + * script preserves that order. Do not sort the output: the matcher takes the + * first hit, and several agents share User-Agent substrings, so reordering + * silently reclassifies real traffic. + */ + +import { writeFileSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const OUT = join(dirname(fileURLToPath(import.meta.url)), '..', 'src', 'bots', 'registry.generated.ts'); + +function readInput() { + const arg = process.argv[2]; + if (arg) return readFileSync(arg, 'utf8'); + return readFileSync(0, 'utf8'); +} + +const raw = readInput().trim(); +if (!raw) { + console.error('gen-bot-registry: no input. Pipe the exporter output or pass a JSON path.'); + process.exit(1); +} + +const { categories, agents } = JSON.parse(raw); + +if (!Array.isArray(agents) || agents.length === 0) { + console.error('gen-bot-registry: input has no agents.'); + process.exit(1); +} +if (!Array.isArray(categories) || categories.length === 0) { + console.error('gen-bot-registry: input has no categories.'); + process.exit(1); +} + +// A pattern that is not already lowercase would never match, because the matcher +// lowercases the User-Agent and compares against the pattern verbatim — same as +// Go's strings.Contains(uaLower, pattern). Catch it here rather than shipping a +// rule that silently never fires. +for (const a of agents) { + for (const p of a.uaPatterns ?? []) { + if (p !== p.toLowerCase()) { + console.error(`gen-bot-registry: agent "${a.id}" has a non-lowercase UA pattern ${JSON.stringify(p)}, which can never match.`); + process.exit(1); + } + } +} + +const q = (s) => JSON.stringify(s); + +const lines = agents.map((a) => { + const patterns = (a.uaPatterns ?? []).map(q).join(', '); + return ( + ` { id: ${q(a.id)}, name: ${q(a.name)}, category: ${q(a.category)}, ` + + `organization: ${q(a.organization)}, baseScore: ${a.baseScore}, ` + + `respectsRobots: ${a.respectsRobots === true}, uaPatterns: [${patterns}] },` + ); +}); + +const out = `/** + * GENERATED FILE — DO NOT EDIT. + * + * Source of truth: WebDecoy/app \`pkg/agents\` (Go). + * Regenerate with \`packages/webdecoy/scripts/gen-bot-registry.mjs\`; see that + * script for the command. + * + * ORDER IS SIGNIFICANT. Entries are in \`agents.MatchUserAgent\` order — AI data + * scrapers first, then AI search crawlers, AI agents, search engines, then the + * remaining categories. \`matchUserAgent()\` returns the first hit, and several + * agents share User-Agent substrings, so sorting this array changes how real + * traffic is classified. + * + * ${agents.length} agents across ${categories.length} categories. + */ + +/** + * Classification categories, mirroring \`agents.Category\` in Go. + * + * Declared here rather than hand-written next door so a category added in Go + * cannot go missing in TypeScript. + */ +export type BotCategory = +${categories.map((c) => ` | ${q(c)}`).join('\n')}; + +/** One entry in the registry. */ +export interface BotAgent { + /** Stable lowercase slug, e.g. \`gptbot\`. */ + id: string; + /** Display name, e.g. \`GPTBot\`. */ + name: string; + category: BotCategory; + /** Operator, e.g. \`OpenAI\`. */ + organization: string; + /** The registry's default threat score for this agent (0-100). */ + baseScore: number; + /** Whether the operator documents honouring robots.txt. */ + respectsRobots: boolean; + /** Lowercase User-Agent substrings that identify it. */ + uaPatterns: readonly string[]; +} + +/** Every declared category, including any with no agents yet. */ +export const BOT_CATEGORIES: readonly BotCategory[] = [ +${categories.map((c) => ` ${q(c)},`).join('\n')} +]; + +/** The agent table, in match order. */ +export const BOT_REGISTRY: readonly BotAgent[] = [ +${lines.join('\n')} +]; +`; + +writeFileSync(OUT, out); +console.error(`gen-bot-registry: wrote ${agents.length} agents, ${categories.length} categories -> ${OUT}`); diff --git a/packages/webdecoy/src/bots/bots.test.ts b/packages/webdecoy/src/bots/bots.test.ts new file mode 100644 index 0000000..958dd6d --- /dev/null +++ b/packages/webdecoy/src/bots/bots.test.ts @@ -0,0 +1,126 @@ +import { + matchUserAgent, + classifyUserAgent, + clearBotCache, + BOT_REGISTRY, + BOT_CATEGORIES, +} from './index'; + +beforeEach(() => clearBotCache()); + +describe('matchUserAgent', () => { + it('identifies AI training crawlers from a real User-Agent', () => { + const gpt = matchUserAgent('Mozilla/5.0 (compatible; GPTBot/1.1; +https://openai.com/gptbot)'); + expect(gpt?.id).toBe('gptbot'); + expect(gpt?.category).toBe('training_crawler'); + expect(gpt?.organization).toBe('OpenAI'); + + const claude = matchUserAgent('Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'); + expect(claude?.id).toBe('claudebot'); + expect(claude?.category).toBe('training_crawler'); + }); + + it('is case-insensitive, because User-Agent casing is not a contract', () => { + expect(matchUserAgent('GPTBOT/1.1')?.id).toBe('gptbot'); + expect(matchUserAgent('gptbot/1.1')?.id).toBe('gptbot'); + }); + + it('returns undefined for an ordinary browser', () => { + expect( + matchUserAgent( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36', + ), + ).toBeUndefined(); + }); + + it('returns undefined for empty and missing input rather than throwing', () => { + expect(matchUserAgent(undefined)).toBeUndefined(); + expect(matchUserAgent(null)).toBeUndefined(); + expect(matchUserAgent('')).toBeUndefined(); + }); + + it('caches both hits and misses', () => { + // A miss caches `null`, which must not be confused with "not cached" — + // otherwise every browser request rescans all 168 agents forever. + const ua = 'Mozilla/5.0 (X11; Linux x86_64) Chrome/120.0.0.0'; + expect(matchUserAgent(ua)).toBeUndefined(); + expect(matchUserAgent(ua)).toBeUndefined(); + expect(matchUserAgent('GPTBot/1.1')?.id).toBe('gptbot'); + expect(matchUserAgent('GPTBot/1.1')?.id).toBe('gptbot'); + }); + + it('bounds the cache against attacker-supplied User-Agents', () => { + // The key is client-controlled. Unbounded growth here would be a memory + // exhaustion vector, so the map must not keep growing past its cap. + for (let i = 0; i < 2500; i++) matchUserAgent(`UniqueAgent/${i}`); + // Still correct after whatever eviction happened. + expect(matchUserAgent('GPTBot/1.1')?.id).toBe('gptbot'); + }); +}); + +describe('registry integrity', () => { + it('carries the full generated table', () => { + expect(BOT_REGISTRY.length).toBe(168); + expect(BOT_CATEGORIES).toContain('training_crawler'); + expect(BOT_CATEGORIES).toContain('search_crawler'); + }); + + it('has only lowercase patterns, which the matcher relies on', () => { + // The matcher lowercases the UA and compares verbatim, mirroring Go's + // strings.Contains(uaLower, pattern). An uppercase pattern would be dead. + const offenders = BOT_REGISTRY.flatMap((a) => + a.uaPatterns.filter((p) => p !== p.toLowerCase()).map((p) => `${a.id}:${p}`), + ); + expect(offenders).toEqual([]); + }); + + it('has no empty pattern, which would match every request', () => { + const offenders = BOT_REGISTRY.filter((a) => a.uaPatterns.some((p) => p === '')); + expect(offenders).toEqual([]); + }); + + it('puts AI data scrapers before other categories, as Go does', () => { + // Match precedence is positional. If the generator ever sorts the table, + // this is what notices. + const firstTraining = BOT_REGISTRY.findIndex((a) => a.category === 'training_crawler'); + const firstGeneric = BOT_REGISTRY.findIndex((a) => a.category === 'generic_scraper'); + expect(firstTraining).toBe(0); + expect(firstGeneric).toBeGreaterThan(firstTraining); + }); +}); + +describe('classifyUserAgent', () => { + it('describes a known agent completely', () => { + const v = classifyUserAgent('Mozilla/5.0 (compatible; GPTBot/1.1; +https://openai.com/gptbot)'); + expect(v).toMatchObject({ + known: true, + id: 'gptbot', + name: 'GPTBot', + category: 'training_crawler', + organization: 'OpenAI', + isAI: true, + }); + expect(v.score).toBeGreaterThan(0); + }); + + it('reports an unknown agent as known:false / category:none, never undefined', () => { + const v = classifyUserAgent('Mozilla/5.0 Chrome/137.0.0.0'); + expect(v.known).toBe(false); + expect(v.category).toBe('none'); + expect(v.score).toBe(0); + expect(v.isAI).toBe(false); + }); + + it('flags every AI category as AI and ordinary crawlers as not', () => { + expect(classifyUserAgent('GPTBot/1.1').isAI).toBe(true); + expect(classifyUserAgent('PerplexityBot/1.0').isAI).toBe(true); + // Googlebot is a search engine, not an AI client. Getting this wrong means a + // customer's `ai: true` rule deindexes their site. + const google = classifyUserAgent( + 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', + ); + expect(google.known).toBe(true); + expect(google.category).toBe('search_crawler'); + expect(google.isAI).toBe(false); + }); +}); diff --git a/packages/webdecoy/src/bots/index.ts b/packages/webdecoy/src/bots/index.ts new file mode 100644 index 0000000..ce00917 --- /dev/null +++ b/packages/webdecoy/src/bots/index.ts @@ -0,0 +1,150 @@ +/** + * Local bot classification (#500). + * + * WHY THIS IS IN THE SDK AT ALL + * + * The scoring pipeline already identifies named agents server-side, but it does + * so *after* the request is reported — which is too late to decide anything. A + * customer who wants "block AI training crawlers, allow search crawlers" needs + * the answer synchronously, in the request path, with no network call. So the Go + * registry is generated into {@link BOT_REGISTRY} and matched here. + * + * WHAT THIS IS NOT + * + * This is a User-Agent lookup. It identifies agents that say who they are, and + * nothing else. An agent that spoofs Chrome is invisible to it, and always will + * be — that is the point of the 1% User-Agent weight in the unified score, and + * nothing here changes that arithmetic. + * + * That is a feature for the use case it serves. GPTBot, ClaudeBot and Googlebot + * all self-identify honestly, and the decision a customer wants to make about + * them ("don't train on my content") is a POLICY choice about a cooperative + * agent, not a threat judgement about a hostile one. Acting on a claim is + * correct when the claim is the thing you care about. + * + * For agents that lie, the answer is the rest of the product: tripwires, + * honeytokens and the edge classifier see behaviour, which cannot be spoofed by + * editing a header. + */ + +import { BOT_REGISTRY, type BotAgent, type BotCategory } from './registry.generated'; + +export { BOT_REGISTRY, BOT_CATEGORIES } from './registry.generated'; +export type { BotAgent, BotCategory } from './registry.generated'; + +/** + * Categories that are some flavour of AI client, collapsed into one flag because + * "is this AI" is the question customers actually ask, and spelling it as a + * four-way comparison in every rule invites getting it wrong by one. + */ +const AI_CATEGORIES: ReadonlySet = new Set([ + 'training_crawler', + 'ai_search_crawler', + 'ai_agent', + 'ai_assistant', +]); + +/** + * What a rule sees about the agent behind a request. + * + * Always populated, never undefined: a rule that has to check whether + * classification ran is a rule people get wrong. When nothing matched, + * `known` is false and `category` is `'none'`. + */ +export interface BotVerdict { + /** Whether the User-Agent matched a known agent. */ + known: boolean; + /** Registry slug, e.g. `gptbot`. Undefined when unknown. */ + id?: string; + /** Display name, e.g. `GPTBot`. Undefined when unknown. */ + name?: string; + /** Customer-facing category — the same vocabulary as the dashboard. */ + category: BotCategory; + /** Operator, e.g. `OpenAI`. Undefined when unknown. */ + organization?: string; + /** The registry's threat score for this agent, 0-100. 0 when unknown. */ + score: number; + /** + * Whether the operator documents honouring robots.txt. Undefined when unknown. + * + * Documented behaviour, not observed behaviour — it is what the operator + * publishes, so treat it as a claim like any other. + */ + respectsRobots?: boolean; + /** True for training crawlers, AI search crawlers, AI agents and assistants. */ + isAI: boolean; +} + +const UNKNOWN: BotVerdict = Object.freeze({ + known: false, + category: 'none' as BotCategory, + score: 0, + isAI: false, +}); + +/** + * Cache of User-Agent -> match, because the same handful of strings repeat + * across essentially every request and the scan is ~250 substring tests. + * + * BOUNDED ON PURPOSE. The key is attacker-controlled: a client sending a unique + * User-Agent per request would otherwise grow this map without limit, turning a + * performance optimisation into a memory-exhaustion vector. On overflow the + * whole map is dropped rather than evicted one-by-one — it costs one rebuild of + * a cache that refills in microseconds, and it needs no bookkeeping to get wrong. + */ +const MAX_CACHE = 1000; +const cache = new Map(); + +/** + * Find the agent behind a User-Agent string, or undefined. + * + * Returns the FIRST match in registry order, which is why {@link BOT_REGISTRY} + * must not be sorted: the table arrives ordered by category priority, mirroring + * `agents.MatchUserAgent` in Go so both sides classify identically. + */ +export function matchUserAgent(userAgent: string | undefined | null): BotAgent | undefined { + if (!userAgent) return undefined; + + const cached = cache.get(userAgent); + if (cached !== undefined) return cached ?? undefined; + + const ua = userAgent.toLowerCase(); + let found: BotAgent | null = null; + + for (const agent of BOT_REGISTRY) { + for (const pattern of agent.uaPatterns) { + if (ua.includes(pattern)) { + found = agent; + break; + } + } + if (found) break; + } + + if (cache.size >= MAX_CACHE) cache.clear(); + cache.set(userAgent, found); + + return found ?? undefined; +} + +/** Classify a User-Agent into the verdict rules evaluate against. */ +export function classifyUserAgent(userAgent: string | undefined | null): BotVerdict { + const agent = matchUserAgent(userAgent); + if (!agent) return UNKNOWN; + + return { + known: true, + id: agent.id, + name: agent.name, + category: agent.category, + organization: agent.organization, + score: agent.baseScore, + respectsRobots: agent.respectsRobots, + isAI: AI_CATEGORIES.has(agent.category), + }; +} + +/** Test seam: drop the memoised matches. */ +export function clearBotCache(): void { + cache.clear(); +} diff --git a/packages/webdecoy/src/bots/parity-vectors.generated.json b/packages/webdecoy/src/bots/parity-vectors.generated.json new file mode 100644 index 0000000..234e8ce --- /dev/null +++ b/packages/webdecoy/src/bots/parity-vectors.generated.json @@ -0,0 +1,2154 @@ +[ + { + "userAgent": "ai2bot", + "id": "ai2bot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; ai2bot/1.0; +http://example.com/bot)", + "id": "ai2bot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "acunetix", + "id": "acunetix", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; acunetix/1.0; +http://example.com/bot)", + "id": "acunetix", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "ahrefsbot", + "id": "ahrefsbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; ahrefsbot/1.0; +http://example.com/bot)", + "id": "ahrefsbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "ahrefs", + "id": "ahrefsbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; ahrefs/1.0; +http://example.com/bot)", + "id": "ahrefsbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "amazonbuyforme", + "id": "amazon-buyforme", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; amazonbuyforme/1.0; +http://example.com/bot)", + "id": "amazon-buyforme", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "buyforme", + "id": "amazon-buyforme", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; buyforme/1.0; +http://example.com/bot)", + "id": "amazon-buyforme", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "amazonbot", + "id": "amazonbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; amazonbot/1.0; +http://example.com/bot)", + "id": "amazonbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "anthropic", + "id": "anthropic", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; anthropic/1.0; +http://example.com/bot)", + "id": "anthropic", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "apache-httpclient", + "id": "apache-httpclient", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; apache-httpclient/1.0; +http://example.com/bot)", + "id": "apache-httpclient", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "apify", + "id": "apify", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; apify/1.0; +http://example.com/bot)", + "id": "apify", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "applebot", + "id": "applebot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; applebot/1.0; +http://example.com/bot)", + "id": "applebot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "applebot-extended", + "id": "applebot-extended", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; applebot-extended/1.0; +http://example.com/bot)", + "id": "applebot-extended", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "archive.org_bot", + "id": "archive-org-bot", + "category": "archiver", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; archive.org_bot/1.0; +http://example.com/bot)", + "id": "archive-org-bot", + "category": "archiver", + "matched": true + }, + { + "userAgent": "axios/", + "id": "axios", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; axios//1.0; +http://example.com/bot)", + "id": "axios", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "baiduspider", + "id": "baiduspider", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; baiduspider/1.0; +http://example.com/bot)", + "id": "baiduspider", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "beautifulsoup", + "id": "beautifulsoup", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; beautifulsoup/1.0; +http://example.com/bot)", + "id": "beautifulsoup", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "betteruptime", + "id": "better-uptime", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; betteruptime/1.0; +http://example.com/bot)", + "id": "better-uptime", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "bingbot", + "id": "bingbot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; bingbot/1.0; +http://example.com/bot)", + "id": "bingbot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "botify", + "id": "botify", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; botify/1.0; +http://example.com/bot)", + "id": "botify", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "bravesearch", + "id": "brave-search", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; bravesearch/1.0; +http://example.com/bot)", + "id": "brave-search", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "brightdata", + "id": "bright-data", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; brightdata/1.0; +http://example.com/bot)", + "id": "bright-data", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "luminati", + "id": "bright-data", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; luminati/1.0; +http://example.com/bot)", + "id": "bright-data", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "browser-use", + "id": "browser-use", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; browser-use/1.0; +http://example.com/bot)", + "id": "browser-use", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "browserbase", + "id": "browserbase", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; browserbase/1.0; +http://example.com/bot)", + "id": "browserbase", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "browserless", + "id": "browserless", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; browserless/1.0; +http://example.com/bot)", + "id": "browserless", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "brozzler", + "id": "brozzler", + "category": "archiver", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; brozzler/1.0; +http://example.com/bot)", + "id": "brozzler", + "category": "archiver", + "matched": true + }, + { + "userAgent": "burp", + "id": "burp", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; burp/1.0; +http://example.com/bot)", + "id": "burp", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "bytespider", + "id": "bytespider", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; bytespider/1.0; +http://example.com/bot)", + "id": "bytespider", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "ccbot", + "id": "ccbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; ccbot/1.0; +http://example.com/bot)", + "id": "ccbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "censys", + "id": "censys", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; censys/1.0; +http://example.com/bot)", + "id": "censys", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "chatgpt-user", + "id": "chatgpt-user", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; chatgpt-user/1.0; +http://example.com/bot)", + "id": "chatgpt-user", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "chatgpt", + "id": "chatgpt-user", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; chatgpt/1.0; +http://example.com/bot)", + "id": "chatgpt-user", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "claude-user", + "id": "claude-user", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; claude-user/1.0; +http://example.com/bot)", + "id": "claude-user", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "claudebot", + "id": "claudebot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; claudebot/1.0; +http://example.com/bot)", + "id": "claudebot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "cohere-ai", + "id": "cohere-ai", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; cohere-ai/1.0; +http://example.com/bot)", + "id": "cohere-ai", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "cohere", + "id": "cohere-ai", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; cohere/1.0; +http://example.com/bot)", + "id": "cohere-ai", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "colly", + "id": "colly", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; colly/1.0; +http://example.com/bot)", + "id": "colly", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "commafeed", + "id": "commafeed", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; commafeed/1.0; +http://example.com/bot)", + "id": "commafeed", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "contentking", + "id": "contentking", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; contentking/1.0; +http://example.com/bot)", + "id": "contentking", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "cortana", + "id": "cortana", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; cortana/1.0; +http://example.com/bot)", + "id": "cortana", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "crawlbase", + "id": "crawlbase", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; crawlbase/1.0; +http://example.com/bot)", + "id": "crawlbase", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "datadog", + "id": "datadog-synthetics", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; datadog/1.0; +http://example.com/bot)", + "id": "datadog-synthetics", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "deepcrawl", + "id": "deepcrawl", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; deepcrawl/1.0; +http://example.com/bot)", + "id": "deepcrawl", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "deepseek", + "id": "deepseek", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; deepseek/1.0; +http://example.com/bot)", + "id": "deepseek", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "detectify", + "id": "detectify", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; detectify/1.0; +http://example.com/bot)", + "id": "detectify", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "diffbot", + "id": "diffbot", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; diffbot/1.0; +http://example.com/bot)", + "id": "diffbot", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "discordbot", + "id": "discordbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; discordbot/1.0; +http://example.com/bot)", + "id": "discordbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "dotbot", + "id": "dotbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; dotbot/1.0; +http://example.com/bot)", + "id": "dotbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "duckduckbot", + "id": "duckduckbot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; duckduckbot/1.0; +http://example.com/bot)", + "id": "duckduckbot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "ecosia", + "id": "ecosia", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; ecosia/1.0; +http://example.com/bot)", + "id": "ecosia", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "embedly", + "id": "embedly", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; embedly/1.0; +http://example.com/bot)", + "id": "embedly", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "exa.ai", + "id": "exa", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; exa.ai/1.0; +http://example.com/bot)", + "id": "exa", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "exabot", + "id": "exa", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; exabot/1.0; +http://example.com/bot)", + "id": "exa", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "facebookexternalhit", + "id": "facebookexternalhit", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; facebookexternalhit/1.0; +http://example.com/bot)", + "id": "facebookexternalhit", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "facebookbot", + "id": "facebookbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; facebookbot/1.0; +http://example.com/bot)", + "id": "facebookbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "faraday", + "id": "faraday", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; faraday/1.0; +http://example.com/bot)", + "id": "faraday", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "feedbin", + "id": "feedbin", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; feedbin/1.0; +http://example.com/bot)", + "id": "feedbin", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "feedly", + "id": "feedly", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; feedly/1.0; +http://example.com/bot)", + "id": "feedly", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "freshrss", + "id": "freshrss", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; freshrss/1.0; +http://example.com/bot)", + "id": "freshrss", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "freshping", + "id": "freshping", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; freshping/1.0; +http://example.com/bot)", + "id": "freshping", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "friendlycrawler", + "id": "friendlycrawler", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; friendlycrawler/1.0; +http://example.com/bot)", + "id": "friendlycrawler", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "gptbot", + "id": "gptbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; gptbot/1.0; +http://example.com/bot)", + "id": "gptbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "gemini", + "id": "gemini", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; gemini/1.0; +http://example.com/bot)", + "id": "gemini", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "gemini-deep-research", + "id": "gemini", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; gemini-deep-research/1.0; +http://example.com/bot)", + "id": "gemini", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "gemini-user", + "id": "gemini", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; gemini-user/1.0; +http://example.com/bot)", + "id": "gemini", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "go-http-client", + "id": "go-http-client", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; go-http-client/1.0; +http://example.com/bot)", + "id": "go-http-client", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "google-extended", + "id": "google-extended", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; google-extended/1.0; +http://example.com/bot)", + "id": "google-extended", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "googleagent-mariner", + "id": "googleagent-mariner", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; googleagent-mariner/1.0; +http://example.com/bot)", + "id": "googleagent-mariner", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "mariner", + "id": "googleagent-mariner", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; mariner/1.0; +http://example.com/bot)", + "id": "googleagent-mariner", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "googlebot", + "id": "googlebot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; googlebot/1.0; +http://example.com/bot)", + "id": "googlebot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "got/", + "id": "got", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; got//1.0; +http://example.com/bot)", + "id": "got", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "guzzlehttp", + "id": "guzzlehttp", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; guzzlehttp/1.0; +http://example.com/bot)", + "id": "guzzlehttp", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "httpx", + "id": "httpx", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; httpx/1.0; +http://example.com/bot)", + "id": "httpx", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "httrack", + "id": "httrack", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; httrack/1.0; +http://example.com/bot)", + "id": "httrack", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "headlesschrome", + "id": "headlesschrome", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; headlesschrome/1.0; +http://example.com/bot)", + "id": "headlesschrome", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "heritrix", + "id": "heritrix", + "category": "archiver", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; heritrix/1.0; +http://example.com/bot)", + "id": "heritrix", + "category": "archiver", + "matched": true + }, + { + "userAgent": "hetrixtools", + "id": "hetrixtools", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; hetrixtools/1.0; +http://example.com/bot)", + "id": "hetrixtools", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "isscyberriskcrawler", + "id": "isscyberriskcrawler", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; isscyberriskcrawler/1.0; +http://example.com/bot)", + "id": "isscyberriskcrawler", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "iframely", + "id": "iframely", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; iframely/1.0; +http://example.com/bot)", + "id": "iframely", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "import.io", + "id": "import-io", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; import.io/1.0; +http://example.com/bot)", + "id": "import-io", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "inoreader", + "id": "inoreader", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; inoreader/1.0; +http://example.com/bot)", + "id": "inoreader", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "ia_archiver", + "id": "ia-archiver", + "category": "archiver", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; ia_archiver/1.0; +http://example.com/bot)", + "id": "ia-archiver", + "category": "archiver", + "matched": true + }, + { + "userAgent": "intruder", + "id": "intruder", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; intruder/1.0; +http://example.com/bot)", + "id": "intruder", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "java/", + "id": "java", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; java//1.0; +http://example.com/bot)", + "id": "java", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "jsoup", + "id": "jsoup", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; jsoup/1.0; +http://example.com/bot)", + "id": "jsoup", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "kagi", + "id": "kagi", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; kagi/1.0; +http://example.com/bot)", + "id": "kagi", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "lwp-", + "id": "lwp", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; lwp-/1.0; +http://example.com/bot)", + "id": "lwp", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "linkedinbot", + "id": "linkedinbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; linkedinbot/1.0; +http://example.com/bot)", + "id": "linkedinbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "mj12bot", + "id": "mj12bot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; mj12bot/1.0; +http://example.com/bot)", + "id": "mj12bot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "mechanize", + "id": "mechanize", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; mechanize/1.0; +http://example.com/bot)", + "id": "mechanize", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "meta-externalagent", + "id": "meta-externalagent", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; meta-externalagent/1.0; +http://example.com/bot)", + "id": "meta-externalagent", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "metaphor", + "id": "metaphor", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; metaphor/1.0; +http://example.com/bot)", + "id": "metaphor", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "copilot", + "id": "copilot", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; copilot/1.0; +http://example.com/bot)", + "id": "copilot", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "miniflux", + "id": "miniflux", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; miniflux/1.0; +http://example.com/bot)", + "id": "miniflux", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "mistral", + "id": "mistral", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; mistral/1.0; +http://example.com/bot)", + "id": "mistral", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "mojeekbot", + "id": "mojeek", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; mojeekbot/1.0; +http://example.com/bot)", + "id": "mojeek", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "multion", + "id": "multion", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; multion/1.0; +http://example.com/bot)", + "id": "multion", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "naver", + "id": "naver", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; naver/1.0; +http://example.com/bot)", + "id": "naver", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "yeti", + "id": "naver", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; yeti/1.0; +http://example.com/bot)", + "id": "naver", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "neevabot", + "id": "neeva", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; neevabot/1.0; +http://example.com/bot)", + "id": "neeva", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "nessus", + "id": "nessus", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; nessus/1.0; +http://example.com/bot)", + "id": "nessus", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "netsparker", + "id": "netsparker", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; netsparker/1.0; +http://example.com/bot)", + "id": "netsparker", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "newrelic", + "id": "newrelic", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; newrelic/1.0; +http://example.com/bot)", + "id": "newrelic", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "newsblur", + "id": "newsblur", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; newsblur/1.0; +http://example.com/bot)", + "id": "newsblur", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "nikto", + "id": "nikto", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; nikto/1.0; +http://example.com/bot)", + "id": "nikto", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "nmap", + "id": "nmap", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; nmap/1.0; +http://example.com/bot)", + "id": "nmap", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "nodeping", + "id": "nodeping", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; nodeping/1.0; +http://example.com/bot)", + "id": "nodeping", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "nuclei", + "id": "nuclei", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; nuclei/1.0; +http://example.com/bot)", + "id": "nuclei", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "oai-searchbot", + "id": "oai-searchbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; oai-searchbot/1.0; +http://example.com/bot)", + "id": "oai-searchbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "zap/", + "id": "zap", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; zap//1.0; +http://example.com/bot)", + "id": "zap", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "okhttp", + "id": "okhttp", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; okhttp/1.0; +http://example.com/bot)", + "id": "okhttp", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "omgili", + "id": "omgili", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; omgili/1.0; +http://example.com/bot)", + "id": "omgili", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "oncrawl", + "id": "oncrawl", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; oncrawl/1.0; +http://example.com/bot)", + "id": "oncrawl", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "operator", + "id": "operator", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; operator/1.0; +http://example.com/bot)", + "id": "operator", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "oxylabs", + "id": "oxylabs", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; oxylabs/1.0; +http://example.com/bot)", + "id": "oxylabs", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "php/", + "id": "php", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; php//1.0; +http://example.com/bot)", + "id": "php", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "perplexitybot", + "id": "perplexitybot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; perplexitybot/1.0; +http://example.com/bot)", + "id": "perplexitybot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "phantomjs", + "id": "phantomjs", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; phantomjs/1.0; +http://example.com/bot)", + "id": "phantomjs", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "phind", + "id": "phind", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; phind/1.0; +http://example.com/bot)", + "id": "phind", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "pingdom", + "id": "pingdom", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; pingdom/1.0; +http://example.com/bot)", + "id": "pingdom", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "pinterest", + "id": "pinterestbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; pinterest/1.0; +http://example.com/bot)", + "id": "pinterestbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "playwright", + "id": "playwright", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; playwright/1.0; +http://example.com/bot)", + "id": "playwright", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "puppeteer", + "id": "puppeteer", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; puppeteer/1.0; +http://example.com/bot)", + "id": "puppeteer", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "python/", + "id": "python", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; python//1.0; +http://example.com/bot)", + "id": "python", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "python-requests", + "id": "python-requests", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; python-requests/1.0; +http://example.com/bot)", + "id": "python-requests", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "python-urllib", + "id": "python-urllib", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; python-urllib/1.0; +http://example.com/bot)", + "id": "python-urllib", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "qualys", + "id": "qualys", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; qualys/1.0; +http://example.com/bot)", + "id": "qualys", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "qwantify", + "id": "qwantify", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; qwantify/1.0; +http://example.com/bot)", + "id": "qwantify", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "qwen", + "id": "qwen", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; qwen/1.0; +http://example.com/bot)", + "id": "qwen", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "reddit", + "id": "reddit", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; reddit/1.0; +http://example.com/bot)", + "id": "reddit", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "rogerbot", + "id": "rogerbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; rogerbot/1.0; +http://example.com/bot)", + "id": "rogerbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "ruby", + "id": "ruby", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; ruby/1.0; +http://example.com/bot)", + "id": "ruby", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "sistrix", + "id": "sistrix", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; sistrix/1.0; +http://example.com/bot)", + "id": "sistrix", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "scrapingant", + "id": "scrapingant", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; scrapingant/1.0; +http://example.com/bot)", + "id": "scrapingant", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "scrapingbee", + "id": "scrapingbee", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; scrapingbee/1.0; +http://example.com/bot)", + "id": "scrapingbee", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "scrapy", + "id": "scrapy", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; scrapy/1.0; +http://example.com/bot)", + "id": "scrapy", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "screaming frog", + "id": "screaming-frog", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; screaming frog/1.0; +http://example.com/bot)", + "id": "screaming-frog", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "searchgpt", + "id": "searchgpt", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; searchgpt/1.0; +http://example.com/bot)", + "id": "searchgpt", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "selenium", + "id": "selenium", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; selenium/1.0; +http://example.com/bot)", + "id": "selenium", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "semrushbot", + "id": "semrushbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; semrushbot/1.0; +http://example.com/bot)", + "id": "semrushbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "semrush", + "id": "semrushbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; semrush/1.0; +http://example.com/bot)", + "id": "semrushbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "sentibot", + "id": "sentibot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; sentibot/1.0; +http://example.com/bot)", + "id": "sentibot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "serpstat", + "id": "serpstat", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; serpstat/1.0; +http://example.com/bot)", + "id": "serpstat", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "seznambot", + "id": "seznam", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; seznambot/1.0; +http://example.com/bot)", + "id": "seznam", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "shodan", + "id": "shodan", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; shodan/1.0; +http://example.com/bot)", + "id": "shodan", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "siri", + "id": "siri", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; siri/1.0; +http://example.com/bot)", + "id": "siri", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "site24x7", + "id": "site24x7", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; site24x7/1.0; +http://example.com/bot)", + "id": "site24x7", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "sitebulb", + "id": "sitebulb", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; sitebulb/1.0; +http://example.com/bot)", + "id": "sitebulb", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "slackbot", + "id": "slackbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; slackbot/1.0; +http://example.com/bot)", + "id": "slackbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "sogou", + "id": "sogou", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; sogou/1.0; +http://example.com/bot)", + "id": "sogou", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "splash", + "id": "splash", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; splash/1.0; +http://example.com/bot)", + "id": "splash", + "category": "headless_browser", + "matched": true + }, + { + "userAgent": "spyfu", + "id": "spyfu", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; spyfu/1.0; +http://example.com/bot)", + "id": "spyfu", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "stagehand", + "id": "stagehand", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; stagehand/1.0; +http://example.com/bot)", + "id": "stagehand", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "statuscake", + "id": "statuscake", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; statuscake/1.0; +http://example.com/bot)", + "id": "statuscake", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "tavily", + "id": "tavily", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; tavily/1.0; +http://example.com/bot)", + "id": "tavily", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "telegrambot", + "id": "telegrambot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; telegrambot/1.0; +http://example.com/bot)", + "id": "telegrambot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "theoldreader", + "id": "theoldreader", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; theoldreader/1.0; +http://example.com/bot)", + "id": "theoldreader", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "timpibot", + "id": "timpibot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; timpibot/1.0; +http://example.com/bot)", + "id": "timpibot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "tumblr", + "id": "tumblr", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; tumblr/1.0; +http://example.com/bot)", + "id": "tumblr", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "twitterbot", + "id": "twitterbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; twitterbot/1.0; +http://example.com/bot)", + "id": "twitterbot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "undici", + "id": "undici", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; undici/1.0; +http://example.com/bot)", + "id": "undici", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "uptimerobot", + "id": "uptimerobot", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; uptimerobot/1.0; +http://example.com/bot)", + "id": "uptimerobot", + "category": "monitoring", + "matched": true + }, + { + "userAgent": "velenpublicwebcrawler", + "id": "velenpublicwebcrawler", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; velenpublicwebcrawler/1.0; +http://example.com/bot)", + "id": "velenpublicwebcrawler", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "wayback", + "id": "wayback", + "category": "archiver", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; wayback/1.0; +http://example.com/bot)", + "id": "wayback", + "category": "archiver", + "matched": true + }, + { + "userAgent": "webscrapingapi", + "id": "webscrapingapi", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; webscrapingapi/1.0; +http://example.com/bot)", + "id": "webscrapingapi", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "webzio", + "id": "webzio", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; webzio/1.0; +http://example.com/bot)", + "id": "webzio", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "wget/", + "id": "wget", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; wget//1.0; +http://example.com/bot)", + "id": "wget", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "whatsapp", + "id": "whatsapp", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; whatsapp/1.0; +http://example.com/bot)", + "id": "whatsapp", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "slurp", + "id": "yahoo-slurp", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; slurp/1.0; +http://example.com/bot)", + "id": "yahoo-slurp", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "yandexbot", + "id": "yandexbot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; yandexbot/1.0; +http://example.com/bot)", + "id": "yandexbot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "youbot", + "id": "youbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; youbot/1.0; +http://example.com/bot)", + "id": "youbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "zyte", + "id": "zyte", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; zyte/1.0; +http://example.com/bot)", + "id": "zyte", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "scrapinghub", + "id": "zyte", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; scrapinghub/1.0; +http://example.com/bot)", + "id": "zyte", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "aiohttp", + "id": "aiohttp", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; aiohttp/1.0; +http://example.com/bot)", + "id": "aiohttp", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "curl/", + "id": "curl", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; curl//1.0; +http://example.com/bot)", + "id": "curl", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "img2dataset", + "id": "img2dataset", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; img2dataset/1.0; +http://example.com/bot)", + "id": "img2dataset", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "libwww-perl", + "id": "libwww-perl", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; libwww-perl/1.0; +http://example.com/bot)", + "id": "libwww-perl", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "node-fetch", + "id": "node-fetch", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; node-fetch/1.0; +http://example.com/bot)", + "id": "node-fetch", + "category": "generic_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36", + "matched": false + }, + { + "userAgent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1", + "matched": false + }, + { + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0", + "matched": false + }, + { + "userAgent": "", + "matched": false + } +] diff --git a/packages/webdecoy/src/bots/parity.test.ts b/packages/webdecoy/src/bots/parity.test.ts new file mode 100644 index 0000000..4433d46 --- /dev/null +++ b/packages/webdecoy/src/bots/parity.test.ts @@ -0,0 +1,80 @@ +/** + * Cross-language parity: this matcher must classify exactly as Go's + * `agents.MatchUserAgent` does. + * + * The two implementations are the reason this test exists. Go scores detections + * after the fact; TypeScript decides policy in the request path. If they + * disagree, a customer blocks GPTBot in their rules and then sees the blocked + * request filed under a different agent — or worse, agrees in testing and + * diverges months later when someone reorders the registry. + * + * The vectors are RECORDED FROM GO, not hand-written: + * + * cd pkg && go run ./cmd/export-agent-registry -golden \ + * > ../../webdecoy-node/packages/webdecoy/src/bots/parity-vectors.generated.json + * + * Regenerate them in the same commit as any registry change. A vector file that + * disagrees with the generated table means the two were exported from different + * revisions. + */ + +import { matchUserAgent, classifyUserAgent } from './index'; +import vectors from './parity-vectors.generated.json'; + +interface Vector { + userAgent: string; + id?: string; + category?: string; + matched: boolean; +} + +const cases = vectors as Vector[]; + +describe('parity with the Go matcher', () => { + it('has a non-trivial corpus', () => { + // Guards against the vector file being emptied or replaced by `[]`, which + // would make every assertion below vacuously pass. + expect(cases.length).toBeGreaterThan(300); + expect(cases.some((v) => v.matched)).toBe(true); + expect(cases.some((v) => !v.matched)).toBe(true); + }); + + it('agrees with Go on every recorded User-Agent', () => { + const disagreements: string[] = []; + + for (const v of cases) { + const got = matchUserAgent(v.userAgent); + + if (!v.matched) { + if (got) disagreements.push(`${JSON.stringify(v.userAgent)}: Go matched nothing, TS matched ${got.id}`); + continue; + } + + if (!got) { + disagreements.push(`${JSON.stringify(v.userAgent)}: Go matched ${v.id}, TS matched nothing`); + continue; + } + if (got.id !== v.id) { + disagreements.push(`${JSON.stringify(v.userAgent)}: Go matched ${v.id}, TS matched ${got.id}`); + continue; + } + if (got.category !== v.category) { + disagreements.push( + `${JSON.stringify(v.userAgent)}: category Go=${v.category} TS=${got.category}`, + ); + } + } + + expect(disagreements).toEqual([]); + }); + + it('never classifies a real browser as a bot', () => { + // Called out separately from the sweep above because this is the failure + // that blocks paying customers rather than merely mislabelling a crawler. + const browsers = cases.filter((v) => v.userAgent.includes('Mozilla/5.0 (') && !v.matched); + expect(browsers.length).toBeGreaterThan(0); + for (const b of browsers) { + expect(classifyUserAgent(b.userAgent).known).toBe(false); + } + }); +}); diff --git a/packages/webdecoy/src/bots/registry.generated.ts b/packages/webdecoy/src/bots/registry.generated.ts new file mode 100644 index 0000000..4af5462 --- /dev/null +++ b/packages/webdecoy/src/bots/registry.generated.ts @@ -0,0 +1,246 @@ +/** + * GENERATED FILE — DO NOT EDIT. + * + * Source of truth: WebDecoy/app `pkg/agents` (Go). + * Regenerate with `packages/webdecoy/scripts/gen-bot-registry.mjs`; see that + * script for the command. + * + * ORDER IS SIGNIFICANT. Entries are in `agents.MatchUserAgent` order — AI data + * scrapers first, then AI search crawlers, AI agents, search engines, then the + * remaining categories. `matchUserAgent()` returns the first hit, and several + * agents share User-Agent substrings, so sorting this array changes how real + * traffic is classified. + * + * 168 agents across 15 categories. + */ + +/** + * Classification categories, mirroring `agents.Category` in Go. + * + * Declared here rather than hand-written next door so a category added in Go + * cannot go missing in TypeScript. + */ +export type BotCategory = + | "training_crawler" + | "ai_search_crawler" + | "ai_agent" + | "ai_assistant" + | "search_crawler" + | "seo_crawler" + | "security_scanner" + | "generic_scraper" + | "archiver" + | "fetcher" + | "monitoring" + | "headless_browser" + | "feed_reader" + | "commercial_scraper" + | "none"; + +/** One entry in the registry. */ +export interface BotAgent { + /** Stable lowercase slug, e.g. `gptbot`. */ + id: string; + /** Display name, e.g. `GPTBot`. */ + name: string; + category: BotCategory; + /** Operator, e.g. `OpenAI`. */ + organization: string; + /** The registry's default threat score for this agent (0-100). */ + baseScore: number; + /** Whether the operator documents honouring robots.txt. */ + respectsRobots: boolean; + /** Lowercase User-Agent substrings that identify it. */ + uaPatterns: readonly string[]; +} + +/** Every declared category, including any with no agents yet. */ +export const BOT_CATEGORIES: readonly BotCategory[] = [ + "training_crawler", + "ai_search_crawler", + "ai_agent", + "ai_assistant", + "search_crawler", + "seo_crawler", + "security_scanner", + "generic_scraper", + "archiver", + "fetcher", + "monitoring", + "headless_browser", + "feed_reader", + "commercial_scraper", + "none", +]; + +/** The agent table, in match order. */ +export const BOT_REGISTRY: readonly BotAgent[] = [ + { id: "gptbot", name: "GPTBot", category: "training_crawler", organization: "OpenAI", baseScore: 85, respectsRobots: true, uaPatterns: ["gptbot"] }, + { id: "chatgpt-user", name: "ChatGPT-User", category: "training_crawler", organization: "OpenAI", baseScore: 85, respectsRobots: true, uaPatterns: ["chatgpt-user", "chatgpt"] }, + { id: "oai-searchbot", name: "OAI-SearchBot", category: "training_crawler", organization: "OpenAI", baseScore: 80, respectsRobots: true, uaPatterns: ["oai-searchbot"] }, + { id: "claudebot", name: "ClaudeBot", category: "training_crawler", organization: "Anthropic", baseScore: 85, respectsRobots: true, uaPatterns: ["claudebot"] }, + { id: "anthropic", name: "Anthropic", category: "training_crawler", organization: "Anthropic", baseScore: 85, respectsRobots: true, uaPatterns: ["anthropic"] }, + { id: "ccbot", name: "CCBot", category: "training_crawler", organization: "Common Crawl", baseScore: 80, respectsRobots: true, uaPatterns: ["ccbot"] }, + { id: "google-extended", name: "Google-Extended", category: "training_crawler", organization: "Google", baseScore: 80, respectsRobots: true, uaPatterns: ["google-extended"] }, + { id: "bytespider", name: "ByteSpider", category: "training_crawler", organization: "ByteDance", baseScore: 75, respectsRobots: false, uaPatterns: ["bytespider"] }, + { id: "amazonbot", name: "Amazonbot", category: "training_crawler", organization: "Amazon", baseScore: 70, respectsRobots: true, uaPatterns: ["amazonbot"] }, + { id: "facebookbot", name: "FacebookBot", category: "training_crawler", organization: "Meta", baseScore: 70, respectsRobots: true, uaPatterns: ["facebookbot"] }, + { id: "meta-externalagent", name: "Meta-ExternalAgent", category: "training_crawler", organization: "Meta", baseScore: 75, respectsRobots: true, uaPatterns: ["meta-externalagent"] }, + { id: "cohere-ai", name: "Cohere", category: "training_crawler", organization: "Cohere", baseScore: 80, respectsRobots: true, uaPatterns: ["cohere-ai", "cohere"] }, + { id: "perplexitybot", name: "PerplexityBot", category: "training_crawler", organization: "Perplexity AI", baseScore: 80, respectsRobots: true, uaPatterns: ["perplexitybot"] }, + { id: "applebot-extended", name: "Applebot-Extended", category: "training_crawler", organization: "Apple", baseScore: 75, respectsRobots: true, uaPatterns: ["applebot-extended"] }, + { id: "youbot", name: "YouBot", category: "training_crawler", organization: "You.com", baseScore: 75, respectsRobots: true, uaPatterns: ["youbot"] }, + { id: "mistral", name: "MistralBot", category: "training_crawler", organization: "Mistral AI", baseScore: 80, respectsRobots: true, uaPatterns: ["mistral"] }, + { id: "gemini", name: "Gemini", category: "training_crawler", organization: "Google", baseScore: 80, respectsRobots: true, uaPatterns: ["gemini"] }, + { id: "ai2bot", name: "AI2Bot", category: "training_crawler", organization: "Allen Institute for AI", baseScore: 75, respectsRobots: true, uaPatterns: ["ai2bot"] }, + { id: "deepseek", name: "DeepSeek", category: "training_crawler", organization: "DeepSeek", baseScore: 80, respectsRobots: true, uaPatterns: ["deepseek"] }, + { id: "qwen", name: "Qwen", category: "training_crawler", organization: "Alibaba", baseScore: 75, respectsRobots: true, uaPatterns: ["qwen"] }, + { id: "webzio", name: "Webz.io", category: "training_crawler", organization: "Webz.io", baseScore: 70, respectsRobots: false, uaPatterns: ["webzio"] }, + { id: "sentibot", name: "Sentibot", category: "training_crawler", organization: "Sentibot", baseScore: 70, respectsRobots: false, uaPatterns: ["sentibot"] }, + { id: "omgili", name: "Omgili", category: "training_crawler", organization: "Webz.io", baseScore: 70, respectsRobots: false, uaPatterns: ["omgili"] }, + { id: "img2dataset", name: "img2dataset", category: "training_crawler", organization: "LAION", baseScore: 75, respectsRobots: false, uaPatterns: ["img2dataset"] }, + { id: "timpibot", name: "Timpibot", category: "training_crawler", organization: "Timpi", baseScore: 70, respectsRobots: true, uaPatterns: ["timpibot"] }, + { id: "velenpublicwebcrawler", name: "VelenPublicWebCrawler", category: "training_crawler", organization: "Velen", baseScore: 70, respectsRobots: false, uaPatterns: ["velenpublicwebcrawler"] }, + { id: "isscyberriskcrawler", name: "ISSCyberRiskCrawler", category: "training_crawler", organization: "ISS", baseScore: 65, respectsRobots: false, uaPatterns: ["isscyberriskcrawler"] }, + { id: "friendlycrawler", name: "FriendlyCrawler", category: "training_crawler", organization: "Unknown", baseScore: 65, respectsRobots: true, uaPatterns: ["friendlycrawler"] }, + { id: "searchgpt", name: "SearchGPT", category: "ai_search_crawler", organization: "OpenAI", baseScore: 75, respectsRobots: true, uaPatterns: ["searchgpt"] }, + { id: "phind", name: "Phind", category: "ai_search_crawler", organization: "Phind", baseScore: 70, respectsRobots: true, uaPatterns: ["phind"] }, + { id: "kagi", name: "Kagi", category: "ai_search_crawler", organization: "Kagi", baseScore: 65, respectsRobots: true, uaPatterns: ["kagi"] }, + { id: "exa", name: "Exa", category: "ai_search_crawler", organization: "Exa AI", baseScore: 70, respectsRobots: true, uaPatterns: ["exa.ai", "exabot"] }, + { id: "metaphor", name: "Metaphor", category: "ai_search_crawler", organization: "Metaphor", baseScore: 70, respectsRobots: true, uaPatterns: ["metaphor"] }, + { id: "tavily", name: "Tavily", category: "ai_search_crawler", organization: "Tavily", baseScore: 70, respectsRobots: true, uaPatterns: ["tavily"] }, + { id: "brave-search", name: "BraveSearch", category: "ai_search_crawler", organization: "Brave", baseScore: 65, respectsRobots: true, uaPatterns: ["bravesearch"] }, + { id: "neeva", name: "NeevaBot", category: "ai_search_crawler", organization: "Neeva", baseScore: 65, respectsRobots: true, uaPatterns: ["neevabot"] }, + { id: "operator", name: "Operator", category: "ai_agent", organization: "OpenAI", baseScore: 75, respectsRobots: true, uaPatterns: ["operator"] }, + { id: "claude-user", name: "Claude-User", category: "ai_agent", organization: "Anthropic", baseScore: 75, respectsRobots: true, uaPatterns: ["claude-user"] }, + { id: "gemini-user", name: "Gemini-User", category: "ai_agent", organization: "Google", baseScore: 75, respectsRobots: true, uaPatterns: ["gemini-user"] }, + { id: "googleagent-mariner", name: "GoogleAgent-Mariner", category: "ai_agent", organization: "Google", baseScore: 75, respectsRobots: true, uaPatterns: ["googleagent-mariner", "mariner"] }, + { id: "amazon-buyforme", name: "AmazonBuyForMe", category: "ai_agent", organization: "Amazon", baseScore: 70, respectsRobots: true, uaPatterns: ["amazonbuyforme", "buyforme"] }, + { id: "browser-use", name: "Browser-Use", category: "ai_agent", organization: "Open Source", baseScore: 70, respectsRobots: false, uaPatterns: ["browser-use"] }, + { id: "stagehand", name: "Stagehand", category: "ai_agent", organization: "Browserbase", baseScore: 70, respectsRobots: false, uaPatterns: ["stagehand"] }, + { id: "multion", name: "MultiOn", category: "ai_agent", organization: "MultiOn", baseScore: 70, respectsRobots: false, uaPatterns: ["multion"] }, + { id: "googlebot", name: "Googlebot", category: "search_crawler", organization: "Google", baseScore: 30, respectsRobots: true, uaPatterns: ["googlebot"] }, + { id: "bingbot", name: "Bingbot", category: "search_crawler", organization: "Microsoft", baseScore: 30, respectsRobots: true, uaPatterns: ["bingbot"] }, + { id: "yandexbot", name: "YandexBot", category: "search_crawler", organization: "Yandex", baseScore: 35, respectsRobots: true, uaPatterns: ["yandexbot"] }, + { id: "baiduspider", name: "Baiduspider", category: "search_crawler", organization: "Baidu", baseScore: 40, respectsRobots: true, uaPatterns: ["baiduspider"] }, + { id: "duckduckbot", name: "DuckDuckBot", category: "search_crawler", organization: "DuckDuckGo", baseScore: 30, respectsRobots: true, uaPatterns: ["duckduckbot"] }, + { id: "applebot", name: "Applebot", category: "search_crawler", organization: "Apple", baseScore: 30, respectsRobots: true, uaPatterns: ["applebot"] }, + { id: "sogou", name: "Sogou Spider", category: "search_crawler", organization: "Sogou", baseScore: 40, respectsRobots: true, uaPatterns: ["sogou"] }, + { id: "qwantify", name: "Qwantify", category: "search_crawler", organization: "Qwant", baseScore: 35, respectsRobots: true, uaPatterns: ["qwantify"] }, + { id: "naver", name: "Naver", category: "search_crawler", organization: "Naver", baseScore: 35, respectsRobots: true, uaPatterns: ["naver", "yeti"] }, + { id: "seznam", name: "SeznamBot", category: "search_crawler", organization: "Seznam", baseScore: 35, respectsRobots: true, uaPatterns: ["seznambot"] }, + { id: "ecosia", name: "Ecosia", category: "search_crawler", organization: "Ecosia", baseScore: 30, respectsRobots: true, uaPatterns: ["ecosia"] }, + { id: "mojeek", name: "MojeekBot", category: "search_crawler", organization: "Mojeek", baseScore: 35, respectsRobots: true, uaPatterns: ["mojeekbot"] }, + { id: "yahoo-slurp", name: "Yahoo! Slurp", category: "search_crawler", organization: "Yahoo", baseScore: 30, respectsRobots: true, uaPatterns: ["slurp"] }, + { id: "ahrefsbot", name: "AhrefsBot", category: "seo_crawler", organization: "Ahrefs", baseScore: 50, respectsRobots: true, uaPatterns: ["ahrefsbot", "ahrefs"] }, + { id: "semrushbot", name: "SemrushBot", category: "seo_crawler", organization: "Semrush", baseScore: 50, respectsRobots: true, uaPatterns: ["semrushbot", "semrush"] }, + { id: "mj12bot", name: "MJ12bot", category: "seo_crawler", organization: "Majestic", baseScore: 50, respectsRobots: true, uaPatterns: ["mj12bot"] }, + { id: "dotbot", name: "DotBot", category: "seo_crawler", organization: "Moz", baseScore: 50, respectsRobots: true, uaPatterns: ["dotbot"] }, + { id: "rogerbot", name: "Rogerbot", category: "seo_crawler", organization: "Moz", baseScore: 50, respectsRobots: true, uaPatterns: ["rogerbot"] }, + { id: "screaming-frog", name: "Screaming Frog", category: "seo_crawler", organization: "Screaming Frog", baseScore: 55, respectsRobots: true, uaPatterns: ["screaming frog"] }, + { id: "sitebulb", name: "Sitebulb", category: "seo_crawler", organization: "Sitebulb", baseScore: 55, respectsRobots: true, uaPatterns: ["sitebulb"] }, + { id: "deepcrawl", name: "DeepCrawl", category: "seo_crawler", organization: "Lumar", baseScore: 50, respectsRobots: true, uaPatterns: ["deepcrawl"] }, + { id: "oncrawl", name: "OnCrawl", category: "seo_crawler", organization: "OnCrawl", baseScore: 50, respectsRobots: true, uaPatterns: ["oncrawl"] }, + { id: "botify", name: "Botify", category: "seo_crawler", organization: "Botify", baseScore: 50, respectsRobots: true, uaPatterns: ["botify"] }, + { id: "contentking", name: "ContentKing", category: "seo_crawler", organization: "ContentKing", baseScore: 50, respectsRobots: true, uaPatterns: ["contentking"] }, + { id: "sistrix", name: "SISTRIX", category: "seo_crawler", organization: "SISTRIX", baseScore: 50, respectsRobots: true, uaPatterns: ["sistrix"] }, + { id: "serpstat", name: "Serpstat", category: "seo_crawler", organization: "Serpstat", baseScore: 50, respectsRobots: true, uaPatterns: ["serpstat"] }, + { id: "spyfu", name: "SpyFu", category: "seo_crawler", organization: "SpyFu", baseScore: 50, respectsRobots: true, uaPatterns: ["spyfu"] }, + { id: "nessus", name: "Nessus", category: "security_scanner", organization: "Tenable", baseScore: 70, respectsRobots: false, uaPatterns: ["nessus"] }, + { id: "qualys", name: "Qualys", category: "security_scanner", organization: "Qualys", baseScore: 70, respectsRobots: false, uaPatterns: ["qualys"] }, + { id: "shodan", name: "Shodan", category: "security_scanner", organization: "Shodan", baseScore: 75, respectsRobots: false, uaPatterns: ["shodan"] }, + { id: "censys", name: "Censys", category: "security_scanner", organization: "Censys", baseScore: 75, respectsRobots: false, uaPatterns: ["censys"] }, + { id: "nmap", name: "Nmap", category: "security_scanner", organization: "Open Source", baseScore: 70, respectsRobots: false, uaPatterns: ["nmap"] }, + { id: "zap", name: "OWASP ZAP", category: "security_scanner", organization: "OWASP", baseScore: 70, respectsRobots: false, uaPatterns: ["zap/"] }, + { id: "burp", name: "Burp Suite", category: "security_scanner", organization: "PortSwigger", baseScore: 70, respectsRobots: false, uaPatterns: ["burp"] }, + { id: "nikto", name: "Nikto", category: "security_scanner", organization: "Open Source", baseScore: 70, respectsRobots: false, uaPatterns: ["nikto"] }, + { id: "nuclei", name: "Nuclei", category: "security_scanner", organization: "ProjectDiscovery", baseScore: 70, respectsRobots: false, uaPatterns: ["nuclei"] }, + { id: "acunetix", name: "Acunetix", category: "security_scanner", organization: "Invicti", baseScore: 70, respectsRobots: false, uaPatterns: ["acunetix"] }, + { id: "netsparker", name: "Netsparker", category: "security_scanner", organization: "Invicti", baseScore: 70, respectsRobots: false, uaPatterns: ["netsparker"] }, + { id: "detectify", name: "Detectify", category: "security_scanner", organization: "Detectify", baseScore: 65, respectsRobots: false, uaPatterns: ["detectify"] }, + { id: "intruder", name: "Intruder", category: "security_scanner", organization: "Intruder", baseScore: 65, respectsRobots: false, uaPatterns: ["intruder"] }, + { id: "python-requests", name: "Python-Requests", category: "generic_scraper", organization: "Open Source", baseScore: 55, respectsRobots: false, uaPatterns: ["python-requests"] }, + { id: "python-urllib", name: "Python-urllib", category: "generic_scraper", organization: "Python", baseScore: 55, respectsRobots: false, uaPatterns: ["python-urllib"] }, + { id: "python", name: "Python", category: "generic_scraper", organization: "Python", baseScore: 50, respectsRobots: false, uaPatterns: ["python/"] }, + { id: "scrapy", name: "Scrapy", category: "generic_scraper", organization: "Scrapy", baseScore: 65, respectsRobots: true, uaPatterns: ["scrapy"] }, + { id: "httpx", name: "HTTPX", category: "generic_scraper", organization: "Open Source", baseScore: 55, respectsRobots: false, uaPatterns: ["httpx"] }, + { id: "aiohttp", name: "aiohttp", category: "generic_scraper", organization: "Open Source", baseScore: 55, respectsRobots: false, uaPatterns: ["aiohttp"] }, + { id: "beautifulsoup", name: "BeautifulSoup", category: "generic_scraper", organization: "Open Source", baseScore: 60, respectsRobots: false, uaPatterns: ["beautifulsoup"] }, + { id: "node-fetch", name: "node-fetch", category: "generic_scraper", organization: "Open Source", baseScore: 55, respectsRobots: false, uaPatterns: ["node-fetch"] }, + { id: "axios", name: "Axios", category: "generic_scraper", organization: "Open Source", baseScore: 55, respectsRobots: false, uaPatterns: ["axios/"] }, + { id: "got", name: "Got", category: "generic_scraper", organization: "Open Source", baseScore: 55, respectsRobots: false, uaPatterns: ["got/"] }, + { id: "undici", name: "Undici", category: "generic_scraper", organization: "Node.js", baseScore: 55, respectsRobots: false, uaPatterns: ["undici"] }, + { id: "go-http-client", name: "Go-HTTP-Client", category: "generic_scraper", organization: "Go", baseScore: 55, respectsRobots: false, uaPatterns: ["go-http-client"] }, + { id: "colly", name: "Colly", category: "generic_scraper", organization: "Open Source", baseScore: 65, respectsRobots: true, uaPatterns: ["colly"] }, + { id: "java", name: "Java", category: "generic_scraper", organization: "Oracle", baseScore: 50, respectsRobots: false, uaPatterns: ["java/"] }, + { id: "okhttp", name: "OkHttp", category: "generic_scraper", organization: "Square", baseScore: 55, respectsRobots: false, uaPatterns: ["okhttp"] }, + { id: "apache-httpclient", name: "Apache-HttpClient", category: "generic_scraper", organization: "Apache", baseScore: 55, respectsRobots: false, uaPatterns: ["apache-httpclient"] }, + { id: "jsoup", name: "Jsoup", category: "generic_scraper", organization: "Open Source", baseScore: 60, respectsRobots: false, uaPatterns: ["jsoup"] }, + { id: "guzzlehttp", name: "GuzzleHttp", category: "generic_scraper", organization: "Open Source", baseScore: 55, respectsRobots: false, uaPatterns: ["guzzlehttp"] }, + { id: "php", name: "PHP", category: "generic_scraper", organization: "PHP", baseScore: 50, respectsRobots: false, uaPatterns: ["php/"] }, + { id: "ruby", name: "Ruby", category: "generic_scraper", organization: "Ruby", baseScore: 50, respectsRobots: false, uaPatterns: ["ruby"] }, + { id: "faraday", name: "Faraday", category: "generic_scraper", organization: "Open Source", baseScore: 55, respectsRobots: false, uaPatterns: ["faraday"] }, + { id: "mechanize", name: "Mechanize", category: "generic_scraper", organization: "Open Source", baseScore: 60, respectsRobots: false, uaPatterns: ["mechanize"] }, + { id: "headlesschrome", name: "HeadlessChrome", category: "headless_browser", organization: "Google", baseScore: 70, respectsRobots: false, uaPatterns: ["headlesschrome"] }, + { id: "puppeteer", name: "Puppeteer", category: "headless_browser", organization: "Google", baseScore: 70, respectsRobots: false, uaPatterns: ["puppeteer"] }, + { id: "playwright", name: "Playwright", category: "headless_browser", organization: "Microsoft", baseScore: 70, respectsRobots: false, uaPatterns: ["playwright"] }, + { id: "selenium", name: "Selenium", category: "headless_browser", organization: "Open Source", baseScore: 70, respectsRobots: false, uaPatterns: ["selenium"] }, + { id: "phantomjs", name: "PhantomJS", category: "headless_browser", organization: "Open Source", baseScore: 75, respectsRobots: false, uaPatterns: ["phantomjs"] }, + { id: "splash", name: "Splash", category: "headless_browser", organization: "Scrapinghub", baseScore: 70, respectsRobots: false, uaPatterns: ["splash"] }, + { id: "browserless", name: "Browserless", category: "headless_browser", organization: "Browserless", baseScore: 65, respectsRobots: false, uaPatterns: ["browserless"] }, + { id: "browserbase", name: "Browserbase", category: "headless_browser", organization: "Browserbase", baseScore: 65, respectsRobots: false, uaPatterns: ["browserbase"] }, + { id: "ia-archiver", name: "Internet Archive", category: "archiver", organization: "Internet Archive", baseScore: 25, respectsRobots: true, uaPatterns: ["ia_archiver"] }, + { id: "archive-org-bot", name: "Archive.org", category: "archiver", organization: "Internet Archive", baseScore: 25, respectsRobots: true, uaPatterns: ["archive.org_bot"] }, + { id: "wayback", name: "Wayback Machine", category: "archiver", organization: "Internet Archive", baseScore: 25, respectsRobots: true, uaPatterns: ["wayback"] }, + { id: "heritrix", name: "Heritrix", category: "archiver", organization: "Internet Archive", baseScore: 30, respectsRobots: true, uaPatterns: ["heritrix"] }, + { id: "brozzler", name: "Brozzler", category: "archiver", organization: "Internet Archive", baseScore: 30, respectsRobots: true, uaPatterns: ["brozzler"] }, + { id: "facebookexternalhit", name: "Facebook", category: "fetcher", organization: "Meta", baseScore: 25, respectsRobots: true, uaPatterns: ["facebookexternalhit"] }, + { id: "twitterbot", name: "Twitterbot", category: "fetcher", organization: "X Corp", baseScore: 25, respectsRobots: true, uaPatterns: ["twitterbot"] }, + { id: "linkedinbot", name: "LinkedInBot", category: "fetcher", organization: "LinkedIn", baseScore: 25, respectsRobots: true, uaPatterns: ["linkedinbot"] }, + { id: "slackbot", name: "Slackbot", category: "fetcher", organization: "Slack", baseScore: 25, respectsRobots: true, uaPatterns: ["slackbot"] }, + { id: "telegrambot", name: "TelegramBot", category: "fetcher", organization: "Telegram", baseScore: 25, respectsRobots: true, uaPatterns: ["telegrambot"] }, + { id: "whatsapp", name: "WhatsApp", category: "fetcher", organization: "Meta", baseScore: 25, respectsRobots: true, uaPatterns: ["whatsapp"] }, + { id: "discordbot", name: "Discordbot", category: "fetcher", organization: "Discord", baseScore: 25, respectsRobots: true, uaPatterns: ["discordbot"] }, + { id: "pinterestbot", name: "Pinterest", category: "fetcher", organization: "Pinterest", baseScore: 30, respectsRobots: true, uaPatterns: ["pinterest"] }, + { id: "tumblr", name: "Tumblr", category: "fetcher", organization: "Automattic", baseScore: 30, respectsRobots: true, uaPatterns: ["tumblr"] }, + { id: "reddit", name: "Reddit", category: "fetcher", organization: "Reddit", baseScore: 30, respectsRobots: true, uaPatterns: ["reddit"] }, + { id: "embedly", name: "Embedly", category: "fetcher", organization: "Medium", baseScore: 30, respectsRobots: true, uaPatterns: ["embedly"] }, + { id: "iframely", name: "Iframely", category: "fetcher", organization: "Iframely", baseScore: 30, respectsRobots: true, uaPatterns: ["iframely"] }, + { id: "curl", name: "cURL", category: "fetcher", organization: "Open Source", baseScore: 45, respectsRobots: false, uaPatterns: ["curl/"] }, + { id: "wget", name: "Wget", category: "fetcher", organization: "GNU", baseScore: 45, respectsRobots: false, uaPatterns: ["wget/"] }, + { id: "libwww-perl", name: "libwww-perl", category: "fetcher", organization: "Perl", baseScore: 50, respectsRobots: false, uaPatterns: ["libwww-perl"] }, + { id: "lwp", name: "LWP", category: "fetcher", organization: "Perl", baseScore: 50, respectsRobots: false, uaPatterns: ["lwp-"] }, + { id: "httrack", name: "HTTrack", category: "fetcher", organization: "Open Source", baseScore: 60, respectsRobots: true, uaPatterns: ["httrack"] }, + { id: "uptimerobot", name: "UptimeRobot", category: "monitoring", organization: "UptimeRobot", baseScore: 20, respectsRobots: true, uaPatterns: ["uptimerobot"] }, + { id: "pingdom", name: "Pingdom", category: "monitoring", organization: "SolarWinds", baseScore: 20, respectsRobots: true, uaPatterns: ["pingdom"] }, + { id: "statuscake", name: "StatusCake", category: "monitoring", organization: "StatusCake", baseScore: 20, respectsRobots: true, uaPatterns: ["statuscake"] }, + { id: "better-uptime", name: "Better Uptime", category: "monitoring", organization: "Better Stack", baseScore: 20, respectsRobots: true, uaPatterns: ["betteruptime"] }, + { id: "datadog-synthetics", name: "Datadog Synthetics", category: "monitoring", organization: "Datadog", baseScore: 25, respectsRobots: true, uaPatterns: ["datadog"] }, + { id: "newrelic", name: "New Relic", category: "monitoring", organization: "New Relic", baseScore: 25, respectsRobots: true, uaPatterns: ["newrelic"] }, + { id: "site24x7", name: "Site24x7", category: "monitoring", organization: "Zoho", baseScore: 20, respectsRobots: true, uaPatterns: ["site24x7"] }, + { id: "freshping", name: "Freshping", category: "monitoring", organization: "Freshworks", baseScore: 20, respectsRobots: true, uaPatterns: ["freshping"] }, + { id: "hetrixtools", name: "HetrixTools", category: "monitoring", organization: "HetrixTools", baseScore: 20, respectsRobots: true, uaPatterns: ["hetrixtools"] }, + { id: "nodeping", name: "NodePing", category: "monitoring", organization: "NodePing", baseScore: 20, respectsRobots: true, uaPatterns: ["nodeping"] }, + { id: "feedly", name: "Feedly", category: "feed_reader", organization: "Feedly", baseScore: 25, respectsRobots: true, uaPatterns: ["feedly"] }, + { id: "newsblur", name: "NewsBlur", category: "feed_reader", organization: "NewsBlur", baseScore: 25, respectsRobots: true, uaPatterns: ["newsblur"] }, + { id: "inoreader", name: "Inoreader", category: "feed_reader", organization: "Inoreader", baseScore: 25, respectsRobots: true, uaPatterns: ["inoreader"] }, + { id: "theoldreader", name: "The Old Reader", category: "feed_reader", organization: "The Old Reader", baseScore: 25, respectsRobots: true, uaPatterns: ["theoldreader"] }, + { id: "feedbin", name: "Feedbin", category: "feed_reader", organization: "Feedbin", baseScore: 25, respectsRobots: true, uaPatterns: ["feedbin"] }, + { id: "miniflux", name: "Miniflux", category: "feed_reader", organization: "Open Source", baseScore: 25, respectsRobots: true, uaPatterns: ["miniflux"] }, + { id: "freshrss", name: "FreshRSS", category: "feed_reader", organization: "Open Source", baseScore: 25, respectsRobots: true, uaPatterns: ["freshrss"] }, + { id: "commafeed", name: "CommaFeed", category: "feed_reader", organization: "Open Source", baseScore: 25, respectsRobots: true, uaPatterns: ["commafeed"] }, + { id: "diffbot", name: "Diffbot", category: "commercial_scraper", organization: "Diffbot", baseScore: 70, respectsRobots: true, uaPatterns: ["diffbot"] }, + { id: "import-io", name: "Import.io", category: "commercial_scraper", organization: "Import.io", baseScore: 65, respectsRobots: true, uaPatterns: ["import.io"] }, + { id: "bright-data", name: "Bright Data", category: "commercial_scraper", organization: "Bright Data", baseScore: 70, respectsRobots: false, uaPatterns: ["brightdata", "luminati"] }, + { id: "oxylabs", name: "Oxylabs", category: "commercial_scraper", organization: "Oxylabs", baseScore: 70, respectsRobots: false, uaPatterns: ["oxylabs"] }, + { id: "scrapingbee", name: "ScrapingBee", category: "commercial_scraper", organization: "ScrapingBee", baseScore: 65, respectsRobots: true, uaPatterns: ["scrapingbee"] }, + { id: "scrapingant", name: "ScrapingAnt", category: "commercial_scraper", organization: "ScrapingAnt", baseScore: 65, respectsRobots: true, uaPatterns: ["scrapingant"] }, + { id: "zyte", name: "Zyte", category: "commercial_scraper", organization: "Zyte", baseScore: 65, respectsRobots: true, uaPatterns: ["zyte", "scrapinghub"] }, + { id: "apify", name: "Apify", category: "commercial_scraper", organization: "Apify", baseScore: 65, respectsRobots: true, uaPatterns: ["apify"] }, + { id: "crawlbase", name: "Crawlbase", category: "commercial_scraper", organization: "Crawlbase", baseScore: 65, respectsRobots: true, uaPatterns: ["crawlbase"] }, + { id: "webscrapingapi", name: "WebScrapingAPI", category: "commercial_scraper", organization: "WebScrapingAPI", baseScore: 65, respectsRobots: true, uaPatterns: ["webscrapingapi"] }, + { id: "gemini-deep-research", name: "Gemini-Deep-Research", category: "ai_assistant", organization: "Google", baseScore: 65, respectsRobots: true, uaPatterns: ["gemini-deep-research"] }, + { id: "copilot", name: "Microsoft Copilot", category: "ai_assistant", organization: "Microsoft", baseScore: 65, respectsRobots: true, uaPatterns: ["copilot"] }, + { id: "cortana", name: "Cortana", category: "ai_assistant", organization: "Microsoft", baseScore: 60, respectsRobots: true, uaPatterns: ["cortana"] }, + { id: "siri", name: "Siri", category: "ai_assistant", organization: "Apple", baseScore: 60, respectsRobots: true, uaPatterns: ["siri"] }, +]; diff --git a/packages/webdecoy/src/index.ts b/packages/webdecoy/src/index.ts index 7b53763..4d17517 100644 --- a/packages/webdecoy/src/index.ts +++ b/packages/webdecoy/src/index.ts @@ -44,17 +44,26 @@ export type { export { readEdgeVerdict, EDGE_CLASS_HEADER, EDGE_CLEARANCE_HEADER } from './edge'; export type { EdgeClass, EdgeVerdict } from './edge'; +// Declared-agent classification (#500). Exported as values for the same reason +// as readEdgeVerdict: code outside protect() — a route handler deciding whether +// to serve a paywall, a robots.txt generator — needs the same answer without +// standing up a rule engine. +export { matchUserAgent, classifyUserAgent, BOT_REGISTRY, BOT_CATEGORIES } from './bots'; +export type { BotVerdict, BotAgent, BotCategory } from './bots'; + // Rules engine exports export { rateLimit, filter, tripwire, + bots, webBotAuth, honeytoken, RuleEngine, RateLimitRule, FilterRule, TripwireRule, + BotRule, WebBotAuthRule, DEFAULT_TRIPWIRE_PATHS, siteHoneytoken, @@ -71,6 +80,7 @@ export type { RateLimitConfig, FilterConfig, TripwireConfig, + BotRuleConfig, WebBotAuthConfig, HoneytokenOptions, Honeytoken, diff --git a/packages/webdecoy/src/rules/bot-rule.test.ts b/packages/webdecoy/src/rules/bot-rule.test.ts new file mode 100644 index 0000000..6f75357 --- /dev/null +++ b/packages/webdecoy/src/rules/bot-rule.test.ts @@ -0,0 +1,141 @@ +import { BotRule } from './bot-rule'; +import { FilterRule } from './filter-rule'; +import { classifyUserAgent } from '../bots'; +import type { RuleContext } from './types'; + +function ctx(userAgent: string): RuleContext { + return { + ip: '203.0.113.9', + path: '/', + method: 'GET', + userAgent, + headers: {}, + timestamp: Date.now(), + bot: classifyUserAgent(userAgent), + }; +} + +const GPTBOT = 'Mozilla/5.0 (compatible; GPTBot/1.1; +https://openai.com/gptbot)'; +const GOOGLEBOT = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'; +const PERPLEXITY = 'Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/bot)'; +const CHROME = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36'; + +describe('BotRule', () => { + it('denies the category it was given', () => { + const rule = new BotRule({ categories: ['training_crawler'] }); + expect(rule.evaluate(ctx(GPTBOT)).action).toBe('DENY'); + }); + + it('leaves search crawlers alone when only training crawlers are blocked', () => { + // The whole reason the two categories are distinct. A rule that blocks + // GPTBot must not take Googlebot with it. + const rule = new BotRule({ categories: ['training_crawler'] }); + expect(rule.evaluate(ctx(GOOGLEBOT)).action).toBe('ALLOW'); + }); + + it('never acts on an unrecognised User-Agent', () => { + const rule = new BotRule({ categories: ['training_crawler'], ai: true }); + expect(rule.evaluate(ctx(CHROME)).action).toBe('ALLOW'); + }); + + it('acts on the whole AI family with ai:true', () => { + const rule = new BotRule({ ai: true }); + expect(rule.evaluate(ctx(GPTBOT)).action).toBe('DENY'); + expect(rule.evaluate(ctx(PERPLEXITY)).action).toBe('DENY'); + // ...but not on a plain search engine. + expect(rule.evaluate(ctx(GOOGLEBOT)).action).toBe('ALLOW'); + }); + + it('lets allow[] override a broader match', () => { + const rule = new BotRule({ ai: true, allow: ['perplexitybot'] }); + expect(rule.evaluate(ctx(PERPLEXITY)).action).toBe('ALLOW'); + expect(rule.evaluate(ctx(GPTBOT)).action).toBe('DENY'); + }); + + it('accepts display names as well as slugs, in either case', () => { + // 'GPTBot' is what the dashboard shows; 'gptbot' is what the registry calls + // it. Requiring the right one is a support ticket per install. + expect(new BotRule({ agents: ['GPTBot'] }).evaluate(ctx(GPTBOT)).action).toBe('DENY'); + expect(new BotRule({ agents: ['gptbot'] }).evaluate(ctx(GPTBOT)).action).toBe('DENY'); + expect(new BotRule({ ai: true, allow: ['GPTBot'] }).evaluate(ctx(GPTBOT)).action).toBe('ALLOW'); + }); + + it('honours THROTTLE and dryRun', () => { + expect(new BotRule({ ai: true, action: 'THROTTLE' }).evaluate(ctx(GPTBOT)).action).toBe( + 'THROTTLE', + ); + const dry = new BotRule({ ai: true, dryRun: true }).evaluate(ctx(GPTBOT)); + expect(dry.action).toBe('ALLOW'); + expect(dry.metadata?.dryRun).toBe(true); + }); + + it('does not claim tripwire-grade confidence', () => { + // A User-Agent match repeats a claim; a tripwire hit proves behaviour. + // Reporting 100 here would let a downstream consumer treat them alike. + const result = new BotRule({ ai: true }).evaluate(ctx(GPTBOT)); + expect(result.metadata?.confidence).toBeLessThan(100); + }); + + it('allows everything when configured with nothing', () => { + // An empty config must be inert, not a default-deny that takes a site down. + const rule = new BotRule(); + expect(rule.evaluate(ctx(GPTBOT)).action).toBe('ALLOW'); + expect(rule.evaluate(ctx(GOOGLEBOT)).action).toBe('ALLOW'); + }); +}); + +describe('bot fields in filter expressions', () => { + it('matches on category using the dashboard vocabulary', () => { + const rule = new FilterRule({ expression: 'bot.category == "training_crawler"' }); + expect(rule.evaluate(ctx(GPTBOT)).action).toBe('DENY'); + expect(rule.evaluate(ctx(GOOGLEBOT)).action).toBe('ALLOW'); + }); + + it('matches on bot.ai', () => { + const rule = new FilterRule({ expression: 'bot.ai' }); + expect(rule.evaluate(ctx(PERPLEXITY)).action).toBe('DENY'); + expect(rule.evaluate(ctx(CHROME)).action).toBe('ALLOW'); + }); + + it('composes with other namespaces', () => { + const rule = new FilterRule({ + expression: 'bot.ai and bot.organization == "OpenAI"', + }); + expect(rule.evaluate(ctx(GPTBOT)).action).toBe('DENY'); + expect(rule.evaluate(ctx(PERPLEXITY)).action).toBe('ALLOW'); + }); + + it('supports in[] over categories', () => { + const rule = new FilterRule({ + expression: 'bot.category in ["training_crawler", "generic_scraper"]', + }); + expect(rule.evaluate(ctx(GPTBOT)).action).toBe('DENY'); + expect(rule.evaluate(ctx(GOOGLEBOT)).action).toBe('ALLOW'); + }); + + it('compares name against undefined rather than "" for an unknown agent', () => { + // If unknown agents reported an empty name, `bot.name != "GPTBot"` would be + // true for every browser — turning a narrow rule into a site-wide block. + const rule = new FilterRule({ expression: 'bot.name != "GPTBot"' }); + expect(rule.evaluate(ctx(CHROME)).action).toBe('ALLOW'); + expect(rule.evaluate(ctx(GOOGLEBOT)).action).toBe('DENY'); + }); + + it('reports bot.known as false, not undefined, for a browser', () => { + const known = new FilterRule({ expression: 'bot.known' }); + expect(known.evaluate(ctx(CHROME)).action).toBe('ALLOW'); + expect(known.evaluate(ctx(GPTBOT)).action).toBe('DENY'); + + const unknown = new FilterRule({ expression: 'not bot.known' }); + expect(unknown.evaluate(ctx(CHROME)).action).toBe('DENY'); + expect(unknown.evaluate(ctx(GPTBOT)).action).toBe('ALLOW'); + }); + + it('compares bot.score numerically', () => { + const rule = new FilterRule({ expression: 'bot.score >= 80' }); + expect(rule.evaluate(ctx(GPTBOT)).action).toBe('DENY'); + // Googlebot's registry score is well below the AI crawlers'. + expect(rule.evaluate(ctx(GOOGLEBOT)).action).toBe('ALLOW'); + }); +}); diff --git a/packages/webdecoy/src/rules/bot-rule.ts b/packages/webdecoy/src/rules/bot-rule.ts new file mode 100644 index 0000000..468b924 --- /dev/null +++ b/packages/webdecoy/src/rules/bot-rule.ts @@ -0,0 +1,86 @@ +/** + * Bot-category rule (#500). + * + * The one-liner for the policy this product category exists to express: + * + * ```typescript + * bots({ categories: ['training_crawler'] }) // no AI training on my content + * ``` + * + * This is expressible as a filter expression too — `bot.category == + * "training_crawler"` — and that is the right tool once the policy has + * conditions. This exists because the common case should not require learning an + * expression language, and because `allow` is fiddly to spell correctly by hand. + * + * WHAT IT CAN AND CANNOT SEE + * + * It matches on the User-Agent, so it only ever acts on agents that declare + * themselves. That is sound for this policy — GPTBot and ClaudeBot identify + * honestly, and the decision is about a cooperative agent's access, not about + * catching a liar. Anything spoofing Chrome passes straight through, and the + * tripwire and edge layers are what catch those. + */ + +import type { Rule, RuleContext, RuleResult, BotRuleConfig } from './types'; + +export class BotRule implements Rule { + readonly name = 'bots'; + private readonly categories: Set; + private readonly agents: Set; + private readonly ai: boolean; + private readonly allow: Set; + private readonly action: 'DENY' | 'THROTTLE'; + private readonly dryRun: boolean; + + constructor(config: BotRuleConfig = {}) { + this.categories = new Set(config.categories ?? []); + // Slugs and display names are both accepted, because `'GPTBot'` is what a + // customer reads in their dashboard and `'gptbot'` is what the registry + // calls it. Requiring the right one would be a support ticket per install. + this.agents = new Set((config.agents ?? []).map((a) => a.toLowerCase())); + this.ai = config.ai ?? false; + this.allow = new Set((config.allow ?? []).map((a) => a.toLowerCase())); + this.action = config.action ?? 'DENY'; + this.dryRun = config.dryRun ?? false; + } + + evaluate(context: RuleContext): RuleResult { + const bot = context.bot; + if (!bot?.known) return { action: 'ALLOW', rule: this.name }; + + // Exemptions win over every other clause. A customer who writes + // `allow: ['googlebot']` means it unconditionally — an SEO outage caused by + // a rule they thought they had scoped is far more expensive than a scraper + // getting through. + const id = bot.id?.toLowerCase(); + const name = bot.name?.toLowerCase(); + if ((id && this.allow.has(id)) || (name && this.allow.has(name))) { + return { action: 'ALLOW', rule: this.name }; + } + + const matched = + (this.ai && bot.isAI) || + this.categories.has(bot.category) || + (id !== undefined && this.agents.has(id)) || + (name !== undefined && this.agents.has(name)); + + if (!matched) return { action: 'ALLOW', rule: this.name }; + + return { + action: this.dryRun ? 'ALLOW' : this.action, + rule: this.name, + reason: `${bot.name ?? 'Bot'} (${bot.category}) matched a bot policy rule`, + metadata: { + bot: bot.id, + botName: bot.name, + category: bot.category, + organization: bot.organization, + dryRun: this.dryRun, + // Deliberately not 100. The match is a self-declared User-Agent, and a + // downstream consumer that treats this like a tripwire hit would be + // wrong: a tripwire proves behaviour, this repeats a claim. + confidence: 70, + }, + }; + } +} diff --git a/packages/webdecoy/src/rules/filter/evaluator.ts b/packages/webdecoy/src/rules/filter/evaluator.ts index bb668c4..b7d432c 100644 --- a/packages/webdecoy/src/rules/filter/evaluator.ts +++ b/packages/webdecoy/src/rules/filter/evaluator.ts @@ -49,6 +49,9 @@ export function evaluate(node: ASTNode, context: RuleContext): any { * ip.asn, ip.asn_org → enrichment.network.* * ip.abuse_score, ip.total_reports, ip.is_high_risk → enrichment.reputation.* * req.path, req.method, req.ip, req.user_agent → context fields + * bot.known, bot.ai, bot.category, bot.name, bot.id, bot.organization, + * bot.score, bot.respects_robots → the declared-agent verdict + * edge.present, edge.class, edge.clearance, ... → the edge validator's verdict */ function resolveProperty(path: string[], context: RuleContext): any { const namespace = path[0]; @@ -112,6 +115,32 @@ function resolveProperty(path: string[], context: RuleContext): any { return undefined; } + // Who the User-Agent claims to be (#500), from the generated agent registry. + // + // `bot.category` uses the customer-facing vocabulary — the same strings as the + // dashboard's ai_scraper_category column — so an expression can be copied from + // what someone is looking at. + // + // Every field is undefined when nothing matched, EXCEPT `known` and `ai`, + // which are false. That asymmetry is deliberate: `bot.known` and `bot.ai` are + // questions with a true answer for an unrecognised agent, while + // `bot.name == "GPTBot"` on an unmatched request should be false rather than + // comparing against an empty string that a typo could satisfy. + if (namespace === 'bot') { + const b = context.bot; + if (!b) return undefined; + if (prop === 'known') return b.known; + if (prop === 'ai') return b.isAI; + if (!b.known) return undefined; + if (prop === 'id') return b.id; + if (prop === 'name') return b.name; + if (prop === 'category') return b.category; + if (prop === 'organization') return b.organization; + if (prop === 'score') return b.score; + if (prop === 'respects_robots') return b.respectsRobots; + return undefined; + } + // Single ident with no namespace — treat as boolean property shorthand if (path.length === 1) return undefined; diff --git a/packages/webdecoy/src/rules/index.ts b/packages/webdecoy/src/rules/index.ts index 21dbab2..025c971 100644 --- a/packages/webdecoy/src/rules/index.ts +++ b/packages/webdecoy/src/rules/index.ts @@ -6,6 +6,7 @@ export { RuleEngine } from './rule-engine'; export { RateLimitRule } from './rate-limit-rule'; export { FilterRule } from './filter-rule'; export { TripwireRule, DEFAULT_TRIPWIRE_PATHS } from './tripwire-rule'; +export { BotRule } from './bot-rule'; export { WebBotAuthRule, webBotAuth } from './web-bot-auth-rule'; export { honeytoken } from './honeytoken'; export { InMemoryRateLimiter } from './rate-limiter'; @@ -18,6 +19,7 @@ export type { RateLimitConfig, FilterConfig, TripwireConfig, + BotRuleConfig, ViolationEvent, IPEnrichmentData, } from './types'; @@ -27,7 +29,14 @@ export type { HoneytokenOptions, Honeytoken } from './honeytoken'; import { RateLimitRule } from './rate-limit-rule'; import { FilterRule } from './filter-rule'; import { TripwireRule } from './tripwire-rule'; -import type { RateLimitConfig, FilterConfig, TripwireConfig, Rule } from './types'; +import { BotRule } from './bot-rule'; +import type { + RateLimitConfig, + FilterConfig, + TripwireConfig, + BotRuleConfig, + Rule, +} from './types'; /** * Factory function to create a rate limit rule @@ -90,6 +99,39 @@ export function filter(config: FilterConfig): Rule { export function tripwire(config: TripwireConfig = {}): Rule { return new TripwireRule(config); } + +/** + * Factory function to create a bot-category rule (#500). + * + * Acts on agents that identify themselves in the User-Agent, matched against the + * registry generated from the same table the scoring pipeline uses — so + * `'training_crawler'` here means what `ai_scraper_category` means in your + * dashboard. + * + * @example + * ```typescript + * import { WebDecoy, bots } from '@webdecoy/node'; + * + * const sdk = new WebDecoy({ + * rules: [ + * // Keep AI models out of your content, keep your search ranking. + * bots({ categories: ['training_crawler'] }), + * + * // Or everything AI, minus the one you want referral traffic from. + * bots({ ai: true, allow: ['perplexitybot'] }), + * + * // Or a specific operator. + * bots({ agents: ['gptbot', 'ClaudeBot'], action: 'THROTTLE' }), + * ], + * }); + * ``` + * + * Only catches agents that declare themselves — anything spoofing a browser + * passes through, by design. Pair with {@link tripwire} for agents that lie. + */ +export function bots(config: BotRuleConfig = {}): Rule { + return new BotRule(config); +} export { siteHoneytoken, injectHoneytokenLink, diff --git a/packages/webdecoy/src/rules/types.ts b/packages/webdecoy/src/rules/types.ts index 4e84092..15a309d 100644 --- a/packages/webdecoy/src/rules/types.ts +++ b/packages/webdecoy/src/rules/types.ts @@ -5,6 +5,7 @@ import type { AgentVerdict } from '../agent/types'; import type { EdgeVerdict } from '../edge'; +import type { BotVerdict, BotCategory } from '../bots'; /** * Context available to rules during evaluation @@ -37,6 +38,15 @@ export interface RuleContext { * `edge.class`, `edge.clearance` and `edge.present`. */ edge?: EdgeVerdict; + /** + * Who the User-Agent says it is (#500), matched against the generated agent + * registry. Always populated — `known: false` when nothing matched. Filter + * expressions read it as `bot.category`, `bot.name`, `bot.ai` and friends. + * + * A self-declared identity, so it is evidence about cooperative agents only. + * See {@link BotVerdict}. + */ + bot?: BotVerdict; } /** @@ -95,6 +105,33 @@ export interface FilterConfig { dryRun?: boolean; } +/** + * Configuration for bot-category rules (#500) + */ +export interface BotRuleConfig { + /** + * Categories to act on, e.g. `['training_crawler']`. Uses the same vocabulary + * as the dashboard's `ai_scraper_category` column. + */ + categories?: BotCategory[]; + /** Specific agents by registry slug or display name, e.g. `['gptbot']`. */ + agents?: string[]; + /** + * Act on every AI client — training crawlers, AI search crawlers, AI agents + * and AI assistants. + */ + ai?: boolean; + /** + * Never act on these, whatever else matches. Applied last, so + * `{ ai: true, allow: ['perplexitybot'] }` reads the way it looks. + */ + allow?: string[]; + /** Action when an agent matches: 'DENY' (default) or 'THROTTLE'. */ + action?: 'DENY' | 'THROTTLE'; + /** Log the violation but don't block. */ + dryRun?: boolean; +} + /** * Configuration for tripwire (honeypot-path) rules */ diff --git a/packages/webdecoy/src/sdk.ts b/packages/webdecoy/src/sdk.ts index a6fc203..2d0b5a0 100644 --- a/packages/webdecoy/src/sdk.ts +++ b/packages/webdecoy/src/sdk.ts @@ -12,6 +12,7 @@ import { IPEnrichmentClient } from './ip-enrichment'; import { AgentVerifier } from './agent/verifier'; import type { AgentRequestInput, AgentVerdict } from './agent/types'; import { readEdgeVerdict } from './edge'; +import { classifyUserAgent } from './bots'; import type { RuleContext, RuleEngineResult, ViolationEvent } from './rules/types'; import { WebDecoyConfig, @@ -186,6 +187,11 @@ export class WebDecoy { // it needs no network, and a rule that has to check whether the edge // verdict was populated is a rule people will get wrong. edge: readEdgeVerdict(metadata.headers), + // Same reasoning as `edge`, and the same cost profile (#500): a memoised + // substring scan over a static table, no network, no async. Populating it + // unconditionally means a rule never has to ask whether classification + // ran. + bot: classifyUserAgent(metadata.user_agent), }; }