diff --git a/backend/src/modules/admin/observability/observability.types.ts b/backend/src/modules/admin/observability/observability.types.ts index 9ec053fb..5956ffe0 100644 --- a/backend/src/modules/admin/observability/observability.types.ts +++ b/backend/src/modules/admin/observability/observability.types.ts @@ -108,6 +108,7 @@ export const ProcessorSnapshotSchema = z.object({ status: z.enum(["ok", "partial"]), timestamp: z.string().datetime(), execution: z.object({ + partial: z.boolean().optional(), runId: z.string().min(1).max(128).nullable().optional().default(null), status: z.enum(["idle", "running", "canceling", "failed", "completed"]), source: z.enum(["manual", "cron"]).nullable(), diff --git a/backend/src/modules/admin/scrapers/scraperClient.ts b/backend/src/modules/admin/scrapers/scraperClient.ts index b8966e65..58197d78 100644 --- a/backend/src/modules/admin/scrapers/scraperClient.ts +++ b/backend/src/modules/admin/scrapers/scraperClient.ts @@ -1,4 +1,4 @@ -import { JobsCountResultSchema } from "./scrapers.types"; +import { JobsCountResultSchema, ScraperStatusSchema } from "./scrapers.types"; import { config } from "../../../config"; import type { GetJobsResult, @@ -10,6 +10,15 @@ import type { const TIMEOUT_MS = 5000; +export class ScraperInvalidSelectionError extends Error { + readonly code = "INVALID_PROVIDER_SELECTION"; +} + +export class ScraperRateLimitedError extends Error { + readonly code = "PROVIDER_RATE_LIMITED"; + constructor(message: string, readonly retryAfter: string | null) { super(message); } +} + /** Lançado quando o Go scraper responde 409 (já em execução) */ export class ScraperAlreadyRunningError extends Error { readonly code = "SCRAPER_ALREADY_RUNNING"; @@ -41,6 +50,14 @@ async function request(path: string, init?: RequestInit): Promise { const body = await response.json().catch(() => null); throw new ScraperAlreadyRunningError(body?.message); } + if (response.status === 400) { + const body = await response.json().catch(() => null); + throw new ScraperInvalidSelectionError(body?.message ?? "Seleção de provedor inválida."); + } + if (response.status === 429) { + const body = await response.json().catch(() => null); + throw new ScraperRateLimitedError(body?.message ?? "Provedor aguardando Retry-After.", response.headers?.get("Retry-After") ?? null); + } if (response.status === 503) { const body = await response.json().catch(() => null); @@ -55,12 +72,15 @@ async function request(path: string, init?: RequestInit): Promise { } export const scraperClient = { - triggerScrape(): Promise { - return request("/admin/scrape", { method: "POST" }); + triggerScrape(sources?: string[]): Promise { + return request("/admin/scrape", { + method: "POST", + ...(sources !== undefined ? { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sources }) } : {}), + }); }, - getStatus(): Promise { - return request("/admin/scrape/status"); + async getStatus(): Promise { + return ScraperStatusSchema.parse(await request("/admin/scrape/status")); }, getJobs(limit?: number): Promise { diff --git a/backend/src/modules/admin/scrapers/scrapers.controller.ts b/backend/src/modules/admin/scrapers/scrapers.controller.ts index b2ecf78b..2658ca64 100644 --- a/backend/src/modules/admin/scrapers/scrapers.controller.ts +++ b/backend/src/modules/admin/scrapers/scrapers.controller.ts @@ -3,8 +3,11 @@ import type { AuditService } from "../audit/audit.service"; import { ScraperAlreadyRunningError, ScraperRunLockUnavailableError, + ScraperInvalidSelectionError, + ScraperRateLimitedError, } from "./scraperClient"; import { ScrapersService } from "./scrapers.service"; +import { TriggerScrapeRequestSchema } from "./scrapers.types"; export class ScrapersController { constructor( @@ -13,13 +16,20 @@ export class ScrapersController { ) {} async trigger(req: Request, res: Response): Promise { + const selection = TriggerScrapeRequestSchema.safeParse(req.body ?? {}); + if (!selection.success) { res.status(400).json({ ok: false, code: "INVALID_PROVIDER_SELECTION" }); return; } try { - const result = await this.scrapersService.triggerScrape(); + const result = await this.scrapersService.triggerScrape(selection.data.sources); this.auditService.fromRequest(req, "scrapers.trigger"); res.status(202).json(result); } catch (error) { + if (error instanceof ScraperRateLimitedError) { + if (error.retryAfter) res.setHeader("Retry-After", error.retryAfter); + res.status(429).json({ ok: false, code: error.code, message: error.message }); return; + } + if (error instanceof ScraperInvalidSelectionError) { res.status(400).json({ ok: false, code: error.code, message: error.message }); return; } if (error instanceof ScraperAlreadyRunningError) { res.status(409).json({ ok: false, @@ -52,6 +62,11 @@ export class ScrapersController { res.status(202).json({ ...result, scraper: scraperName }); } catch (error) { + if (error instanceof ScraperRateLimitedError) { + if (error.retryAfter) res.setHeader("Retry-After", error.retryAfter); + res.status(429).json({ ok: false, code: error.code, message: error.message }); return; + } + if (error instanceof ScraperInvalidSelectionError) { res.status(400).json({ ok: false, code: error.code, message: error.message }); return; } if (error instanceof ScraperAlreadyRunningError) { res.status(409).json({ ok: false, diff --git a/backend/src/modules/admin/scrapers/scrapers.service.ts b/backend/src/modules/admin/scrapers/scrapers.service.ts index 7c8c262b..ee1b7248 100644 --- a/backend/src/modules/admin/scrapers/scrapers.service.ts +++ b/backend/src/modules/admin/scrapers/scrapers.service.ts @@ -1,6 +1,8 @@ import { ScraperAlreadyRunningError, ScraperRunLockUnavailableError, + ScraperInvalidSelectionError, + ScraperRateLimitedError, scraperClient, } from "./scraperClient"; import type { @@ -11,15 +13,16 @@ import type { ScraperStatus, TriggerScrapeResult, } from "./scrapers.types"; +import { TriggerScrapeRequestSchema } from "./scrapers.types"; export class ScrapersService { - async triggerScrape(): Promise { + async triggerScrape(sources?: string[]): Promise { try { - return await scraperClient.triggerScrape(); + return await scraperClient.triggerScrape(sources); } catch (error) { if ( error instanceof ScraperAlreadyRunningError || - error instanceof ScraperRunLockUnavailableError + error instanceof ScraperRunLockUnavailableError || error instanceof ScraperInvalidSelectionError || error instanceof ScraperRateLimitedError ) { // repropaga para o controller decidir o status HTTP (409) throw error; @@ -31,7 +34,9 @@ export class ScrapersService { async triggerScraper(name: string): Promise { const normalized = name.trim().toLowerCase(); if (normalized !== "go-scraper") { - throw new Error(`scraper desconhecido: ${name}`); + const selection = TriggerScrapeRequestSchema.safeParse({ sources: [normalized] }); + if (!selection.success) throw new Error(`scraper desconhecido: ${name}`); + return this.triggerScrape(selection.data.sources); } return this.triggerScrape(); diff --git a/backend/src/modules/admin/scrapers/scrapers.types.ts b/backend/src/modules/admin/scrapers/scrapers.types.ts index cab253bf..a342c58a 100644 --- a/backend/src/modules/admin/scrapers/scrapers.types.ts +++ b/backend/src/modules/admin/scrapers/scrapers.types.ts @@ -25,12 +25,18 @@ export const TriggerScrapeResultSchema = z.object({ }); export type TriggerScrapeResult = z.infer; +export const TriggerScrapeRequestSchema = z.object({ + sources: z.array(z.enum(["linkedin", "adzuna", "gupy", "inhire", "greenhouse", "lever", "jooble", "themuse"])).max(8).optional(), +}).strict(); + export const TriggerScrapeErrorSchema = z.object({ ok: z.literal(false), code: z.enum([ "SCRAPER_ALREADY_RUNNING", "SCRAPER_RUN_LOCK_UNAVAILABLE", "SCRAPER_RUN_LOCK_LOST", + "INVALID_PROVIDER_SELECTION", + "PROVIDER_RATE_LIMITED", ]), message: z.string(), }); @@ -40,8 +46,27 @@ export type TriggerScrapeError = z.infer; export const ScraperStatusSchema = z.object({ name: z.string().optional(), running: z.boolean(), - lastRunAt: z.string().optional(), + lastRunAt: z.string().nullable().optional(), jobsCollected: z.number().int().min(0).optional(), + lastRun: z.object({ + runId: z.string(), + providers: z.array(z.string()).nullable().optional(), + status: z.enum(["success", "partial", "failed", "timeout", "canceled"]), + finishedAt: z.string().datetime(), + jobsSaved: z.number().int().nonnegative(), + jobsIndexed: z.number().int().nonnegative(), + errorType: z.string(), + }).nullable().optional(), + providerWindows: z.array(z.union([ + z.object({ + provider: z.string(), id: z.string().optional(), timezone: z.literal("UTC").optional(), + startMinuteUTC: z.number().int().min(0).max(1439), endMinuteUTC: z.number().int().min(1).max(1440), + }), + z.object({ + provider: z.string(), id: z.string(), timezone: z.string().min(1), + startMinuteLocal: z.number().int().min(0).max(1439), endMinuteLocal: z.number().int().min(1).max(1440), + }), + ])).nullable().optional(), }); export type ScraperStatus = z.infer; diff --git a/backend/src/scripts/seed.ts b/backend/src/scripts/seed.ts index 98af8f7e..047a88d7 100644 --- a/backend/src/scripts/seed.ts +++ b/backend/src/scripts/seed.ts @@ -1,4 +1,5 @@ import * as argon2 from "argon2"; +import { assertLocalSeedAllowed } from "./seedPolicy"; import "dotenv/config"; import { and, eq } from "drizzle-orm"; import { db, pool } from "../db/client"; @@ -237,6 +238,7 @@ async function upsertApplicationEvent( async function main() { + assertLocalSeedAllowed(); console.log("Iniciando seed de desenvolvimento local..."); const devUserSpec = SEED_USERS.find((spec) => spec.role === "user")!; diff --git a/backend/src/scripts/seedCatalogJobs.ts b/backend/src/scripts/seedCatalogJobs.ts index 83ac4f32..922b43af 100644 --- a/backend/src/scripts/seedCatalogJobs.ts +++ b/backend/src/scripts/seedCatalogJobs.ts @@ -1,4 +1,5 @@ import type { ProfessionalFamily } from "../modules/jobs/types/professionalTaxonomy"; +import { assertLocalSeedAllowed } from "./seedPolicy"; import { createClient } from "redis"; export type SeedTechnologyExperience = { @@ -391,6 +392,7 @@ async function connectSeedCache(url: string) { } export async function seedCatalogJobs(): Promise { + assertLocalSeedAllowed(); const url = process.env.VALKEY_URL; if (!url) { diff --git a/backend/src/scripts/seedPolicy.ts b/backend/src/scripts/seedPolicy.ts new file mode 100644 index 00000000..7e6ae524 --- /dev/null +++ b/backend/src/scripts/seedPolicy.ts @@ -0,0 +1,13 @@ +// Demo users, saved jobs and catalog fixtures are only permitted in an +// explicitly opted-in local environment. NODE_ENV=development also occurs in DEV. +export function assertLocalSeedAllowed(env = process.env): void { + if ( + env.APP_ENV !== "local" || + env.ALLOW_DEMO_SEED !== "true" || + env.NODE_ENV === "production" + ) { + throw new Error( + "Seed fictício bloqueado: exige APP_ENV=local, ALLOW_DEMO_SEED=true e NODE_ENV diferente de production.", + ); + } +} diff --git a/backend/tests/unit/modules/admin/providerDispatch.test.ts b/backend/tests/unit/modules/admin/providerDispatch.test.ts new file mode 100644 index 00000000..213d1566 --- /dev/null +++ b/backend/tests/unit/modules/admin/providerDispatch.test.ts @@ -0,0 +1,38 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { scraperClient, ScraperInvalidSelectionError, ScraperRateLimitedError } from "../../../../src/modules/admin/scrapers/scraperClient"; +import { ScrapersService } from "../../../../src/modules/admin/scrapers/scrapers.service"; +import { ScrapersController } from "../../../../src/modules/admin/scrapers/scrapers.controller"; +import type { Request, Response } from "express"; + +afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); }); + +it("preserves Retry-After when a provider is temporarily blocked", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 429, headers: { get: () => "120" }, json: async () => ({ message: "rate limited" }) })); + await expect(new ScrapersService().triggerScraper("greenhouse")).rejects.toBeInstanceOf(ScraperRateLimitedError); + const res = { status: vi.fn().mockReturnThis(), json: vi.fn(), setHeader: vi.fn() } as unknown as Response; + await new ScrapersController(new ScrapersService(), { fromRequest: vi.fn() } as never).triggerOne({ params: { id: "greenhouse" } } as unknown as Request, res); + expect(res.status).toHaveBeenCalledWith(429); + expect(res.setHeader).toHaveBeenCalledWith("Retry-After", "120"); +}); + +it("forwards an individual ProviderID to Go without a global fallback", async () => { + const fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ ok: true, message: "started" }) }); + vi.stubGlobal("fetch", fetch); + await new ScrapersService().triggerScraper("greenhouse"); + expect(JSON.parse(fetch.mock.calls[0][1].body)).toEqual({ sources: ["greenhouse"] }); +}); + +it("preserves a Go rejection when the provider is disabled", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 400, json: async () => ({ message: "provider is not configured" }) })); + await expect(new ScrapersService().triggerScraper("greenhouse")).rejects.toBeInstanceOf(ScraperInvalidSelectionError); +}); + +it("rejects invalid selection before sending any request to Go", async () => { + const trigger = vi.spyOn(scraperClient, "triggerScrape"); + const service = new ScrapersService(); + const res = { status: vi.fn().mockReturnThis(), json: vi.fn() } as unknown as Response; + const controller = new ScrapersController(service, { fromRequest: vi.fn() } as never); + await controller.trigger({ body: { sources: ["unknown"] } } as Request, res); + expect(res.status).toHaveBeenCalledWith(400); + expect(trigger).not.toHaveBeenCalled(); +}); diff --git a/backend/tests/unit/modules/admin/providerSchedule.test.ts b/backend/tests/unit/modules/admin/providerSchedule.test.ts new file mode 100644 index 00000000..f7bf3f45 --- /dev/null +++ b/backend/tests/unit/modules/admin/providerSchedule.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; +import { ScraperStatusSchema } from "../../../../src/modules/admin/scrapers/scrapers.types"; + +describe("provider window status contract", () => { + it("preserves legacy UTC windows and manual status responses", () => { + expect(ScraperStatusSchema.parse({ running: false })).toEqual({ running: false }); + const window = { provider: "gupy", startMinuteUTC: 60, endMinuteUTC: 120 }; + expect(ScraperStatusSchema.parse({ running: false, providerWindows: [window] }).providerWindows).toEqual([window]); + }); + + it("preserves local window identity and timezone", () => { + const window = { provider: "greenhouse", id: "greenhouse-1", timezone: "America/Sao_Paulo", startMinuteLocal: 0, endMinuteLocal: 60 }; + expect(ScraperStatusSchema.parse({ running: true, providerWindows: [window] }).providerWindows).toEqual([window]); + }); + + it("rejects incomplete local windows and invalid minute bounds", () => { + for (const window of [ + { provider: "gupy", startMinuteLocal: 0, endMinuteLocal: 60 }, + { provider: "gupy", id: "gupy-1", timezone: "America/Sao_Paulo", startMinuteLocal: 1440, endMinuteLocal: 1500 }, + ]) { + expect(ScraperStatusSchema.safeParse({ running: false, providerWindows: [window] }).success).toBe(false); + } + }); +}); diff --git a/backend/tests/unit/modules/admin/scraperResult.test.ts b/backend/tests/unit/modules/admin/scraperResult.test.ts new file mode 100644 index 00000000..a7389a0e --- /dev/null +++ b/backend/tests/unit/modules/admin/scraperResult.test.ts @@ -0,0 +1,14 @@ +import { expect, it } from "vitest"; +import { ScraperStatusSchema } from "../../../../src/modules/admin/scrapers/scrapers.types"; + +it("retains partial execution counts and accepts an empty initial history", () => { + expect(ScraperStatusSchema.parse({ running: false, lastRunAt: null, lastRun: null }).lastRun).toBeNull(); + const result = ScraperStatusSchema.parse({ + running: false, + lastRunAt: "2026-11-01T00:00:00Z", + jobsCollected: 210, + lastRun: { runId: "test", status: "partial", finishedAt: "2026-11-01T00:00:00Z", jobsSaved: 210, jobsIndexed: 210, errorType: "timeout" }, + }); + expect(result.lastRun?.status).toBe("partial"); + expect(result.lastRun?.jobsIndexed).toBe(210); +}); diff --git a/backend/tests/unit/modules/admin/scrapers.test.ts b/backend/tests/unit/modules/admin/scrapers.test.ts index 95969aa4..badbbf04 100644 --- a/backend/tests/unit/modules/admin/scrapers.test.ts +++ b/backend/tests/unit/modules/admin/scrapers.test.ts @@ -211,7 +211,9 @@ describe("ScrapersService", () => { ok: true, message: "started", }); - await expect(service.triggerScraper("lever")).rejects.toThrow( + await expect(service.triggerScraper("lever")).resolves.toEqual({ ok: true, message: "started" }); + expect(scraperClient.triggerScrape).toHaveBeenLastCalledWith(["lever"]); + await expect(service.triggerScraper("invalid")).rejects.toThrow( "scraper desconhecido", ); }); diff --git a/backend/tests/unit/scripts/seed.test.ts b/backend/tests/unit/scripts/seed.test.ts index 5ccf7746..4c26a65e 100644 --- a/backend/tests/unit/scripts/seed.test.ts +++ b/backend/tests/unit/scripts/seed.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const seedMocks = vi.hoisted(() => ({ applicationEventsFindFirst: vi.fn(), @@ -34,6 +34,9 @@ const originalArgv = [...process.argv]; describe("seed script", () => { beforeEach(() => { + vi.stubEnv("APP_ENV", "local"); + vi.stubEnv("ALLOW_DEMO_SEED", "true"); + vi.stubEnv("NODE_ENV", "test"); vi.resetModules(); vi.clearAllMocks(); process.argv = [...originalArgv]; @@ -56,6 +59,22 @@ describe("seed script", () => { vi.spyOn(console, "error").mockImplementation(() => undefined); }); + afterEach(() => { + vi.unstubAllEnvs(); + process.exitCode = 0; + }); + + it("blocks DEV fixtures before any database or catalog write", async () => { + vi.stubEnv("APP_ENV", "dev"); + await import("../../../src/scripts/seed"); + await vi.waitFor(() => expect(seedMocks.poolEnd).toHaveBeenCalledOnce()); + expect(seedMocks.credentialsFindFirst).not.toHaveBeenCalled(); + expect(seedMocks.update).not.toHaveBeenCalled(); + expect(seedMocks.insert).not.toHaveBeenCalled(); + expect(seedMocks.seedCatalogJobs).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); + it("preserves existing records while refreshing profile and catalog data", async () => { await import("../../../src/scripts/seed"); diff --git a/backend/tests/unit/scripts/seedPolicy.test.ts b/backend/tests/unit/scripts/seedPolicy.test.ts new file mode 100644 index 00000000..d2c0252a --- /dev/null +++ b/backend/tests/unit/scripts/seedPolicy.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { assertLocalSeedAllowed } from "../../../src/scripts/seedPolicy"; + +describe("demo seed policy", () => { + it.each([undefined, "dev", "development", "staging", "production"])( + "rejects deployment environment %s despite opt-in", + (APP_ENV) => { + expect(() => assertLocalSeedAllowed({ APP_ENV, ALLOW_DEMO_SEED: "true" })).toThrow(); + }, + ); + it("requires local opt-in and rejects production runtime", () => { + expect(() => assertLocalSeedAllowed({ APP_ENV: "local" })).toThrow(); + expect(() => assertLocalSeedAllowed({ APP_ENV: "local", ALLOW_DEMO_SEED: "true", NODE_ENV: "production" })).toThrow(); + expect(() => assertLocalSeedAllowed({ APP_ENV: "local", ALLOW_DEMO_SEED: "true", NODE_ENV: "development" })).not.toThrow(); + }); +}); diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 999b574f..1a6a693f 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2022", "module": "CommonJS", "moduleResolution": "Node", - "ignoreDeprecations": "6.0", + "ignoreDeprecations": "5.0", "outDir": "./dist", "rootDir": "./src", "strict": true, diff --git a/docker/docker-compose.app.yml b/docker/docker-compose.app.yml index b92079b9..4119015f 100644 --- a/docker/docker-compose.app.yml +++ b/docker/docker-compose.app.yml @@ -13,6 +13,9 @@ services: environment: - GO_SCRAPER_ADDR=:8081 - VALKEY_URL=redis://valkey:6379/0 + - SCRAPER_SCRAPE_TIMEOUT=${SCRAPER_SCRAPE_TIMEOUT-50m} + - SCRAPER_PROVIDER_WINDOWS=${SCRAPER_PROVIDER_WINDOWS-} + - SCRAPER_GREENHOUSE_BATCH_SIZE=${SCRAPER_GREENHOUSE_BATCH_SIZE-1000} - SCRAPER_MAX_CONCURRENCY=${SCRAPER_MAX_CONCURRENCY-12} - SCRAPER_PROVIDER_MAX_CONCURRENCY=${SCRAPER_PROVIDER_MAX_CONCURRENCY-2} - SCRAPER_PROVIDER_CONCURRENCY_OVERRIDES=${SCRAPER_PROVIDER_CONCURRENCY_OVERRIDES-} diff --git a/docker/docker-compose.migrate.yml b/docker/docker-compose.migrate.yml index b79f34cf..88b69fb7 100644 --- a/docker/docker-compose.migrate.yml +++ b/docker/docker-compose.migrate.yml @@ -11,7 +11,7 @@ services: environment: DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB} VALKEY_URL: redis://valkey:6379/0 - command: sh -c "npm run db:migrate && npm run db:seed && npm run security:backfill-user-pii -- --write" + command: sh -c "npm run db:migrate && npm run security:backfill-user-pii -- --write" depends_on: postgres: condition: service_healthy @@ -39,4 +39,4 @@ services: networks: vagas-net: - external: true \ No newline at end of file + external: true diff --git a/docs/PAV-178-PR.md b/docs/PAV-178-PR.md new file mode 100644 index 00000000..463c179a --- /dev/null +++ b/docs/PAV-178-PR.md @@ -0,0 +1,107 @@ +# Conteúdo sugerido para o PR + +## Linear + +Issue: PAV-178 +Destino: `develop`. Não foi aberto PR nem realizada operação remota. + +## Objetivo + +Evitar perda de resultados válidos no encerramento da coleta e preparar a publicação controlada do Processor em DEV, com execuções por provedor sem sobreposição e sem introduzir vagas fictícias. + +## Resumo das alterações + +- Preserva o hotfix de resultados parciais, a seleção por ProviderID e o desligamento do scheduler. +- Centraliza `SCRAPER_SCRAPE_TIMEOUT` com padrão de 50 minutos e validação de override. +- Atualiza o resumo e a observabilidade quando há vagas persistidas após timeout/falha parcial. +- Remove o seed demonstrativo do fluxo de migrations e bloqueia fixtures fora de opt-in local explícito. +- Encaminha seleção administrativa Node → Go, com validação e propagação dos erros 400/409/429/503. +- Adiciona janelas diárias UTC configuráveis, reserva por janela e reutilização do lock global. +- Limita o despacho individual de Greenhouse a até 1.000 adapters e grava checkpoint de lote somente após conclusão confirmada, com proteção de propriedade do lock. +- Respeita Retry-After do Greenhouse entre execuções e após reinício; renova os caches de Greenhouse/Lever por execução. +- Mantém o registry completo e a busca nos documentos versionados do Valkey. +- Ajusta o teste de configuração ao Compose modular e o typecheck à versão 5.9 do TypeScript utilizada pelo backend. + +## Validação realizada + +- Suíte Go: 310 testes principais aprovados e 14 testes de integração pulados sem configuração externa; regressão adicional do registry completo aprovada. +- Integrações Go: 14 testes aprovados em PostgreSQL/Redis descartáveis, incluindo commit, falha de commit e recuperação de indexação. +- Regressões Node.js: 395 testes aprovados em 32 arquivos, incluindo rotas, RBAC, busca, contratos e bloqueio de seeds. +- Busca versionada: 18 testes aprovados com Redis descartável. +- Detector de corridas de dados: aprovado em scheduler e pipeline. +- Typecheck do backend, build Go e validação estática do Compose: aprovados. + +As instâncias descartáveis foram encerradas. Nenhum scrape real, seed real, deploy ou alteração de produção foi executado nesta rodada. O teste real de 300 segundos informado no contexto permanece como evidência anterior, não foi repetido. + +## Banco, contratos e configuração + +Nenhuma migration nova. O catálogo PostgreSQL, IDs estáveis, transações e revisões de indexação existentes foram mantidos. A mudança no Compose remove somente a chamada ao seed demonstrativo; migrations e backfill legítimo continuam. + +O status administrativo ganha `lastRun` e `providerWindows`; o snapshot operacional ganha o indicador `partial`. O endpoint administrativo aceita `sources`, preservando o disparo sem seleção. Headers Retry-After e conflitos são propagados ao cliente. + +Novas configurações: `SCRAPER_SCRAPE_TIMEOUT`, `SCRAPER_PROVIDER_WINDOWS` e `SCRAPER_GREENHOUSE_BATCH_SIZE`. Começar DEV com o scheduler desabilitado, validar um provider e medir duração/recursos antes de habilitar janelas definitivas. + +## Pendências e riscos + +- **Bloqueador condicional:** destino com índice legado não vazio e sem versão ativa exige plano explícito de migração/reconstrução antes da publicação. O código não apaga nem reconstrói índices automaticamente. +- **Bloqueador de ativação automática do Greenhouse:** validar a saúde das empresas e dimensionar o lote. Falhas permanentes ou um lote que nunca conclui impedem avançar seu checkpoint. +- **Não bloqueador do deploy inicial com scheduler desligado:** não há checkpoint por empresa/página dentro de lote parcial, nem histórico SQL de execuções. A repetição é idempotente, mas pode refazer trabalho. +- **Não bloqueador:** os demais providers mantêm suas políticas próprias de retry; o novo tratamento de Retry-After é específico do Greenhouse. +- A retenção de claims/checkpoints depende da persistência do Valkey. O prazo adicional de processamento é limitado a 30 segundos e não garante drenar qualquer volume. +- Fixtures já existentes não foram removidas. O bloqueio impede novas execuções automáticas do seed. + +Consultar [o roteiro de publicação e validação](PAV-178-dev.md) para configuração, comandos e checklist. Nenhuma configuração temporária, `.env`, binário ou `cspell.json` faz parte desta entrega. Nenhum arquivo do frontend/PAV-181 foi alterado. + +## Arquivos alterados nesta finalização + +- `backend/src/modules/admin/observability/observability.types.ts` +- `backend/src/modules/admin/scrapers/scraperClient.ts` +- `backend/src/modules/admin/scrapers/scrapers.controller.ts` +- `backend/src/modules/admin/scrapers/scrapers.service.ts` +- `backend/src/modules/admin/scrapers/scrapers.types.ts` +- `backend/src/scripts/seed.ts` +- `backend/src/scripts/seedCatalogJobs.ts` +- `backend/src/scripts/seedPolicy.ts` +- `backend/tests/unit/modules/admin/providerDispatch.test.ts` +- `backend/tests/unit/modules/admin/scraperResult.test.ts` +- `backend/tests/unit/modules/admin/scrapers.test.ts` +- `backend/tests/unit/scripts/seed.test.ts` +- `backend/tests/unit/scripts/seedPolicy.test.ts` +- `backend/tsconfig.json` +- `docker/docker-compose.app.yml` +- `docker/docker-compose.migrate.yml` +- `docs/PAV-178-PR.md` +- `docs/PAV-178-dev.md` +- `scraper-go/cmd/server/admin_handlers.go` +- `scraper-go/cmd/server/main.go` +- `scraper-go/cmd/server/provider_manual_test.go` +- `scraper-go/cmd/server/server.go` +- `scraper-go/internal/adapters/adapterutil/retry_after.go` +- `scraper-go/internal/adapters/adapterutil/retry_after_test.go` +- `scraper-go/internal/adapters/greenhouse/adapter.go` +- `scraper-go/internal/adapters/greenhouse/retry_after_test.go` +- `scraper-go/internal/adapters/lever/adapter.go` +- `scraper-go/internal/adapters/lever/run_cache_test.go` +- `scraper-go/internal/adapters/registry_test.go` +- `scraper-go/internal/config/config.go` +- `scraper-go/internal/config/config_test.go` +- `scraper-go/internal/config/provider_schedule.go` +- `scraper-go/internal/config/provider_schedule_test.go` +- `scraper-go/internal/config/scrape_timeout_test.go` +- `scraper-go/internal/cronjob/collection_timeout_test.go` +- `scraper-go/internal/cronjob/cronjob.go` +- `scraper-go/internal/cronjob/provider_batch.go` +- `scraper-go/internal/cronjob/provider_dispatch_test.go` +- `scraper-go/internal/cronjob/provider_retry.go` +- `scraper-go/internal/cronjob/provider_retry_test.go` +- `scraper-go/internal/cronjob/provider_schedule.go` +- `scraper-go/internal/domain/job.go` +- `scraper-go/internal/metrics/operational.go` +- `scraper-go/internal/metrics/partial_run_test.go` +- `scraper-go/internal/metrics/state.go` +- `scraper-go/internal/pipeline/pipeline.go` +- `scraper-go/internal/pipeline/process.go` +- `scraper-go/internal/pipeline/provider_selection.go` +- `scraper-go/internal/pipeline/scrape.go` +- `scraper-go/internal/ports/rate_limit.go` +- `scraper-go/internal/ports/rate_limit_test.go` diff --git a/docs/PAV-178-dev.md b/docs/PAV-178-dev.md new file mode 100644 index 00000000..5dc36161 --- /dev/null +++ b/docs/PAV-178-dev.md @@ -0,0 +1,102 @@ +# PAV-178 — publicação controlada em DEV + +## Diagnóstico e comportamento entregue + +O scheduler, a coleta e o processamento já compartilhavam o catálogo PostgreSQL e a projeção versionada do Valkey. O hotfix preserva resultados devolvidos com erro e separa o prazo da coleta de até 30 segundos adicionais para processamento, respeitando shutdown e perda do lock. Transações confirmadas permanecem no catálogo quando há falha de indexação; `indexed_revision < revision` identifica pendências recuperáveis pelos comandos existentes de manutenção. + +O resumo `GET /admin/scrape/status` agora inclui `lastRun`, com provedores, resultado, vagas salvas/indexadas e categoria do erro. Um timeout com persistência retorna `lastRun.status=partial`; a observabilidade mantém `execution.status=failed`, `lastStatus=timeout` e `partial=true`. O histórico detalhado permanece em memória e reinicia com o processo. + +O deploy executava `db:seed` junto às migrations. Esse script cria contas e vagas de demonstração, sem dados estruturais necessários à aplicação. Sua chamada automática foi removida; migrations e backfill de PII foram preservados. As duas entradas de fixtures exigem `APP_ENV=local`, `ALLOW_DEMO_SEED=true` e runtime fora de produção. Não executar seeds na publicação em DEV. + +O registry mantém os oito provedores e seus controles originais por flags/credenciais. Não foi transferido o registry temporário limitado ao LinkedIn. Greenhouse e Lever agora reutilizam seu cache somente na mesma execução, evitando dados ou erros presos durante toda a vida do processo. + +## Configuração de operação + +| Variável | Padrão | Comportamento | +|---|---|---| +| `SCRAPER_SCHEDULER_ENABLED` | `true` | Usar `false` na primeira publicação; disparo manual continua disponível. | +| `SCRAPER_SCRAPE_TIMEOUT` | `50m` | Duração Go positiva, por exemplo `300s`; valores inválidos impedem a inicialização. | +| `SCRAPER_PROVIDER_WINDOWS` | vazio | Janelas diárias UTC no formato `provider=HH:MM-HH:MM`, separadas por vírgulas. | +| `SCRAPER_GREENHOUSE_BATCH_SIZE` | `1000` | Inteiro de 1 a 1000; começar com lote menor e medir. | +| `SCRAPER_MAX_CONCURRENCY` | `12` | Limite global já existente. | +| `SCRAPER_PROVIDER_MAX_CONCURRENCY` | `2` | Limite por provedor já existente, sujeito aos overrides existentes. | + +Exemplo de planejamento, **não uma recomendação de horários definitivos**: + +```text +SCRAPER_SCHEDULER_ENABLED=false +SCRAPER_SCRAPE_TIMEOUT=50m +SCRAPER_PROVIDER_WINDOWS=linkedin=00:00-01:00,gupy=01:00-02:00,greenhouse=02:00-03:00 +SCRAPER_GREENHOUSE_BATCH_SIZE=100 +``` + +`SCRAPER_SCRAPE_TIMEOUT` governa o scheduler e os disparos administrativos assíncronos. O endpoint público síncrono `/scrape` mantém seu limite anterior de 15 minutos; ele não é o caminho recomendado para a coleta operacional de DEV. + +As janelas são em UTC, com início inclusivo e fim exclusivo. `24:00` é aceito como fim. Janelas que cruzam a meia-noite devem ser divididas. Sobreposição, provider desconhecido ou provider indisponível no registry impedem a inicialização. Não existe cálculo automático pelo fuso do host. + +Com janelas configuradas, o scheduler executa somente o provider da janela ativa. Uma reserva diária no Valkey impede repetir a janela depois de reinício ou por outra réplica. O lock global existente impede sobreposição com cron, disparos manuais e buscas públicas. Se o lock estiver ocupado, a janela tenta novamente enquanto ainda houver tempo. Os últimos 35 segundos ficam reservados para processamento/liberação; o contexto limita a execução ao fim da janela. + +**Sem janelas, permanece o modo legado de execução conjunta a cada três horas. Não habilitar esse modo em DEV sem decisão operacional explícita.** Uma execução manual também pode ocupar uma janela; nesse caso a exclusão global prevalece. + +## Disparo individual e contratos + +Com autenticação e permissão administrativa existentes no backend: + +```http +POST /admin/scrapers/linkedin/run +POST /admin/scrapers/greenhouse/run +``` + +Os mesmos endpoints têm o prefixo `/api/v1` disponível. Também é possível enviar `{"sources":["linkedin"]}` em `POST /admin/scrapers/run`. A integração Node → Go usa `POST /admin/scrape` com o mesmo corpo. Ausência de seleção preserva a execução conjunta. ID desconhecido ou provider desabilitado retorna erro; nunca há fallback silencioso para execução global. + +Respostas relevantes: 400 para seleção inválida, 409 para lock ocupado, 429 com `Retry-After` quando o provider está bloqueado e 503 para indisponibilidade do lock. O disparo é assíncrono: consultar status/observabilidade para conhecer o resultado final, pois aceitação HTTP não significa coleta completa. + +## Greenhouse: lotes, retomada e rate limit + +O despacho individual de Greenhouse ordena os adapters por identidade e executa até o tamanho configurado. O checkpoint `scraper:provider:greenhouse:checkpoint` guarda offset e hash da lista. A gravação usa comparação atômica com o proprietário do lock global. O offset só avança quando coleta, persistência e indexação terminam sem erro. Alterar a lista reinicia o ciclo com segurança; os IDs estáveis tornam a repetição idempotente. + +A retomada é por lote, não por página ou empresa concluída dentro de lote parcial. Timeout e erros não avançam o checkpoint: resultados confirmados ficam disponíveis e o próximo disparo repete esse lote. Uma empresa permanentemente inválida pode impedir seu avanço; medir e revisar o registry antes de habilitar varreduras grandes. Não há descarte automático dessas empresas. + +Greenhouse interpreta `Retry-After` em segundos ou data HTTP; sem cabeçalho válido usa um minuto. O maior prazo informado pelos adapters é gravado no Valkey, condicionado à propriedade do lock. O bloqueio sobrevive ao reinício e vale para disparos individuais e conjuntos. Uma janela bloqueada por rate limit não é consumida; poderá executar após o prazo, se ainda estiver aberta. Fora dela, aguarda uma nova janela. Os demais providers mantêm suas políticas anteriores; não houve unificação geral de retries. + +## Busca e recuperação + +O backend consulta a versão ativa em `scraper:jobs:index-version` e hidrata documentos em `scraper:jobs:ns::job:`. Há fallback legado quando não existe versão ativa. O catálogo SQL continua sendo a fonte durável para reconstrução da projeção. IDs de um índice sem documentos não permitem recriar o conteúdo das vagas. + +O Processor recusa inicializar com índice legado não vazio e sem versão ativa, até uma migração/reconstrução explícita. Verificar essa condição no destino é **bloqueador do deploy**; não remover a proteção, apagar índices ou executar reconstrução automaticamente. Uma reconstrução eventual exige autorização e procedimento separado. + +## Validação local + +Sem chamar providers ou iniciar o servidor: + +```bash +cd scraper-go +go test ./... -count=1 -timeout=25s +go test -race ./internal/cronjob ./internal/pipeline -run 'Test(Collection|ProviderRetry|ManualProvider|ProviderWindowClaim|Greenhouse|RunWithConcurrencyCancel)' -count=1 -timeout=20s +``` + +Os testes com miniredis precisam de sockets locais. Integrações SQL e Redis/Valkey reais são opt-in: configurar `PAV125_TEST_DATABASE_URL` e `PAV125_TEST_VALKEY_URL` **somente para instâncias descartáveis dedicadas**. A suíte cria seus próprios schemas/chaves. Nunca apontar essas variáveis para DEV compartilhado ou produção. + +```bash +cd backend +npx --no-install tsc --noEmit -p tsconfig.json +npx --no-install vitest run tests/unit/modules/admin tests/unit/modules/jobs tests/unit/libs/cache.test.ts tests/unit/scripts/seed.test.ts tests/unit/scripts/seedPolicy.test.ts tests/unit/scripts/seedCatalogJobs.test.ts tests/integration/routes/admin.routes.test.ts tests/integration/routes/searchJobs.routes.test.ts +# Somente com Redis/Valkey descartável explicitamente configurado: +npx --no-install vitest run tests/integration/jobs.valkey.test.ts +``` + +## Checklist de publicação em DEV + +- [ ] Revisar o PR único para `develop`, incluindo os commits anteriores da PAV-178. +- [ ] Confirmar ausência de `.env`, `compose.pav178-test.yml`, binários e `cspell.json` no diff do PR. +- [ ] Confirmar o registry completo e as flags/credenciais dos providers selecionados. +- [ ] Verificar catálogo, versão ativa e compatibilidade das migrations no destino, sem alterar dados. +- [ ] Publicar inicialmente com `SCRAPER_SCHEDULER_ENABLED=false`; não executar seeds. +- [ ] Após publicação autorizada, executar somente um provider e conferir salvos/indexados, documentos reais na busca e liberação do lock. +- [ ] Medir CPU, memória, requisições, duração e progresso de cada lote. Definir janelas com margem a partir dessas medições. +- [ ] Reduzir Greenhouse se um lote não concluir dentro da janela; tratar empresas com falha permanente antes de habilitar sua varredura automática. +- [ ] Configurar janelas sem sobreposição e somente então habilitar o scheduler. +- [ ] Acompanhar resumo parcial, erros, Retry-After e checkpoint nas primeiras execuções. +- [ ] Manter rollback restrito à imagem/versão de DEV e ao desligamento do scheduler; preservar catálogo e volumes. + +A publicação, validação com providers reais e definição dos horários finais não foram executadas nesta rodada. Não há garantia de que 30 segundos bastem para drenar qualquer backlog, nem isolamento por processo para adapters que desrespeitem cancelamento. O estado de janelas/checkpoints depende da retenção configurada no Valkey. diff --git a/scraper-go/cmd/server/admin_handlers.go b/scraper-go/cmd/server/admin_handlers.go index 302c29d6..0f1d7a42 100644 --- a/scraper-go/cmd/server/admin_handlers.go +++ b/scraper-go/cmd/server/admin_handlers.go @@ -8,6 +8,7 @@ import ( "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/metrics" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "io" "log/slog" "net/http" "sort" @@ -16,6 +17,7 @@ import ( "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/cronjob" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/jobstore" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/pipeline" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" ) @@ -23,9 +25,39 @@ import ( // retorna 409 se já houver uma execução em andamento. func handleTriggerScrape(scheduler *cronjob.Scheduler, executionCtx context.Context) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var request struct { + Sources []string `json:"sources"` + } + if r.Body != nil { + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&request); err != nil && !errors.Is(err, io.EOF) { + writeScraperError(w, http.StatusBadRequest, "INVALID_PROVIDER_SELECTION", "invalid request body") + return + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + writeScraperError(w, http.StatusBadRequest, "INVALID_PROVIDER_SELECTION", "unexpected trailing request data") + return + } + } // A execução manual precisa sobreviver ao fim da request HTTP; se usarmos // r.Context(), o scraper nasce com contexto cancelado assim que respondemos. - if err := scheduler.RunNow(executionCtx); err != nil { + if err := scheduler.RunProvidersNow(executionCtx, request.Sources); err != nil { + if writeGlobalPauseError(w, err) { + return + } + var limit *ports.RateLimitError + if errors.As(err, &limit) { + seconds := max(1, int((time.Until(limit.NotBefore)+time.Second-1)/time.Second)) + w.Header().Set("Retry-After", strconv.Itoa(seconds)) + writeScraperError(w, http.StatusTooManyRequests, "PROVIDER_RATE_LIMITED", err.Error()) + return + } + if errors.Is(err, pipeline.ErrInvalidProviderSelection) { + writeScraperError(w, http.StatusBadRequest, "INVALID_PROVIDER_SELECTION", err.Error()) + return + } if errors.Is(err, cronjob.ErrAlreadyRunning) { writeScraperError( w, @@ -72,6 +104,9 @@ func writeScraperError(w http.ResponseWriter, status int, code, message string) } func mapRunLockError(w http.ResponseWriter, err error) bool { + if writeGlobalPauseError(w, err) { + return true + } switch { case errors.Is(err, runlock.ErrAlreadyHeld): writeScraperError( @@ -102,6 +137,16 @@ func mapRunLockError(w http.ResponseWriter, err error) bool { } } +func writeGlobalPauseError(w http.ResponseWriter, err error) bool { + var pause *runlock.CooldownError + if !errors.As(err, &pause) { + return false + } + w.Header().Set("Retry-After", strconv.Itoa(max(1, int((time.Until(pause.NextAllowedAt)+time.Second-1)/time.Second)))) + writeScraperError(w, http.StatusConflict, "SCRAPER_ALREADY_RUNNING", pause.Error()) + return true +} + // handleScraperStatus retorna o estado atual do scheduler via GET /admin/scrape/status func handleScraperStatus(scheduler *cronjob.Scheduler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -114,10 +159,12 @@ func handleScraperStatus(scheduler *cronjob.Scheduler) http.HandlerFunc { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]any{ - "name": "go-scraper", - "running": running, - "lastRunAt": lastRunAtValue, - "jobsCollected": jobsCollected, + "name": "go-scraper", + "running": running, + "lastRunAt": lastRunAtValue, + "jobsCollected": jobsCollected, + "lastRun": scheduler.LastRun(), + "providerWindows": scheduler.ProviderWindows(), }) } } diff --git a/scraper-go/cmd/server/admin_handlers_test.go b/scraper-go/cmd/server/admin_handlers_test.go index 88aa46a2..d60ea265 100644 --- a/scraper-go/cmd/server/admin_handlers_test.go +++ b/scraper-go/cmd/server/admin_handlers_test.go @@ -111,7 +111,7 @@ func newHandlerTestScheduler( client := redis.NewClient(&redis.Options{Addr: server.Addr()}) t.Cleanup(func() { _ = client.Close() }) - manager, err := runlock.New(runlock.NewValkeyStore(client), runlock.Config{ + manager, err := runlock.New(runlock.NewValkeyStore(client, 0), runlock.Config{ TTL: 2 * time.Second, RenewInterval: 500 * time.Millisecond, }) diff --git a/scraper-go/cmd/server/handlers.go b/scraper-go/cmd/server/handlers.go index ceb4b020..adfa0f19 100644 --- a/scraper-go/cmd/server/handlers.go +++ b/scraper-go/cmd/server/handlers.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "net/http" "time" @@ -70,6 +71,10 @@ func handleScrape( "public_endpoint", ) if err != nil { + if errors.Is(err, pipeline.ErrInvalidProviderSelection) { + writeScraperError(w, http.StatusBadRequest, "INVALID_PROVIDER_SELECTION", err.Error()) + return + } if mapRunLockError(w, err) { return } diff --git a/scraper-go/cmd/server/main.go b/scraper-go/cmd/server/main.go index 77e5728a..0aca91e1 100644 --- a/scraper-go/cmd/server/main.go +++ b/scraper-go/cmd/server/main.go @@ -8,6 +8,7 @@ import ( "strconv" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/pipeline" ) func main() { @@ -35,6 +36,12 @@ func main() { } adapterList := buildAdapters(rdb) + for _, window := range runtimeCfg.ProviderWindows { + if _, err := pipeline.SelectProviders(adapterList, []string{string(window.Provider)}); err != nil { + slog.Error("provider window references an unavailable provider", "error", err) + os.Exit(1) + } + } slog.Info("servidor inicializado", "adapters_total", len(adapterList)) run(adapterList, runtimeCfg) @@ -47,6 +54,7 @@ func logRuntimeConfig(cfg config.RuntimeConfig) { memLimit := debug.SetMemoryLimit(-1) slog.Info("scraper runtime configurado", + "scrape_timeout", cfg.ScrapeTimeout, "max_concurrency", cfg.MaxConcurrency, "max_concurrency_source", cfg.MaxConcurrencySource, "provider_max_concurrency", cfg.ProviderMaxConcurrency, diff --git a/scraper-go/cmd/server/provider_manual_test.go b/scraper-go/cmd/server/provider_manual_test.go new file mode 100644 index 00000000..d52181e1 --- /dev/null +++ b/scraper-go/cmd/server/provider_manual_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/cache" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/cronjob" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/jobstore" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/keywords" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" +) + +type manualSource struct { + provider ports.ProviderID + calls *atomic.Int32 +} + +func (s manualSource) SourceName() string { return string(s.provider) } +func (s manualSource) Capabilities() ports.SourceCapabilities { + return ports.SourceCapabilities{Provider: s.provider, Mode: ports.DiscoveryKeyword} +} +func (s manualSource) Search(context.Context, string, domain.ScrapeRequest) ([]domain.Job, error) { + s.calls.Add(1) + return nil, nil +} + +func TestAdminManualProviderBodyIsValidatedAndIsolated(t *testing.T) { + _, manager, client, _ := newHandlerTestScheduler(t) + var linkedin, gupy atomic.Int32 + cfg := cronjob.DefaultConfig() + cfg.ScrapeTimeout = time.Second + scheduler := cronjob.New(cfg, keywords.NewStore(cache.NewRedisCache(client)), jobstore.New(client), []ports.JobSource{manualSource{ports.ProviderLinkedIn, &linkedin}, manualSource{ports.ProviderGupy, &gupy}}, client, manager) + for _, body := range []string{`{"sources":["unknown"]}`, `{"sources":[1]}`, `{"other":true}`, `{"sources":["linkedin"]} {}`} { + w := httptest.NewRecorder() + handleTriggerScrape(scheduler, context.Background()).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/admin/scrape", strings.NewReader(body))) + require.Equal(t, http.StatusBadRequest, w.Code) + } + require.Zero(t, linkedin.Load()) + require.Zero(t, gupy.Load()) + w := httptest.NewRecorder() + handleTriggerScrape(scheduler, context.Background()).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/admin/scrape", strings.NewReader(`{"sources":["linkedin"]}`))) + require.Equal(t, http.StatusOK, w.Code) + require.Eventually(t, func() bool { return scheduler.LastRun() != nil }, time.Second, time.Millisecond) + require.EqualValues(t, 1, linkedin.Load()) + require.Zero(t, gupy.Load()) + require.Equal(t, []string{"linkedin"}, scheduler.LastRun().Providers) +} diff --git a/scraper-go/cmd/server/provider_selection_test.go b/scraper-go/cmd/server/provider_selection_test.go new file mode 100644 index 00000000..2cdc3ce0 --- /dev/null +++ b/scraper-go/cmd/server/provider_selection_test.go @@ -0,0 +1,22 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/cache" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/stretchr/testify/require" +) + +func TestScrapeRejectsInvalidProviderSelectionWithBadRequest(t *testing.T) { + for _, provider := range []string{"invalid", "gupy"} { + r := httptest.NewRequest(http.MethodPost, "/scrape", strings.NewReader(`{"keywords":["go"],"sources":["`+provider+`"]}`)) + w := httptest.NewRecorder() + handleScrape(nil, nil, cache.NewMemoryCache(), nil, nil, config.RuntimeConfig{MaxConcurrency: 4}).ServeHTTP(w, r) + require.Equal(t, http.StatusBadRequest, w.Code) + require.Contains(t, w.Body.String(), "INVALID_PROVIDER_SELECTION") + } +} diff --git a/scraper-go/cmd/server/server.go b/scraper-go/cmd/server/server.go index 7c7dbd8e..5b9e626a 100644 --- a/scraper-go/cmd/server/server.go +++ b/scraper-go/cmd/server/server.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "strings" "syscall" "time" @@ -111,6 +112,11 @@ func run(adapterList []ports.JobSource, runtimeCfg config.RuntimeConfig) { // ── Scheduler (cronjob) ── schedulerCfg := cronjob.DefaultConfig() + + schedulerCfg.ScrapeTimeout = runtimeCfg.ScrapeTimeout + schedulerCfg.ProviderWindows = runtimeCfg.ProviderWindows + schedulerCfg.GreenhouseBatchSize = runtimeCfg.GreenhouseBatchSize + schedulerCfg.MaxConcurrency = runtimeCfg.MaxConcurrency schedulerCfg.ProviderMaxConcurrency = runtimeCfg.ProviderMaxConcurrency schedulerCfg.ProviderConcurrencyOverrides = runtimeCfg.ProviderConcurrencyOverrides @@ -164,7 +170,28 @@ func run(adapterList []ports.JobSource, runtimeCfg config.RuntimeConfig) { } // ── Background: inicia o scheduler ── - scheduler.Start(bgCtx) + //scheduler.Start(bgCtx) + schedulerEnabled := true + + if value, exists := os.LookupEnv("SCRAPER_SCHEDULER_ENABLED"); exists { + enabled, err := strconv.ParseBool(value) + if err != nil { + slog.Error( + "valor inválido para SCRAPER_SCHEDULER_ENABLED", + "value", value, + "error", err, + ) + return + } + schedulerEnabled = enabled + } + + if schedulerEnabled { + scheduler.Start(bgCtx) + slog.Info("scheduler automático habilitado") + } else { + slog.Info("scheduler automático desabilitado; execução manual disponível") + } // ── Graceful shutdown ── stop := make(chan os.Signal, 1) diff --git a/scraper-go/docs/PAV-178_JOBS_PROCESSOR_IMPLEMENTATION_PLAN.md b/scraper-go/docs/PAV-178_JOBS_PROCESSOR_IMPLEMENTATION_PLAN.md new file mode 100644 index 00000000..61ccf121 --- /dev/null +++ b/scraper-go/docs/PAV-178_JOBS_PROCESSOR_IMPLEMENTATION_PLAN.md @@ -0,0 +1,448 @@ +# PAV-178 — Jobs Processor Performance & Staging Implementation Plan + +## 1. Objetivo + +Evoluir o Jobs Processor para reduzir de forma significativa o tempo total de processamento, indexação e disponibilização de vagas, preservando os contratos atuais de consistência, revisão, lifecycle, retry, recovery e projeção. + +A implementação será feita por fases pequenas e verificáveis. Cada fase deve: + +1. preservar os contratos existentes; +2. possuir testes próprios; +3. ser validada com benchmark antes/depois; +4. gerar um commit isolado; +5. somente liberar a próxima fase após validação. + +O Greenhouse será utilizado como provider principal de benchmark por ser atualmente o maior cenário conhecido, com aproximadamente 3.339 adapters/fontes. + +## 2. Diagnóstico confirmado + +A investigação técnica da PAV-178 confirmou os seguintes pontos. + +### 2.1 Indexação síncrona gera backpressure + +O fluxo atual bloqueia o consumidor principal durante a indexação: + +```text +enqueuePersisted + -> flushIndexBuffer + -> indexPersistedChunk + -> espera concluir + -> continua processamento +``` + +Enquanto a indexação ocorre, o pipeline deixa de consumir novos jobs, classificação e persistência seguintes aguardam, `incoming` cresce, `results` pode encher, workers de coleta ficam bloqueados e a coleta perde vazão. + +### 2.2 `IndexKeys` é caro + +Medições locais com as funções reais mostraram que, para 250 vagas, 445 keywords e descrição sintética de 16 KiB: + +- preparação total: ~5,2 s; +- quase todo o custo está em `IndexKeys`; +- Build/metadata + serialização: ~42 ms; +- alocação cumulativa: ~9,5 GiB. + +O custo está fortemente associado a repetição de normalização, buscas textuais repetidas, concatenação/cópia de strings e execução de matching para muitas keywords/subtermos. + +### 2.3 Lua/Valkey não explica sozinho os 14–24 s observados + +O Lua durável isolado em Redis descartável apresentou ordens de grandeza menores que a latência total observada no pipeline. Portanto, não se deve assumir que o Valkey sozinho é o gargalo. + +### 2.4 Revision churn causa reprojeção desnecessária + +Hoje um `ON CONFLICT` no catálogo gera nova revisão mesmo quando o conteúdo não mudou semanticamente. + +```text +vaga reencontrada + -> merge + -> ON CONFLICT + -> nova revision + -> job_updated + -> IndexKeys + -> full apply no índice +``` + +Isso significa que vagas idênticas podem pagar novamente cálculo completo de chaves, serialização, remoção/reinserção de memberships, atualização de documentos, metadata e invalidação de geração. + +### 2.5 `commands=0` não representa ausência de trabalho + +No caminho durável, a variável `commands` não representa corretamente o trabalho executado pelo `jobindex.Apply`. Os logs atuais precisam ser corrigidos antes de serem usados como base de decisão. + +### 2.6 Timeout global + +O scheduler possui timeout global de 40 minutos. O timeout deve ser mantido inicialmente como proteção. Aumentá-lo antes das otimizações apenas esconderia capacidade insuficiente. + +## 3. Princípios da nova arquitetura + +### 3.1 PostgreSQL continua sendo Source of Truth + +Valkey nunca deve ser tratado como a fonte durável definitiva das vagas. Uma vaga somente é considerada persistida após commit confirmado no PostgreSQL. + +### 3.2 Valkey poderá exercer dois papéis distintos + +#### Valkey Staging + +Responsável por absorver bursts, deduplicar por `StableID`, consolidar múltiplas ocorrências, manter o estado mais recente durante uma janela e reduzir trabalho redundante antes das etapas caras. + +#### Valkey Projection + +Responsável por índices de busca, documentos de consulta, memberships, TTLs e projeção derivada do catálogo durável. + +Os dois papéis devem permanecer conceitualmente separados, mesmo se inicialmente utilizarem a mesma infraestrutura. + +### 3.3 O sistema deve processar mudanças, não ocorrências + +O foco do pipeline deve mudar de quantidade de eventos coletados para: + +```text +jobs únicos consolidados + -> create + -> real update + -> refresh +``` + +### 3.4 Backpressure continua obrigatório + +Filas, staging e workers devem possuir limites claros de jobs, bytes, tempo, retries e shutdown/drain. + +### 3.5 Complexidade somente entra após ganho comprovado + +Não implementar múltiplos workers, filas distribuídas ou aumento agressivo de concorrência antes de provar que as fases anteriores não são suficientes. + +## 4. Arquitetura-alvo + +```text +Providers + | + v ++-----------------------------+ +| 1. Collection | +| HTTP / adapters | +| provider concurrency | +| normalizacao minima | +| StableID | ++-------------+---------------+ + | + v ++-----------------------------+ +| 2. Valkey Staging | +| dedup por StableID | +| coalescing | +| latest state | +| burst absorption | +| dirty/pending set | ++-------------+---------------+ + | + v ++-----------------------------+ +| 3. Processing | +| classification | +| canonicalization | +| change detection | +| CREATE / UPDATE / REFRESH | ++-------------+---------------+ + | + v ++-----------------------------+ +| 4. PostgreSQL Catalog | +| SOURCE OF TRUTH | +| batch persistence | +| revision / lifecycle | +| projection pending state | ++-------------+---------------+ + | + | COMMIT OK + v ++-----------------------------+ +| 5. Projection | +| Valkey search/index | +| full apply | +| diff apply | +| light refresh | ++-----------------------------+ +``` + +## 5. Estratégia de implementação por fases + +### Fase 0 — Baseline e observabilidade correta + +**Objetivo:** criar uma linha de base confiável antes de alterar comportamento. + +Implementar métricas/logs separados para collection, classification, persistence, `IndexKeys`, Build/metadata, serialização, Redis/Lua, `MarkIndexed`, retries, backpressure, create, real update, refresh, full apply, refresh apply, skipped, fallback full, batch bytes e keys por job. + +Corrigir a semântica de `commands=0`, `indexed=len(jobs)` e profundidade do buffer durante trabalho em andamento. + +**Validação:** executar baseline Greenhouse com concorrência 6. + +**Commit sugerido:** + +```text +feat(PAV-178): adiciona baseline e observabilidade detalhada do pipeline +``` + +### Fase 1 — Otimização de IndexKeys + +**Objetivo:** reduzir CPU, alocações e tempo de preparação sem alterar o resultado do índice. + +Direção técnica: + +- preprocessar keywords uma única vez por execução; +- normalizar texto uma única vez por job; +- evitar `" "+text+" "` repetido em loops internos; +- reduzir concatenações e cópias; +- reutilizar estruturas imutáveis de matching; +- reduzir normalizações repetidas de aliases/subtermos; +- preservar token boundaries, aliases e regras atuais. + +**Meta:** >= 50% de redução no tempo e nas alocações cumulativas da preparação em fixtures representativas. + +**Commit sugerido:** + +```text +perf(PAV-178): otimiza preparação e geração de chaves de índice +``` + +### Fase 2 — Create / Real Update / Refresh + +**Objetivo:** parar de tratar todo reencontro como alteração real. + +Novo modelo: + +- CREATE: novo job -> persistência completa -> nova revision -> full projection. +- REAL UPDATE: conteúdo relevante alterado -> nova revision -> full/diff projection. +- REFRESH: conteúdo equivalente -> renovar freshness -> evitar full reprojection. + +Não simplesmente manter `revision` igual sem protocolo adicional. O modelo atual usa revision para lifecycle, fencing, retry, crash recovery, ordenação de tombstones e checkpoint de indexação. + +Refresh leve deve atualizar ZSETs de expiração, TTL dos documentos, metadata/membership necessários, token de publicação e checkpoint SQL após confirmação. Se a projeção estiver ausente ou incompatível, fazer fallback para full apply. + +**Commit sugerido:** + +```text +feat(PAV-178): distingue create update e refresh no catálogo e índice +``` + +### Fase 3 — Coalescing antes do trabalho caro + +**Objetivo:** consolidar múltiplas ocorrências do mesmo `StableID`. + +Primeiro provar o conceito dentro do Processor antes de introduzir staging distribuído. + +Medir eventos coletados, jobs únicos, taxa de redução por coalescing, duplicatas tardias, memória utilizada e latência introduzida. + +Preservar merge de Sources, Keywords, latest valid state, lifecycle, cancellation e limites de memória. + +**Commit sugerido:** + +```text +perf(PAV-178): adiciona coalescing de jobs antes do processamento durável +``` + +### Fase 4 — Valkey Staging + +**Objetivo:** absorver bursts de providers e desacoplar a velocidade da coleta da velocidade de persistência. + +Contrato do staging: + +```text +"job coletado e aguardando consolidação/persistência" +``` + +Nunca: + +```text +"job já persistido" +``` + +Responsabilidades: chave lógica por `StableID`, latest state, dirty/pending set, dedup, coalescing, TTL, limite de jobs, limite de bytes, cleanup, retry e recovery definido. + +**Commit sugerido:** + +```text +feat(PAV-178): adiciona staging de jobs no Valkey +``` + +### Fase 5 — Persistência consolidada em PostgreSQL + +**Objetivo:** entregar ao PostgreSQL somente jobs consolidados e classificados corretamente. + +Fluxo esperado: + +```text +250 jobs consolidados + -> N creates + -> N real updates + -> N refreshes +``` + +Usar operações em batch sempre que possível e reduzir revision churn, updates inúteis, round-trips, lock contention e reprojeção posterior. + +**Commit sugerido:** + +```text +perf(PAV-178): consolida persistência em batch no catálogo durável +``` + +### Fase 6 — Projection Worker opcional + +**Objetivo:** somente se necessário após as fases anteriores, desacoplar persistência e projeção. + +Pré-requisito: a pendência deve ser durável. Não usar uma goroutine/fila apenas em memória como mecanismo de garantia. + +Usar ou evoluir o conceito `indexed_revision < revision` ou equivalente que também represente corretamente refresh/freshness. + +Worker deve possuir bounded queue, retry, recovery, shutdown/drain, lease/ownership, ordenação, stale revision fencing, erro propagável e confirmação de sucesso. + +**Commit sugerido:** + +```text +feat(PAV-178): adiciona worker durável de projeção +``` + +### Fase 7 — Lua e batch tuning + +**Objetivo:** otimizar o script somente após reduzir o volume desnecessário. + +Candidatos: + +- aplicar apenas diferenças entre memberships; +- deduplicar validações `TYPE`; +- evitar `PERSIST` redundante; +- reduzir metadata duplicada; +- evitar SREM/SADD de associações inalteradas; +- calibrar batch por benchmark. + +Comparar batches 50, 100, 125, 250 e 500. Não assumir que batch maior é melhor. + +**Commit sugerido:** + +```text +perf(PAV-178): otimiza projeção Valkey e calibra index batches +``` + +### Fase 8 — Benchmark Greenhouse e concurrency tuning + +Ordem: 6 -> 8 -> 10 -> eventualmente 12. + +Medir duração total, tasks produced/completed, jobs coletados/únicos, create/update/refresh, stage durations, queue depths, backpressure, CPU, RSS, GC, throttling, Valkey latency, PostgreSQL latency, lock renewal, HTTP errors, 429, timeouts e search latency. + +**Critério inicial:** Greenhouse concurrency 6 deve terminar com margem antes dos 40 minutos. Meta inicial desejada: <= 30 minutos em execuções comparáveis. + +**Commit sugerido, se houver tuning de configuração:** + +```text +perf(PAV-178): calibra concorrência do Greenhouse após otimizações +``` + +### Fase 9 — Scheduling dos providers + +Depois de medir cada provider, classificar como heavy, medium ou light e definir janelas de execução/coexistência conforme duração, volume, CPU, memória, downstream pressure e concorrência ótima. + +## 6. Greenhouse como cenário de referência + +O Greenhouse possui aproximadamente 3.339 adapters/fontes. + +Cada adapter é uma tarefa e recebe o conjunto de keywords. Não são 3.339 × keywords tarefas HTTP, mas o processamento local ainda possui custo proporcional a jobs × keywords em partes do matching. + +O Greenhouse deve ser tratado como cenário de stress principal da PAV-178. + +### Atenção ao cache interno + +O adapter mantém estado em memória como `loaded`, `cachedJobs` e `cacheErr`. Benchmarks devem controlar execução fria/quente para não comparar cenários diferentes. + +## 7. Critérios de sucesso globais + +A implementação só será considerada bem-sucedida se: + +- IDs e documentos continuarem corretos; +- memberships continuarem corretos; +- reconciliação não encontrar novas divergências; +- refresh renovar todos os TTLs/expirações; +- revisões antigas não sobrescreverem estado novo; +- crash/retry não perder update nem refresh; +- busca não sofrer regressão relevante; +- distributed lock continuar renovando corretamente; +- IndexKeys reduzir significativamente custo; +- jobs idênticos deixarem de pagar full reprojection; +- p95 de `stage=index` cair; +- tempo de backpressure cair; +- Greenhouse 6 terminar antes de 40 min com margem; +- aumentar concurrency reduzir tempo total, não apenas aumentar bursts. + +## 8. Estratégia de commits + +Uma fase = um commit. + +Antes de cada commit: + +```bash +git diff --check +go test ./... +go test -race ./... +go vet ./... +``` + +Executar também testes/benchmarks específicos da fase. + +Se um teste global falhar por dívida preexistente, registrar explicitamente e provar que não foi causado pela fase atual. + +## 9. Regra de progressão + +Após cada fase: + +1. revisar diff; +2. executar testes; +3. executar benchmark específico; +4. comparar baseline; +5. registrar resultado; +6. somente então criar commit; +7. somente então iniciar próxima fase. + +Se uma fase não produzir benefício mensurável ou introduzir complexidade desproporcional, parar, revisar arquitetura, simplificar ou reverter. + +## 10. Fora de escopo inicial + +Não fazer antes de necessidade comprovada: + +- aumentar timeout como solução principal; +- múltiplos projection workers; +- batch 500 por padrão; +- remover estruturas de compatibilidade; +- transformar revision em revisão exclusivamente de conteúdo sem mecanismo equivalente de freshness/lifecycle; +- trocar PostgreSQL como source of truth; +- alterações no front_admin; +- alterações no backend Node não necessárias ao Processor. + +## 11. Arquivos prováveis + +```text +scraper-go/internal/pipeline/index.go +scraper-go/internal/pipeline/pipeline.go +scraper-go/internal/pipeline/process.go +scraper-go/internal/catalog/store.go +scraper-go/internal/jobstore/jobstore.go +scraper-go/internal/jobindex/index.go +scraper-go/internal/domain/job.go +scraper-go/internal/catalogops/maintenance.go +scraper-go/internal/metrics/* +scraper-go/internal/config/config.go +scraper-go/internal/cronjob/cronjob.go +scraper-go/cmd/server/server.go +``` + +O adapter Greenhouse poderá receber correção separada relacionada ao lifecycle do cache. + +## 12. Resultado esperado + +A arquitetura deve evoluir de um pipeline no qual a indexação bloqueia o restante do processamento para um sistema em que: + +```text +coleta + -> staging/coalescing + -> processamento de jobs únicos + -> PostgreSQL durável + -> projeção eficiente +``` + +Cada estágio deverá operar com backpressure controlado, mas sem transformar a latência de indexação em bloqueio desnecessário da coleta. + +O objetivo final da PAV-178 não é apenas reduzir o tempo de um cron. É aumentar throughput de vagas, frescor do catálogo, número de providers suportados, previsibilidade operacional e capacidade de escalabilidade do Jobs Processor. diff --git a/scraper-go/internal/adapters/adapterutil/keyword_coverage.go b/scraper-go/internal/adapters/adapterutil/keyword_coverage.go new file mode 100644 index 00000000..e6dfd61f --- /dev/null +++ b/scraper-go/internal/adapters/adapterutil/keyword_coverage.go @@ -0,0 +1,58 @@ +package adapterutil + +import ( + "context" + "errors" + "log/slog" + "strings" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" +) + +// SearchAllKeywords executes each configured query at most once in input order. +// Pagination/retries remain in the adapter. A source rate limit stops this batch; +// its typed error reaches the existing provider Retry-After coordination. +func SearchAllKeywords(ctx context.Context, keywords []string, req domain.ScrapeRequest, provider ports.ProviderID, + search func(context.Context, string, domain.ScrapeRequest) ([]domain.Job, error), +) (jobs []domain.Job, runErr error) { + queries := make([]string, 0, len(keywords)) + seen := make(map[string]bool, len(keywords)) + for _, keyword := range keywords { + if keyword = strings.TrimSpace(keyword); keyword != "" && !seen[keyword] { + seen[keyword] = true + queries = append(queries, keyword) + } + } + started, completed, failed := 0, 0, 0 + defer func() { + status := "complete" + if runErr != nil || completed != len(queries) { + status = "partial" + } + slog.Info("provider keyword coverage", "provider", provider, "run_id", req.RunID, "window_id", req.WindowID, + "planned", len(queries), "started", started, "completed", completed, "failed", failed, "not_executed", len(queries)-started, "status", status) + }() + for _, keyword := range queries { + if cause := context.Cause(ctx); cause != nil { + return jobs, errors.Join(runErr, cause) + } + started++ + found, err := search(ctx, keyword, req) + jobs = append(jobs, found...) + if err == nil { + completed++ + continue + } + failed++ + runErr = errors.Join(runErr, err) + var limit *ports.RateLimitError + if errors.As(err, &limit) { + return jobs, runErr + } + if cause := context.Cause(ctx); cause != nil { + return jobs, errors.Join(runErr, cause) + } + } + return jobs, runErr +} diff --git a/scraper-go/internal/adapters/adapterutil/keyword_coverage_test.go b/scraper-go/internal/adapters/adapterutil/keyword_coverage_test.go new file mode 100644 index 00000000..7e0549a4 --- /dev/null +++ b/scraper-go/internal/adapters/adapterutil/keyword_coverage_test.go @@ -0,0 +1,38 @@ +package adapterutil + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" +) + +func TestKeywordCoverageLogsPlannedCompletedFailedAndNotExecuted(t *testing.T) { + var output bytes.Buffer + old := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, nil))) + defer slog.SetDefault(old) + calls := 0 + limit := &ports.RateLimitError{Provider: ports.ProviderAdzuna, NotBefore: time.Now().Add(time.Hour)} + jobs, err := SearchAllKeywords(context.Background(), []string{" go ", "go", "java", "rust", "python", " "}, domain.ScrapeRequest{RunID: "run-1", WindowID: "adzuna-1"}, ports.ProviderAdzuna, func(context.Context, string, domain.ScrapeRequest) ([]domain.Job, error) { + calls++ + if calls == 2 { + return nil, limit + } + return []domain.Job{{Title: "Developer"}}, nil + }) + require.ErrorIs(t, err, limit) + require.Len(t, jobs, 1) + require.Equal(t, 2, calls) + var entry map[string]any + require.NoError(t, json.Unmarshal(output.Bytes(), &entry)) + for key, value := range map[string]any{"planned": float64(4), "started": float64(2), "completed": float64(1), "failed": float64(1), "not_executed": float64(2), "status": "partial", "window_id": "adzuna-1", "run_id": "run-1"} { + require.Equal(t, value, entry[key], key) + } +} diff --git a/scraper-go/internal/adapters/adapterutil/retry_after.go b/scraper-go/internal/adapters/adapterutil/retry_after.go new file mode 100644 index 00000000..926548ff --- /dev/null +++ b/scraper-go/internal/adapters/adapterutil/retry_after.go @@ -0,0 +1,22 @@ +package adapterutil + +import ( + "net/http" + "strconv" + "strings" + "time" +) + +func ParseRetryAfter(value string, now time.Time) (time.Time, bool) { + value = strings.TrimSpace(value) + if seconds, err := strconv.ParseInt(value, 10, 64); err == nil && seconds >= 0 && seconds <= int64((1<<63-1)/time.Second) { + return now.Add(time.Duration(seconds) * time.Second), true + } + if at, err := http.ParseTime(value); err == nil { + if at.Before(now) { + return now, true + } + return at, true + } + return time.Time{}, false +} diff --git a/scraper-go/internal/adapters/adapterutil/retry_after_test.go b/scraper-go/internal/adapters/adapterutil/retry_after_test.go new file mode 100644 index 00000000..475b5068 --- /dev/null +++ b/scraper-go/internal/adapters/adapterutil/retry_after_test.go @@ -0,0 +1,21 @@ +package adapterutil + +import ( + "github.com/stretchr/testify/require" + "net/http" + "testing" + "time" +) + +func TestRetryAfterDeltaAndHTTPDate(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + for value, want := range map[string]time.Time{"120": now.Add(2 * time.Minute), "0": now, now.Add(time.Hour).Format(http.TimeFormat): now.Add(time.Hour), now.Add(-time.Hour).Format(http.TimeFormat): now} { + at, ok := ParseRetryAfter(value, now) + require.True(t, ok) + require.Equal(t, want, at) + } + for _, value := range []string{"", "-1", "invalid", "999999999999999999999"} { + _, ok := ParseRetryAfter(value, now) + require.False(t, ok) + } +} diff --git a/scraper-go/internal/adapters/adzuna/adapter.go b/scraper-go/internal/adapters/adzuna/adapter.go index ca482650..69198b0f 100644 --- a/scraper-go/internal/adapters/adzuna/adapter.go +++ b/scraper-go/internal/adapters/adzuna/adapter.go @@ -27,6 +27,14 @@ const ( type adzunaStatusError struct { statusCode int + retryAfter time.Time +} + +func (e adzunaStatusError) Unwrap() error { + if e.statusCode == http.StatusTooManyRequests { + return &ports.RateLimitError{Provider: ports.ProviderAdzuna, NotBefore: e.retryAfter} + } + return nil } func (e adzunaStatusError) Error() string { @@ -111,6 +119,9 @@ func (a *AdzunaAdapter) Search(ctx context.Context, keyword string, req domain.S } func (a *AdzunaAdapter) SearchBatch(ctx context.Context, keywords []string, req domain.ScrapeRequest) ([]domain.Job, error) { + if req.FullKeywordCoverage { + return adapterutil.SearchAllKeywords(ctx, keywords, req, ports.ProviderAdzuna, a.searchKeyword) + } slot := a.nextKeywordSlot(keywords, adzunaKeywordSlotSize()) if len(slot) == 0 { return nil, nil @@ -201,10 +212,13 @@ func (a *AdzunaAdapter) searchKeyword(ctx context.Context, keyword string, req d endpoint := a.buildURL(keyword, req, page) pageCtx, cancel := context.WithTimeout(ctx, pageTimeout) - jobs, err := a.fetchPageWithRetry(pageCtx, endpoint, keyword) + jobs, err := a.fetchPageWithRetry(pageCtx, endpoint, keyword, req.FullKeywordCoverage) cancel() if err != nil { + if req.FullKeywordCoverage { + return allJobs, fmt.Errorf("adzuna erro na página %d: %w", page, err) + } if len(allJobs) > 0 && shouldKeepAdzunaPartialResults(err) { return allJobs, nil } @@ -221,6 +235,9 @@ func (a *AdzunaAdapter) searchKeyword(ctx context.Context, keyword string, req d allJobs = append(allJobs, jobs...) if err := adapterutil.Wait(ctx, waitDuration); err != nil { + if req.FullKeywordCoverage { + return allJobs, err + } return nil, err } } @@ -228,7 +245,7 @@ func (a *AdzunaAdapter) searchKeyword(ctx context.Context, keyword string, req d return allJobs, nil } -func (a *AdzunaAdapter) fetchPageWithRetry(ctx context.Context, endpoint, keyword string) ([]domain.Job, error) { +func (a *AdzunaAdapter) fetchPageWithRetry(ctx context.Context, endpoint, keyword string, fullCoverage bool) ([]domain.Job, error) { var lastErr error for attempt := 0; attempt < 3; attempt++ { @@ -238,6 +255,12 @@ func (a *AdzunaAdapter) fetchPageWithRetry(ctx context.Context, endpoint, keywor } lastErr = err + if fullCoverage { + var limit *ports.RateLimitError + if errors.As(err, &limit) { + return nil, err + } + } if !isTransientAdzunaError(err) { return nil, err } @@ -298,7 +321,15 @@ func (a *AdzunaAdapter) fetchPage(ctx context.Context, endpoint, keyword string) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, adzunaStatusError{statusCode: resp.StatusCode} + var at time.Time + if resp.StatusCode == http.StatusTooManyRequests { + var ok bool + at, ok = adapterutil.ParseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + if !ok { + at = time.Now().Add(90 * time.Second) + } + } + return nil, adzunaStatusError{statusCode: resp.StatusCode, retryAfter: at} } var data adzunaResponse diff --git a/scraper-go/internal/adapters/adzuna/keyword_coverage_test.go b/scraper-go/internal/adapters/adzuna/keyword_coverage_test.go new file mode 100644 index 00000000..9629b7d1 --- /dev/null +++ b/scraper-go/internal/adapters/adzuna/keyword_coverage_test.go @@ -0,0 +1,119 @@ +package adzuna + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/adapters/testutil" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" +) + +const coverageJobJSON = `{"results":[{"title":"Backend Engineer","company":{"display_name":"Acme"},"redirect_url":"https://example.com/jobs/1"}]}` + +func TestAdzunaFullCoveragePerWindowPreservesLegacyCursor(t *testing.T) { + t.Setenv("ADZUNA_KEYWORD_SLOT_SIZE", "2") + a := NewAdzuna("app", "key", "br") + a.nextOffset = 5 + var keywords []string + for i := 0; i < 37; i++ { + keywords = append(keywords, fmt.Sprintf("query-%02d", i)) + } + var seen []string + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + require.Equal(t, "app", r.URL.Query().Get("app_id")) + require.Equal(t, "key", r.URL.Query().Get("app_key")) + seen = append(seen, r.URL.Query().Get("what")) + return testutil.Response(`{"results":[]}`), nil + }) + for _, window := range []string{"adzuna-1", "adzuna-2"} { + seen = nil + _, err := a.SearchBatch(context.Background(), keywords, domain.ScrapeRequest{FullKeywordCoverage: true, WindowID: window, MaxPagesPerKeyword: 1, WaitBetweenSearchesMs: 1}) + require.NoError(t, err) + require.Equal(t, keywords, seen) + require.Equal(t, 5, a.nextOffset) + } + seen = nil + _, err := a.SearchBatch(context.Background(), keywords, domain.ScrapeRequest{MaxPagesPerKeyword: 1, WaitBetweenSearchesMs: 1}) + require.NoError(t, err) + require.Equal(t, keywords[5:7], seen) + require.Equal(t, 7, a.nextOffset) +} + +func TestAdzunaFullCoverageStopsAtRetryAfterAndKeepsCompletedResults(t *testing.T) { + a := NewAdzuna("app", "key", "br") + calls := 0 + retry := time.Now().UTC().Add(time.Hour).Truncate(time.Second) + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return testutil.Response(coverageJobJSON), nil + } + resp := testutil.Response("") + resp.StatusCode = 429 + resp.Header.Set("Retry-After", retry.Format(http.TimeFormat)) + return resp, nil + }) + jobs, err := a.SearchBatch(context.Background(), []string{"go", "java", "rust"}, domain.ScrapeRequest{FullKeywordCoverage: true, MaxPagesPerKeyword: 1, WaitBetweenSearchesMs: 1}) + var limit *ports.RateLimitError + require.ErrorAs(t, err, &limit) + require.Equal(t, retry, limit.NotBefore) + require.Equal(t, 2, calls) + require.Len(t, jobs, 1) +} + +func TestAdzunaFullCoverageCancellationKeepsCurrentPage(t *testing.T) { + a := NewAdzuna("app", "key", "br") + calls := 0 + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + calls++ + cancel() + return testutil.Response(coverageJobJSON), nil + }) + jobs, err := a.SearchBatch(ctx, []string{"go", "java", "rust"}, domain.ScrapeRequest{FullKeywordCoverage: true, MaxPagesPerKeyword: 3, WaitBetweenSearchesMs: 1}) + require.ErrorIs(t, err, context.Canceled) + require.Len(t, jobs, 1) + require.Equal(t, 1, calls) +} + +func TestAdzunaFullCoverageReportsLaterPageFailure(t *testing.T) { + a := NewAdzuna("app", "key", "br") + calls := 0 + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + calls++ + resp := testutil.Response(coverageJobJSON) + if calls == 2 { + resp.StatusCode = 400 + } + return resp, nil + }) + jobs, err := a.SearchBatch(context.Background(), []string{"go"}, domain.ScrapeRequest{FullKeywordCoverage: true, MaxPagesPerKeyword: 3, WaitBetweenSearchesMs: 1}) + require.Error(t, err) + require.Len(t, jobs, 1) + require.Equal(t, 2, calls) +} + +func TestAdzunaFullCoverageDeadlineKeepsEarlierKeywords(t *testing.T) { + a := NewAdzuna("app", "key", "br") + calls := 0 + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return testutil.Response(coverageJobJSON), nil + } + <-r.Context().Done() + return nil, r.Context().Err() + }) + jobs, err := a.SearchBatch(ctx, []string{"go", "java", "rust"}, domain.ScrapeRequest{FullKeywordCoverage: true, MaxPagesPerKeyword: 1, WaitBetweenSearchesMs: 1}) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Len(t, jobs, 1) + require.Equal(t, 2, calls) +} diff --git a/scraper-go/internal/adapters/greenhouse/adapter.go b/scraper-go/internal/adapters/greenhouse/adapter.go index 5cf04ac4..8f81cd0c 100644 --- a/scraper-go/internal/adapters/greenhouse/adapter.go +++ b/scraper-go/internal/adapters/greenhouse/adapter.go @@ -115,6 +115,7 @@ type GreenhouseAdapter struct { cachedJobs []greenhouseListJob cacheErr error loaded bool + cacheRunID string } func NewGreenhouse(boardToken, companyName string) *GreenhouseAdapter { @@ -129,6 +130,10 @@ func (a *GreenhouseAdapter) SourceName() string { return fmt.Sprintf("Green House:%s", a.companyName) } +func (a *GreenhouseAdapter) SourceID() string { + return strings.TrimSpace(a.boardToken) +} + func (a *GreenhouseAdapter) Capabilities() ports.SourceCapabilities { return ports.SourceCapabilities{ Provider: ports.ProviderGreenhouse, @@ -193,11 +198,12 @@ func (a *GreenhouseAdapter) SearchBatch(ctx context.Context, keywords []string, func (a *GreenhouseAdapter) fetchJobs(ctx context.Context, req domain.ScrapeRequest) ([]greenhouseListJob, error) { a.mu.Lock() - if a.loaded { + if a.loaded && a.cacheRunID == req.RunID { defer a.mu.Unlock() return a.cachedJobs, a.cacheErr } defer a.mu.Unlock() + a.cacheRunID, a.loaded, a.cachedJobs, a.cacheErr = req.RunID, false, nil, nil pageTimeout := time.Duration(req.PageTimeoutMs) * time.Millisecond if pageTimeout <= 0 { @@ -227,7 +233,14 @@ func (a *GreenhouseAdapter) fetchJobs(ctx context.Context, req domain.ScrapeRequ defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - err := fmt.Errorf("greenhouse: status inesperado %d para board '%s'", resp.StatusCode, a.boardToken) + var err error = fmt.Errorf("greenhouse: status inesperado %d para board '%s'", resp.StatusCode, a.boardToken) + if resp.StatusCode == http.StatusTooManyRequests { + at, valid := adapterutil.ParseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + if !valid { + at = time.Now().Add(time.Minute) + } + err = &ports.RateLimitError{Provider: ports.ProviderGreenhouse, NotBefore: at} + } a.cachedJobs = nil a.cacheErr = err a.loaded = true diff --git a/scraper-go/internal/adapters/greenhouse/identity_test.go b/scraper-go/internal/adapters/greenhouse/identity_test.go new file mode 100644 index 00000000..9d8f45ec --- /dev/null +++ b/scraper-go/internal/adapters/greenhouse/identity_test.go @@ -0,0 +1,17 @@ +package greenhouse + +import "testing" + +func TestGreenhouseSourceIdentityUsesBoardTokenNotDisplayName(t *testing.T) { + a := NewGreenhouse("stable-board", "Company Before Rename") + b := NewGreenhouse("stable-board", "Company After Rename") + if a.SourceName() == b.SourceName() { + t.Fatal("fixture must rename display name") + } + if a.SourceID() != b.SourceID() || a.SourceID() != "stable-board" { + t.Fatal("identity must remain the board token") + } + if NewGreenhouse("another-board", "Company Before Rename").SourceID() == a.SourceID() { + t.Fatal("display names must not collapse different boards") + } +} diff --git a/scraper-go/internal/adapters/greenhouse/retry_after_test.go b/scraper-go/internal/adapters/greenhouse/retry_after_test.go new file mode 100644 index 00000000..4fa94595 --- /dev/null +++ b/scraper-go/internal/adapters/greenhouse/retry_after_test.go @@ -0,0 +1,38 @@ +package greenhouse + +import ( + "context" + "errors" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" + "net/http" + "testing" + "time" +) + +func TestGreenhouseRetryAfterAndCacheAreScopedToExecution(t *testing.T) { + a := NewGreenhouse("test", "test") + calls := 0 + a.client = testHTTPClient(func(*http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return &http.Response{StatusCode: 429, Header: http.Header{"Retry-After": []string{"120"}}, Body: http.NoBody}, nil + } + return testResponse(`{"jobs":[]}`), nil + }) + start := time.Now() + _, err := a.fetchJobs(context.Background(), domain.ScrapeRequest{RunID: "first"}) + var limit *ports.RateLimitError + require.True(t, errors.As(err, &limit)) + require.False(t, limit.NotBefore.Before(start.Add(120*time.Second))) + _, err = a.fetchJobs(context.Background(), domain.ScrapeRequest{RunID: "first"}) + require.Error(t, err) + require.Equal(t, 1, calls) + _, err = a.fetchJobs(context.Background(), domain.ScrapeRequest{RunID: "next"}) + require.NoError(t, err) + require.Equal(t, 2, calls) + _, err = a.fetchJobs(context.Background(), domain.ScrapeRequest{RunID: "last"}) + require.NoError(t, err) + require.Equal(t, 3, calls) +} diff --git a/scraper-go/internal/adapters/gupy/adapter.go b/scraper-go/internal/adapters/gupy/adapter.go index 390c277f..5cdb0bed 100644 --- a/scraper-go/internal/adapters/gupy/adapter.go +++ b/scraper-go/internal/adapters/gupy/adapter.go @@ -105,7 +105,7 @@ func (a *GupyAdapter) Search(ctx context.Context, keyword string, req domain.Scr return a.SearchBatch(ctx, []string{keyword}, req) } -func (a *GupyAdapter) SearchBatch(ctx context.Context, keywords []string, req domain.ScrapeRequest) ([]domain.Job, error) { +func (a *GupyAdapter) SearchBatch(ctx context.Context, keywords []string, req domain.ScrapeRequest) (jobs []domain.Job, runErr error) { queryKeywords := gupyExpandedQueries(keywords, req) if len(queryKeywords) == 0 { return nil, nil @@ -116,7 +116,40 @@ func (a *GupyAdapter) SearchBatch(ctx context.Context, keywords []string, req do queries = append(queries, query) } sort.Strings(queries) - queries = a.nextQuerySlot(queries, gupyQueryLimit()) + allQueries := queries + var budget *runBudget + var manifest string + offset := 0 + if req.GupyWindow { + if req.QueryCursor == nil { + return nil, fmt.Errorf("gupy automatic window requires persistent cursor") + } + queryLimit, err := positiveEnv("GUPY_AUTO_QUERY_LIMIT", 120) + if err != nil { + return nil, err + } + maxPages, err := positiveEnv("GUPY_AUTO_MAX_PAGES", 5) + if err != nil { + return nil, err + } + attempts, err := positiveEnv("GUPY_AUTO_MAX_HTTP_ATTEMPTS", 600) + if err != nil { + return nil, err + } + manifest = queryManifest(allQueries) + offset, err = req.QueryCursor.Load(ctx, manifest, len(allQueries)) + if err != nil { + return nil, err + } + queries = make([]string, 0, min(queryLimit, len(allQueries))) + for i := 0; i < min(queryLimit, len(allQueries)); i++ { + queries = append(queries, allQueries[(offset+i)%len(allQueries)]) + } + budget = &runBudget{initial: attempts, remaining: attempts, maxPages: maxPages} + ctx = context.WithValue(ctx, budgetKey{}, budget) + } else { + queries = a.nextQuerySlot(queries, gupyQueryLimit()) + } type collectedJob struct { raw gupyJob @@ -125,20 +158,33 @@ func (a *GupyAdapter) SearchBatch(ctx context.Context, keywords []string, req do collected := make(map[string]collectedJob) order := make([]string, 0) - for _, query := range queries { + started, completed, failed := 0, 0, 0 + defer func() { + jobs = make([]domain.Job, 0, len(order)) + for _, key := range order { + entry := collected[key] + kw := adapterutil.UniqueTrimmedStrings(entry.keywords) + if len(kw) > 0 { + jobs = append(jobs, gupyToJob(entry.raw, kw)) + } + } + slog.Info("gupy query coverage", "run_id", req.RunID, "window_id", req.WindowID, "planned", len(queries), "started", started, "completed", completed, "failed", failed, "not_executed", len(queries)-started, "partial", runErr != nil) + if budget != nil { + slog.Info("gupy HTTP budget", "run_id", req.RunID, "window_id", req.WindowID, "attempts", budget.initial-budget.remaining, "attempt_limit", budget.initial, "max_pages_per_query", budget.maxPages) + } + }() + for i, query := range queries { if cause := context.Cause(ctx); cause != nil { - return nil, cause + return nil, errors.Join(runErr, cause) + } + if budget != nil { + budget.started = false + index := (offset + i) % len(allQueries) + budget.onStart = func(c context.Context) error { return req.QueryCursor.Advance(c, manifest, len(allQueries), index) } } rawJobs, err := a.fetchAll(ctx, query, req) - if err != nil { - if cause := context.Cause(ctx); cause != nil { - return nil, cause - } - slog.Warn("gupy: query ignorada por erro", - "query", query, - "error", err, - ) - continue + if budget == nil || budget.started { + started++ } for _, raw := range rawJobs { @@ -169,19 +215,22 @@ func (a *GupyAdapter) SearchBatch(ctx context.Context, keywords []string, req do current.keywords = adapterutil.UniqueTrimmedStrings(append(current.keywords, queryKeywords[query]...)) collected[key] = current } - } - - jobs := make([]domain.Job, 0, len(order)) - for _, key := range order { - collectedJob := collected[key] - keywords := adapterutil.UniqueTrimmedStrings(collectedJob.keywords) - if len(keywords) == 0 { - continue + if err != nil { + if budget == nil || budget.started { + failed++ + } + runErr = errors.Join(runErr, err) + var limit *ports.RateLimitError + var checkpoint cursorError + if errors.As(err, &limit) || errors.As(err, &checkpoint) || errors.Is(err, errHTTPBudget) || context.Cause(ctx) != nil { + return nil, runErr + } + slog.Warn("gupy query partial", "query", query, "error", err) + } else { + completed++ } - jobs = append(jobs, gupyToJob(collectedJob.raw, keywords)) } - - return jobs, nil + return nil, runErr } func gupyQueryLimit() int { @@ -235,21 +284,38 @@ func (a *GupyAdapter) fetchAll(ctx context.Context, keyword string, req domain.S } all := make([]gupyJob, 0, limit) + seen := map[string]bool{} + pages := 0 + budget, _ := ctx.Value(budgetKey{}).(*runBudget) for offset := 0; offset <= maxOffset; offset += limit { if cause := context.Cause(ctx); cause != nil { - return nil, cause + return all, cause + } + if budget != nil && pages >= budget.maxPages { + return all, errPageBudget } page, err := a.fetchPageWithRetry(ctx, keyword, offset, limit, req) if err != nil { if gupyIsFullSweepEnd(err, keyword, offset) { break } - return nil, err + return all, err + } + pages++ + if len(page) > 0 { + fingerprint := pageFingerprint(page) + if fingerprint != "" && seen[fingerprint] { + break + } + seen[fingerprint] = true } all = append(all, page...) if len(page) < limit { break } + if budget != nil && offset+limit > maxOffset { + return all, errPageBudget + } } return all, nil @@ -332,12 +398,24 @@ func (a *GupyAdapter) fetchPage(ctx context.Context, keyword string, offset, lim httpReq.Header.Set("Accept", "application/json") httpReq.Header.Set("User-Agent", "JobsScraper/1.0") + if budget, _ := ctx.Value(budgetKey{}).(*runBudget); budget != nil { + if err := budget.reserve(ctx); err != nil { + return nil, err + } + } resp, err := a.client.Do(httpReq) if err != nil { return nil, fmt.Errorf("gupy: http do: %w", err) } defer resp.Body.Close() + if resp.StatusCode == http.StatusTooManyRequests { + at, ok := adapterutil.ParseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + if !ok { + at = time.Now().Add(time.Minute) + } + return nil, &ports.RateLimitError{Provider: ports.ProviderGupy, NotBefore: at} + } if resp.StatusCode != http.StatusOK { return nil, &gupyHTTPError{statusCode: resp.StatusCode, keyword: keyword, offset: offset} } diff --git a/scraper-go/internal/adapters/gupy/adapter_test.go b/scraper-go/internal/adapters/gupy/adapter_test.go index c6230cf2..90f21499 100644 --- a/scraper-go/internal/adapters/gupy/adapter_test.go +++ b/scraper-go/internal/adapters/gupy/adapter_test.go @@ -230,7 +230,7 @@ func TestGupySearchHandlesNonOKStatus(t *testing.T) { if err == nil { t.Fatal("expected error for non-OK response") } - if !strings.Contains(err.Error(), "status inesperado 429") { + if !strings.Contains(err.Error(), "rate limit (429)") { t.Fatalf("unexpected error: %v", err) } } diff --git a/scraper-go/internal/adapters/gupy/budget.go b/scraper-go/internal/adapters/gupy/budget.go new file mode 100644 index 00000000..2c4fdffc --- /dev/null +++ b/scraper-go/internal/adapters/gupy/budget.go @@ -0,0 +1,76 @@ +package gupy + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "strconv" +) + +var errHTTPBudget = errors.New("gupy HTTP attempt budget exhausted") +var errPageBudget = errors.New("gupy query pagination truncated by page budget") + +type cursorError struct{ error } + +func (e cursorError) Unwrap() error { return e.error } + +type budgetKey struct{} +type runBudget struct { + initial int + remaining int + maxPages int + started bool + onStart func(context.Context) error +} + +func (b *runBudget) reserve(ctx context.Context) error { + if err := context.Cause(ctx); err != nil { + return err + } + if b.remaining <= 0 { + return errHTTPBudget + } + if !b.started { + if b.onStart != nil { + if err := b.onStart(ctx); err != nil { + return cursorError{err} + } + } + b.started = true + } + b.remaining-- + return nil +} +func positiveEnv(name string, fallback int) (int, error) { + value := os.Getenv(name) + if value == "" { + return fallback, nil + } + n, err := strconv.Atoi(value) + if err != nil || n <= 0 { + return 0, fmt.Errorf("invalid %s", name) + } + return n, nil +} +func queryManifest(queries []string) string { + raw, _ := json.Marshal(queries) + h := sha256.Sum256(raw) + return hex.EncodeToString(h[:]) +} +func pageFingerprint(jobs []gupyJob) string { + ids := make([]string, 0, len(jobs)) + for _, job := range jobs { + id := gupyDedupeKey(job) + if id == "" { + return "" + } + ids = append(ids, id) + } + sort.Strings(ids) + return queryManifest(ids) +} diff --git a/scraper-go/internal/adapters/gupy/budget_test.go b/scraper-go/internal/adapters/gupy/budget_test.go new file mode 100644 index 00000000..de8973df --- /dev/null +++ b/scraper-go/internal/adapters/gupy/budget_test.go @@ -0,0 +1,174 @@ +package gupy + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/adapters/testutil" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" +) + +type fakeCursor struct { + offset int + manifest string +} + +func (c *fakeCursor) Load(_ context.Context, hash string, _ int) (int, error) { + if c.manifest != "" && c.manifest != hash { + return 0, fmt.Errorf("changed manifest") + } + return c.offset, nil +} +func (c *fakeCursor) Advance(_ context.Context, hash string, count, index int) error { + if index != c.offset { + return fmt.Errorf("cursor conflict") + } + c.manifest = hash + c.offset = (index + 1) % count + return nil +} +func automaticGupy(t *testing.T) (*GupyAdapter, domain.ScrapeRequest) { + t.Helper() + t.Setenv("GUPY_RAW_DISCOVERY_ENABLED", "false") + t.Setenv("GUPY_FULL_SWEEP_ENABLED", "false") + return NewGupy(), domain.ScrapeRequest{GupyWindow: true, QueryCursor: &fakeCursor{}, WindowID: "gupy-1"} +} +func rawGupyPage(count, offset int) string { + data := make([]gupyJob, 0, count) + for i := 0; i < count; i++ { + id := offset + i + data = append(data, gupyJob{ID: id + 1, Name: "Software Engineer", CareerPageName: "Acme", JobURL: fmt.Sprintf("https://example.com/jobs/%d", id), Description: "Build Go APIs"}) + } + raw, _ := json.Marshal(gupyResponse{Data: data}) + return string(raw) +} + +func TestGupyAutomaticSelects120AndRestartsAtPersistentPosition(t *testing.T) { + a, req := automaticGupy(t) + var keywords []string + for i := 0; i < 200; i++ { + keywords = append(keywords, fmt.Sprintf("query-%03d", i)) + } + var seen []string + client := testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + seen = append(seen, r.URL.Query().Get("jobName")) + return testutil.Response(`{"data":[]}`), nil + }) + a.client = client + _, err := a.SearchBatch(context.Background(), keywords, req) + require.NoError(t, err) + require.Len(t, seen, 120) + require.Equal(t, 120, req.QueryCursor.(*fakeCursor).offset) + a = NewGupy() + a.client = client + seen = nil + req.WindowID = "gupy-2" + _, err = a.SearchBatch(context.Background(), keywords, req) + require.NoError(t, err) + require.Equal(t, "query-120", seen[0]) + require.Len(t, seen, 120) +} + +func TestGupyHTTPBudgetDoesNotAdvanceUnstartedQueries(t *testing.T) { + a, req := automaticGupy(t) + t.Setenv("GUPY_AUTO_MAX_HTTP_ATTEMPTS", "2") + calls := 0 + a.client = testutil.HTTPClient(func(*http.Request) (*http.Response, error) { calls++; return testutil.Response(`{"data":[]}`), nil }) + _, err := a.SearchBatch(context.Background(), []string{"a", "b", "c", "d"}, req) + require.ErrorIs(t, err, errHTTPBudget) + require.Equal(t, 2, calls) + require.Equal(t, 2, req.QueryCursor.(*fakeCursor).offset) +} + +func TestGupyPageBudgetReturnsPartialAndRepeatedPageEndsNaturally(t *testing.T) { + for _, repeated := range []bool{false, true} { + t.Run(fmt.Sprint(repeated), func(t *testing.T) { + a, req := automaticGupy(t) + calls := 0 + a.client = testutil.HTTPClient(func(*http.Request) (*http.Response, error) { + offset := calls * 100 + if repeated { + offset = 0 + } + calls++ + return testutil.Response(rawGupyPage(100, offset)), nil + }) + jobs, err := a.SearchBatch(context.Background(), []string{"unique-query"}, req) + if repeated { + require.NoError(t, err) + require.Equal(t, 2, calls) + require.Len(t, jobs, 100) + } else { + require.ErrorIs(t, err, errPageBudget) + require.Equal(t, 5, calls) + require.Len(t, jobs, 500) + } + }) + } +} + +func TestGupy429AndCancellationKeepEarlierJobs(t *testing.T) { + for _, cancelled := range []bool{false, true} { + t.Run(fmt.Sprint(cancelled), func(t *testing.T) { + a, req := automaticGupy(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls := 0 + retry := time.Now().Add(time.Hour).UTC().Truncate(time.Second) + a.client = testutil.HTTPClient(func(*http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return testutil.Response(rawGupyPage(1, 0)), nil + } + if cancelled { + cancel() + return nil, ctx.Err() + } + r := testutil.StatusResponse(429, "") + r.Header.Set("Retry-After", retry.Format(http.TimeFormat)) + return r, nil + }) + jobs, err := a.SearchBatch(ctx, []string{"a", "b", "c"}, req) + require.Len(t, jobs, 1) + require.Equal(t, 2, calls) + if cancelled { + require.ErrorIs(t, err, context.Canceled) + } else { + var limit *ports.RateLimitError + require.ErrorAs(t, err, &limit) + require.Equal(t, retry, limit.NotBefore) + } + }) + } +} + +func TestGupyRetriesConsumeAttemptBudget(t *testing.T) { + a, req := automaticGupy(t) + t.Setenv("GUPY_AUTO_MAX_HTTP_ATTEMPTS", "2") + calls := 0 + a.client = testutil.HTTPClient(func(*http.Request) (*http.Response, error) { calls++; return testutil.StatusResponse(500, ""), nil }) + _, err := a.SearchBatch(context.Background(), []string{"a", "b", "c"}, req) + require.ErrorIs(t, err, errHTTPBudget) + require.Equal(t, 2, calls) + require.Equal(t, 1, req.QueryCursor.(*fakeCursor).offset) +} + +func TestGupyLegacyKeeps60QuerySelection(t *testing.T) { + a, _ := automaticGupy(t) + t.Setenv("GUPY_QUERY_LIMIT", "60") + var kw []string + for i := 0; i < 150; i++ { + kw = append(kw, fmt.Sprintf("query-%03d", i)) + } + calls := 0 + a.client = testutil.HTTPClient(func(*http.Request) (*http.Response, error) { calls++; return testutil.Response(`{"data":[]}`), nil }) + _, err := a.SearchBatch(context.Background(), kw, domain.ScrapeRequest{}) + require.NoError(t, err) + require.Equal(t, 60, calls) +} diff --git a/scraper-go/internal/adapters/inhire/adapter.go b/scraper-go/internal/adapters/inhire/adapter.go index d2f9d949..0aa81744 100644 --- a/scraper-go/internal/adapters/inhire/adapter.go +++ b/scraper-go/internal/adapters/inhire/adapter.go @@ -3,6 +3,7 @@ package inhire import ( "context" "encoding/json" + "errors" "fmt" "html" "io" @@ -112,16 +113,10 @@ func (a *InHireAdapter) SearchBatch(ctx context.Context, keywords []string, req return nil, nil } - rawJobs := a.fetchAllTenants(ctx, tenants, req) - if cause := context.Cause(ctx); cause != nil { - return nil, cause - } + rawJobs, collectionErr := a.fetchAllTenants(ctx, tenants, req) jobs := make([]domain.Job, 0, len(rawJobs)) var skippedStatus, skippedRemote int for _, job := range rawJobs { - if cause := context.Cause(ctx); cause != nil { - return nil, cause - } if status := strings.TrimSpace(job.raw.Status); status != "" && !strings.EqualFold(status, "published") { skippedStatus++ continue @@ -143,7 +138,7 @@ func (a *InHireAdapter) SearchBatch(ctx context.Context, keywords []string, req if inhireDetailEnrichmentEnabled() { jobs = a.enrichJobsWithDetails(ctx, jobs, keywords) if cause := context.Cause(ctx); cause != nil { - return nil, cause + return jobs, errors.Join(collectionErr, cause) } } @@ -155,7 +150,7 @@ func (a *InHireAdapter) SearchBatch(ctx context.Context, keywords []string, req "jobs_returned", len(jobs), ) - return jobs, nil + return jobs, errors.Join(collectionErr, context.Cause(ctx)) } func (a *InHireAdapter) enrichJobsWithDetails(ctx context.Context, jobs []domain.Job, keywords []string) []domain.Job { @@ -381,16 +376,18 @@ type inhireTenantJob struct { raw inhireJob } -func (a *InHireAdapter) fetchAllTenants(ctx context.Context, tenants []inhireTenant, req domain.ScrapeRequest) []inhireTenantJob { +func (a *InHireAdapter) fetchAllTenants(ctx context.Context, tenants []inhireTenant, req domain.ScrapeRequest) ([]inhireTenantJob, error) { jobs := make([]inhireTenantJob, 0) var fetched, failed, withJobs int + var collectedErr error for _, tenant := range tenants { if context.Cause(ctx) != nil { - return jobs + return jobs, errors.Join(collectedErr, context.Cause(ctx)) } page, err := a.fetchTenant(ctx, tenant, req) if err != nil { failed++ + collectedErr = errors.Join(collectedErr, err) continue } if page.TenantName != "" { @@ -413,7 +410,7 @@ func (a *InHireAdapter) fetchAllTenants(ctx context.Context, tenants []inhireTen "jobs_raw", len(jobs), ) - return jobs + return jobs, errors.Join(collectedErr, context.Cause(ctx)) } func (a *InHireAdapter) fetchTenant(ctx context.Context, tenant inhireTenant, req domain.ScrapeRequest) (inhirePageResponse, error) { diff --git a/scraper-go/internal/adapters/inhire/catalog_coverage_test.go b/scraper-go/internal/adapters/inhire/catalog_coverage_test.go new file mode 100644 index 00000000..03c7f17c --- /dev/null +++ b/scraper-go/internal/adapters/inhire/catalog_coverage_test.go @@ -0,0 +1,26 @@ +package inhire + +import ( + "context" + "net/http" + "testing" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestPAV178InHireSelectsAll496ConfiguredTenants(t *testing.T) { + t.Setenv("INHIRE_TENANTS_FILE", "../../interfaces/inhireTenants.json") + t.Setenv("INHIRE_ENRICH_DETAILS", "false") + a := NewInHire() + seen := map[string]bool{} + a.client = testHTTPClient(func(r *http.Request) (*http.Response, error) { + slug := r.Header.Get("X-Tenant") + require.False(t, seen[slug]) + seen[slug] = true + return testResponse(`{"jobsPage":[]}`), nil + }) + _, err := a.SearchCatalog(context.Background(), []string{"go"}, domain.ScrapeRequest{}) + require.NoError(t, err) + require.Len(t, seen, 496) +} diff --git a/scraper-go/internal/adapters/inhire/partial_results_test.go b/scraper-go/internal/adapters/inhire/partial_results_test.go new file mode 100644 index 00000000..28b87eac --- /dev/null +++ b/scraper-go/internal/adapters/inhire/partial_results_test.go @@ -0,0 +1,42 @@ +package inhire + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestInHireReturnsCollectedTenantWhenAnotherFailsOrCancels(t *testing.T) { + for _, cancelled := range []bool{false, true} { + t.Run(fmt.Sprint(cancelled), func(t *testing.T) { + file := writeInHireTenantsFile(t, `[{"slug":"a","tenantName":"Acme"},{"slug":"b","tenantName":"Other"}]`) + t.Setenv("INHIRE_TENANTS_FILE", file) + t.Setenv("INHIRE_ENRICH_DETAILS", "false") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + a := NewInHire() + a.client = testHTTPClient(func(r *http.Request) (*http.Response, error) { + if r.Header.Get("X-Tenant") == "a" { + return testResponse(`{"jobsPage":[{"displayName":"Desenvolvedor Go","jobId":"one","status":"published","workplaceType":"Remote","location":"Brasil"}]}`), nil + } + if cancelled { + cancel() + return nil, ctx.Err() + } + return nil, errors.New("tenant unavailable") + }) + jobs, err := a.SearchCatalog(ctx, []string{"go"}, domain.ScrapeRequest{}) + require.Error(t, err) + require.Len(t, jobs, 1) + require.NotEmpty(t, jobs[0].URL) + if cancelled { + require.ErrorIs(t, err, context.Canceled) + } + }) + } +} diff --git a/scraper-go/internal/adapters/jooble/adapter.go b/scraper-go/internal/adapters/jooble/adapter.go index 0fae4853..7e8cc437 100644 --- a/scraper-go/internal/adapters/jooble/adapter.go +++ b/scraper-go/internal/adapters/jooble/adapter.go @@ -21,7 +21,7 @@ import ( const ( joobleQuotaKey = "jooble:quota:daily" // contador no Valkey - joobleQuotaLimit = 300 // chamadas máximas por dia + joobleQuotaLimit = 150 // chamadas máximas por dia joobleSlotSize = 37 // keywords por execução joobleRotationKey = "jooble:rotation:offset" // offset rotativo no Valkey joobleDefaultPagesPerKeyword = 3 @@ -53,46 +53,80 @@ func (a *JoobleAdapter) Capabilities() ports.SourceCapabilities { } } -// quotaRemaining retorna quantas chamadas ainda cabem hoje. -// Se Valkey estiver indisponível, segue com o limite local para não zerar o adapter silenciosamente. -func (a *JoobleAdapter) quotaRemaining(ctx context.Context) int { +// Quota reservations fail closed and are shared by manual and scheduled runs. +func joobleDailyLimit() (int, error) { + value := strings.TrimSpace(os.Getenv("JOOBLE_DAILY_HTTP_LIMIT")) + if value == "" { + return joobleQuotaLimit, nil + } + limit, err := strconv.Atoi(value) + if err != nil || limit <= 0 || limit > joobleQuotaLimit { + return 0, fmt.Errorf("JOOBLE_DAILY_HTTP_LIMIT must be between 1 and 150") + } + return limit, nil +} +func (a *JoobleAdapter) quotaRemaining(ctx context.Context) (int, error) { + limit, err := joobleDailyLimit() + if err != nil { + return 0, err + } if a.rdb == nil { - slog.Warn("jooble: Valkey indisponível, seguindo sem contador distribuído de cota") - return joobleQuotaLimit + return 0, fmt.Errorf("jooble quota store unavailable") } - used, err := a.rdb.Get(ctx, joobleQuotaKey).Int() if err == redis.Nil { - return joobleQuotaLimit // chave não existe ainda → dia novo + return limit, nil } if err != nil { - slog.Warn("jooble: falha ao ler cota no Valkey, seguindo sem bloquear adapter", "error", err) - return joobleQuotaLimit + return 0, err } - remaining := joobleQuotaLimit - used - if remaining < 0 { - return 0 + if used < 0 { + return 0, fmt.Errorf("invalid jooble quota counter") } - return remaining + return max(0, limit-used), nil } -// incrementQuota registra 1 chamada usada. -// Na primeira chamada do dia, define TTL até meia-noite. -func (a *JoobleAdapter) incrementQuota(ctx context.Context) { +var reserveJoobleQuota = redis.NewScript(` +local used=tonumber(redis.call('GET',KEYS[1]) or '0') +if used < 0 then return -1 end +local clock=redis.call('TIME') +local now=tonumber(clock[1])*1000+math.floor(tonumber(clock[2])/1000) +if tonumber(ARGV[2]) <= now then return -2 end +if used >= tonumber(ARGV[1]) then return 0 end +redis.call('INCR',KEYS[1]) +redis.call('PEXPIREAT',KEYS[1],ARGV[2]) +return 1 +`) + +func (a *JoobleAdapter) reserveQuota(ctx context.Context) error { + limit, err := joobleDailyLimit() + if err != nil { + return err + } if a.rdb == nil { - return + return fmt.Errorf("jooble quota store unavailable") } - - pipe := a.rdb.Pipeline() - pipe.Incr(ctx, joobleQuotaKey) - - // TTL dinâmico: segundos restantes até meia-noite - now := time.Now() - midnight := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, now.Location()) - ttl := time.Until(midnight) - pipe.Expire(ctx, joobleQuotaKey, ttl) - - pipe.Exec(ctx) //nolint:errcheck — falha aqui não deve parar o scraper + now, err := a.rdb.Time(ctx).Result() + if err != nil { + return err + } + loc, err := time.LoadLocation("America/Sao_Paulo") + if err != nil { + return err + } + local := now.In(loc) + midnight := time.Date(local.Year(), local.Month(), local.Day()+1, 0, 0, 0, 0, loc) + reserved, err := reserveJoobleQuota.Run(ctx, a.rdb, []string{joobleQuotaKey}, limit, midnight.UnixMilli()).Int() + if err != nil { + return err + } + if reserved < 0 { + return fmt.Errorf("jooble quota counter/period changed; admission rejected") + } + if reserved != 1 { + return &ports.RateLimitError{Provider: ports.ProviderJooble, NotBefore: midnight} + } + return nil } // nextSlot retorna o subconjunto rotacionado de keywords para essa execução. @@ -132,10 +166,13 @@ func (a *JoobleAdapter) SearchBatch(ctx context.Context, allKeywords []string, r a.mu.Lock() defer a.mu.Unlock() - remaining := a.quotaRemaining(ctx) + remaining, quotaErr := a.quotaRemaining(ctx) + if quotaErr != nil { + return nil, quotaErr + } if remaining <= 0 { slog.Warn("jooble: cota diária esgotada, adapter não fará novas chamadas", "limit", joobleQuotaLimit) - return nil, nil // cota esgotada → para silenciosamente + return nil, fmt.Errorf("jooble daily HTTP quota exhausted") } slotSize := joobleSlotSize @@ -152,11 +189,7 @@ func (a *JoobleAdapter) SearchBatch(ctx context.Context, allKeywords []string, r for _, keyword := range slot { jobs, err := a.search(ctx, keyword, req) if err != nil { - if len(allJobs) == 0 { - return nil, err - } - slog.Warn("jooble: busca interrompida após resultados parciais", "keyword", keyword, "error", err) - break + return append(allJobs, jobs...), err } allJobs = append(allJobs, jobs...) } @@ -194,14 +227,11 @@ func (a *JoobleAdapter) search(ctx context.Context, keyword string, req domain.S if err != nil { if len(allJobs) > 0 { - slog.Warn("jooble: página falhou após resultados parciais", "keyword", keyword, "page", page, "error", err) - break + return allJobs, err } return nil, err } - a.incrementQuota(ctx) - if len(jobs) == 0 { break } @@ -262,6 +292,9 @@ func (a *JoobleAdapter) fetchPage( httpReq.Header.Set("Accept", "application/json") httpReq.Header.Set("User-Agent", "JobsScraper/1.0") + if err := a.reserveQuota(ctx); err != nil { + return nil, 0, err + } resp, err := a.client.Do(httpReq) if err != nil { return nil, 0, err @@ -269,7 +302,11 @@ func (a *JoobleAdapter) fetchPage( defer resp.Body.Close() if resp.StatusCode == http.StatusTooManyRequests { - return nil, 0, fmt.Errorf("jooble: 429") + at, ok := adapterutil.ParseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + if !ok { + at = time.Now().Add(time.Minute) + } + return nil, 0, &ports.RateLimitError{Provider: ports.ProviderJooble, NotBefore: at} } if resp.StatusCode == http.StatusForbidden { return nil, 0, fmt.Errorf("jooble: 403 acesso negado, verifique JOOBLE_API_KEY") diff --git a/scraper-go/internal/adapters/jooble/pagination_test.go b/scraper-go/internal/adapters/jooble/pagination_test.go index 9473375b..62994024 100644 --- a/scraper-go/internal/adapters/jooble/pagination_test.go +++ b/scraper-go/internal/adapters/jooble/pagination_test.go @@ -12,7 +12,7 @@ import ( func TestJoobleSearchUsesRestPayloadAndPagination(t *testing.T) { calls := 0 - adapter := NewJooble("test-key", nil) + adapter := newQuotaTestAdapter(t) adapter.client = testutil.HTTPClient(func(req *http.Request) (*http.Response, error) { calls++ diff --git a/scraper-go/internal/adapters/jooble/quota_test.go b/scraper-go/internal/adapters/jooble/quota_test.go new file mode 100644 index 00000000..338612a4 --- /dev/null +++ b/scraper-go/internal/adapters/jooble/quota_test.go @@ -0,0 +1,83 @@ +package jooble + +import ( + "context" + "errors" + "net/http" + "sync" + "sync/atomic" + "testing" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/adapters/testutil" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func newQuotaTestAdapter(t *testing.T) *JoobleAdapter { + t.Helper() + server := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: server.Addr()}) + t.Cleanup(func() { _ = client.Close() }) + return NewJooble("test-key", client) +} +func TestJoobleAtomicQuotaAllowsExactly150Reservations(t *testing.T) { + t.Setenv("JOOBLE_DAILY_HTTP_LIMIT", "150") + a := newQuotaTestAdapter(t) + var allowed atomic.Int32 + var wg sync.WaitGroup + for range 200 { + wg.Add(1) + go func() { + defer wg.Done() + if a.reserveQuota(context.Background()) == nil { + allowed.Add(1) + } + }() + } + wg.Wait() + require.EqualValues(t, 150, allowed.Load()) + require.Equal(t, "150", a.rdb.Get(context.Background(), joobleQuotaKey).Val()) +} +func TestJoobleCountsFailuresAndCannotSendPastConfiguredLimit(t *testing.T) { + t.Setenv("JOOBLE_DAILY_HTTP_LIMIT", "2") + a := newQuotaTestAdapter(t) + calls := 0 + a.client = testutil.HTTPClient(func(*http.Request) (*http.Response, error) { calls++; return nil, errors.New("transport failed") }) + for range 4 { + _, _, err := a.fetchPage(context.Background(), "go", domain.ScrapeRequest{}, 1, 20) + require.Error(t, err) + } + require.Equal(t, 2, calls) + require.Equal(t, "2", a.rdb.Get(context.Background(), joobleQuotaKey).Val()) +} +func TestJoobleQuotaFailsClosedWithoutStore(t *testing.T) { + a := NewJooble("test-key", nil) + require.Error(t, a.reserveQuota(context.Background())) +} + +func TestJooble150AttemptsAreSharedAcrossAdapterInstances(t *testing.T) { + t.Setenv("JOOBLE_DAILY_HTTP_LIMIT", "150") + a := newQuotaTestAdapter(t) + b := NewJooble("test-key", a.rdb) + calls := 0 + client := testutil.HTTPClient(func(*http.Request) (*http.Response, error) { calls++; return testutil.Response(`{"jobs":[]}`), nil }) + a.client = client + b.client = client + for _, adapter := range []*JoobleAdapter{a, b} { + for range 100 { + _, _, _ = adapter.fetchPage(context.Background(), "go", domain.ScrapeRequest{}, 2, 20) + } + } + require.Equal(t, 150, calls) + require.Equal(t, "150", a.rdb.Get(context.Background(), joobleQuotaKey).Val()) +} + +func TestJoobleQuotaRejectsInvalidConfiguration(t *testing.T) { + for _, value := range []string{"0", "-1", "151", "bad"} { + t.Setenv("JOOBLE_DAILY_HTTP_LIMIT", value) + _, err := joobleDailyLimit() + require.Error(t, err) + } +} diff --git a/scraper-go/internal/adapters/lever/adapter.go b/scraper-go/internal/adapters/lever/adapter.go index 7a8ec76c..2fa195c5 100644 --- a/scraper-go/internal/adapters/lever/adapter.go +++ b/scraper-go/internal/adapters/lever/adapter.go @@ -28,6 +28,7 @@ type LeverAdapter struct { cachedJobs []leverPosting cacheErr error loaded bool + cacheRunID string } type leverCompany struct { @@ -198,11 +199,12 @@ func (a *LeverAdapter) SearchBatch(ctx context.Context, keywords []string, req d func (a *LeverAdapter) fetchPostings(ctx context.Context, req domain.ScrapeRequest) ([]leverPosting, error) { a.mu.Lock() - if a.loaded { + if a.loaded && a.cacheRunID == req.RunID { defer a.mu.Unlock() return a.cachedJobs, a.cacheErr } defer a.mu.Unlock() + a.cacheRunID, a.loaded, a.cachedJobs, a.cacheErr = req.RunID, false, nil, nil pageTimeout := time.Duration(req.PageTimeoutMs) * time.Millisecond if pageTimeout <= 0 { diff --git a/scraper-go/internal/adapters/lever/catalog_coverage_test.go b/scraper-go/internal/adapters/lever/catalog_coverage_test.go new file mode 100644 index 00000000..2a12d4b5 --- /dev/null +++ b/scraper-go/internal/adapters/lever/catalog_coverage_test.go @@ -0,0 +1,28 @@ +package lever + +import ( + "context" + "net/http" + "testing" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestPAV178LeverQueries44CompaniesInBothWindows(t *testing.T) { + t.Setenv("LEVER_COMPANIES_FILE", "../../interfaces/leverCompanies.json") + sources, err := BuildLeverAdapters(context.Background()) + require.NoError(t, err) + require.Len(t, sources, 44) + calls := 0 + for _, source := range sources { + source.(*LeverAdapter).client = testHTTPClient(func(*http.Request) (*http.Response, error) { calls++; return testResponse(`[]`), nil }) + } + for _, runID := range []string{"lever-1", "lever-2"} { + for _, source := range sources { + _, err := source.(*LeverAdapter).SearchCatalog(context.Background(), []string{"go"}, domain.ScrapeRequest{RunID: runID}) + require.NoError(t, err) + } + } + require.Equal(t, 88, calls) +} diff --git a/scraper-go/internal/adapters/lever/run_cache_test.go b/scraper-go/internal/adapters/lever/run_cache_test.go new file mode 100644 index 00000000..b7d48f19 --- /dev/null +++ b/scraper-go/internal/adapters/lever/run_cache_test.go @@ -0,0 +1,20 @@ +package lever + +import ( + "context" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/stretchr/testify/require" + "net/http" + "testing" +) + +func TestLeverCacheExpiresWhenExecutionChanges(t *testing.T) { + a := NewLever("test", "test") + calls := 0 + a.client = testHTTPClient(func(*http.Request) (*http.Response, error) { calls++; return testResponse(`[]`), nil }) + for _, run := range []string{"first", "first", "next"} { + _, err := a.fetchPostings(context.Background(), domain.ScrapeRequest{RunID: run}) + require.NoError(t, err) + } + require.Equal(t, 2, calls) +} diff --git a/scraper-go/internal/adapters/linkedin/adapter.go b/scraper-go/internal/adapters/linkedin/adapter.go index 45a32b33..e5d75d09 100644 --- a/scraper-go/internal/adapters/linkedin/adapter.go +++ b/scraper-go/internal/adapters/linkedin/adapter.go @@ -111,6 +111,13 @@ func (a *LinkedInAdapter) fetchJobsChunk(ctx context.Context, keyword string, re defer resp.Body.Close() if resp.StatusCode == http.StatusTooManyRequests { + if req.FullKeywordCoverage { + at, ok := adapterutil.ParseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + if !ok { + at = time.Now().Add(15 * time.Second) + } + return nil, &ports.RateLimitError{Provider: ports.ProviderLinkedIn, NotBefore: at} + } return nil, fmt.Errorf("status 429") } if resp.StatusCode != http.StatusOK { @@ -271,6 +278,10 @@ func (a *LinkedInAdapter) Search(ctx context.Context, keyword string, req domain } func (a *LinkedInAdapter) SearchBatch(ctx context.Context, keywords []string, req domain.ScrapeRequest) ([]domain.Job, error) { + if req.FullKeywordCoverage { + jobs, err := adapterutil.SearchAllKeywords(ctx, keywords, req, ports.ProviderLinkedIn, a.searchKeyword) + return dedupeLinkedIn(jobs), err + } slot := a.nextKeywordSlot(keywords, linkedinKeywordSlotSize()) if len(slot) == 0 { return nil, nil @@ -287,7 +298,7 @@ func (a *LinkedInAdapter) SearchBatch(ctx context.Context, keywords []string, re var firstErr error for _, keyword := range slot { if cause := context.Cause(ctx); cause != nil { - return nil, cause + return dedupeLinkedIn(allJobs), cause } keyword = strings.TrimSpace(keyword) if keyword == "" { @@ -295,16 +306,16 @@ func (a *LinkedInAdapter) SearchBatch(ctx context.Context, keywords []string, re } jobs, err := a.searchKeyword(ctx, keyword, req) + allJobs = append(allJobs, jobs...) if err != nil { if firstErr == nil { firstErr = err } continue } - allJobs = append(allJobs, jobs...) } if len(allJobs) > 0 { - return dedupeLinkedIn(allJobs), nil + return dedupeLinkedIn(allJobs), firstErr } return nil, firstErr @@ -355,11 +366,14 @@ func (a *LinkedInAdapter) searchKeyword(ctx context.Context, keyword string, req jobs, err := a.fetchJobsChunk(ctx, keyword, req, start) if err != nil { + if req.FullKeywordCoverage { + return dedupeLinkedIn(allJobs), err + } // 429 → back-off e tenta a mesma página uma vez if strings.Contains(err.Error(), "429") { backoff := 15 * time.Second if err := adapterutil.Wait(ctx, backoff); err != nil { - return nil, err + return dedupeLinkedIn(allJobs), err } jobs, err = a.fetchJobsChunk(ctx, keyword, req, start) if err != nil { @@ -390,7 +404,7 @@ func (a *LinkedInAdapter) searchKeyword(ctx context.Context, keyword string, req if pageIndex < maxPages-1 { if err := adapterutil.Wait(ctx, waitBetween); err != nil { - return nil, err + return dedupeLinkedIn(allJobs), err } } } diff --git a/scraper-go/internal/adapters/linkedin/keyword_coverage_test.go b/scraper-go/internal/adapters/linkedin/keyword_coverage_test.go new file mode 100644 index 00000000..048b0d22 --- /dev/null +++ b/scraper-go/internal/adapters/linkedin/keyword_coverage_test.go @@ -0,0 +1,101 @@ +package linkedin + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/adapters/testutil" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" +) + +const coverageJobHTML = `

Backend Engineer

Acme

` + +func TestLinkedInFullCoveragePerWindowPreservesLegacyCursor(t *testing.T) { + t.Setenv("LINKEDIN_KEYWORD_SLOT_SIZE", "2") + a := NewLinkedIn() + a.nextOffset = 5 + var keywords []string + for i := 0; i < 37; i++ { + keywords = append(keywords, fmt.Sprintf("query-%02d", i)) + } + var seen []string + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + seen = append(seen, r.URL.Query().Get("keywords")) + return testutil.Response(""), nil + }) + for _, window := range []string{"linkedin-1", "linkedin-2", "linkedin-3"} { + seen = nil + _, err := a.SearchBatch(context.Background(), keywords, domain.ScrapeRequest{FullKeywordCoverage: true, WindowID: window, MaxPagesPerKeyword: 1, WaitBetweenSearchesMs: 1}) + require.NoError(t, err) + require.Equal(t, keywords, seen) + require.Equal(t, 5, a.nextOffset) + } + seen = nil + _, err := a.SearchBatch(context.Background(), keywords, domain.ScrapeRequest{MaxPagesPerKeyword: 1, WaitBetweenSearchesMs: 1}) + require.NoError(t, err) + require.Equal(t, keywords[5:7], seen) + require.Equal(t, 7, a.nextOffset) +} + +func TestLinkedInFullCoverageStopsAtRateLimitAndKeepsCompletedResults(t *testing.T) { + a := NewLinkedIn() + calls := 0 + retry := time.Now().UTC().Add(time.Hour).Truncate(time.Second) + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return testutil.Response(coverageJobHTML), nil + } + resp := testutil.Response("") + resp.StatusCode = 429 + resp.Header.Set("Retry-After", retry.Format(http.TimeFormat)) + return resp, nil + }) + jobs, err := a.SearchBatch(context.Background(), []string{"go", "java", "rust"}, domain.ScrapeRequest{FullKeywordCoverage: true, MaxPagesPerKeyword: 1, WaitBetweenSearchesMs: 1}) + var limit *ports.RateLimitError + require.ErrorAs(t, err, &limit) + require.Equal(t, retry, limit.NotBefore) + require.Equal(t, 2, calls) + require.Len(t, jobs, 1) +} + +func TestLinkedInFullCoverageCancellationKeepsEarlierKeywords(t *testing.T) { + a := NewLinkedIn() + calls := 0 + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return testutil.Response(coverageJobHTML), nil + } + cancel() + return nil, ctx.Err() + }) + jobs, err := a.SearchBatch(ctx, []string{"go", "java", "rust"}, domain.ScrapeRequest{FullKeywordCoverage: true, MaxPagesPerKeyword: 1, WaitBetweenSearchesMs: 1}) + require.ErrorIs(t, err, context.Canceled) + require.Len(t, jobs, 1) + require.Equal(t, 2, calls) +} + +func TestLinkedInFullCoverageReportsLaterPageFailure(t *testing.T) { + a := NewLinkedIn() + calls := 0 + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + calls++ + resp := testutil.Response(coverageJobHTML) + if calls == 2 { + resp.StatusCode = 500 + } + return resp, nil + }) + jobs, err := a.SearchBatch(context.Background(), []string{"go"}, domain.ScrapeRequest{FullKeywordCoverage: true, MaxPagesPerKeyword: 3, WaitBetweenSearchesMs: 1}) + require.Error(t, err) + require.Len(t, jobs, 1) + require.Equal(t, 2, calls) +} diff --git a/scraper-go/internal/adapters/linkedin/partial_results_test.go b/scraper-go/internal/adapters/linkedin/partial_results_test.go new file mode 100644 index 00000000..4d8d28d6 --- /dev/null +++ b/scraper-go/internal/adapters/linkedin/partial_results_test.go @@ -0,0 +1,26 @@ +package linkedin + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/adapters/testutil" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestLinkedInKeepsPartialBatchWhenCollectionIsCanceled(t *testing.T) { + t.Setenv("LINKEDIN_KEYWORD_SLOT_SIZE", "2") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + adapter := NewLinkedIn() + adapter.client = testutil.HTTPClient(func(*http.Request) (*http.Response, error) { + cancel() + return testutil.Response(`

Backend Engineer

Acme

`), nil + }) + jobs, err := adapter.SearchBatch(ctx, []string{"go", "java"}, domain.ScrapeRequest{MaxPagesPerKeyword: 2, WaitBetweenSearchesMs: 1}) + require.True(t, errors.Is(err, context.Canceled)) + require.Len(t, jobs, 1) +} diff --git a/scraper-go/internal/adapters/registry_test.go b/scraper-go/internal/adapters/registry_test.go new file mode 100644 index 00000000..a822e9ce --- /dev/null +++ b/scraper-go/internal/adapters/registry_test.go @@ -0,0 +1,35 @@ +package adapters_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/adapters" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" +) + +func TestRegistryIncludesEveryConfiguredProviderWithoutScraping(t *testing.T) { + for _, key := range []string{"GUPY_ENABLED", "INHIRE_ENABLED", "GREENHOUSE_ENABLED", "LEVER_ENABLED"} { + t.Setenv(key, "true") + } + for _, key := range []string{"ADZUNA_APP_ID", "ADZUNA_APP_KEY", "JOOBLE_API_KEY"} { + t.Setenv(key, "test-credential") + } + dir := t.TempDir() + greenhouse := filepath.Join(dir, "greenhouse.json") + lever := filepath.Join(dir, "lever.json") + require.NoError(t, os.WriteFile(greenhouse, []byte(`["test-board"]`), 0600)) + require.NoError(t, os.WriteFile(lever, []byte(`[{"site":"test-board","companyName":"Test"}]`), 0600)) + t.Setenv("GREENHOUSE_COMPANIES_FILE", greenhouse) + t.Setenv("LEVER_COMPANIES_FILE", lever) + providers := map[ports.ProviderID]int{} + for _, source := range adapters.GetAdapters(nil) { + providers[ports.CapabilitiesOf(source).Provider]++ + } + require.Len(t, providers, 8) + for _, id := range []ports.ProviderID{ports.ProviderLinkedIn, ports.ProviderAdzuna, ports.ProviderGupy, ports.ProviderInHire, ports.ProviderGreenhouse, ports.ProviderLever, ports.ProviderJooble, ports.ProviderTheMuse} { + require.Equal(t, 1, providers[id], id) + } +} diff --git a/scraper-go/internal/adapters/themuse/adapter.go b/scraper-go/internal/adapters/themuse/adapter.go index d692b095..c5e5d681 100644 --- a/scraper-go/internal/adapters/themuse/adapter.go +++ b/scraper-go/internal/adapters/themuse/adapter.go @@ -3,6 +3,7 @@ package themuse import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/url" @@ -125,12 +126,9 @@ type theMuseDetailResponse struct { func (a *TheMuseAdapter) Search(ctx context.Context, keyword string, req domain.ScrapeRequest) ([]domain.Job, error) { feed, err := a.fetchFeed(ctx, req) - if err != nil { + if err != nil && len(feed) == 0 { return nil, err } - if cause := context.Cause(ctx); cause != nil { - return nil, cause - } matches := make([]domain.Job, 0) for _, job := range feed { @@ -142,7 +140,7 @@ func (a *TheMuseAdapter) Search(ctx context.Context, keyword string, req domain. matches = append(matches, job) } - return matches, nil + return matches, errors.Join(err, context.Cause(ctx)) } func (a *TheMuseAdapter) SearchCatalog( @@ -151,15 +149,12 @@ func (a *TheMuseAdapter) SearchCatalog( req domain.ScrapeRequest, ) ([]domain.Job, error) { feed, err := a.fetchFeed(ctx, req) - if err != nil { + if err != nil && len(feed) == 0 { return nil, err } jobs := make([]domain.Job, 0, len(feed)) for _, job := range feed { - if cause := context.Cause(ctx); cause != nil { - return nil, cause - } matchedKeywords := make([]string, 0, len(keywords)) for _, keyword := range keywords { @@ -176,7 +171,7 @@ func (a *TheMuseAdapter) SearchCatalog( job.Keywords = matchedKeywords jobs = append(jobs, job) } - return jobs, nil + return jobs, errors.Join(err, context.Cause(ctx)) } func (a *TheMuseAdapter) fetchFeed(ctx context.Context, req domain.ScrapeRequest) ([]domain.Job, error) { @@ -225,7 +220,7 @@ func (a *TheMuseAdapter) fetchFeed(ctx context.Context, req domain.ScrapeRequest cancel() if err != nil { - return nil, err + return allJobs, err } if len(jobs) == 0 { @@ -240,14 +235,14 @@ func (a *TheMuseAdapter) fetchFeed(ctx context.Context, req domain.ScrapeRequest if page < maxPages { if err := adapterutil.Wait(ctx, waitBetween); err != nil { - return nil, err + return allJobs, err } } } allJobs = a.enrichJobs(ctx, allJobs) if cause := context.Cause(ctx); cause != nil { - return nil, cause + return allJobs, cause } a.mu.Lock() diff --git a/scraper-go/internal/adapters/themuse/catalog_coverage_test.go b/scraper-go/internal/adapters/themuse/catalog_coverage_test.go new file mode 100644 index 00000000..d58656f7 --- /dev/null +++ b/scraper-go/internal/adapters/themuse/catalog_coverage_test.go @@ -0,0 +1,26 @@ +package themuse + +import ( + "context" + "fmt" + "net/http" + "testing" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/adapters/testutil" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestPAV178TheMuseDefaultStopsAt50ListingPages(t *testing.T) { + a := NewTheMuse() + calls := 0 + a.client = testutil.HTTPClient(func(*http.Request) (*http.Response, error) { + calls++ + require.LessOrEqual(t, calls, 50) + return testutil.Response(fmt.Sprintf(`{"results":[{"name":"Go Developer %d","company":{"name":"Acme"},"refs":{"landing_page":"https://example.test/job/%d"}}]}`, calls, calls)), nil + }) + jobs, err := a.SearchCatalog(context.Background(), []string{"go"}, domain.ScrapeRequest{WaitBetweenSearchesMs: 1}) + require.NoError(t, err) + require.Len(t, jobs, 50) + require.Equal(t, 50, calls) +} diff --git a/scraper-go/internal/adapters/themuse/partial_results_test.go b/scraper-go/internal/adapters/themuse/partial_results_test.go new file mode 100644 index 00000000..75de1e97 --- /dev/null +++ b/scraper-go/internal/adapters/themuse/partial_results_test.go @@ -0,0 +1,42 @@ +package themuse + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/adapters/testutil" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestTheMuseReturnsFirstPageWhenLaterPageFailsOrCancels(t *testing.T) { + for _, cancelled := range []bool{false, true} { + t.Run(fmt.Sprint(cancelled), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + a := NewTheMuse() + calls := 0 + a.client = testutil.HTTPClient(func(r *http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return testutil.Response(`{"results":[{"name":"Go Developer","company":{"name":"Acme"},"refs":{"landing_page":"https://example.com/job"},"contents":"Build Go APIs"}]}`), nil + } + if cancelled { + cancel() + return nil, ctx.Err() + } + return nil, errors.New("page unavailable") + }) + jobs, err := a.SearchCatalog(ctx, []string{"go"}, domain.ScrapeRequest{WaitBetweenSearchesMs: 1}) + require.Error(t, err) + require.Len(t, jobs, 1) + require.Equal(t, 2, calls) + if cancelled { + require.ErrorIs(t, err, context.Canceled) + } + }) + } +} diff --git a/scraper-go/internal/catalog/source_counts_test.go b/scraper-go/internal/catalog/source_counts_test.go index e49db6b7..5f00929a 100644 --- a/scraper-go/internal/catalog/source_counts_test.go +++ b/scraper-go/internal/catalog/source_counts_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" "github.com/stretchr/testify/require" "os" "testing" @@ -68,3 +69,40 @@ func TestSourceCountsFullActiveCatalog(t *testing.T) { require.Empty(t, counts) require.NotNil(t, counts) } + +func TestContentEqualityPreservesPersistedSemantics(t *testing.T) { + base := domain.Job{ID: "id", Title: "title", Company: "company", Location: "location", URL: "url", Sources: []string{"a", "b"}, Keywords: []string{"first", "second"}, Classification: &domain.Classification{PrimaryFamily: "backend", RelatedFamilies: []string{"platform", "data"}, Technologies: []string{"go", "sql"}, Reasons: []string{"one", "two"}, InScope: true, Confidence: .8}} + other := base + other.CatalogRevision = 100 + other.CatalogChange = "job_refreshed" + other.CatalogExpiresAt = time.Now() + require.True(t, sameContent(base, other)) + changes := map[string]func(*domain.Job){ + "title": func(j *domain.Job) { j.Title = "other" }, "company": func(j *domain.Job) { j.Company = "other" }, + "description": func(j *domain.Job) { j.Description = "other" }, "location": func(j *domain.Job) { j.Location = "other" }, + "url": func(j *domain.Job) { j.URL = "other" }, "salary": func(j *domain.Job) { j.Salary = "other" }, + "modality": func(j *domain.Job) { j.Modality = "remote" }, "posted": func(j *domain.Job) { j.PostedAt = "date" }, + "source": func(j *domain.Job) { j.Source = "other" }, "keyword": func(j *domain.Job) { j.Keyword = "second" }, + "sources_order": func(j *domain.Job) { j.Sources = []string{"b", "a"} }, "keywords_order": func(j *domain.Job) { j.Keywords = []string{"second", "first"} }, + "family": func(j *domain.Job) { j.Classification.PrimaryFamily = "data" }, + "related_order": func(j *domain.Job) { j.Classification.RelatedFamilies = []string{"data", "platform"} }, + "technologies_order": func(j *domain.Job) { j.Classification.Technologies = []string{"sql", "go"} }, + "reasons_order": func(j *domain.Job) { j.Classification.Reasons = []string{"two", "one"} }, + "seniority": func(j *domain.Job) { j.Classification.Seniority = "senior" }, + "scope": func(j *domain.Job) { j.Classification.InScope = false }, "confidence": func(j *domain.Job) { j.Classification.Confidence = .9 }, + } + for name, mutate := range changes { + t.Run(name, func(t *testing.T) { + changed := base + c := *base.Classification + changed.Classification = &c + mutate(&changed) + require.False(t, sameContent(base, changed)) + }) + } + a := domain.Job{Classification: &domain.Classification{}} + b := domain.Job{Classification: &domain.Classification{Reasons: []string{}, Technologies: []string{}, RelatedFamilies: []string{}}} + require.True(t, sameContent(a, b)) + b.Sources = []string{} + require.False(t, sameContent(a, b)) +} diff --git a/scraper-go/internal/catalog/store.go b/scraper-go/internal/catalog/store.go index 949f2663..0a09e9fa 100644 --- a/scraper-go/internal/catalog/store.go +++ b/scraper-go/internal/catalog/store.go @@ -2,11 +2,14 @@ package catalog import ( + "bytes" "context" "database/sql" "database/sql/driver" "encoding/json" "fmt" + "log/slog" + "reflect" "sort" "time" @@ -41,6 +44,18 @@ func (s *Store) Import(ctx context.Context, jobs []domain.Job) (jobstore.SaveRes } func (s *Store) save(ctx context.Context, jobs []domain.Job, importing bool) (result jobstore.SaveResult, returnErr error) { started := time.Now() + var preparation, begin, lock, readExisting, decode, mergeCompare, serialization, refreshUpdate, upsert, returningDecode, commit time.Duration + defer func() { + slog.Info("catalog persist attempt", "stage", "catalog_persist", "importing", importing, + "received", len(jobs), "created", result.Inserted, "real_updated", result.RealUpdated, + "refreshed", result.Refreshed, "reactivated", result.Reactivated, + "status", metrics.BatchStatus(returnErr), "preparation_duration", preparation, + "begin_duration", begin, "lock_duration", lock, "read_existing_duration", readExisting, + "decode_duration", decode, "merge_compare_duration", mergeCompare, + "serialization_duration", serialization, "refresh_update_duration", refreshUpdate, + "upsert_duration", upsert, "returning_decode_duration", returningDecode, + "commit_duration", commit, "duration", time.Since(started)) + }() defer func() { metrics.ObservePersistence(started, len(jobs), result.Inserted, result.Updated, len(jobs)-len(result.Persisted), returnErr) }() @@ -74,7 +89,10 @@ func (s *Store) save(ctx context.Context, jobs []domain.Job, importing bool) (re ids = append(ids, id) } sort.Strings(ids) + preparation = time.Since(started) + phase := time.Now() tx, err := s.DB.BeginTx(ctx, nil) + begin = time.Since(phase) if err != nil { return result, fmt.Errorf("catalog begin: %w", err) } @@ -84,28 +102,55 @@ func (s *Store) save(ctx context.Context, jobs []domain.Job, importing bool) (re metrics.PersistenceRollbacks.WithLabelValues(metrics.PersistenceErrorReason(returnErr)).Inc() } }() + phase = time.Now() // Fence absent IDs as well, so concurrent inserts preserve merge semantics. - if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended(id,125)) FROM (SELECT unnest($1::text[]) AS id ORDER BY 1) AS ordered_ids`, pq.Array(ids)); err != nil { + _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended(id,125)) FROM (SELECT unnest($1::text[]) AS id ORDER BY 1) AS ordered_ids`, pq.Array(ids)) + lock = time.Since(phase) + if err != nil { return result, err } + phase = time.Now() // Sorted locking avoids deadlocks between overlapping collection batches. - rows, err := tx.QueryContext(ctx, `SELECT id,payload FROM job_catalog WHERE id=ANY($1) ORDER BY id FOR UPDATE`, pq.Array(ids)) + rows, err := tx.QueryContext(ctx, `SELECT id,payload,expires_at FROM job_catalog WHERE id=ANY($1) ORDER BY id FOR UPDATE`, pq.Array(ids)) + readExisting = time.Since(phase) if err != nil { return result, fmt.Errorf("catalog read batch: %w", err) } existing := map[string]domain.Job{} - for rows.Next() { + comparable := map[string]bool{} + for { + phase = time.Now() + more := rows.Next() + readExisting += time.Since(phase) + if !more { + break + } var id string var raw []byte - if err = rows.Scan(&id, &raw); err != nil { + var expires time.Time + phase = time.Now() + if err = rows.Scan(&id, &raw, &expires); err != nil { rows.Close() return result, err } + readExisting += time.Since(phase) + phase = time.Now() var j domain.Job - if err = json.Unmarshal(raw, &j); err != nil { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + comparable[id] = decoder.Decode(&j) == nil + // Unknown persisted fields must force a full rewrite, never a false refresh. + if !comparable[id] { + err = json.Unmarshal(raw, &j) + } else { + err = nil + } + if err != nil { rows.Close() return result, fmt.Errorf("catalog decode: %w", err) } + decode += time.Since(phase) + j.CatalogExpiresAt = expires existing[id] = j } err = rows.Err() @@ -116,10 +161,27 @@ func (s *Store) save(ctx context.Context, jobs []domain.Job, importing bool) (re now := s.Now().UTC() payloads := make([]json.RawMessage, 0, len(ids)) expiry := make([]string, 0, len(ids)) + refreshIDs := make([]string, 0, len(ids)) + changes := make(map[string]string, len(ids)) for _, id := range ids { + phase = time.Now() job := byID[id] + change := "job_created" if old, ok := existing[id]; ok && !importing { job = jobstore.MergeStored(old, job) + change = "job_updated" + if !old.CatalogExpiresAt.After(now) { + change = "job_reactivated" + } else if comparable[id] && sameContent(old, job) { + change = "job_refreshed" + } + } + mergeCompare += time.Since(phase) + changes[id] = change + if change == "job_refreshed" { + byID[id] = job + refreshIDs = append(refreshIDs, id) + continue } end := now.Add(s.Lifetime) if importing { @@ -129,46 +191,134 @@ func (s *Store) save(ctx context.Context, jobs []domain.Job, importing bool) (re continue } } + phase = time.Now() raw, e := json.Marshal(job) + serialization += time.Since(phase) if e != nil { return result, e } payloads = append(payloads, raw) expiry = append(expiry, end.Format(time.RFC3339Nano)) } - if len(payloads) == 0 { - return jobstore.SaveResult{Invalid: result.Invalid}, nil - } - raw, _ := json.Marshal(payloads) - conflict := `DO UPDATE SET payload=EXCLUDED.payload,last_seen_at=EXCLUDED.last_seen_at,updated_at=EXCLUDED.updated_at,expires_at=EXCLUDED.expires_at,revision=nextval('job_catalog_revision_seq')` - if importing { - conflict = `DO NOTHING` + // Refresh keeps a new mutation revision (and durable pending checkpoint), + // but neither serializes nor assigns payload. updated_at retains its existing + // meaning: time of the latest catalog mutation, including freshness. + persisted := make([]domain.Job, 0, len(ids)) + if len(refreshIDs) > 0 { + phase = time.Now() + rows, err = tx.QueryContext(ctx, `UPDATE job_catalog SET last_seen_at=$2,updated_at=$2,expires_at=$3,revision=nextval('job_catalog_revision_seq') WHERE id=ANY($1) RETURNING id,revision,expires_at`, pq.Array(refreshIDs), now, now.Add(s.Lifetime)) + if err != nil { + refreshUpdate = time.Since(phase) + return result, fmt.Errorf("catalog refresh: %w", err) + } + for rows.Next() { + var id string + var revision int64 + var end time.Time + if err = rows.Scan(&id, &revision, &end); err != nil { + rows.Close() + return result, err + } + job := byID[id] + job.CatalogRevision, job.CatalogExpiresAt = revision, end + persisted = append(persisted, job) + } + err = rows.Err() + rows.Close() + refreshUpdate = time.Since(phase) + if err != nil { + return result, err + } + if len(persisted) != len(refreshIDs) { + return result, fmt.Errorf("catalog refresh returned incomplete batch") + } } - rows, err = tx.QueryContext(ctx, `INSERT INTO job_catalog(id,payload,first_seen_at,last_seen_at,updated_at,expires_at) + if len(payloads) > 0 { + phase = time.Now() + raw, e := json.Marshal(payloads) + serialization += time.Since(phase) + if e != nil { + return result, e + } + conflict := `DO UPDATE SET payload=EXCLUDED.payload,last_seen_at=EXCLUDED.last_seen_at,updated_at=EXCLUDED.updated_at,expires_at=EXCLUDED.expires_at,revision=nextval('job_catalog_revision_seq')` + if importing { + conflict = `DO NOTHING` + } + phase = time.Now() + rows, err = tx.QueryContext(ctx, `INSERT INTO job_catalog(id,payload,first_seen_at,last_seen_at,updated_at,expires_at) SELECT p->>'id',p,$2,$2,$2,e::timestamptz FROM jsonb_array_elements($1::jsonb) WITH ORDINALITY AS t(p,n) JOIN unnest($3::text[]) WITH ORDINALITY AS x(e,n) USING(n) ON CONFLICT(id) `+conflict+` RETURNING payload,revision,expires_at`, string(raw), now, pq.Array(expiry)) - if err != nil { - return result, fmt.Errorf("catalog upsert: %w", err) + upsert = time.Since(phase) + if err != nil { + return result, fmt.Errorf("catalog upsert: %w", err) + } + phase = time.Now() + applied, e := decodeRows(rows) + returningDecode = time.Since(phase) + if e != nil { + return result, e + } + persisted = append(persisted, applied...) } - persisted, err := decodeRows(rows) + phase = time.Now() + err = tx.Commit() + commit = time.Since(phase) if err != nil { - return result, err - } - if err = tx.Commit(); err != nil { return jobstore.SaveResult{Invalid: result.Invalid}, fmt.Errorf("catalog commit: %w", err) } + result.ContentChangeDetectionAvailable = !importing for i, j := range persisted { - if _, ok := existing[j.ID]; ok { - result.Updated++ - persisted[i].CatalogChange = "job_updated" - } else { + change := changes[j.ID] + persisted[i].CatalogChange = change + switch change { + case "job_created": result.Inserted++ - persisted[i].CatalogChange = "job_created" + case "job_updated": + result.Updated++ + result.RealUpdated++ + case "job_refreshed": + result.Updated++ + result.Refreshed++ + case "job_reactivated": + result.Updated++ + result.Reactivated++ } } + // Keep a deterministic batch result independent of SQL RETURNING order. + sort.Slice(persisted, func(i, j int) bool { return persisted[i].ID < persisted[j].ID }) result.Persisted = persisted return result, nil } + +// sameContent compares all modeled JSON content after MergeStored. Only the +// json:"-" catalog metadata is excluded. Sources/Keywords (including nil vs +// empty) and every classification array retain order: no sorting or dedup of +// semantic contributions. Empty omitempty classification arrays serialize alike. +// The strict reader above forces updates for unknown persisted JSON fields. +func sameContent(a, b domain.Job) bool { + a.CatalogChange, b.CatalogChange = "", "" + a.CatalogRevision, b.CatalogRevision = 0, 0 + a.CatalogExpiresAt, b.CatalogExpiresAt = time.Time{}, time.Time{} + canonical := func(c *domain.Classification) *domain.Classification { + if c == nil { + return nil + } + copy := *c + if len(copy.RelatedFamilies) == 0 { + copy.RelatedFamilies = nil + } + if len(copy.Technologies) == 0 { + copy.Technologies = nil + } + if len(copy.Reasons) == 0 { + copy.Reasons = nil + } + return © + } + a.Classification, b.Classification = canonical(a.Classification), canonical(b.Classification) + return reflect.DeepEqual(a, b) +} + func decodeRows(rows *sql.Rows) ([]domain.Job, error) { defer rows.Close() jobs := []domain.Job{} diff --git a/scraper-go/internal/catalogops/maintenance_integration_test.go b/scraper-go/internal/catalogops/maintenance_integration_test.go index 76ca91d0..a85e2d09 100644 --- a/scraper-go/internal/catalogops/maintenance_integration_test.go +++ b/scraper-go/internal/catalogops/maintenance_integration_test.go @@ -14,11 +14,12 @@ import ( "github.com/stretchr/testify/require" "os" "strings" + "sync" "testing" "time" ) -func integration(t *testing.T) (*Maintenance, *redis.Client) { +func integration(t testing.TB) (*Maintenance, *redis.Client) { t.Helper() dsn := os.Getenv("PAV125_TEST_DATABASE_URL") if dsn == "" { @@ -36,7 +37,9 @@ func integration(t *testing.T) (*Maintenance, *redis.Client) { _, e = db.Exec(string(migration)) require.NoError(t, e) t.Cleanup(func() { db.Close(); root.Exec(`DROP SCHEMA ` + schema + ` CASCADE`); root.Close() }) - mr := miniredis.RunT(t) + mr, e := miniredis.Run() + require.NoError(t, e) + t.Cleanup(mr.Close) r := redis.NewClient(&redis.Options{Addr: mr.Addr()}) t.Cleanup(func() { r.Close() }) store := catalog.New(db, catalog.DefaultLifetime) @@ -294,3 +297,247 @@ func TestReadOnlyCatalogStreamCountAndLimit(t *testing.T) { cancel() require.Error(t, m.Store.StreamActive(canceled, 0, func(int64) error { return nil }, func(domain.Job) error { return nil })) } + +func TestCatalogContentOutcomesAndRefreshRecovery(t *testing.T) { + m, r := integration(t) + ctx := context.Background() + first := m.Store.Now() + j := sample("outcomes", "backend") + j.Company = "Company" + j.Location = "São Paulo" + j.Sources = []string{"test"} + j.Keywords = []string{"sql"} + j.Description = strings.Repeat("description ", 400) + created, e := m.Store.SaveBatch(ctx, []domain.Job{j}) + require.NoError(t, e) + require.Equal(t, 1, created.Inserted) + require.True(t, created.ContentChangeDetectionAvailable) + old := created.Persisted[0] + require.Equal(t, "job_created", old.CatalogChange) + _, e = m.Index.Apply(ctx, created.Persisted, m.keys) + require.NoError(t, e) + require.NoError(t, m.Store.MarkIndexed(ctx, created.Persisted)) + // Column-target trigger proves refresh never assigns payload, even an equal value. + _, e = m.Store.DB.Exec(`CREATE FUNCTION forbid_payload() RETURNS trigger AS $$ BEGIN RAISE EXCEPTION 'payload assigned'; END; $$ LANGUAGE plpgsql; CREATE TRIGGER forbid_payload BEFORE UPDATE OF payload ON job_catalog FOR EACH ROW EXECUTE FUNCTION forbid_payload();`) + require.NoError(t, e) + m.Store.Now = func() time.Time { return first.Add(time.Hour) } + refreshed, e := m.Store.SaveBatch(ctx, []domain.Job{j}) + require.NoError(t, e) + require.Equal(t, 1, refreshed.Refreshed) + require.Zero(t, refreshed.RealUpdated) + require.Equal(t, "job_refreshed", refreshed.Persisted[0].CatalogChange) + require.Greater(t, refreshed.Persisted[0].CatalogRevision, old.CatalogRevision) + var revision, indexed int64 + var last, expiry time.Time + require.NoError(t, m.Store.DB.QueryRow(`SELECT revision,indexed_revision,last_seen_at,expires_at FROM job_catalog WHERE id=$1`, old.ID).Scan(&revision, &indexed, &last, &expiry)) + require.Less(t, indexed, revision) + require.WithinDuration(t, m.Store.Now(), last, time.Microsecond) + require.WithinDuration(t, m.Store.Now().Add(catalog.DefaultLifetime), expiry, time.Microsecond) + // Stale acknowledgement cannot clear a newer durable refresh. + require.NoError(t, m.Store.MarkIndexed(ctx, created.Persisted)) + require.NoError(t, m.Store.DB.QueryRow(`SELECT indexed_revision FROM job_catalog WHERE id=$1`, old.ID).Scan(&indexed)) + require.Less(t, indexed, revision) + // Simulated crash: absent projection must rebuild via full fallback on reconcile. + require.NoError(t, r.FlushDB(ctx).Err()) + report, e := m.Reconcile(ctx, true) + require.NoError(t, e) + require.Positive(t, report.Active) + require.NoError(t, m.Store.DB.QueryRow(`SELECT indexed_revision FROM job_catalog WHERE id=$1`, old.ID).Scan(&indexed)) + require.Equal(t, revision, indexed) + require.Positive(t, r.TTL(ctx, jobindex.Prefix(r.Get(ctx, jobindex.ActiveKey).Val())+"job:"+old.ID).Val()) + _, e = m.Store.DB.Exec(`DROP TRIGGER forbid_payload ON job_catalog`) + require.NoError(t, e) + // Each relevant effective field change advances revision and requests full apply. + updates := []func(*domain.Job){func(j *domain.Job) { j.Description = "changed" }, func(j *domain.Job) { j.Location = "SAO PAULO" }, func(j *domain.Job) { j.Classification = &domain.Classification{PrimaryFamily: "data", InScope: true} }, func(j *domain.Job) { j.Salary = "100" }, func(j *domain.Job) { j.URL = "https://example.test/changed" }} + for _, mutate := range updates { + mutate(&j) + result, e := m.Store.SaveBatch(ctx, []domain.Job{j}) + require.NoError(t, e) + require.Equal(t, 1, result.RealUpdated) + require.Equal(t, old.ID, result.Persisted[0].ID) + require.Greater(t, result.Persisted[0].CatalogRevision, revision) + revision = result.Persisted[0].CatalogRevision + } + // Current merge intentionally retains stored Title/Company for the same identity. + j.Title = strings.ToUpper(j.Title) + result, e := m.Store.SaveBatch(ctx, []domain.Job{j}) + require.NoError(t, e) + require.Equal(t, 1, result.Refreshed) + // A normalized title change changes stable identity and is a CREATE, not UPDATE. + newIdentity := j + newIdentity.Title = "Different job" + result, e = m.Store.SaveBatch(ctx, []domain.Job{newIdentity}) + require.NoError(t, e) + require.Equal(t, 1, result.Inserted) + _, e = m.Store.Deactivate(ctx, []string{old.ID}) + require.NoError(t, e) + result, e = m.Store.SaveBatch(ctx, []domain.Job{j}) + require.NoError(t, e) + require.Equal(t, 1, result.Reactivated) + require.Zero(t, result.Refreshed) + require.Equal(t, "job_reactivated", result.Persisted[0].CatalogChange) + // Unknown JSON fields are conservatively rewritten, never classified as refresh. + _, e = m.Store.DB.Exec(`UPDATE job_catalog SET payload=payload || '{"unknownFutureField":true}'::jsonb WHERE id=$1`, old.ID) + require.NoError(t, e) + result, e = m.Store.SaveBatch(ctx, []domain.Job{j}) + require.NoError(t, e) + require.Equal(t, 1, result.RealUpdated) +} + +func TestCatalogConcurrentRefreshAndUpdate(t *testing.T) { + m, _ := integration(t) + ctx := context.Background() + j := sample("concurrent", "backend") + j.Sources = []string{"test"} + j.Keywords = []string{"sql"} + j.Company = "company" + j.Description = "initial" + initial, e := m.Store.SaveBatch(ctx, []domain.Job{j}) + require.NoError(t, e) + run := func(inputs []domain.Job) []jobstore.SaveResult { + results := make([]jobstore.SaveResult, len(inputs)) + errors := make([]error, len(inputs)) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range inputs { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + results[i], errors[i] = m.Store.SaveBatch(ctx, []domain.Job{inputs[i]}) + }(i) + } + close(start) + wg.Wait() + for _, e := range errors { + require.NoError(t, e) + } + return results + } + results := run([]domain.Job{j, j}) + require.Equal(t, 2, results[0].Refreshed+results[1].Refreshed) + require.NotEqual(t, results[0].Persisted[0].CatalogRevision, results[1].Persisted[0].CatalogRevision) + update := j + update.Description = "changed" + // A sparse refresh does not overwrite concurrent nonempty update content. + refresh := j + refresh.Description = "" + results = run([]domain.Job{refresh, update}) + require.Equal(t, 1, results[0].Refreshed+results[1].Refreshed) + require.Equal(t, 1, results[0].RealUpdated+results[1].RealUpdated) + stored, e := m.Store.GetByIDs(ctx, []string{initial.Persisted[0].ID}) + require.NoError(t, e) + require.Equal(t, "changed", stored[0].Description) +} + +func TestCatalogRefreshCommitFailureDoesNotPublish(t *testing.T) { + m, _ := integration(t) + ctx := context.Background() + j := sample("refresh-commit", "backend") + j.Sources = []string{"test"} + j.Keywords = []string{"sql"} + initial, e := m.Store.SaveBatch(ctx, []domain.Job{j}) + require.NoError(t, e) + _, e = m.Store.DB.Exec(`CREATE FUNCTION fail_refresh_commit() RETURNS trigger AS $$ BEGIN RAISE EXCEPTION 'forced refresh commit failure'; END; $$ LANGUAGE plpgsql; CREATE CONSTRAINT TRIGGER fail_refresh_commit AFTER UPDATE ON job_catalog DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION fail_refresh_commit();`) + require.NoError(t, e) + result, e := m.Store.SaveBatch(ctx, []domain.Job{j}) + require.ErrorContains(t, e, "catalog commit") + require.Empty(t, result.Persisted) + require.Zero(t, result.Refreshed) + var revision int64 + require.NoError(t, m.Store.DB.QueryRow(`SELECT revision FROM job_catalog WHERE id=$1`, initial.Persisted[0].ID).Scan(&revision)) + require.Equal(t, initial.Persisted[0].CatalogRevision, revision) +} + +func TestCatalogOrderedMergeAndMixedBatchRollback(t *testing.T) { + m, _ := integration(t) + ctx := context.Background() + j := sample("ordered", "backend") + j.Sources = []string{"a", "b"} + j.Source = "a, b" + j.Keywords = []string{"first", "second"} + j.Keyword = "first" + initial, e := m.Store.SaveBatch(ctx, []domain.Job{j}) + require.NoError(t, e) + incoming := j + incoming.Sources = []string{"b", "a"} + incoming.Keywords = []string{"second", "first"} + result, e := m.Store.SaveBatch(ctx, []domain.Job{incoming}) + require.NoError(t, e) + require.Equal(t, 1, result.Refreshed) + require.Equal(t, j.Sources, result.Persisted[0].Sources) + require.Equal(t, j.Keywords, result.Persisted[0].Keywords) + require.Equal(t, "first", result.Persisted[0].Keyword) + incoming.Sources = []string{"c"} + incoming.Keywords = []string{"third"} + result, e = m.Store.SaveBatch(ctx, []domain.Job{incoming}) + require.NoError(t, e) + require.Equal(t, 1, result.RealUpdated) + require.Equal(t, []string{"a", "b", "c"}, result.Persisted[0].Sources) + require.Equal(t, "a, b, c", result.Persisted[0].Source) + require.Equal(t, "first", result.Persisted[0].Keyword) + lastRevision := result.Persisted[0].CatalogRevision + _, e = m.Store.DB.Exec(`ALTER TABLE job_catalog ADD CONSTRAINT reject_batch CHECK (payload->>'title' <> 'fail')`) + require.NoError(t, e) + bad := sample("bad-mixed", "backend") + bad.Title = "fail" + result, e = m.Store.SaveBatch(ctx, []domain.Job{incoming, bad}) + require.Error(t, e) + require.Empty(t, result.Persisted) + require.Zero(t, result.Refreshed) + var revision int64 + require.NoError(t, m.Store.DB.QueryRow(`SELECT revision FROM job_catalog WHERE id=$1`, initial.Persisted[0].ID).Scan(&revision)) + require.Equal(t, lastRevision, revision) + // Expiration itself, without explicit deactivation, also requires reactivation. + m.Store.Now = func() time.Time { return time.Now().Add(2 * catalog.DefaultLifetime) } + result, e = m.Store.SaveBatch(ctx, []domain.Job{incoming}) + require.NoError(t, e) + require.Equal(t, 1, result.Reactivated) +} + +func BenchmarkCatalogSaveBatch(b *testing.B) { + for _, size := range []int{4096, 16384} { + for _, scenario := range []string{"unchanged", "mixed", "updates", "creates"} { + b.Run(fmt.Sprintf("%s/%dB", scenario, size), func(b *testing.B) { + m, _ := integration(b) + ctx := context.Background() + base := make([]domain.Job, 100) + for i := range base { + base[i] = sample(fmt.Sprint(i), "backend") + base[i].Company = "benchmark" + base[i].Description = strings.Repeat("Backend SQL Go product requirements. ", size/36+1)[:size] + base[i].Sources = []string{"test"} + base[i].Keywords = []string{"sql"} + } + _, e := m.Store.SaveBatch(ctx, base) + require.NoError(b, e) + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + jobs := append([]domain.Job(nil), base...) + for i := range jobs { + switch scenario { + case "mixed": + if i < 10 { + jobs[i].Description += fmt.Sprint(n) + } + case "updates": + jobs[i].Description += fmt.Sprint(n) + case "creates": + jobs[i].Company = fmt.Sprintf("new-%d", n) + } + } + result, e := m.Store.SaveBatch(ctx, jobs) + if e != nil { + b.Fatal(e) + } + if len(result.Persisted) != 100 { + b.Fatal("incomplete batch") + } + } + b.StopTimer() + b.ReportMetric(100, "jobs/batch") + }) + } + } +} diff --git a/scraper-go/internal/classifier/classifier.go b/scraper-go/internal/classifier/classifier.go index 6bfa3dd3..0490eb04 100644 --- a/scraper-go/internal/classifier/classifier.go +++ b/scraper-go/internal/classifier/classifier.go @@ -56,8 +56,9 @@ func Classify(job domain.Job) (classification domain.Classification) { }, " ")) title := normalizeText(job.Title) - scores, exclusions := scoreFamilies(text, title) - titleScores, _ := scoreFamilies(title, title) + bodyMatches, titleMatches := newMatchedText(text), newMatchedText(title) + scores, exclusions := scoreFamilyMatches(bodyMatches, titleMatches) + titleScores, _ := scoreFamilyMatches(titleMatches, titleMatches) // Title-supported technical candidates survive conflicting description evidence. for _, ts := range titleScores { if ts.evidence == "" { @@ -80,10 +81,10 @@ func Classify(job domain.Job) (classification domain.Classification) { scores = append(scores, ts) } } - technologies := detectTechnologies(text) - seniority := detectSeniority(text) + technologies := detectTechnologyMatches(bodyMatches) + seniority := detectSeniorityMatches(bodyMatches) - if blocked, reason := blockedOpeningReason(job); blocked { + if blocked, reason := blockedOpeningReasonMatched(titleMatches); blocked { return domain.Classification{ PrimaryFamily: "other", Technologies: technologies, @@ -150,39 +151,43 @@ func Classify(job domain.Job) (classification domain.Classification) { } } +var nonConcreteTerms = []string{ + "banco de talentos", + "talent pool", + "cadastro reserva", +} +var administrativeTerms = []string{ + "assistente central de reservas", + "central de reservas", + "assistente de negocios", + "central de relacionamento", + "assistente contabil", + "assistente financeiro", + "assistente de autorizacao", + "motorista", + "motorista de van", + "atendente", + "analista qualidade", + "analista de qualidade", +} + func blockedOpeningReason(job domain.Job) (bool, string) { - title := normalizeText(job.Title) - if title == "" { + return blockedOpeningReasonMatched(newMatchedText(normalizeText(job.Title))) +} + +func blockedOpeningReasonMatched(title *matchedText) (bool, string) { + if title.text == "" { return false, "" } - nonConcreteTerms := []string{ - "banco de talentos", - "talent pool", - "cadastro reserva", - } for _, term := range nonConcreteTerms { - if containsTokenOrPhrase(title, term) { + if title.contains(term, false) { return true, "vaga nao concreta: " + term } } - administrativeTerms := []string{ - "assistente central de reservas", - "central de reservas", - "assistente de negocios", - "central de relacionamento", - "assistente contabil", - "assistente financeiro", - "assistente de autorizacao", - "motorista", - "motorista de van", - "atendente", - "analista qualidade", - "analista de qualidade", - } for _, term := range administrativeTerms { - if containsTokenOrPhrase(title, term) { + if title.contains(term, false) { return true, "vaga administrativa: " + term } } @@ -368,6 +373,10 @@ func topCountLabels(values map[string]int, limit int) []string { } func scoreFamilies(text, title string) ([]familyScore, []string) { + return scoreFamilyMatches(newMatchedText(text), newMatchedText(title)) +} + +func scoreFamilyMatches(text, title *matchedText) ([]familyScore, []string) { scores := make([]familyScore, 0, len(familyRules)) var exclusions []string @@ -379,8 +388,8 @@ func scoreFamilies(text, title string) ([]familyScore, []string) { strongText = title blocked := false for _, term := range rule.NegativeTerms { - if containsWholePhrase(title, term) { - if hasTitleFamilyContext(title, rule.Family) { + if title.contains(term, true) { + if hasTitleFamilyContextMatched(title, rule.Family) { exclusions = append(exclusions, "exclusao "+rule.Family+": "+term) } blocked = true @@ -395,9 +404,9 @@ func scoreFamilies(text, title string) ([]familyScore, []string) { for _, term := range rule.StrongTerms { var matches bool if rule.TitleRequired { - matches = containsWholePhrase(strongText, term) + matches = strongText.contains(term, true) } else { - matches = containsTokenOrPhrase(strongText, term) + matches = strongText.contains(term, false) } if matches { score += 4 @@ -412,7 +421,7 @@ func scoreFamilies(text, title string) ([]familyScore, []string) { } if !rule.TitleRequired { for _, term := range rule.StrongTerms { - if containsTokenOrPhrase(title, term) { + if title.contains(term, false) { evidence = term priority = titlePriorityTechnical break @@ -427,7 +436,7 @@ func scoreFamilies(text, title string) ([]familyScore, []string) { } complementary := 0 for _, term := range rule.TechnologyTerms { - if containsTokenOrPhrase(text, term) { + if text.contains(term, false) { complementary++ } } @@ -437,7 +446,7 @@ func scoreFamilies(text, title string) ([]familyScore, []string) { score += complementary if !rule.TitleRequired { for _, term := range rule.NegativeTerms { - if containsTokenOrPhrase(text, term) { + if text.contains(term, false) { score -= 3 } } @@ -454,22 +463,30 @@ func scoreFamilies(text, title string) ([]familyScore, []string) { // Both ends of multi-word titles must be whole words: product != production, // product manager != product management, and designer != designership. func containsWholePhrase(text, term string) bool { - return strings.Contains(" "+text+" ", " "+normalizeText(term)+" ") + return containsNormalized(text, normalizedFixedTerm(term), true) } func hasTitleFamilyContext(title, family string) bool { + return hasTitleFamilyContextMatched(newMatchedText(title), family) +} + +func hasTitleFamilyContextMatched(title *matchedText, family string) bool { if family == "product" { - return containsAny(title, "product", "produto", "produtos", "production", "producao") + return title.any(productContextTerms...) } - return containsAny(title, "designer", "design", "ux", "ui") + return title.any(designContextTerms...) } func detectTechnologies(text string) []string { + return detectTechnologyMatches(newMatchedText(text)) +} + +func detectTechnologyMatches(text *matchedText) []string { technologies := make([]string, 0) for technology, aliases := range technologyAliases { for _, alias := range aliases { - if containsTokenOrPhrase(text, alias) { + if text.contains(alias, false) { technologies = append(technologies, technology) break } @@ -482,13 +499,17 @@ func detectTechnologies(text string) []string { } func detectSeniority(text string) string { - if containsAny(text, "estagio", "estagiario", "intern", "internship", "trainee") { + return detectSeniorityMatches(newMatchedText(text)) +} + +func detectSeniorityMatches(text *matchedText) string { + if text.any(internTerms...) { return "estagio" } - if containsAny(text, "senior", "sr", "especialista", "lead", "principal", "staff") { + if text.any(seniorTerms...) { return "senior" } - if containsAny(text, "junior", "jr", "entry level", "assistente") { + if text.any(juniorTerms...) { return "junior" } return "pleno" diff --git a/scraper-go/internal/classifier/classify_batch_test.go b/scraper-go/internal/classifier/classify_batch_test.go index 5e871385..22d92e3c 100644 --- a/scraper-go/internal/classifier/classify_batch_test.go +++ b/scraper-go/internal/classifier/classify_batch_test.go @@ -1,7 +1,19 @@ package classifier import ( + "fmt" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/metrics" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/taxonomy" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" + "math" + "math/rand" + "sort" + "strings" + "sync" "testing" + "time" + "unicode" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" "github.com/stretchr/testify/assert" @@ -64,3 +76,434 @@ func TestCurrentFamiliesPreservedAndProductFamiliesAdded(t *testing.T) { "product_design", }, got) } + +// Frozen classifier from 7f3ac75, kept only as a differential test oracle. +func referenceClassify(job domain.Job) (classification domain.Classification) { + started := time.Now() + defer func() { metrics.Classification(job, classification, started) }() + text := referenceNormalizeText(strings.Join([]string{ + job.Title, + job.Company, + job.Location, + job.Modality, + job.Description, + }, " ")) + + title := referenceNormalizeText(job.Title) + scores, exclusions := referenceScoreFamilies(text, title) + titleScores, _ := referenceScoreFamilies(title, title) + // Title-supported technical candidates survive conflicting description evidence. + for _, ts := range titleScores { + if ts.evidence == "" { + continue + } + found := false + for i := range scores { + if ts.family != scores[i].family { + continue + } + found = true + scores[i].titlePriority = ts.titlePriority + if ts.titlePriority <= titlePriorityLeadership { + scores[i].score = ts.score + } + scores[i].evidence = ts.evidence + break + } + if !found { + scores = append(scores, ts) + } + } + technologies := referenceDetectTechnologies(text) + seniority := referenceDetectSeniority(text) + + if blocked, reason := referenceBlockedOpeningReason(job); blocked { + return domain.Classification{ + PrimaryFamily: "other", + Technologies: technologies, + Seniority: seniority, + InScope: false, + Confidence: 0, + Reasons: []string{reason}, + } + } + + if len(scores) == 0 { + return domain.Classification{ + PrimaryFamily: "other", + Technologies: technologies, + Seniority: seniority, + InScope: false, + Confidence: 0, + Reasons: append([]string{"nenhuma familia reconhecida"}, exclusions...), + } + } + + sort.Slice(scores, func(i, j int) bool { + if scores[i].titlePriority != scores[j].titlePriority { + return scores[i].titlePriority > scores[j].titlePriority + } + if scores[i].score == scores[j].score { + return scores[i].family < scores[j].family + } + return scores[i].score > scores[j].score + }) + + primary := scores[0] + related := make([]string, 0, len(scores)-1) + for _, candidate := range scores[1:] { + if candidate.score >= 3 { + related = append(related, candidate.family) + } + } + if primary.family == "mobile" && referenceContainsString(technologies, "react-native") { + related = append(related, "frontend") + } + + if primary.family == "fullstack" { + related = append(related, "backend", "frontend") + } + confidence := math.Min(0.99, 0.35+(float64(primary.score)*0.08)) + + reason := "classificacao local por titulo descricao tecnologias" + if primary.evidence != "" { + reason = "titulo: " + primary.evidence + } + if primary.score < 2 { + reason = "score abaixo do minimo" + } + + return domain.Classification{ + PrimaryFamily: primary.family, + RelatedFamilies: taxonomy.Related(primary.family, related), + Technologies: technologies, + Seniority: seniority, + InScope: primary.score >= 2, + Confidence: math.Round(confidence*100) / 100, + Reasons: append([]string{reason}, exclusions...), + } +} + +func referenceBlockedOpeningReason(job domain.Job) (bool, string) { + title := referenceNormalizeText(job.Title) + if title == "" { + return false, "" + } + + nonConcreteTerms := []string{ + "banco de talentos", + "talent pool", + "cadastro reserva", + } + for _, term := range nonConcreteTerms { + if referenceContainsTokenOrPhrase(title, term) { + return true, "vaga nao concreta: " + term + } + } + + administrativeTerms := []string{ + "assistente central de reservas", + "central de reservas", + "assistente de negocios", + "central de relacionamento", + "assistente contabil", + "assistente financeiro", + "assistente de autorizacao", + "motorista", + "motorista de van", + "atendente", + "analista qualidade", + "analista de qualidade", + } + for _, term := range administrativeTerms { + if referenceContainsTokenOrPhrase(title, term) { + return true, "vaga administrativa: " + term + } + } + + return false, "" +} + +func referenceContainsString(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func referenceScoreFamilies(text, title string) ([]familyScore, []string) { + scores := make([]familyScore, 0, len(familyRules)) + var exclusions []string + + for _, rule := range familyRules { + score := 0 + evidence := "" + strongText := text + if rule.TitleRequired { + strongText = title + blocked := false + for _, term := range rule.NegativeTerms { + if referenceContainsWholePhrase(title, term) { + if referenceHasTitleFamilyContext(title, rule.Family) { + exclusions = append(exclusions, "exclusao "+rule.Family+": "+term) + } + blocked = true + break + } + } + if blocked { + continue + } + } + + for _, term := range rule.StrongTerms { + var matches bool + if rule.TitleRequired { + matches = referenceContainsWholePhrase(strongText, term) + } else { + matches = referenceContainsTokenOrPhrase(strongText, term) + } + if matches { + score += 4 + if rule.TitleRequired && evidence == "" { + evidence = term + } + } + } + priority := rule.TitlePriority + if rule.TitleRequired { + priority += titlePriorityProductOffset + } + if !rule.TitleRequired { + for _, term := range rule.StrongTerms { + if referenceContainsTokenOrPhrase(title, term) { + evidence = term + priority = titlePriorityTechnical + break + } + } + if evidence != "" && rule.Family == "leadership" { + priority = titlePriorityLeadership + } + } + if rule.TitleRequired && evidence == "" { + continue + } + complementary := 0 + for _, term := range rule.TechnologyTerms { + if referenceContainsTokenOrPhrase(text, term) { + complementary++ + } + } + if rule.TitleRequired { + complementary = min(complementary, 3) + } + score += complementary + if !rule.TitleRequired { + for _, term := range rule.NegativeTerms { + if referenceContainsTokenOrPhrase(text, term) { + score -= 3 + } + } + } + + if score > 0 { + scores = append(scores, familyScore{family: rule.Family, score: score, titlePriority: priority, evidence: evidence}) + } + } + + return scores, exclusions +} + +func referenceContainsWholePhrase(text, term string) bool { + return strings.Contains(" "+text+" ", " "+referenceNormalizeText(term)+" ") +} + +func referenceHasTitleFamilyContext(title, family string) bool { + if family == "product" { + return referenceContainsAny(title, "product", "produto", "produtos", "production", "producao") + } + return referenceContainsAny(title, "designer", "design", "ux", "ui") +} + +func referenceDetectTechnologies(text string) []string { + technologies := make([]string, 0) + + for technology, aliases := range technologyAliases { + for _, alias := range aliases { + if referenceContainsTokenOrPhrase(text, alias) { + technologies = append(technologies, technology) + break + } + } + } + + technologies = referenceUnique(technologies) + sort.Strings(technologies) + return technologies +} + +func referenceDetectSeniority(text string) string { + if referenceContainsAny(text, "estagio", "estagiario", "intern", "internship", "trainee") { + return "estagio" + } + if referenceContainsAny(text, "senior", "sr", "especialista", "lead", "principal", "staff") { + return "senior" + } + if referenceContainsAny(text, "junior", "jr", "entry level", "assistente") { + return "junior" + } + return "pleno" +} + +func referenceContainsAny(text string, needles ...string) bool { + for _, needle := range needles { + if referenceContainsTokenOrPhrase(text, needle) { + return true + } + } + return false +} + +func referenceNormalizeText(value string) string { + t := transform.Chain(norm.NFD, transform.RemoveFunc(func(r rune) bool { + return unicode.Is(unicode.Mn, r) + }), norm.NFC) + + result, _, _ := transform.String(t, value) + + var b strings.Builder + for _, r := range strings.ToLower(result) { + if unicode.IsLetter(r) || unicode.IsNumber(r) { + b.WriteRune(r) + } else { + b.WriteRune(' ') + } + } + + return strings.Join(strings.Fields(b.String()), " ") +} + +func referenceContainsTokenOrPhrase(text, needle string) bool { + needle = referenceNormalizeText(needle) + if needle == "" { + return false + } + if strings.Contains(needle, " ") { + return strings.Contains(text, needle) + } + return strings.Contains(" "+text+" ", " "+needle+" ") +} + +func referenceUnique(values []string) []string { + seen := make(map[string]struct{}, len(values)) + result := make([]string, 0, len(values)) + + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + + return result +} + +func classifierBenchJobs(size int) []domain.Job { + titles := []string{"Backend Engineer", "Back-end Developer", "Front-end Developer", "Full-stack Engineer", "Platform Engineer", "DevOps Engineer", "Data Scientist", "QA Engineer", "Security Engineer", "Engineering Manager", "Software Engineer", "Product Manager", "Product Designer", "UX Researcher", "Graphic Designer", "Motorista", "Banco de Talentos"} + paragraph := "Responsibilities: build distributed systems, APIs and microservices using Go, Node.js, C#, .NET, Java and PostgreSQL. Work with React, TypeScript, Kubernetes, AWS, Kafka and Redis. Collaborate with product managers and UX researchers on roadmap, discovery, usability and design systems. Senior remote role; junior applicants and internship experience welcome. Comunicação e operações de produto. " + text := strings.Repeat(paragraph, size/len(paragraph)+1)[:size] + jobs := make([]domain.Job, 100) + for i := range jobs { + jobs[i] = domain.Job{Title: titles[i%len(titles)], Company: fmt.Sprintf("Acme %d", i), Location: "São Paulo, Brasil", Modality: "Remote", Description: text, Source: "greenhouse"} + } + return jobs +} + +func TestClassifyEquivalentToCommittedClassifier(t *testing.T) { + jobs := classifierBenchJobs(4096) + jobs = append(jobs, classifierBenchJobs(16384)...) + for _, text := range []string{"", "back end", "back-end", "front end", "front-end", "full stack", "full-stack", "C# .NET Node.js", "anode json", "Product Manager", "Product Management", "Product Designer", "Product Design Manager", "Production Manager", "Graphic Designer", "UX/UI Designer", "Talent Pool", "Assistente Financeiro", "Senior", "Júnior", "Estágio", "BACK-END ENGINEER", "operações de produto", "operações de produto", "Backend\tEngineer\nGo", "DevOps Platform Engineer", "Engineering Manager Backend", "Product Security Engineer"} { + jobs = append(jobs, domain.Job{Title: text, Description: text, Company: "Acme", Location: "Brasil"}) + } + rng := rand.New(rand.NewSource(178)) + words := []string{"back-end", "back end", "front-end", "full-stack", "go", "gopher", "node.js", "C#", ".NET", "product manager", "product designer", "production", "graphic designer", "senior", "estágio", "remote", "marketing", "engineer", "platform", "devops", "software", "react native", "data scientist", "banco de talentos", "operações"} + for i := 0; i < 150; i++ { + parts := make([]string, 10) + for n := range parts { + parts[n] = words[rng.Intn(len(words))] + } + jobs = append(jobs, domain.Job{Title: strings.Join(parts[:3], " "), Company: parts[3], Location: parts[4], Modality: parts[5], Description: strings.Join(parts, " / ")}) + } + for i, job := range jobs { + t.Run(fmt.Sprint(i), func(t *testing.T) { require.Equal(t, referenceClassify(job), Classify(job)) }) + } +} + +func TestScoringRetainsEquivalentTermContributions(t *testing.T) { + for _, title := range []string{"back end", "front end", "full stack", "backend product designer", "platform devops engineer", "product manager marketing", "graphic designer"} { + normalized := referenceNormalizeText(title) + want, exclusions := referenceScoreFamilies(normalized, normalized) + got, actualExclusions := scoreFamilies(normalized, normalized) + require.Equal(t, want, got) + require.Equal(t, exclusions, actualExclusions) + } + scores, _ := scoreFamilies("back end", "back end") + for _, score := range scores { + if score.family == "backend" { + require.Equal(t, 8, score.score) + return + } + } + t.Fatal("backend score missing") +} + +func TestClassifierMatchingBoundariesEquivalent(t *testing.T) { + for _, text := range []string{"", " ", "go", "gopher go", "go-go", "go\tgo", "anode json", "product manager", "aproduct managerx", "c net", "a b", "é go é"} { + for _, term := range []string{"", "go", "node js", "product manager", "C#", ".NET", "---", "a b", "é"} { + require.Equal(t, referenceContainsTokenOrPhrase(text, term), containsTokenOrPhrase(text, term), "%q / %q", text, term) + require.Equal(t, referenceContainsWholePhrase(text, term), containsWholePhrase(text, term), "whole %q / %q", text, term) + } + } +} + +func TestClassifierPreparedRulesAreSafeForConcurrentUse(t *testing.T) { + job := classifierBenchJobs(4096)[0] + want := referenceClassify(job) + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for n := 0; n < 10; n++ { + assert.Equal(t, want, Classify(job)) + } + }() + } + wg.Wait() +} + +func BenchmarkClassifyBatch(b *testing.B) { + for _, size := range []int{4096, 16384} { + jobs := classifierBenchJobs(size) + for _, job := range jobs { + require.Equal(b, referenceClassify(job), Classify(job)) + } + b.Run(fmt.Sprintf("%dKiB", size/1024), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + for _, job := range jobs { + Classify(job) + } + } + }) + } +} diff --git a/scraper-go/internal/classifier/normalize.go b/scraper-go/internal/classifier/normalize.go index 2fb90c2f..08586160 100644 --- a/scraper-go/internal/classifier/normalize.go +++ b/scraper-go/internal/classifier/normalize.go @@ -27,15 +27,101 @@ func normalizeText(value string) string { return strings.Join(strings.Fields(b.String()), " ") } +// These inputs are immutable rule data. Only their normalized representation +// is shared; scoring still visits every original term, including duplicates. +var productContextTerms = []string{"product", "produto", "produtos", "production", "producao"} +var designContextTerms = []string{"designer", "design", "ux", "ui"} +var internTerms = []string{"estagio", "estagiario", "intern", "internship", "trainee"} +var seniorTerms = []string{"senior", "sr", "especialista", "lead", "principal", "staff"} +var juniorTerms = []string{"junior", "jr", "entry level", "assistente"} +var normalizedTerms = prepareFixedTerms() + +func prepareFixedTerms() map[string]string { + terms := make(map[string]string) + add := func(values []string) { + for _, term := range values { + if _, exists := terms[term]; !exists { + terms[term] = normalizeText(term) + } + } + } + for _, rule := range familyRules { + add(rule.StrongTerms) + add(rule.TechnologyTerms) + add(rule.NegativeTerms) + } + for _, aliases := range technologyAliases { + add(aliases) + } + for _, values := range [][]string{nonConcreteTerms, administrativeTerms, productContextTerms, designContextTerms, internTerms, seniorTerms, juniorTerms} { + add(values) + } + return terms +} + +func normalizedFixedTerm(term string) string { + if normalized, ok := normalizedTerms[term]; ok { + return normalized + } + return normalizeText(term) +} + +type matchQuery struct { + needle string + whole bool +} +type matchedText struct { + text string + results map[matchQuery]bool +} + +func newMatchedText(text string) *matchedText { + return &matchedText{text: text, results: make(map[matchQuery]bool)} +} +func (text *matchedText) contains(term string, whole bool) bool { + query := matchQuery{needle: normalizedFixedTerm(term), whole: whole} + if result, ok := text.results[query]; ok { + return result + } + result := containsNormalized(text.text, query.needle, whole) + text.results[query] = result + return result +} +func (text *matchedText) any(terms ...string) bool { + for _, term := range terms { + if text.contains(term, false) { + return true + } + } + return false +} + func containsTokenOrPhrase(text, needle string) bool { - needle = normalizeText(needle) + return containsNormalized(text, normalizedFixedTerm(needle), false) +} + +func containsNormalized(text, needle string, whole bool) bool { if needle == "" { - return false + // The former whole-phrase padding could match two adjacent spaces, + // including the artificial pair for empty text. Keep that edge case. + return whole && (text == "" || strings.HasPrefix(text, " ") || strings.HasSuffix(text, " ") || strings.Contains(text, " ")) } - if strings.Contains(needle, " ") { + if !whole && strings.Contains(needle, " ") { return strings.Contains(text, needle) } - return strings.Contains(" "+text+" ", " "+needle+" ") + for offset := 0; offset <= len(text)-len(needle); { + relative := strings.Index(text[offset:], needle) + if relative < 0 { + return false + } + start := offset + relative + end := start + len(needle) + if (start == 0 || text[start-1] == ' ') && (end == len(text) || text[end] == ' ') { + return true + } + offset = start + 1 + } + return false } func unique(values []string) []string { diff --git a/scraper-go/internal/config/config.go b/scraper-go/internal/config/config.go index ebf19af1..a72a8a8b 100644 --- a/scraper-go/internal/config/config.go +++ b/scraper-go/internal/config/config.go @@ -11,6 +11,7 @@ import ( ) const ( + DefaultScrapeTimeout = 50 * time.Minute DefaultCatalogLifetime = 9 * 24 * time.Hour DefaultMaxConcurrency = 12 DefaultProviderMaxConcurrency = 2 @@ -27,6 +28,7 @@ const ( SourceInternalDefault = "internal_default" ScraperMaxConcurrencyEnv = "SCRAPER_MAX_CONCURRENCY" + ScraperScrapeTimeoutEnv = "SCRAPER_SCRAPE_TIMEOUT" ScraperProviderMaxConcurrencyEnv = "SCRAPER_PROVIDER_MAX_CONCURRENCY" ScraperProviderConcurrencyOverridesEnv = "SCRAPER_PROVIDER_CONCURRENCY_OVERRIDES" ScraperRunLockTTLEnv = "SCRAPER_RUN_LOCK_TTL" @@ -37,6 +39,9 @@ const ( ) type RuntimeConfig struct { + ProviderWindows []ProviderWindow + GreenhouseBatchSize int + ScrapeTimeout time.Duration ApplicationVersion string CatalogLifetime time.Duration MaxConcurrency int @@ -115,6 +120,21 @@ func LoadRuntimeConfigFromLookup(lookup func(string) (string, bool)) (RuntimeCon } var err error + if value, ok := lookup(ProviderWindowsEnv); ok { + timezone, _ := lookup(ProviderScheduleTimezoneEnv) + cfg.ProviderWindows, err = ParseProviderWindowsInTimezone(value, timezone) + if err != nil { + return RuntimeConfig{}, err + } + } + cfg.GreenhouseBatchSize, err = boundedPositiveIntFromLookup(lookup, "SCRAPER_GREENHOUSE_BATCH_SIZE", 1000, 1000) + if err != nil { + return RuntimeConfig{}, err + } + cfg.ScrapeTimeout, err = durationFromLookup(lookup, ScraperScrapeTimeoutEnv, DefaultScrapeTimeout) + if err != nil { + return RuntimeConfig{}, err + } cfg.CatalogLifetime, err = durationFromLookup(lookup, "SCRAPER_CATALOG_LIFETIME", DefaultCatalogLifetime) if err != nil { return RuntimeConfig{}, err diff --git a/scraper-go/internal/config/config_test.go b/scraper-go/internal/config/config_test.go index b0ca3368..6d7dbcda 100644 --- a/scraper-go/internal/config/config_test.go +++ b/scraper-go/internal/config/config_test.go @@ -252,7 +252,7 @@ func TestLoadRuntimeConfigRejectsInvalidRunLockDurations(t *testing.T) { } func TestDockerComposeUsesUnsetOnlyDefaultsForRunLock(t *testing.T) { - composePath, ok := findMonorepoFile(t, "docker-compose.yml") + composePath, ok := findMonorepoFile(t, "docker/docker-compose.app.yml") if !ok { t.Skip("docker-compose.yml not available outside the monorepo checkout") } @@ -261,6 +261,8 @@ func TestDockerComposeUsesUnsetOnlyDefaultsForRunLock(t *testing.T) { require.NoError(t, err) compose := string(content) + assert.Contains(t, compose, "SCRAPER_SCRAPE_TIMEOUT=${SCRAPER_SCRAPE_TIMEOUT-50m}") + assert.Contains(t, compose, "SCRAPER_GREENHOUSE_BATCH_SIZE=${SCRAPER_GREENHOUSE_BATCH_SIZE-1000}") assert.Contains(t, compose, "SCRAPER_MAX_CONCURRENCY=${SCRAPER_MAX_CONCURRENCY-12}") assert.Contains(t, compose, "SCRAPER_PROVIDER_MAX_CONCURRENCY=${SCRAPER_PROVIDER_MAX_CONCURRENCY-2}") assert.Contains(t, compose, "SCRAPER_PROVIDER_CONCURRENCY_OVERRIDES=${SCRAPER_PROVIDER_CONCURRENCY_OVERRIDES-}") diff --git a/scraper-go/internal/config/provider_schedule.go b/scraper-go/internal/config/provider_schedule.go new file mode 100644 index 00000000..fd8c311b --- /dev/null +++ b/scraper-go/internal/config/provider_schedule.go @@ -0,0 +1,160 @@ +package config + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "time" + _ "time/tzdata" // Runtime images need not install the IANA timezone database. + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" +) + +const ProviderWindowsEnv = "SCRAPER_PROVIDER_WINDOWS" +const ProviderScheduleTimezoneEnv = "SCRAPER_SCHEDULE_TIMEZONE" +const ProviderSchedulePreset = "pav178" +const ProviderScheduleTimezone = "America/Sao_Paulo" + +// ProviderWindow uses civil minutes in Timezone. An empty timezone retains UTC. +// Overnight windows must be split explicitly. ID is stable across config ordering. +type ProviderWindow struct { + Provider ports.ProviderID `json:"provider"` + Start int `json:"-"` + End int `json:"-"` + ID string `json:"id"` + Timezone string `json:"timezone"` + // Only the PAV-178 preset enables fixed groups. Custom and UTC schedules + // retain their existing batching behavior, even if they use the same IDs. + GreenhouseFixedGroup bool `json:"-"` + FullKeywordCoverage bool `json:"-"` + GupyWindow bool `json:"-"` +} + +func (w ProviderWindow) WindowID() string { + if w.ID != "" { + return w.ID + } + return fmt.Sprintf("%s-%02d%02d", w.Provider, w.Start/60, w.Start%60) +} + +func (w ProviderWindow) Location() *time.Location { + if w.Timezone == "" || w.Timezone == "UTC" { + return time.UTC + } + loc, err := time.LoadLocation(w.Timezone) + if err != nil { + panic("provider window timezone was not validated: " + w.Timezone) + } + return loc +} + +// Preserve legacy UTC response fields and expose unambiguous local fields. +func (w ProviderWindow) MarshalJSON() ([]byte, error) { + value := map[string]any{"provider": w.Provider, "id": w.WindowID(), "timezone": w.Location().String()} + if w.Location() == time.UTC { + value["startMinuteUTC"], value["endMinuteUTC"] = w.Start, w.End + } else { + value["startMinuteLocal"], value["endMinuteLocal"] = w.Start, w.End + } + return json.Marshal(value) +} + +func ParseProviderWindows(value string) ([]ProviderWindow, error) { + return ParseProviderWindowsInTimezone(value, "") +} + +// Custom syntax: [window-id@]provider=HH:MM-HH:MM. The preset is opt-in; +// neither an empty configuration nor this parser enables the scheduler. +func ParseProviderWindowsInTimezone(value, timezone string) ([]ProviderWindow, error) { + preset := strings.TrimSpace(value) == ProviderSchedulePreset + if preset { + if timezone != "" && timezone != ProviderScheduleTimezone { + return nil, fmt.Errorf("%s: preset requires %s", ProviderScheduleTimezoneEnv, ProviderScheduleTimezone) + } + timezone = ProviderScheduleTimezone + value = strings.Join([]string{ + "greenhouse-1@greenhouse=00:00-01:00", "gupy-1@gupy=01:00-02:00", + "inhire-1@inhire=02:00-03:00", "linkedin-1@linkedin=03:00-04:00", + "adzuna-1@adzuna=04:00-05:00", "lever-1@lever=05:00-06:00", + "greenhouse-2@greenhouse=06:00-07:00", "gupy-2@gupy=07:00-08:00", + "inhire-2@inhire=08:00-09:00", "linkedin-2@linkedin=09:00-10:00", + "themuse-1@themuse=10:00-11:00", "greenhouse-3@greenhouse=11:00-12:00", + "gupy-3@gupy=12:00-13:00", "inhire-3@inhire=13:00-14:00", + "adzuna-2@adzuna=14:00-15:00", "lever-2@lever=15:00-16:00", + "linkedin-3@linkedin=16:00-17:00", "greenhouse-4@greenhouse=17:00-18:00", + "gupy-4@gupy=18:00-19:00", "inhire-4@inhire=19:00-20:00", + "jooble-1@jooble=20:00-21:00", + }, ",") + } + if timezone == "" { + timezone = "UTC" + } + if _, err := time.LoadLocation(timezone); err != nil { + return nil, fmt.Errorf("%s: %w", ProviderScheduleTimezoneEnv, err) + } + if strings.TrimSpace(value) == "" { + return nil, nil + } + var windows []ProviderWindow + for _, entry := range strings.Split(value, ",") { + parts := strings.Split(strings.TrimSpace(entry), "=") + if len(parts) != 2 { + return nil, fmt.Errorf("%s: expected provider=HH:MM-HH:MM", ProviderWindowsEnv) + } + identity := strings.Split(strings.TrimSpace(parts[0]), "@") + if len(identity) > 2 { + return nil, fmt.Errorf("%s: invalid window identity", ProviderWindowsEnv) + } + id := "" + if len(identity) == 2 { + id = identity[0] + if !windowIDPattern.MatchString(id) { + return nil, fmt.Errorf("%s: invalid window ID %q", ProviderWindowsEnv, id) + } + } + provider, known := ports.ParseProviderID(identity[len(identity)-1]) + bounds := strings.Split(parts[1], "-") + if !known || len(bounds) != 2 { + return nil, fmt.Errorf("%s: invalid provider/window %q", ProviderWindowsEnv, entry) + } + parse := func(s string) (int, error) { + if s == "24:00" { + return 1440, nil + } + t, err := time.Parse("15:04", s) + if err != nil { + return 0, err + } + return t.Hour()*60 + t.Minute(), nil + } + start, e1 := parse(bounds[0]) + end, e2 := parse(bounds[1]) + if e1 != nil || e2 != nil || start >= end || start >= 1440 { + return nil, fmt.Errorf("%s: invalid bounds %q", ProviderWindowsEnv, entry) + } + windows = append(windows, ProviderWindow{ + Provider: provider, Start: start, End: end, ID: id, Timezone: timezone, + GreenhouseFixedGroup: preset && provider == ports.ProviderGreenhouse, + FullKeywordCoverage: preset && (provider == ports.ProviderLinkedIn || provider == ports.ProviderAdzuna), + GupyWindow: preset && provider == ports.ProviderGupy, + }) + } + ids := make(map[string]bool) + for _, w := range windows { + if ids[w.WindowID()] { + return nil, fmt.Errorf("%s: duplicate window ID %q", ProviderWindowsEnv, w.WindowID()) + } + ids[w.WindowID()] = true + } + sort.Slice(windows, func(i, j int) bool { return windows[i].Start < windows[j].Start }) + for i := 1; i < len(windows); i++ { + if windows[i].Start < windows[i-1].End { + return nil, fmt.Errorf("%s: overlapping windows", ProviderWindowsEnv) + } + } + return windows, nil +} + +var windowIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,63}$`) diff --git a/scraper-go/internal/config/provider_schedule_p0_test.go b/scraper-go/internal/config/provider_schedule_p0_test.go new file mode 100644 index 00000000..e4531f25 --- /dev/null +++ b/scraper-go/internal/config/provider_schedule_p0_test.go @@ -0,0 +1,81 @@ +package config + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestProviderSchedulePresetHas21IndependentWindows(t *testing.T) { + windows, err := ParseProviderWindows(ProviderSchedulePreset) + require.NoError(t, err) + require.Len(t, windows, 21) + counts := map[string]int{} + ids := map[string]bool{} + expected := []string{"greenhouse", "gupy", "inhire", "linkedin", "adzuna", "lever", "greenhouse", "gupy", "inhire", "linkedin", "themuse", "greenhouse", "gupy", "inhire", "adzuna", "lever", "linkedin", "greenhouse", "gupy", "inhire", "jooble"} + for i, w := range windows { + require.Equal(t, expected[i], string(w.Provider)) + require.Equal(t, i*60, w.Start) + require.Equal(t, (i+1)*60, w.End) + require.Equal(t, ProviderScheduleTimezone, w.Location().String()) + require.Equal(t, string(w.Provider) == "greenhouse", w.GreenhouseFixedGroup) + require.Equal(t, string(w.Provider) == "linkedin" || string(w.Provider) == "adzuna", w.FullKeywordCoverage) + require.Equal(t, string(w.Provider) == "gupy", w.GupyWindow) + require.False(t, ids[w.WindowID()]) + ids[w.WindowID()] = true + counts[string(w.Provider)]++ + } + require.Equal(t, map[string]int{"greenhouse": 4, "gupy": 4, "inhire": 4, "linkedin": 3, "adzuna": 2, "lever": 2, "themuse": 1, "jooble": 1}, counts) + values := map[string]string{ProviderWindowsEnv: ProviderSchedulePreset} + cfg, err := LoadRuntimeConfigFromLookup(func(k string) (string, bool) { v, ok := values[k]; return v, ok }) + require.NoError(t, err) + require.Equal(t, windows, cfg.ProviderWindows) + defaults, err := LoadRuntimeConfigFromLookup(func(string) (string, bool) { return "", false }) + require.NoError(t, err) + require.Empty(t, defaults.ProviderWindows) + custom, err := ParseProviderWindowsInTimezone("greenhouse-1@greenhouse=00:00-01:00", ProviderScheduleTimezone) + require.NoError(t, err) + require.False(t, custom[0].GreenhouseFixedGroup) + custom, err = ParseProviderWindowsInTimezone("linkedin-1@linkedin=03:00-04:00", ProviderScheduleTimezone) + require.NoError(t, err) + require.False(t, custom[0].FullKeywordCoverage) +} + +func TestProviderWindowIDsAreStableAndTimezoneIsExplicit(t *testing.T) { + a, err := ParseProviderWindows("gupy=01:00-02:00,gupy=03:00-04:00") + require.NoError(t, err) + b, err := ParseProviderWindows("gupy=03:00-04:00,gupy=01:00-02:00") + require.NoError(t, err) + require.Equal(t, a, b) + require.NotEqual(t, a[0].WindowID(), a[1].WindowID()) + local, err := ParseProviderWindowsInTimezone("gupy-1@gupy=23:00-24:00", ProviderScheduleTimezone) + require.NoError(t, err) + require.Equal(t, "gupy-1", local[0].WindowID()) + utcJSON, err := json.Marshal(a[0]) + require.NoError(t, err) + require.Contains(t, string(utcJSON), `"startMinuteUTC":60`) + localJSON, err := json.Marshal(local[0]) + require.NoError(t, err) + require.Contains(t, string(localJSON), `"startMinuteLocal":1380`) + require.NotContains(t, string(localJSON), "startMinuteUTC") +} + +func TestProviderScheduleRejectsDuplicateIDsInvalidTimezoneAndOverlap(t *testing.T) { + for _, value := range []string{ + "same@gupy=01:00-02:00,same@linkedin=03:00-04:00", + "@gupy=01:00-02:00", "bad/id@gupy=01:00-02:00", + "a@b@gupy=01:00-02:00", "a@gupy=01:00-03:00,b@gupy=02:00-04:00", + } { + _, err := ParseProviderWindows(value) + require.Error(t, err, value) + } + _, err := ParseProviderWindowsInTimezone("gupy=01:00-02:00", "Invalid/Zone") + require.Error(t, err) + _, err = ParseProviderWindowsInTimezone(ProviderSchedulePreset, "UTC") + require.Error(t, err) + values := map[string]string{ProviderWindowsEnv: "gupy=01:00-02:00", ProviderScheduleTimezoneEnv: ProviderScheduleTimezone} + cfg, err := LoadRuntimeConfigFromLookup(func(k string) (string, bool) { v, ok := values[k]; return v, ok }) + require.NoError(t, err) + require.Equal(t, ProviderScheduleTimezone, cfg.ProviderWindows[0].Timezone) +} diff --git a/scraper-go/internal/config/provider_schedule_test.go b/scraper-go/internal/config/provider_schedule_test.go new file mode 100644 index 00000000..a389654e --- /dev/null +++ b/scraper-go/internal/config/provider_schedule_test.go @@ -0,0 +1,34 @@ +package config + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestProviderWindowsRejectOverlapAndInvalidConfiguration(t *testing.T) { + for _, value := range []string{"gupy", "unknown=01:00-02:00", "gupy=bad-02:00", "gupy=02:00-01:00", "gupy=01:00-01:00", "gupy=24:00-24:00", "gupy=00:00-25:00", "gupy=01:00-03:00,linkedin=02:00-04:00", "gupy=00:00-01:00,"} { + _, err := ParseProviderWindows(value) + require.Error(t, err, value) + } + windows, err := ParseProviderWindows("greenhouse=23:00-24:00, LinkedIn=01:00-02:00,gupy=02:00-03:00") + require.NoError(t, err) + require.Len(t, windows, 3) + require.EqualValues(t, "linkedin", windows[0].Provider) + require.Equal(t, 1440, windows[2].End) + empty, err := ParseProviderWindows("") + require.NoError(t, err) + require.Empty(t, empty) +} + +func TestProviderWindowEnvironmentAndBatchLimits(t *testing.T) { + values := map[string]string{ProviderWindowsEnv: "greenhouse=01:00-02:00", "SCRAPER_GREENHOUSE_BATCH_SIZE": "20"} + cfg, err := LoadRuntimeConfigFromLookup(func(k string) (string, bool) { v, ok := values[k]; return v, ok }) + require.NoError(t, err) + require.Len(t, cfg.ProviderWindows, 1) + require.Equal(t, 20, cfg.GreenhouseBatchSize) + for _, value := range []string{"0", "1001", "bad"} { + values["SCRAPER_GREENHOUSE_BATCH_SIZE"] = value + _, err = LoadRuntimeConfigFromLookup(func(k string) (string, bool) { v, ok := values[k]; return v, ok }) + require.Error(t, err) + } +} diff --git a/scraper-go/internal/config/scrape_timeout_test.go b/scraper-go/internal/config/scrape_timeout_test.go new file mode 100644 index 00000000..48647768 --- /dev/null +++ b/scraper-go/internal/config/scrape_timeout_test.go @@ -0,0 +1,23 @@ +package config + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestScrapeTimeoutDefaultAndOverride(t *testing.T) { + cfg, err := LoadRuntimeConfigFromLookup(func(string) (string, bool) { return "", false }) + require.NoError(t, err) + require.Equal(t, 50*time.Minute, cfg.ScrapeTimeout) + for value, want := range map[string]time.Duration{"300s": 5 * time.Minute, " 2m ": 2 * time.Minute} { + cfg, err = LoadRuntimeConfigFromLookup(func(key string) (string, bool) { return value, key == ScraperScrapeTimeoutEnv }) + require.NoError(t, err) + require.Equal(t, want, cfg.ScrapeTimeout) + } + for _, value := range []string{"", " ", "0s", "-1m", "300", "invalid", "999999999999999h"} { + _, err = LoadRuntimeConfigFromLookup(func(key string) (string, bool) { return value, key == ScraperScrapeTimeoutEnv }) + require.Error(t, err, value) + } +} diff --git a/scraper-go/internal/cronjob/collection_timeout_test.go b/scraper-go/internal/cronjob/collection_timeout_test.go new file mode 100644 index 00000000..1fcd66f3 --- /dev/null +++ b/scraper-go/internal/cronjob/collection_timeout_test.go @@ -0,0 +1,32 @@ +package cronjob + +import ( + "context" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/stretchr/testify/require" +) + +func TestCollectionTimeoutPersistsPartialJobsBeforeReleasingGlobalLock(t *testing.T) { + scheduler, _, client := newConcurrentTestScheduler(t) + scheduler.cfg.ScrapeTimeout = 20 * time.Millisecond + completed := false + scheduler.OnComplete = func([]string, int, int, time.Duration) { completed = true } + scheduler.runCron(context.Background()) + count, err := scheduler.jobStore.Count(context.Background()) + require.NoError(t, err) + require.Greater(t, count, int64(0)) + require.False(t, completed) + require.False(t, scheduler.IsRunning()) + summary := scheduler.LastRun() + require.NotNil(t, summary) + require.Equal(t, "partial", summary.Status) + require.Positive(t, summary.JobsSaved) + require.Positive(t, summary.JobsIndexed) + _, lastRunAt, jobs := scheduler.Snapshot() + require.False(t, lastRunAt.IsZero()) + require.Positive(t, jobs) + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) +} diff --git a/scraper-go/internal/cronjob/cronjob.go b/scraper-go/internal/cronjob/cronjob.go index 14928ae0..9d35747c 100644 --- a/scraper-go/internal/cronjob/cronjob.go +++ b/scraper-go/internal/cronjob/cronjob.go @@ -21,6 +21,8 @@ import ( ) type Config struct { + ProviderWindows []config.ProviderWindow + GreenhouseBatchSize int Interval time.Duration ScrapeTimeout time.Duration SearchLocation string @@ -37,8 +39,9 @@ type Config struct { func DefaultConfig() Config { return Config{ + GreenhouseBatchSize: 1000, Interval: 3 * time.Hour, - ScrapeTimeout: 40 * time.Minute, + ScrapeTimeout: config.DefaultScrapeTimeout, SearchLocation: "Brasil", JobTypes: "C,F", TimeFilter: "r604800", @@ -53,22 +56,25 @@ func DefaultConfig() Config { } type Scheduler struct { - cfg Config - kwStore *keywords.Store - jobStore *jobstore.Store - adapterList []ports.JobSource - rdb *redis.Client - runLock *runlock.Manager - BeforeRun func(context.Context) error - OnComplete func(keywords []string, scraped, saved int, duration time.Duration) - mu sync.Mutex - running bool - stopped bool - lastRunAt time.Time - lastJobs int - stop chan struct{} - stopOnce sync.Once - active sync.WaitGroup + admissionClock func() time.Time + failedAdmissions map[string]string + lastSummary *RunSummary + cfg Config + kwStore *keywords.Store + jobStore *jobstore.Store + adapterList []ports.JobSource + rdb *redis.Client + runLock *runlock.Manager + BeforeRun func(context.Context) error + OnComplete func(keywords []string, scraped, saved int, duration time.Duration) + mu sync.Mutex + running bool + stopped bool + lastRunAt time.Time + lastJobs int + stop chan struct{} + stopOnce sync.Once + active sync.WaitGroup } func New( @@ -92,6 +98,11 @@ func New( } func (s *Scheduler) Start(ctx context.Context) { + if len(s.cfg.ProviderWindows) > 0 { + metrics.CronInterval.Set((24 * time.Hour).Seconds()) + go s.startProviderWindows(ctx) + return + } metrics.CronInterval.Set(s.cfg.Interval.Seconds()) slog.Info("cronjob: scheduler iniciado", "interval", s.cfg.Interval) @@ -147,6 +158,17 @@ func (s *Scheduler) Shutdown(ctx context.Context) error { } func (s *Scheduler) RunNow(ctx context.Context) error { + return s.RunProvidersNow(ctx, nil) +} + +func (s *Scheduler) RunProvidersNow(ctx context.Context, sources []string) error { + if _, err := pipeline.SelectProviders(s.adapterList, sources); err != nil { + return err + } + sources = append([]string(nil), sources...) + if err := s.checkProviderDelay(ctx, sources); err != nil { + return err + } lease, err := s.acquire(ctx, "admin_manual") if err != nil { return err @@ -154,10 +176,9 @@ func (s *Scheduler) RunNow(ctx context.Context) error { // Contabiliza a execução antes do yield da goroutine para o Shutdown não // retornar enquanto o lease ainda estiver ativo. - s.active.Add(1) go func() { defer s.active.Done() - if runErr := s.runWithLease(lease); runErr != nil { + if runErr := s.runSelectedWithLease(lease, sources, s.cfg.ScrapeTimeout); runErr != nil { slog.Error("cronjob: execução manual falhou", "source", "admin_manual", "error", runErr, @@ -198,7 +219,6 @@ func (s *Scheduler) runCron(ctx context.Context) { return } - s.active.Add(1) defer s.active.Done() if err := s.runWithLease(lease); err != nil { slog.Error("cronjob: execução falhou", "source", "cron", "error", err) @@ -208,18 +228,38 @@ func (s *Scheduler) runCron(ctx context.Context) { func (s *Scheduler) acquire(ctx context.Context, source string) (*runlock.Lease, error) { s.mu.Lock() stopped := s.stopped + if !stopped { + s.active.Add(1) + } s.mu.Unlock() if stopped { return nil, fmt.Errorf("%w: scheduler is shutting down", runlock.ErrUnavailable) } if s.runLock == nil { + s.active.Done() return nil, fmt.Errorf("%w: lock service is not configured", runlock.ErrUnavailable) } - return s.runLock.Acquire(ctx, source) + lease, err := s.runLock.Acquire(ctx, source) + if err != nil { + s.active.Done() + } + return lease, err } func (s *Scheduler) runWithLease(lease *runlock.Lease) (runErr error) { - defer func() { metrics.FinishRun(runErr, lease.RunID()) }() + return s.runSelectedWithLease(lease, nil, s.cfg.ScrapeTimeout) +} + +func (s *Scheduler) runSelectedWithLease(lease *runlock.Lease, sources []string, timeout time.Duration) (runErr error) { + return s.runSelectedWindowWithLease(lease, sources, timeout, nil) +} + +func (s *Scheduler) runSelectedWindowWithLease(lease *runlock.Lease, sources []string, timeout time.Duration, window *config.ProviderWindow) (runErr error) { + var stats pipeline.ProcessStats + defer func() { + s.recordResult(lease.RunID(), stats, runErr, sources) + metrics.FinishRun(runErr, lease.RunID()) + }() s.mu.Lock() s.running = true s.mu.Unlock() @@ -239,7 +279,11 @@ func (s *Scheduler) runWithLease(lease *runlock.Lease) (runErr error) { }() start := time.Now() - scrapeCtx, cancel := context.WithTimeout(lease.Context(), s.cfg.ScrapeTimeout) + if err := s.checkProviderDelay(lease.Context(), sources); err != nil { + return err + } + collectionDeadline := lease.CollectionDeadline(timeout) + scrapeCtx, cancel := context.WithDeadline(lease.Context(), collectionDeadline) defer cancel() if s.BeforeRun != nil { @@ -260,6 +304,31 @@ func (s *Scheduler) runWithLease(lease *runlock.Lease) (runErr error) { } config := s.searchConfig(kws, lease.RunID()) + config.Sources = sources + config.CollectionTimeout = timeout + config.CollectionDeadline = collectionDeadline + if window != nil { + config.FullKeywordCoverage = window.FullKeywordCoverage + config.WindowID = window.WindowID() + config.GupyWindow = window.GupyWindow + if window.GupyWindow { + config.QueryCursor = &gupyCursor{scheduler: s, lease: lease} + } + } + var adapters []ports.JobSource + var checkpoint func(*runlock.Lease) error + if window != nil && window.GreenhouseFixedGroup { + if len(sources) != 1 || sources[0] != string(ports.ProviderGreenhouse) || window.Provider != ports.ProviderGreenhouse { + return fmt.Errorf("fixed greenhouse group requires exclusive greenhouse selection") + } + adapters, err = s.greenhouseFixedBatch(scrapeCtx, lease, window.WindowID()) + } else { + adapters, checkpoint, err = s.providerBatch(scrapeCtx, sources) + } + if err != nil { + slog.Error("cronjob: adapter selection failed", "run_id", lease.RunID(), "sources", sources, "error", err) + return err + } slog.Info("scraper execução iniciada", "source", lease.State().Source, "run_id", lease.RunID(), @@ -267,17 +336,26 @@ func (s *Scheduler) runWithLease(lease *runlock.Lease) (runErr error) { "max_concurrency_effective", config.MaxConcurrency, "provider_max_concurrency", config.ProviderMaxConcurrency, "keywords", len(kws), - "adapters", len(s.adapterList), + "adapters", len(adapters), + "sources", sources, ) - jobs, stats, err := pipeline.ScrapeAllSources(scrapeCtx, config, s.adapterList, s.rdb) + jobs, stats, err := pipeline.ScrapeAllSources(lease.Context(), config, adapters, s.rdb) + if retryErr := s.rememberRetryAfter(lease, sources, stats.CollectionError); retryErr != nil { + err = errors.Join(err, retryErr) + } if err != nil { slog.Error("cronjob: scrape falhou", "error", err) return fmt.Errorf("cronjob: scrape: %w", err) } - if err := executionError(scrapeCtx); err != nil { + if err := executionError(lease.Context()); err != nil { return err } + if checkpoint != nil { + if err := checkpoint(lease); err != nil { + return err + } + } scraped := stats.Received saved := stats.Saved() @@ -305,8 +383,42 @@ func (s *Scheduler) runWithLease(lease *runlock.Lease) (runErr error) { return nil } +type RunSummary struct { + Providers []string `json:"providers"` + RunID string `json:"runId"` + Status string `json:"status"` + FinishedAt time.Time `json:"finishedAt"` + JobsSaved int `json:"jobsSaved"` + JobsIndexed int `json:"jobsIndexed"` + ErrorType string `json:"errorType"` +} + +func (s *Scheduler) recordResult(runID string, stats pipeline.ProcessStats, err error, sources []string) { + status := metrics.Status(err) + if err != nil && stats.Saved() > 0 { + status = "partial" + } + now := time.Now().UTC() + s.mu.Lock() + defer s.mu.Unlock() + s.lastRunAt, s.lastJobs = now, stats.Saved() + s.lastSummary = &RunSummary{Providers: append([]string(nil), sources...), RunID: runID, Status: status, FinishedAt: now, JobsSaved: stats.Saved(), JobsIndexed: stats.Indexed, ErrorType: metrics.ErrorType(err)} +} + +func (s *Scheduler) LastRun() *RunSummary { + s.mu.Lock() + defer s.mu.Unlock() + if s.lastSummary == nil { + return nil + } + copy := *s.lastSummary + copy.Providers = append([]string(nil), s.lastSummary.Providers...) + return © +} + func (s *Scheduler) searchConfig(kws []string, runID string) pipeline.SearchConfig { return pipeline.SearchConfig{ + CollectionTimeout: s.cfg.ScrapeTimeout, Store: s.jobStore, Keywords: kws, SearchLocation: s.cfg.SearchLocation, diff --git a/scraper-go/internal/cronjob/cronjob_test.go b/scraper-go/internal/cronjob/cronjob_test.go index 209a87a8..e19210db 100644 --- a/scraper-go/internal/cronjob/cronjob_test.go +++ b/scraper-go/internal/cronjob/cronjob_test.go @@ -273,7 +273,7 @@ func newConcurrentTestScheduler(t *testing.T) (*Scheduler, *blockingAdapter, *re client := redis.NewClient(&redis.Options{Addr: server.Addr()}) t.Cleanup(func() { _ = client.Close() }) - lockManager, err := runlock.New(runlock.NewValkeyStore(client), runlock.Config{ + lockManager, err := runlock.New(runlock.NewValkeyStore(client, 0), runlock.Config{ TTL: 2 * time.Second, RenewInterval: 500 * time.Millisecond, }) diff --git a/scraper-go/internal/cronjob/greenhouse_manifest.go b/scraper-go/internal/cronjob/greenhouse_manifest.go new file mode 100644 index 00000000..e4d5e0ca --- /dev/null +++ b/scraper-go/internal/cronjob/greenhouse_manifest.go @@ -0,0 +1,135 @@ +package cronjob + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "log/slog" + "strings" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/redis/go-redis/v9" +) + +// The reviewed manifest is the persistent assignment, not a slice recomputed +// from the live registry. Catalog drift fails closed. To add/remove boards, +// edit explicit memberships, bump the version and review the diff. Preserve +// every retained board's group; never regenerate all groups from a new sort. +// A version already pinned in Valkey is immutable, including across replicas. +// +//go:embed greenhouse_manifest_v1.json +var greenhouseManifestJSON []byte + +type greenhouseManifest struct { + Version string `json:"version"` + Groups map[string][]string `json:"groups"` +} + +func parseGreenhouseManifest(data []byte) (greenhouseManifest, string, error) { + var m greenhouseManifest + if err := json.Unmarshal(data, &m); err != nil { + return m, "", fmt.Errorf("greenhouse manifest: %w", err) + } + if !strings.HasPrefix(m.Version, "greenhouse-v") || len(m.Version) <= len("greenhouse-v") || len(m.Groups) != 4 { + return m, "", fmt.Errorf("invalid greenhouse manifest version/groups") + } + seen := map[string]bool{} + for i := 1; i <= 4; i++ { + group, ok := m.Groups[fmt.Sprintf("greenhouse-%d", i)] + if !ok { + return m, "", fmt.Errorf("missing greenhouse group %d", i) + } + for _, id := range group { + if id == "" || id != strings.TrimSpace(id) || seen[id] { + return m, "", fmt.Errorf("invalid/duplicate greenhouse board identity %q", id) + } + seen[id] = true + } + } + if len(seen) == 0 { + return m, "", fmt.Errorf("empty greenhouse manifest") + } + // Canonical JSON avoids treating formatting or map order as a migration. + canonical, _ := json.Marshal(m) + digest := sha256.Sum256(canonical) + return m, hex.EncodeToString(digest[:]), nil +} + +func (m greenhouseManifest) selectGroup(sources []ports.JobSource, windowID string) ([]ports.JobSource, error) { + group, ok := m.Groups[windowID] + if !ok { + return nil, fmt.Errorf("unknown greenhouse group %q", windowID) + } + byID := map[string]ports.JobSource{} + for _, source := range sources { + if ports.CapabilitiesOf(source).Provider != ports.ProviderGreenhouse { + continue + } + stable, ok := source.(ports.StableJobSource) + if !ok || stable.SourceID() == "" { + return nil, fmt.Errorf("greenhouse adapter lacks stable board identity") + } + id := stable.SourceID() + if _, duplicate := byID[id]; duplicate { + return nil, fmt.Errorf("duplicate greenhouse adapter identity %q", id) + } + byID[id] = source + } + // Validate the complete catalog before starting any group. A removal is not + // silently filled by another company, and an addition is never auto-assigned. + count := 0 + for _, ids := range m.Groups { + for _, id := range ids { + if _, ok := byID[id]; !ok { + return nil, fmt.Errorf("greenhouse catalog drift: missing board %q; review manifest %s", id, m.Version) + } + count++ + } + } + if len(byID) != count { + return nil, fmt.Errorf("greenhouse catalog drift: unassigned boards; review manifest %s", m.Version) + } + batch := make([]ports.JobSource, 0, len(group)) + for _, id := range group { + batch = append(batch, byID[id]) + } + return batch, nil +} + +var pinGreenhouseManifest = redis.NewScript(` +if redis.call('GET',KEYS[1]) ~= ARGV[1] then return -1 end +local current = redis.call('GET',KEYS[2]) +if current and current ~= ARGV[2] then return 0 end +if not current then redis.call('SET',KEYS[2],ARGV[2]) end +return 1 +`) + +func (s *Scheduler) greenhouseFixedBatch(ctx context.Context, lease *runlock.Lease, windowID string) ([]ports.JobSource, error) { + manifest, digest, err := parseGreenhouseManifest(greenhouseManifestJSON) + if err != nil { + return nil, err + } + batch, err := manifest.selectGroup(s.adapterList, windowID) + if err != nil { + return nil, err + } + if s.rdb == nil { + return nil, ErrLockUnavailable + } + written, err := pinGreenhouseManifest.Run(ctx, s.rdb, []string{runlock.LockKey, "scraper:provider:greenhouse:manifest:" + manifest.Version}, lease.Token(), digest).Int() + if err != nil { + return nil, fmt.Errorf("pin greenhouse manifest: %w", err) + } + if written == -1 { + return nil, fmt.Errorf("%w: greenhouse manifest ownership changed", runlock.ErrLost) + } + if written != 1 { + return nil, fmt.Errorf("greenhouse manifest %s changed without a version migration", manifest.Version) + } + slog.Info("greenhouse fixed group selected", "manifest_version", manifest.Version, "manifest_hash", digest, "window_id", windowID, "adapters", len(batch), "run_id", lease.RunID()) + return batch, nil +} diff --git a/scraper-go/internal/cronjob/greenhouse_manifest_test.go b/scraper-go/internal/cronjob/greenhouse_manifest_test.go new file mode 100644 index 00000000..964d850e --- /dev/null +++ b/scraper-go/internal/cronjob/greenhouse_manifest_test.go @@ -0,0 +1,267 @@ +package cronjob + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/stretchr/testify/require" +) + +type fixedGroupTestSource struct { + id string + search func(context.Context, string) ([]domain.Job, error) +} + +func (a fixedGroupTestSource) SourceID() string { return a.id } +func (a fixedGroupTestSource) SourceName() string { return "same display name for every company" } +func (a fixedGroupTestSource) Capabilities() ports.SourceCapabilities { + return ports.SourceCapabilities{Provider: ports.ProviderGreenhouse, Mode: ports.DiscoveryKeyword} +} +func (a fixedGroupTestSource) Search(ctx context.Context, _ string, _ domain.ScrapeRequest) ([]domain.Job, error) { + if a.search != nil { + return a.search(ctx, a.id) + } + return nil, nil +} + +func fixedGroupTestManifest(t *testing.T) greenhouseManifest { + t.Helper() + m, _, err := parseGreenhouseManifest(greenhouseManifestJSON) + require.NoError(t, err) + return m +} +func fixedGroupTestRegistry(t *testing.T, search func(context.Context, string) ([]domain.Job, error)) []ports.JobSource { + t.Helper() + m := fixedGroupTestManifest(t) + var sources []ports.JobSource + for i := 1; i <= 4; i++ { + for _, id := range m.Groups[fmt.Sprintf("greenhouse-%d", i)] { + sources = append(sources, fixedGroupTestSource{id: id, search: search}) + } + } + return sources +} + +func TestGreenhouseFixedManifestCoversCatalogOnceAndSurvivesReordering(t *testing.T) { + m := fixedGroupTestManifest(t) + raw, err := os.ReadFile("../interfaces/greenhouseCompanies.json") + require.NoError(t, err) + var catalog []string + require.NoError(t, json.Unmarshal(raw, &catalog)) + sources := fixedGroupTestRegistry(t, nil) + seen := map[string]bool{} + counts := []int{850, 850, 850, 789} + for i, count := range counts { + id := fmt.Sprintf("greenhouse-%d", i+1) + batch, err := m.selectGroup(sources, id) + require.NoError(t, err) + require.Len(t, batch, count) + for _, source := range batch { + board := source.(ports.StableJobSource).SourceID() + require.False(t, seen[board], board) + seen[board] = true + } + // New process, reordered adapters and identical display names preserve IDs. + restarted := fixedGroupTestManifest(t) + sort.Slice(sources, func(a, b int) bool { + return sources[a].(ports.StableJobSource).SourceID() > sources[b].(ports.StableJobSource).SourceID() + }) + other, err := restarted.selectGroup(sources, id) + require.NoError(t, err) + require.Equal(t, batch, other) + } + require.Len(t, seen, 3339) + require.Len(t, catalog, len(seen)) + for _, id := range catalog { + require.True(t, seen[id], id) + } +} + +func TestGreenhouseFixedManifestRejectsCatalogDriftAndInvalidIdentities(t *testing.T) { + m := fixedGroupTestManifest(t) + sources := fixedGroupTestRegistry(t, nil) + for _, changed := range [][]ports.JobSource{ + sources[1:], + append(append([]ports.JobSource(nil), sources...), fixedGroupTestSource{id: "new-unassigned-company"}), + append(append([]ports.JobSource(nil), sources...), sources[0]), + append([]ports.JobSource{dispatchSource{provider: ports.ProviderGreenhouse}}, sources[1:]...), + } { + _, err := m.selectGroup(changed, "greenhouse-1") + require.Error(t, err) + } + _, err := m.selectGroup(sources, "greenhouse-5") + require.Error(t, err) + for _, raw := range []string{ + `{}`, `{"version":"greenhouse-v1","groups":{"greenhouse-1":["a"],"greenhouse-2":["a"],"greenhouse-3":[],"greenhouse-4":[]}}`, + `{"version":"greenhouse-v1","groups":{"greenhouse-1":[" "],"greenhouse-2":[],"greenhouse-3":[],"greenhouse-4":[]}}`, + } { + _, _, err := parseGreenhouseManifest([]byte(raw)) + require.Error(t, err) + } + // An explicitly reviewed new manifest can have another catalog size. + updated := greenhouseManifest{Version: "greenhouse-v2", Groups: map[string][]string{"greenhouse-1": {"a"}, "greenhouse-2": {"b"}, "greenhouse-3": {"c"}, "greenhouse-4": {"d", "e"}}} + raw, err := json.Marshal(updated) + require.NoError(t, err) + parsed, _, err := parseGreenhouseManifest(raw) + require.NoError(t, err) + var next []ports.JobSource + for _, id := range []string{"a", "b", "c", "d", "e"} { + next = append(next, fixedGroupTestSource{id: id}) + } + batch, err := parsed.selectGroup(next, "greenhouse-4") + require.NoError(t, err) + require.Len(t, batch, 2) +} + +func TestGreenhouseFixedManifestPinIsImmutableAndFenced(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + s.adapterList = fixedGroupTestRegistry(t, nil) + lease, err := s.runLock.Acquire(context.Background(), "cron") + require.NoError(t, err) + t.Cleanup(func() { _ = lease.Release(context.Background()) }) + batch, err := s.greenhouseFixedBatch(context.Background(), lease, "greenhouse-1") + require.NoError(t, err) + require.Len(t, batch, 850) + m, digest, err := parseGreenhouseManifest(greenhouseManifestJSON) + require.NoError(t, err) + key := "scraper:provider:greenhouse:manifest:" + m.Version + require.Equal(t, digest, client.Get(context.Background(), key).Val()) + restarted := New(s.cfg, s.kwStore, s.jobStore, s.adapterList, client, s.runLock) + _, err = restarted.greenhouseFixedBatch(context.Background(), lease, "greenhouse-2") + require.NoError(t, err) + require.NoError(t, client.Set(context.Background(), key, "different-content-same-version", 0).Err()) + _, err = s.greenhouseFixedBatch(context.Background(), lease, "greenhouse-1") + require.ErrorContains(t, err, "version migration") + require.NoError(t, client.Set(context.Background(), key, digest, 0).Err()) + require.NoError(t, client.Set(context.Background(), runlock.LockKey, "replacement-owner", time.Minute).Err()) + _, err = s.greenhouseFixedBatch(context.Background(), lease, "greenhouse-1") + require.ErrorIs(t, err, runlock.ErrLost) + require.Equal(t, digest, client.Get(context.Background(), key).Val()) +} + +func TestGreenhouseFixedGroupsPreservePartialJobsAndNeverTransferFailures(t *testing.T) { + for _, failure := range []string{"adapter-error", "timeout"} { + t.Run(failure, func(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + m := fixedGroupTestManifest(t) + first := m.Groups["greenhouse-1"][0] + last := m.Groups["greenhouse-1"][849] + visits := map[string]int{} + var mu sync.Mutex + var failing atomic.Bool + failing.Store(true) + s.cfg.ScrapeTimeout = 300 * time.Millisecond + s.cfg.ClassificationBatchSize = 1 + s.cfg.PersistBatchSize = 1 + s.cfg.IndexBatchSize = 1 + s.adapterList = fixedGroupTestRegistry(t, func(ctx context.Context, id string) ([]domain.Job, error) { + mu.Lock() + visits[id]++ + mu.Unlock() + if id != first { + return nil, nil + } + job := schedulerTestJob() + job.Source = "greenhouse" + if !failing.Load() { + return []domain.Job{job}, nil + } + if failure == "timeout" { + <-ctx.Done() + return []domain.Job{job}, ctx.Err() + } + return []domain.Job{job}, errors.New("board unavailable") + }) + windows, err := config.ParseProviderWindows(config.ProviderSchedulePreset) + require.NoError(t, err) + s.cfg.ProviderWindows = windows + require.NoError(t, client.Set(context.Background(), greenhouseCheckpointKey, "legacy-state-must-not-change", 0).Err()) + day := time.Now().In(mustProviderLocation(t)).AddDate(0, 0, 1) + start, _ := windowBounds(day, windows[0]) + require.Error(t, s.dispatchProviderWindow(context.Background(), start)) + require.Equal(t, "partial", s.LastRun().Status) + require.Positive(t, s.LastRun().JobsSaved) + require.Positive(t, s.LastRun().JobsIndexed) + mu.Lock() + before := visits[first] + if failure == "timeout" { + require.Zero(t, visits[last], "unstarted board must remain in its own group") + } + mu.Unlock() + require.Equal(t, 1, before) + // The consumed occurrence is not replayed, including after restart. + restarted := New(s.cfg, s.kwStore, s.jobStore, s.adapterList, client, s.runLock) + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), start)) + second := windows[6] // greenhouse-2 at 06:00 local. + require.Equal(t, "greenhouse-2", second.WindowID()) + secondStart, _ := windowBounds(day, second) + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), secondStart)) + mu.Lock() + require.Equal(t, 1, visits[first]) + for _, id := range m.Groups["greenhouse-2"] { + require.Equal(t, 1, visits[id], id) + } + for _, id := range m.Groups["greenhouse-3"] { + require.Zero(t, visits[id], id) + } + if failure == "timeout" { + require.Zero(t, visits[last], "next window must not receive unfinished boards") + } + mu.Unlock() + failing.Store(false) + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), start.AddDate(0, 0, 1))) + mu.Lock() + require.Equal(t, 2, visits[first]) + if failure == "timeout" { + require.Equal(t, 1, visits[last], "unfinished board returns in the same window on the next day") + } + for _, id := range m.Groups["greenhouse-2"] { + require.Equal(t, 1, visits[id]) + } + mu.Unlock() + require.Equal(t, "legacy-state-must-not-change", client.Get(context.Background(), greenhouseCheckpointKey).Val()) + count, err := s.jobStore.Count(context.Background()) + require.NoError(t, err) + require.Positive(t, count) + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) + }) + } +} + +func TestGreenhouseFixedPresetKeepsManualAndCustomWindowLegacyBatching(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + windows, err := config.ParseProviderWindows(config.ProviderSchedulePreset) + require.NoError(t, err) + s.cfg.ProviderWindows = windows + s.cfg.GreenhouseBatchSize = 2 + var calls atomic.Int32 + // Legacy sources intentionally have no stable-identity interface and a + // different catalog. Manual execution must still use the existing offset. + s.adapterList = []ports.JobSource{ + dispatchSource{provider: ports.ProviderGreenhouse, name: "a", calls: &calls}, + dispatchSource{provider: ports.ProviderGreenhouse, name: "b", calls: &calls}, + dispatchSource{provider: ports.ProviderGreenhouse, name: "c", calls: &calls}, + } + runOneProvider(t, s, "greenhouse") + require.EqualValues(t, 2, calls.Load()) + require.NotEmpty(t, client.Get(context.Background(), greenhouseCheckpointKey).Val()) + custom, err := config.ParseProviderWindowsInTimezone("greenhouse-1@greenhouse=00:00-24:00", config.ProviderScheduleTimezone) + require.NoError(t, err) + s.cfg.ProviderWindows = custom + start, _ := windowBounds(time.Now().UTC().Add(24*time.Hour), custom[0]) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start)) + require.EqualValues(t, 3, calls.Load()) + require.Empty(t, client.Keys(context.Background(), "scraper:provider:greenhouse:manifest:*").Val()) +} diff --git a/scraper-go/internal/cronjob/greenhouse_manifest_v1.json b/scraper-go/internal/cronjob/greenhouse_manifest_v1.json new file mode 100644 index 00000000..34227883 --- /dev/null +++ b/scraper-go/internal/cronjob/greenhouse_manifest_v1.json @@ -0,0 +1,3352 @@ +{ + "version": "greenhouse-v1", + "groups": { + "greenhouse-1": [ + "103644278", + "10alabs", + "10beauty", + "10xgenomics", + "12jlkfsk", + "12twenty", + "1456754456yhgbhfg", + "1800contacts", + "1800gotjunk", + "1pyra)mid_health&care", + "204951305985924", + "21shares", + "2k", + "2kearlycareers", + "2kmadrid", + "2u", + "2uinlinepromotions", + "31stunion", + "3cloud", + "3dayblindscorporate", + "3mgroofing", + "3redpartners", + "5364856uhdfnvbkldfnbhrpkdfgbdvtyhro", + "540", + "5minlab", + "5wpr", + "60decibelsinc", + "66degrees", + "6sense", + "7shifts", + "8451", + "8thlightrebuild", + "a3c41b8b71eff8c4", + "aacpnw", + "abacusfinancegroup", + "abacusinsights", + "abbyy", + "abcellera", + "abilitypath", + "abinbev", + "able", + "absherconstruction", + "absolutesecurityintl", + "absurdventures", + "accela", + "accelerationpartners", + "acceleronfusion", + "accelschools", + "accenturefederalservices", + "accesshealthca", + "accessholdingsmanagementfirm", + "accesso", + "accordion", + "accreditedlabs", + "accuweather", + "aceable", + "acentria", + "acilearning", + "aclu", + "acluinternships", + "aclunc", + "acog", + "acommerce", + "acornhealth", + "acquia", + "acrisureinnovation", + "acryldata", + "action", + "actpowerservices", + "acuitymd", + "acumen", + "acutechgroupinc", + "ada18", + "adamsclinical", + "adapter", + "addepar1", + "adelphigrouplimited", + "adelphiresearch", + "adfinternational", + "aditumbio", + "adjustjobs", + "admios", + "adswerveinc", + "advancedtechnologyservices", + "advertisingspecialtyinstitute", + "advocateconstruction", + "advocatesforchildrenofnewyork", + "adyen", + "aechelontechnology", + "aegisventures", + "aegworldwide", + "aerospike", + "aestudio", + "aevexaerospace", + "affinidi", + "affinitiv", + "affirm", + "affirmedrxpbc", + "afresh", + "ag1", + "agebold", + "agecareers", + "agency", + "agencywithin", + "ageoflearninginc", + "agibank", + "agilesix", + "agoda", + "ahrefsjobs", + "aift", + "air", + "aircapture", + "aircompany", + "airnorth", + "airsculpt", + "airship", + "airspace", + "airtable", + "airtamejobs", + "airtrunk", + "airwave", + "aizerhealth", + "ajboggs", + "akidolabs", + "akko", + "akuity", + "alabamatitleloansinc", + "alamarbiosciences", + "alarmcom", + "albertmackenziellp", + "alertmedia", + "algolia", + "align", + "align46", + "alkujobs", + "allbooked", + "allcareers", + "allencontrolsystems", + "alliancedefendingfreedom", + "alliedmaker", + "allinc", + "alltrna", + "allwebleads", + "alma31", + "aloyoga", + "alpaca", + "alpha9oncology", + "alphaalternatives", + "alphafmcroles", + "alphagrepsecurities", + "alphapublicschools", + "alphasense", + "alphasensehelsinki", + "alphasenseindia", + "alpineinternships", + "alpineinternships-private", + "alt", + "altanaai", + "altentechnologyusa", + "altium", + "altoslabs", + "altruist", + "altscore", + "alu", + "alumis", + "alumniventures", + "alveole", + "alxafrica", + "amaehealth", + "ambercharterschools", + "ambiqmicroinc", + "ameelio", + "amenitiz", + "americanfloodcoalition", + "americaninstitutesforresearch", + "amiralearning", + "amity", + "amount", + "amperesand", + "amperon", + "amplemarket", + "amplitude", + "ampm", + "ampsortation", + "amtechsoftware", + "amwell", + "amylyx", + "amyrisinc", + "analyst1", + "analyticservicesinc", + "anaplan", + "anchanto", + "andesite", + "andurilindustries", + "animalmedicalcenter", + "aninebing", + "annexonbioscience", + "anodize", + "antenna", + "anteriad", + "anteristech", + "anthropic", + "antora", + "aoti", + "apartmentlife", + "aperaaiinc", + "aperiasolutions", + "aperiatechnologies", + "apexcompanies", + "apexcompaniescsw", + "apexit", + "apiiro", + "apiphani", + "apogeetherapeutics", + "apolloio", + "apothecom", + "apparatus", + "appdirect", + "appfire", + "appian", + "appier", + "applovin", + "applytobambi", + "applytogreenspark", + "applytoslabstack", + "applytosuno", + "applytowhoosh", + "appnovation", + "appodeal", + "appomni", + "appspace", + "apptronik", + "aptoslabs", + "aputah", + "aquaticcapitalmanagement", + "aquia", + "arborenergy", + "arcadiacareers", + "arcaea", + "arcanaanalytics", + "arcboatcompany", + "arceeai", + "arcesiumllc", + "archer56", + "archera", + "arcinstitute", + "arcoeducacao", + "ardentmc", + "arenaclub", + "arenaim", + "arevonenergyimpltest", + "arielinvestments", + "arizeai", + "arkestroinc", + "arkoselabs", + "arlosolutionsllc", + "armada", + "armamentsresearchcompany", + "armissecurity", + "armorcode", + "array", + "arrayeducation", + "arspharmaceuticalsoperationsinc", + "artefact", + "artefactlinkedin", + "articlegroup", + "arvaintelligence", + "arvinas", + "arxroboticsgmbh", + "asana", + "ascend21", + "asgjobs", + "ashfieldadvisory", + "ashfieldmedcomms", + "asigovernment", + "aspectbiosystems", + "aspirehealthalliance", + "assemblyai", + "assetliving", + "assetwatch", + "assystinc", + "asteralabs", + "astoundcommercesandbox", + "astranis", + "astspacemobile", + "atalantatherapeutics", + "atariinc", + "atbayjobs", + "atek", + "athleticsbaseballops", + "athleticsbusinessops", + "atlassand", + "atlasxhm", + "atomai", + "atomiccartoons", + "atomicmachines", + "atriumcampus", + "attain", + "attainpartners", + "attainsports", + "attaintalent", + "attentionarc", + "attentive", + "attivopartners", + "attn", + "attotrading", + "attune", + "atwellgroup", + "auctane", + "audaxgroup", + "audibenehearcom", + "audioeye", + "augmentcomputing", + "auldwhitetalentcommunity", + "aura798", + "aurosglobal", + "authenticbrandsgroup", + "authenticinsurance", + "autods", + "autogenai", + "automatticcareers", + "automox", + "autoproff", + "autoscout24", + "autotradercanada", + "avantium", + "avantus", + "avebykormancommunities", + "aviationinstituteofmaintenance", + "avidhealth", + "avidxchangeinc", + "avomdincdbaavo", + "avride", + "awin", + "axiad", + "axicorpfinancialservicesptyltd", + "axios", + "axle", + "axon", + "axonag", + "axontalentcommunity", + "axq", + "axs", + "axsometherapeutics", + "axuall", + "aypapower", + "azragames", + "azuritypharmaceuticals", + "azuritypharmaceuticalsindia", + "b12", + "babylist", + "backblaze", + "baidu", + "bamboohr17", + "bancopan", + "bandwidth", + "banyancanopygroup", + "banyaninfrastructure", + "banyansoftware", + "barbarian", + "barbaricum", + "baringa", + "barkleyokrp", + "barrfoundation", + "basejobs", + "bathworksmichigan", + "baton", + "baublebar", + "bauerhockeycascademaveriklacrosse", + "bayada", + "bayasystems", + "bbyo", + "bdainc", + "beaconbiosignals", + "beam", + "beamtherapeutics", + "beamup", + "bearing", + "beatboxbeveragesllc", + "beautifulai", + "beelinemedicines", + "behavox", + "bellcabinetry", + "bellroy", + "benchprep", + "bennie", + "berkadiatalentpool", + "berkshiregroupllc", + "berlinrosen", + "bertramcapitalmanagement", + "berylls", + "bestpass", + "besxar", + "bethesdahealthgroup", + "betsson", + "betterhelp", + "betterhelpcom", + "bettinghero", + "bettygamingca", + "bettyjobboard", + "bevicareers", + "bevlabvet", + "beyondfinance", + "beyondtrust", + "bgbx", + "bgbxconsulting", + "bgeinc", + "bgeinccampus", + "bidease", + "billiontoone", + "biohub", + "biolumina", + "biomechanicsconsultingandresearchllc", + "biomedrealty", + "bird", + "birgo", + "bitcoindepot", + "bitgo", + "bitly", + "bitmex", + "bitpanda", + "blab", + "blackbirdhealth", + "blackcanyonconsulting", + "blackduck", + "blackedgecapital", + "blackforestlabs", + "blacklane", + "blackthorn", + "blankstreet", + "blastpoint", + "blend", + "blenheimchalcot", + "blenheimchalcotindia", + "blinkhealth", + "blip-global", + "blockchain", + "blockrenovation", + "bloombergorg", + "bloomerang", + "bloomreach", + "bluecherry", + "blueconic", + "bluecrestcapitalmanagement", + "bluecubeservices", + "bluefishai", + "bluehole", + "bluelabsanalyticsinc", + "bluemoonmetals", + "blueprintmedicines", + "blueroseresearch", + "bluestarfamilies", + "bluevineindia", + "bluevineus", + "bluewaterthinking", + "blumira", + "blytheco", + "bmnt", + "bobbie", + "bobtail", + "boingo", + "boku", + "boldmetrics", + "boloai", + "bombas", + "bondora", + "bondvet", + "boomentertainment", + "boomilp", + "boostedai", + "boostlingo", + "bosapropertiesinc", + "bottomlinetechnologies", + "bouldercare", + "boulevard", + "boxinc", + "bpcs", + "bpd", + "bracebridgecapital", + "braeburn", + "brainlabs", + "brainpop", + "brainstation", + "braintrusttutors", + "branch", + "brandtechplus", + "brandwatch", + "braskem", + "brave", + "braveheartbio", + "bravo", + "braze", + "breakwatercorp", + "breezeairways", + "breezecash", + "breezeway", + "brevium", + "bridgebio", + "bridgewater89", + "bridgewaterassociatescampusrecruitingreferral", + "brightai", + "brightai1", + "brightcoreenergy", + "brightsign", + "brightstonetherapy", + "brillapubliccharterschools", + "bringg", + "britishasiantrust", + "britive", + "brkz", + "broadsign", + "broadvoice", + "broadwayventures", + "brookecharterschools", + "brooklinen", + "brph", + "brunswickgroup", + "bruntworkwear", + "bswift", + "btgpactualchile", + "btig27", + "bubbleskincare", + "bugcrowd", + "builder", + "buildingdecarbonizationcoalition", + "buildkite", + "buncha", + "bungie", + "businessoffashion", + "butcherbox", + "butlr", + "butternutbox", + "buyersedgeplatformrecruiting", + "buynomics", + "buzzsolutions", + "bvnk", + "bwreferrals", + "c3el", + "c6bank", + "cabify", + "cadencehealth", + "cadrehospice", + "cafortune", + "cais", + "cakeai", + "calahealth", + "calendly", + "callrail", + "calm", + "calyxinstitute", + "calyxo", + "cameo", + "camp", + "campuscompact", + "cancostileandstone", + "candid", + "candidaturasdirecionadasxpinc", + "candidly", + "cannabisandglass", + "cannondale", + "canonical", + "canopyconnect", + "canopytax", + "canopyworks", + "capco", + "capintel", + "capitalbank", + "capitalfarmcredit", + "capitalgymnasticscedarpark", + "capitalgymnasticsroundrock", + "capitalontap", + "capitaltg", + "capstoneinvestmentadvisors", + "captivation", + "carbon", + "carbonchain", + "carbondirect", + "carbonfuture", + "carbonrobotics", + "cardata", + "cardinalpoint", + "careaccess", + "careerteam", + "cargomatic", + "cariadinc", + "caribou", + "cariboubiosciencesinc", + "carmichaellynch", + "carolinatitleloansinc", + "carrotfertility", + "carta", + "cascadeloans", + "casechek", + "caseguard", + "casestatus", + "cashcowlouisiana", + "catamountconstructors", + "catapultsports", + "catawiki", + "catchcreationllc", + "catdaddy", + "caylent", + "cayuse", + "cbemllc", + "cbinsights", + "cc", + "ccah", + "ccahremote", + "cclfg", + "cclim", + "cdbabyjobs", + "celerocommunicationsinc", + "celigo", + "cellanome", + "cellsignalingtechnology79", + "celonis", + "censys", + "centerforemploymentopportunities", + "centessapharmaceuticalsinc", + "centralreach", + "centriaautism", + "centriahealthcare", + "centrumhealth", + "centuracollege", + "cerebral", + "ceribell", + "ceros", + "certifiedgroup", + "chainguard", + "championhq", + "championsgroupholdings", + "chanzuckerberginitiative", + "chaosindustries", + "chaparralmedicalgroup", + "chariotdefense", + "charlesriverassociates", + "chartbeatinc", + "charterup", + "checkalt", + "checkbook", + "checkr", + "cheddar", + "chefman", + "chenmoore", + "chicagotradingcampus", + "childrenstreehouse", + "chile", + "chime", + "chorusinnovations", + "chowbus", + "christfellowship", + "cie", + "circleso", + "cision", + "cityoffortworth", + "citytherapeutics", + "cityvetinc", + "civicactions", + "civisanalytics", + "clara", + "clarishealth62", + "clariticloudinc", + "clarityinnovates", + "claros", + "claudecorps", + "cleancroptech", + "clear", + "clearfield", + "clearlinktechnologiesllc", + "clearscoretechnologylimited", + "clearstreet", + "clearviewhealthcarepartners", + "clearwayjobs", + "clenera", + "cleo", + "cleo-emea", + "cleoindia", + "clevelandguardiansbops", + "clickhouse", + "clicktherapeutics", + "climateai", + "climatecabinet", + "clockworksystems", + "closure-tech", + "cloudbeds", + "cloudbedsthirdpartyboard", + "cloudchamberen", + "cloudflare", + "cloudsek", + "cloverhealth", + "cloverly", + "clubcolors", + "clubmonaco", + "clutch", + "cmt", + "coactive", + "coast", + "cobaltio", + "cobaltservicepartners", + "cobblestoneenergy", + "cobblestoneenergy4", + "cobre", + "coconutsoftware", + "cocoon", + "codazen", + "code3", + "codeorg", + "codepath", + "coderoad", + "coefficient", + "cofertility", + "cofraholding", + "cogentbiosciences", + "cognite", + "cognitiv", + "cogstateinc", + "coherehealth", + "coherusbiosciences", + "coinbase", + "colabsoftware", + "colehourcoheninc", + "colemanresearch", + "collectively", + "collegetrack", + "colovore", + "commerceiq", + "commercetools", + "commonthreadcollective", + "communitymanager", + "commvault", + "comparaja", + "compeerfinancial", + "compliancygroupllc", + "comstock", + "concentric", + "concerthealth", + "conga", + "conifersaicareers", + "connectder", + "connectedcannabis", + "connectedcannabisco", + "connectwise", + "consensys", + "constantcontact", + "constellationsoftwareinc", + "constructionresources", + "consumeredge", + "consumerreports", + "contentful", + "convene", + "convenientmd", + "cooksys", + "copperco", + "cordance", + "cordellcordell", + "cordial81", + "corelight", + "coreone", + "coretelligent", + "coretrustpurchasinggroupllc", + "coreview", + "correlationone", + "cortex", + "cortica", + "corticamelmed", + "cortland", + "cosmoslabs", + "costar", + "cottinghambutlerinsuranceservicesinc", + "couchbaseinc", + "counterpart", + "coupanginternal", + "courierhealth", + "coursera", + "covar", + "coverahealth", + "cpisecurity", + "cpm", + "craftsmansocials", + "cranialtechnologies", + "creativefabrica", + "credible", + "crescent", + "crescolabs", + "cresta", + "crestwoodcareers", + "crexi", + "crfamilyofcompanies", + "criminaljusticeagency", + "crisprecruit", + "criticalmass", + "criticalmassgroup", + "crocodilecloth", + "croihealth", + "crossbeam", + "crunchyroll", + "crystaldynamics", + "csciconsulting", + "csgconsultants", + "css", + "cssmerge", + "culthealth", + "cultureamp", + "curaleaf", + "curative", + "curicapital", + "current", + "customcomputerspecialists", + "customerio", + "cuyana", + "cvx", + "cybersheath", + "cymulate", + "dagsterlabs", + "daiyafoodsinc", + "darkhorseemergency", + "darkwolfsolutions", + "dashlane", + "databento", + "databricks", + "datacamp", + "datagrail", + "dataiku", + "dataikujobs", + "datakindinc", + "datarails", + "datasocietyresearchinstitute", + "datasystemsanalystsinc", + "datavant2", + "datsolutions", + "davidzwirnergallery", + "day1academies", + "daybreakhealth", + "daylight", + "daymarkhealth", + "dbeaver", + "dcard", + "ddbhealth", + "ddome", + "debtbook", + "debutbiotech25", + "decimainternational", + "dedicatedit", + "deepintent", + "deeplocal", + "deepmind", + "defcon", + "defenseunicorns", + "definitivehc", + "definitivehcindia" + ], + "greenhouse-2": [ + "definiumtherapeutics", + "degreed", + "deliveryassociates", + "democracyforward", + "democracypreppublicschools", + "densityai", + "dental365", + "denverbroncosteamllc", + "dept", + "descope", + "descript", + "designbridge", + "designedconveyorsystems", + "detect94", + "detroitlions", + "developmentseed", + "devfuturetalent", + "devrev", + "devries", + "devtechnology", + "dfinity", + "dhigroupinc", + "dhpace", + "dialpad", + "dianahealth94", + "dianthustherapeutics", + "dicefm-careers", + "digicert", + "digimarc", + "digisecuritysystems", + "digitalbiology", + "digitalbridge", + "digitalcurrencygroup", + "digitalextremes", + "diligentcorporation", + "diligentrobotics", + "dimagi", + "discmedicine", + "discord", + "distantjob", + "districtofcolumbiainternationalschool", + "distrokid", + "divcowest", + "divergent", + "dkatalislabs", + "dkbcodefactory", + "dlhcorporation", + "dlpbank", + "dlrgroup", + "dmgevents", + "dna", + "dnsfilter", + "doctolib", + "docugami", + "dodgshunmedlin", + "doitintl", + "dollarshaveclub", + "doma", + "donorbox", + "donorschoose", + "donorschoosefellowship", + "doordashaustralia", + "doordashcanada", + "doordashinternational", + "doordashmexico", + "doordashusa", + "dots", + "doublegood", + "doubleverify", + "downtownmusic", + "doximity", + "dragos", + "drdansanimalhospital", + "droit", + "droplet", + "drsquatch", + "drweng", + "dsi", + "duettoresearch", + "durable", + "durinmining", + "dv01", + "dvtrading", + "dxacirca", + "dynetherapeutics", + "dyopath", + "eamesinstitute", + "earlycareerprograms", + "earnin", + "eastharlemtutorialprogram", + "eastsidedermatology", + "easygo", + "ebanx", + "ebury", + "echodynecorp", + "eclinicalsolutions", + "eclipse", + "eclipsetrading", + "ecoatmgazelle", + "edelmanfinancialenginesllc", + "edgewoodpartnersinsurancecenter", + "edmentum", + "edo", + "edobestsandbox", + "educate", + "education", + "efficientcomputer", + "eikontherapeutics", + "eko", + "eldersburgveterinaryhospital", + "electrasteel", + "elementalimpact", + "elementbiosciences", + "elend", + "eleoshealth", + "eleventhhourgames", + "eliotcommunityhumanservices", + "elitedentalpartnersllc", + "elitetechnology", + "elliginthealth", + "elwoodtechnologies", + "emarketer", + "embroker", + "emergentlabsinc", + "emotainizioengage", + "emplifimonster", + "employerdirecthealthcare", + "empowerbrands", + "emslinqinc", + "enavatecareers", + "enchargeai36", + "encora10", + "encoura", + "endeavourinspiredinfrastructure", + "endorlabs", + "energage", + "energyexemplarllc", + "energyhub", + "energyimpactpartnerslp", + "energysolutions", + "energysolutionsinternships", + "enforce", + "engageseniortherapy", + "engine", + "engineersgate", + "englishcanada", + "enigmaio", + "enlacehealth", + "ennoblecare", + "enova", + "ensco", + "enscoskillbridge", + "entera", + "enterpret", + "entersekt", + "entradatherapeutics", + "entreehealth", + "envisionconsulting", + "enviva", + "envoyglobalinc", + "envoymortgage", + "eolapower", + "eonio", + "epickids", + "episodesix", + "episodesixlinkedin", + "eplusinc", + "eqtcorporation", + "equalexperts", + "equatic", + "equilibriumenergy", + "equipmentsharecom", + "eqvilentjobs", + "erasca", + "ergeon", + "ernesta", + "ernestpackagingsolutions", + "escribers", + "espirita", + "essential", + "etchedai", + "etchinc", + "etec", + "etelligentgroup", + "ethernovia", + "ethos", + "ethoslife", + "eucalyptus", + "eudia", + "eulerity", + "eve", + "eventbriteinc", + "eventsandinterns", + "ever", + "everagtester", + "evergreennephrology", + "evergreenservicesgroup", + "everlane", + "everlaw", + "everway", + "evgspecialtynetwork", + "evio", + "evismart", + "evolutionaryscale", + "evolutioniq", + "evolvevacationrental", + "exabeam", + "exadelinc", + "exame", + "excelsportsmanagement", + "exodus54", + "exoduspoint", + "experigreen", + "expertnetwork", + "explorasolutions", + "extend", + "extenteamcareers", + "externaljobboards", + "extrahopnetworks", + "eyeo", + "ezcaterinc", + "ezcatertalentcommunity", + "factored", + "factorialenergy", + "faeththerapeutics", + "faire", + "fairlife", + "fairmarkit", + "fairsteadescllc", + "falconx", + "fambrands", + "familyoffice", + "familywell", + "fanaticscollectibles", + "fanaticsfbg", + "fanaticsinc", + "faradayfuture", + "farmsanctuary", + "faropoint", + "fartherfinance", + "fashionnova", + "fastautoloansinc", + "fastpaydayloansfloridainc", + "fay", + "fccincinnati", + "federato", + "felixandfido", + "fender", + "fermat", + "ferocia", + "feverup", + "fgsglobal", + "fictiv", + "fieldstonebio", + "figma", + "figment", + "figure", + "figureai", + "filescom", + "filson", + "financialtimes33", + "finepointconsulting", + "finitestate", + "firaxis", + "fireworksai", + "firmpilotailawfirmmarketing", + "firmus", + "firstconnectinsurance", + "firsthand", + "firstnationalbankofamerica", + "firstprinciples", + "firststepsforkids", + "five9", + "fiveringsllc", + "fixify", + "flagshippioneeringinc", + "flash", + "flatironenergy", + "flaunt", + "fleetio", + "fletcherjonesautomotivegroup", + "flex", + "flexport", + "flighthub", + "flip", + "flipdish", + "flockhomes", + "flodesk", + "flohealth", + "flourish", + "flowtraders", + "fluxon", + "fluxx", + "flyr", + "flywheeldigital", + "focalsystems", + "focusfinancialpartners", + "focuspartnerswealth", + "foliahealth", + "follettsoftware", + "folxhealth", + "forafinancial", + "forbes", + "forcetherapeutics", + "foresitelabs", + "forgebiologics", + "forgeglobal", + "forgehealth", + "formaaiinc", + "formaaiinc3", + "formationbio", + "formhealth", + "formic", + "forta", + "forter", + "forthea", + "fortisfiresafety", + "fortra", + "fortrobotics", + "forumone", + "forwardnetworks", + "fosphamarketing", + "fossainc", + "found", + "foundationriskpartners", + "foundrydigital", + "fourhands", + "fourkites", + "fourthline", + "foxen", + "foxholetechnology", + "fractile", + "fractylhealthinc", + "freedassociates", + "freedomtogether", + "freeformfuturecorp", + "freenome", + "freenow", + "freestonecapitalmanagement", + "freshprints", + "fronterahealth", + "frontierdermatologyprovidercareers", + "fsastorecom", + "fsg", + "fueledcareers", + "fulfil", + "funga", + "fuseglobalpartners", + "fusionworldwide", + "future", + "futurhealth", + "g2vp", + "gaintheory", + "gaithersburg", + "galaxydigitalservices", + "galaxyservicepartners", + "galileo", + "gallaghereveliusjonesllp", + "gallup", + "galvanizeclimatesolutions", + "gametimeunited", + "gapinternational", + "garagedoormedics", + "gardacp", + "garnerhealth", + "gassouth", + "gatesventures", + "gather", + "gatherai", + "gatikaiinc", + "gavindebeckerassociates", + "gavindebeckerassociatesindeed", + "gcmgrosvenor", + "gearbox", + "gearboxquebec", + "gelbergroup", + "gelberhandshake", + "gelfandrennertfeldman", + "gemechanicalasp", + "gemini", + "genea", + "generalassembly", + "generalatlantic", + "generalcatalyst", + "generalmatter", + "generalproximity", + "genetixbiotherapeutics", + "genevatrading", + "genezenlabs", + "geniantllc", + "genius", + "geniussports", + "geniussportssn", + "genscript", + "gensyn", + "geocgi", + "georgiaautopawninc", + "geotab", + "geotekoperationslimited", + "getbuilt", + "getwhys", + "gibsondunn", + "gigaenergy", + "gillig", + "gingerlabsinc", + "ginkgobioworks", + "girleffect", + "gitai", + "gitlab", + "givecampus", + "givedirectly", + "givewell", + "glance", + "gleanwork", + "globalaccelerator", + "globalenergyallianceforpeopleandplanetgeappllc", + "globalhealthcareexchangeinc", + "globalizationpartners", + "globalli", + "globalwebindex", + "glossgenius", + "glossier", + "glyphicbiotechnologies", + "go1au", + "go1eu", + "go1us", + "goalcast", + "goalhonduras", + "goalsyria", + "goatgroup", + "gocardless", + "godfreydadichpartners", + "gofundme", + "goguardian", + "gokenamericallc", + "goldenstate", + "golf", + "gomotive", + "gongio", + "goodbysilversteinpartners", + "goodfire", + "goodhouse", + "goodinside", + "goodjobgames", + "goodnotes", + "goodr", + "goodsservices", + "goodwaygroup", + "goop", + "goremutualinsurance", + "gorjana", + "gotion", + "govtechbarbados", + "gr8tech", + "gradial", + "gradientai", + "grafanalabs", + "grahamcapitalmanagement", + "gramgamescareers", + "granum", + "graphcore", + "grasshopperasia", + "grassi", + "grassrootsanalytics", + "grayscaleinvestments", + "greatstate", + "greenbrookmedical", + "greenhouse", + "greenirony", + "greenpeace", + "greenplaces", + "greenpointtechnologies", + "greenthumbindustries", + "greenworkssunriseglobalmarketing", + "greyus", + "gridmaticinc", + "groma", + "groomecareers", + "group14", + "group14korea", + "grovecollaborative", + "grovelane", + "grover", + "growe", + "growetalents", + "gruns", + "grvty", + "gsgcareers", + "gsrmarkets", + "gtv", + "guardianrestoration", + "guardsquare", + "guerrilla-games", + "guidelighthealth", + "guidepoint", + "guidepointsecurity", + "guidepostmontessori", + "guildgaragegroup", + "gulfwindtechnology", + "gumgum", + "gusto", + "gwkinvestmentmanagementllc", + "gympass", + "gymshark", + "habitathealth", + "hackerrank", + "haigroup", + "haizelabs", + "hala", + "halcyon", + "hangar13", + "hankooktireamericacorp", + "hanover", + "hanwhaenergyusa", + "hanwharenewables", + "harbingermotors", + "harborglobal", + "harmonic", + "harnessinc", + "harpergroup", + "harrison&star", + "harrowhealth", + "harrys", + "hasbro", + "hatchcareers", + "haven", + "havenenglish", + "hazel", + "hbstudios", + "hc360", + "hcg", + "headlandsresearch", + "headlandstechnologiesllc", + "headoutcareers", + "headoutlinkedin", + "headoutreferrals", + "headspaceproviders", + "healthjoy", + "healthlink", + "hearcom", + "hearcomin", + "heartaerospace", + "heartflowinc", + "heartpaw", + "hebrewpublic", + "heliosx", + "helium", + "helium10", + "hellobackpack", + "hellofresh", + "hellommc", + "help", + "helpinghandsfamily", + "hereio", + "herselfhealth", + "hexagonbio", + "hexium", + "heygen", + "hiddenlayer", + "highdive", + "higherlogic", + "highmetric", + "highnote", + "hightouch", + "highwire", + "hillandknowlton", + "hillel", + "hillhousehome", + "hillpointe", + "hiper", + "hippo70", + "hiro", + "hivewatch", + "hlb90067", + "holder", + "holderconstruction", + "holisticindustries", + "hologram", + "homechef", + "homeconstructionregulatoryauthorityvolunteer", + "homeinstead", + "homelight", + "homemarketfoods", + "homesolutions", + "hometap", + "hometapjobs", + "homeward", + "honeathome", + "honehealth", + "honeycomb", + "hoodhp", + "hootsuite", + "hopscotchprimarycare", + "hopskipdrive", + "horacemannagents", + "horacemannservicecorporation", + "horizenlabs", + "horizenlabstalentpool", + "horizonindustrieslimited", + "hotmartcareersbr", + "hourglasscosmetics", + "housemarque", + "housinganywhere", + "hovercraft", + "hoyoverse", + "hpiq", + "hrpliving", + "hs", + "huddleup", + "hudl", + "hugeinc", + "humanagency", + "humaninterest", + "humanrightswatch", + "humansignal", + "humeai", + "hummingbirdregtech", + "hungryroot", + "huntress", + "hut8", + "hydritechemicalco", + "hyliion", + "hyphenconnect", + "ians", + "ibanfirst", + "icapitalnetwork", + "iconcareers", + "iconiq", + "idahotitleloansinc", + "ideo", + "idme", + "idmeuniversityrecruiting", + "idnow", + "ie", + "ifoodcarreiras", + "iftother", + "iherb", + "ilia", + "imaginepediatrics", + "imagineworldwide", + "imbue", + "immersive", + "immunefi", + "immunomeinc", + "impact", + "impinjexternal", + "impiricus", + "implicit", + "imvtcorporation", + "inalabconsulting", + "incadigitalinc", + "inceptive", + "inchargeenergy", + "incode", + "incognia", + "indicacoesifoodinterno", + "indigenouspactpbcinc", + "indigo", + "industrialelectricmanufacturing", + "industriouslabs", + "infinitumelectric", + "inflectionai", + "infotrust", + "infuse", + "ingenious", + "inhometherapy", + "inizio", + "iniziomedical", + "inkind", + "inmobi", + "innodatainc", + "insomniac", + "inspiraeducation", + "inspire11", + "inspiremedicalsystemsinc", + "inspiren", + "instabase", + "instawork", + "instead", + "instiglio", + "instride", + "instridehealth", + "insurify", + "insurityindia", + "intecrowd", + "integra", + "integralmolecular", + "intelluminc", + "inter", + "interbrand", + "intercom", + "interdependence", + "internaljobsatlush", + "interstellarlab", + "intersystems", + "interviewengineering", + "interwellhealth", + "interworks", + "inthepocket", + "intradiem", + "intrinsicrobotics", + "inversionspace", + "invivyd", + "inyova", + "ionq", + "ionqcontractors", + "iovancebiotherapeutics", + "iovino", + "iris", + "irradianttechnologies", + "isaac", + "isaraerospace", + "isccareers", + "isidor", + "ism", + "isomorphiclabs", + "ispottv", + "itd", + "iterable", + "iterativehealth", + "its", + "itslogisticsllc", + "ivxhealth", + "ixllearning", + "jadebiosciences", + "jamfilled", + "janestreet", + "jennikayne", + "jensenhughes", + "jjsnackfoods", + "jobsatphamily", + "jobsforthefuture", + "johnmcorcorancompany", + "joinaffect", + "joinforage", + "joinparadigm", + "jomboymedia", + "jordanparkgroup", + "joskoasp", + "journey", + "joya", + "jrmconstructionmanagementllc", + "jshiddenevents", + "jukeboxhealth", + "julieproducts", + "jumio", + "jumpcrypto", + "jumptrading", + "juno", + "just-global", + "justanswer", + "justfoodcompany", + "justfund", + "justprotein", + "justworks", + "juullabs", + "k2spacecorporation", + "kailera", + "kairospower", + "kalcon", + "kalepa", + "kalshi", + "karat", + "karbon", + "kardfinancialinc", + "kardigan", + "karya", + "kasa", + "katalyst", + "kateschwartzphysicaltherapyjobs", + "kayak", + "kayzen", + "keeleyconstruction", + "keelinfrastructure", + "keepersecurity", + "kellerpostman", + "kenjyatrusantgroup", + "kensingtoncorporate", + "kensingtontours", + "keplergroup", + "kernalbio", + "ketryx", + "keyfactorinc", + "khaerospace", + "khaite", + "khanacademy", + "khealthcareers", + "kickstarter", + "kidscountry", + "kikoff", + "kincellbio", + "kindbridgecorporation", + "kindsnacks", + "kinexus", + "kivaorg", + "kiwicoinc", + "knak", + "knightdivisiontactical", + "knit", + "knithealth", + "knock", + "knowbe4", + "knowde", + "knowledgecity", + "known", + "koboldmetals", + "koboldmetalsdrc", + "kodiak", + "kodiaksolutions", + "koleyjessen", + "kolmacintegratedbehavioralhealth", + "komodohealth", + "konovo", + "konux", + "koodoo", + "kostelanetzllp", + "krafton", + "kraftonamericas", + "kraftonindia", + "kreativekids", + "krollbondratingagency", + "kronosresearch", + "kunai", + "kuraoncology", + "kurosbiosciencesinc", + "kyowakirinusa90", + "la28careers", + "labelbox", + "labviva", + "lacarguy", + "ladder33", + "lakeasbury", + "lakefrontbiotherapeuticsinc", + "landdesign", + "landor", + "langanengineeringandenvironmentalservicesllc", + "laportefr", + "laporteusa", + "lasenza", + "lastpass", + "later", + "latitude", + "launch2", + "launchdarkly", + "launchpadtechnologiesinc", + "lawmatics", + "lawzero", + "layerhealth", + "layerzerolabs", + "lazarusenterprises" + ], + "greenhouse-3": [ + "leaflink", + "leagueinc", + "learneo", + "learningnetwork", + "learnlux", + "learnupon", + "ledgy", + "legalservicesnyc", + "legatosecurity", + "legendcareers", + "legion", + "leit", + "lemurianlabs", + "lendingtree", + "leolabsinc", + "levanta", + "levelaccess", + "leveltenenergy", + "levelworks", + "levio", + "levitate", + "lexingtonmedical", + "lgairesearch", + "lgelectronics", + "li-thermal-works", + "liberate", + "licor", + "life360", + "lifelinkiii", + "lifeskillsautismacademy", + "lightfeatheriollc", + "lightforceorthodontics", + "lighthouse", + "lighthousebehavioralhealthsolutions", + "lightningai", + "lightspeeddms", + "lightspeedsystems", + "lilasciences", + "lineate", + "link", + "lionsheadprecisionmetals", + "liontree", + "lisc", + "lithic", + "litify", + "litmos", + "littlebutterflies", + "littlewordsproject", + "livecareer", + "livefront", + "lively43", + "liveparalleljobs", + "liveperson", + "liveviewtechnologiesinc", + "localcoin", + "localitymediallcdbafirstdue", + "lockwood", + "locusrobotics", + "loenbro", + "loftfederal", + "logicalintelligence", + "logicgate", + "logos", + "lokainc", + "looneyrickskiss", + "loonshotgames", + "lovable", + "loyal36", + "lp-professionalstaff", + "lpc", + "lucidbots", + "lucidmotors", + "lucidsoftware", + "luckybeverageco", + "ludorobotics", + "lumahealth", + "lumbermens", + "lumimeds", + "lunarenergy", + "luno", + "lush", + "lusternational", + "lyellimmunopharma", + "lynxanalytics", + "m0dbathenextthingltd", + "m2ingredients", + "m2xenergyinc", + "m9solutions", + "mabl", + "machinifyinc", + "mackaysposito", + "madano", + "maddoxindustrialtransformer", + "madisonlogicinc", + "maev", + "magnolia", + "magrathea", + "mainstreethealth", + "makeawishamerica", + "malbon", + "mammothbrands", + "mandm", + "manifoldai", + "manifoldbio", + "manscaped", + "mantl", + "mantrahealth", + "manychat", + "map", + "maravailifesciences", + "margaux", + "mariadbplc", + "markforged", + "markmorrisdancegroup", + "marksman", + "markspainrealestate-corp", + "marqeta", + "marqvision", + "marscousg", + "marshallwace", + "martellgrowthsolutions", + "maslanskycareers", + "masterclass", + "matchpointtx", + "materialbank", + "matherheadquarters", + "matherintysons", + "matherplace", + "mathersplendido", + "matik", + "matteprojects", + "mattermost", + "matx", + "mavenclinic", + "mavenrobotics", + "mavenscareers", + "mavensecuritiesholdingltd", + "maxcessinternational", + "maxmanufacturingcareers", + "maymobility", + "mbooth", + "mboothhealth", + "mcadams", + "mcclureoilcorporation", + "mcghealth", + "mcneeswallacenurickllc", + "mechanicallicensingcollective", + "medelitellc", + "medeloop", + "mediabrands", + "mediasmart", + "medispend", + "medistrava", + "meditelecare", + "medium", + "medrio", + "medsien", + "mejuri", + "melio", + "memic", + "mentalhealthcenterofdenver", + "mentimeter", + "merceradvisors", + "mercury", + "mercyforanimals", + "merge", + "mergeworld", + "meridianpartners", + "meruhealth", + "mesh", + "meshopticaltechnologies", + "messari", + "metalab", + "metapack", + "method", + "methodco", + "metisinc", + "metoxinternationalinc", + "metron", + "metronome", + "metropolis", + "mgproperties", + "mhi", + "michaelanthonycontractingcorp", + "microblink", + "midihealth", + "midpenhousing", + "midpointmarkets", + "mighty", + "mightynetworks", + "mill", + "millerremick", + "mindgrub", + "mindgym", + "mindtheproduct", + "mineralystherapeutics", + "minimal", + "minio", + "minitab", + "minno", + "miopartners", + "miqdigital", + "mirakl", + "mirakllabs", + "miris", + "misfitsmarket", + "mishimoto", + "missionhealthcare", + "missionlane", + "missionlanellc", + "mississippititleloansinc", + "missourititleloansinc", + "mithril", + "mitratech", + "mitsogoinc", + "mitsubishimotorsna", + "mixbook", + "mixpanel", + "mlbnetwork", + "mm", + "mntn", + "mobentertainment", + "mobiik", + "mobileanesthesiologists", + "mobilityware", + "mochihealth", + "moderawealthmanagement", + "modernhealth", + "modulrfinance", + "mogli", + "mojangab", + "moloco", + "momentenergy", + "momentous", + "momentumcompany3", + "momentumfinancialservicesgroup", + "momentus", + "monetserviceinc", + "moneyherogroup", + "moneymart", + "moneysmart", + "mongodb", + "monks", + "monocl", + "monsterenergy", + "monstro", + "monumentalsports", + "monzo", + "moodhealth", + "moon", + "moonlite", + "morganmorganjobsapplynow", + "morrisonmaierle", + "mos", + "mossnewyorkllc", + "motifneurotech", + "motional", + "motivity", + "movementstrategy", + "moveonorg", + "mozilla", + "mqreferrals", + "mrapplecareers", + "mrbeastyoutube", + "msfcareers", + "mthreerecruitingportal", + "muckrack", + "muonspace", + "murj", + "mwinternshipprogram", + "mx51", + "mxtechnologiesinc", + "myfitnesspal", + "myfundedfutures", + "myriad360", + "mythicalgames", + "n26", + "n2publishingglassdoor", + "nabis", + "nadiacare", + "nakedfarmer", + "nametag", + "nanit", + "nanonets", + "nanopathinc", + "nansen", + "narvar", + "nascompany", + "nashvillezoo", + "natera", + "national", + "nationalbusinesscapital", + "nationallutheraninc", + "nationalpublicradioinc", + "naturesbakery", + "naughtydog", + "navapbc", + "navierboat", + "navtechnologies", + "navvis", + "nearform", + "nearspacelabs", + "neo4j", + "neocybernetica", + "neonaerospace", + "neoris", + "neptunebio", + "neptunemedical", + "nerdy", + "nerostechnologies", + "netbrain", + "netdocuments", + "neteasegames", + "netlify", + "neuehealth", + "neuraflash", + "neurahealth", + "neuralink", + "nevadatitleandpaydayloansinc", + "newarkacademy", + "newengeninc", + "neweratech", + "neweratechnology", + "newglobesandbox", + "newlabcareers", + "newleafenergy", + "newlimit", + "newrelic", + "newsbreak", + "newsela", + "newsrevenuehub", + "newsweek", + "nexightgroup", + "nextinsurance66", + "nextroll", + "nextrollinc", + "nflcareers", + "ngmbiopharmaceuticals", + "nift", + "nightdivestudios", + "nimblerobotics", + "nimbus", + "ninedotholdingsinc", + "ninjatrader", + "ninjatradercontractors", + "nisc", + "nitricity", + "nlcventures", + "nmcareers", + "nmi", + "noahmedical", + "noble", + "nobuhoteltoronto", + "noctrixhealth", + "noctuatechnology", + "nomadhealth", + "nomina", + "nonprofitfinancefund", + "northbeam", + "northeastanimalclinic", + "northern911", + "northmarq", + "northpointrecoveryholdingsllc", + "northpointtechnology", + "northspyre", + "northwestadministratorsinc", + "notexternal", + "nova401", + "novacredit", + "novoed", + "noyocareers", + "nozominetworks", + "ntconcepts", + "nuancelabs", + "nubank", + "nucleus", + "numa", + "nurix", + "nutrafol", + "nutscom", + "nuvem", + "nve", + "nycedc", + "nysonian", + "o2epcminc", + "oafkenya", + "oasishealthpartners", + "oasissecurity", + "obexp", + "obrienveterinarygroup", + "obsidiansecurity", + "obsidiantherapeutics", + "oceanx", + "ocrolusinc", + "octaura", + "octus", + "oddball", + "odeko", + "odlesalescareers", + "offerup", + "offerzen", + "officehours", + "officespacesoftware", + "offshorelaunch", + "ogilvyhealthuk", + "ogilvyhealthusa", + "ogilvymena", + "ogroup", + "ohalogenetics", + "oklo", + "okx", + "oldcastlebuildingenvelope", + "olema", + "olipop", + "oliver", + "oliverseapac", + "oliverusa", + "olly", + "olsson", + "omadahealth", + "omenai", + "omidyarnetwork", + "omnicomhealth", + "onapsis", + "onbe", + "oncoverycare", + "oneacrefundethiopia", + "oneacrefundglobal", + "oneacrefundmalawi", + "oneacrefundnigeria", + "oneacrefundrwanda", + "oneacrefundtanzania", + "oneacrefunduganda", + "oneacrefundzambia", + "oneearthfuture", + "oneenergyrenewables", + "oneimaging", + "onemodel", + "onenergy", + "oneoncology", + "onepath", + "onesixsolutions27", + "onetrust", + "onexgeneral", + "onnitlabs", + "onxmaps", + "ooma", + "openap", + "opencoreventures", + "openeye", + "openfx", + "openly", + "opentable", + "openwork", + "operantai", + "operationscareers", + "ophelia", + "oportun", + "opploans", + "optimadermatologycareers", + "optimal", + "optimalcare", + "optimaldynamics", + "optimecare", + "optimove", + "optiverus", + "oralsurgerypartners", + "orangetwist", + "orchard", + "orchestra", + "orderly", + "orennia", + "oriongroup", + "orium", + "orkes", + "orthosportsmedphysicaltherapyjobs", + "oruka", + "osano", + "oscar", + "oshihealth", + "osmosis", + "otrcapital1", + "otter", + "ouihelp", + "oulahealth", + "ounceofcare", + "oura", + "outerspace", + "outpostspace", + "outrider", + "outschool", + "outsetmedical", + "outshine", + "overline", + "overstory", + "owenscompaniesasp", + "ownwell", + "oxosmedical", + "p72pi", + "pacaso", + "pacificfusion", + "pacificlegalfoundation", + "packardculliganwater", + "pacnyc", + "pacvue", + "pagaya", + "pagayais", + "pagerduty", + "paidyinc", + "pairteam", + "palaktakidsacademy", + "pallet", + "palmettocleantech", + "palosverdes", + "pandadoc", + "pandvil", + "panthalassa", + "pantheonpublic", + "pantherlabs", + "papa", + "papaya", + "paperlessparts", + "parachutehealth", + "parachutehome", + "parallel", + "parallellearning", + "paretocaptiveservicesllc", + "parloa", + "parrishdevaughn", + "parsecautomationcorp", + "parsleyhealth", + "particle41llc", + "passportbranddesign", + "pathrobotics", + "pathstream", + "pathward", + "pathwaysforlife", + "patientpoint", + "patients&purpose", + "patterndata", + "paveakatroveinformationtechnologies", + "pawapay", + "paxlabs", + "pay2dc", + "paynearmeinc", + "paypay", + "paypaycard", + "paytient", + "pdi", + "pdtpartners", + "peachpilot", + "pearceservices", + "pei", + "pelago", + "pendo", + "penninteractive", + "peregrinetechnologies", + "perfectserve", + "perform-careers", + "perionnetworkltd", + "perpay", + "perscholashires", + "persefoniaiinc", + "personalisinc", + "personalizedbeautydiscoveryincdbaipsy", + "petag", + "pfm", + "phaidra", + "phamily", + "phantomai", + "phasev", + "philadelphiaeagles", + "philliesbaseballoperations", + "philo", + "phoenixcontact", + "phonepe", + "phynetdermatology", + "physicsx", + "picarroinc", + "picoquantitativetrading", + "pieinsurance", + "piermontbank", + "pilothq", + "pineadvisorsolutions", + "pingidentity", + "pinn", + "pinterest", + "pinwheelapi", + "pipetechnologies", + "pirateship", + "pistontechnologies", + "pitchbookdata", + "pivotbio", + "pixability", + "placementsio", + "planetlabs", + "planetscale", + "planningcenter", + "platacard", + "platformscience", + "platinumdermphysicians", + "playsports", + "plootocareers", + "plos", + "plscareers", + "pluspower", + "pma", + "pmg", + "pmguk", + "podium81", + "point72", + "pointc", + "pointwild", + "pokemoncareers", + "polyai", + "polychaincapital", + "polygonus", + "pomelocare", + "pontera", + "poppulo", + "porschesouthbay", + "portable", + "porternovelli", + "portland-communications", + "poshmark", + "post-gazette", + "postman", + "postmanlaw", + "postpartumsupportinternational", + "postscript", + "powerdigitalmarketing", + "powerfinance", + "powerhousearts", + "practicebetter", + "practisinglawinstitute", + "prathaminternational", + "praxent", + "praxis", + "praxisprecisionmedicines", + "precisionaq", + "precisionmedicinegroup", + "precisionvehicleholdings", + "predictiveindex", + "presencelearning", + "presidents", + "presidentssummit", + "prevail", + "prezzee", + "pricefox", + "primemedicine", + "primerai", + "primexbt", + "prisma6", + "prismatic", + "privateequityinsights", + "privatehealthmanagement", + "procaresolutions", + "processstreet", + "prodigal", + "productpeople", + "profluent", + "project44", + "project44opportunities", + "projectaservicesgmbhcokg", + "prokidney", + "prolaio", + "prolific", + "pronto", + "prophecysimpledatalabs", + "prophero", + "prophet", + "propublica", + "prosek", + "proskill", + "prosperhealth", + "proteinqureinc", + "protillionbiosciences", + "protonai", + "prove", + "psibufet", + "pt", + "pubgmadison", + "public", + "publiclabel", + "pulley", + "pulse", + "pulsebiosciences", + "pulumicorporation", + "pumpcareers", + "purestorage", + "purplestrategies", + "purposemed", + "pursuit", + "pushpay", + "putnamassociatesllc", + "pyramidroofing", + "pzenainvestmentmanagement", + "qgenda", + "qohash", + "qphox", + "quadbridge", + "quadraturecapital", + "quaise", + "qualia", + "qualifieddigital", + "qualio", + "quanata", + "quansight", + "quantifind", + "quantumsi25", + "quantumspacellc", + "quartzbio", + "quberesearchandtechnologies", + "queracomputinginc", + "questbridge", + "quillbot", + "quilt", + "quince", + "quintoandar", + "quip", + "rabinmartin", + "racapitalmanagementllc", + "rackner", + "radar", + "radiantsecurity", + "radiclehealth", + "radixark", + "radixexperienced", + "radixuniversity", + "raft", + "railsware", + "raisin", + "ramosmarbleandgranite", + "rangeviewinc", + "rapidfortinc", + "rapidsos", + "rapp", + "rayeitconsulting", + "razorpaysoftwareprivatelimited", + "rclco", + "rcxsports", + "rdccareers", + "rdsourcing", + "rdstation", + "readysettechnologyinc", + "realcapital", + "realchemistry", + "reallygreatreading", + "realm", + "rebag", + "rebelliondefense", + "rebtel", + "rebuildmanufacturing", + "recall", + "recidiviz", + "recordedfuture", + "recruitingprograms", + "rectanglehealth", + "recursionpharmaceuticals", + "redcellpartners", + "reddit", + "redpeak", + "redventures", + "redwoodmaterials", + "redwoodsoftware", + "referralsuseonly", + "reflective", + "reformation", + "regscale", + "relationalai", + "relativity", + "relaygraduateschoolofeducation", + "relaypayments", + "relaypro", + "relaytherapeutics", + "released", + "relishworks", + "reltio", + "remedyhomehealthcare", + "remixtherapeutics", + "remodelhealth", + "remoracarbon", + "remotasks", + "remotecom", + "remotereferralboardinternaluseonly", + "renaissancelearning-emea", + "renaissancelearning-nam", + "renewedvision", + "renpsg", + "renttherunway", + "repisodic", + "reproductivefreedomforall", + "reprofreedomforallinternships", + "res", + "researchpartnership", + "residenthome", + "residential", + "resilience", + "resolvetosavelives", + "resortpass", + "restaurantsupply", + "resultsforamerica", + "revero", + "revivn", + "revloncorporate", + "rewardsnetwork", + "rexfordindustrial", + "rhombuspower", + "rhythmx-ai", + "rialtic", + "ribolifamilywines", + "ridgeline", + "rimestechnologies", + "ringtherapeutics", + "ripcpc", + "rise8", + "rithum", + "rithumliboard", + "ritterandbrogdenorthodontics", + "ritual", + "rivainternationalinc", + "rivaltechnologies", + "riverai", + "riversidenaturalfoodsltd", + "riviamind1", + "roadie", + "robertrauschenbergfoundation", + "robinhood", + "roblox", + "roboforce", + "robotsandpencils", + "rockbot", + "rocketlab", + "rocketlawyer", + "rocketmiles", + "rockstargames", + "roivantsciences", + "roku", + "roller", + "rondoenergy", + "roo", + "roofr", + "roofstock", + "root", + "rothesaygraduates", + "rothesaylife", + "route06", + "rpa", + "rubiconcarbon", + "ruelala" + ], + "greenhouse-4": [ + "ruggable", + "ruggedrobotics", + "rumble", + "runwise", + "rushdownstudios", + "rushstreetinteractive", + "russett", + "rvi", + "rvohcontentfreelance", + "rvohealth", + "rxsense", + "rzero", + "rzr", + "safebreach", + "safetyworxs", + "sage49", + "sagebionetworks", + "salientmotion", + "saltbox", + "saltxc", + "samainc", + "samayaai", + "samsungresearchamerica", + "samsungresearchamericainternship", + "samsungsemiconductor", + "sandscapitalmanagementllc", + "sandstonecarecastlerock", + "saraworks", + "saxbys", + "saxllp", + "sayari", + "sbigrowth", + "sbusa", + "scaleai", + "scanner", + "schonfeld", + "schrdinger", + "science&purpose", + "science37", + "scileads", + "scm", + "scopely", + "scorpionenterprisesllc", + "scotch", + "scoutai", + "scoutmotors", + "scoutspace", + "scowtt", + "scsfinancial", + "sdcpinternshipprogram", + "seafireresortltd", + "seaporttherapeutics", + "seatown", + "seattlesoundersfc", + "secondharvest", + "secretariatadvisorsllc", + "securitize", + "securityscorecard", + "sedna", + "seed", + "seesaw", + "seisandbox", + "selffinancial", + "selinicapital", + "semafor", + "sendabiosciences", + "sendcloudnew", + "sensei", + "sensiblecare", + "seoulrobotics", + "septerna", + "serhant", + "serif", + "sertis", + "servicechampions", + "serviceexpertsllc", + "servicewizard", + "sesai", + "sesolabor", + "setpoint", + "setsales", + "sevenresearch", + "sezzle", + "sfox", + "shakepay", + "shapercapital", + "sharebite", + "sharkninjaoperatingllc", + "sharpelectronics", + "shearwater", + "shein", + "shennonbiotechnologies", + "shieldshealthsolutions", + "shift5", + "shifttechnology", + "shinola", + "shinolaretail", + "shipbobinc", + "shipmonk", + "shopltk", + "shopmy", + "showpad", + "shyftsolutionsllc", + "si", + "siboneinc", + "sidecarhealth", + "siei", + "sierrallc", + "sightlinemediagroup", + "sigmacomputing", + "sigmoid", + "signerscareers", + "signifyd95", + "silananotechnologies", + "silverado", + "silvr", + "silvus", + "silvus-international-opportunites", + "similarweb", + "simplesense", + "simpletechnologysolutions", + "simplextrading", + "simplifed", + "simplisafe", + "simpluris", + "simpplr", + "simtrabps", + "singlestore", + "singlestore-linkedin", + "sirenopt", + "sirum", + "sixfold", + "sixgeninc", + "sixspeed", + "sixthstreet", + "skedda", + "skildai-careers", + "skilledwoundcare", + "skinlaundry", + "skyepointdecisionsinc", + "skylighthq", + "skyryse", + "skysafe", + "sleepdoctor", + "slingshotaerospace", + "slingshotbiosciences", + "smaamerica", + "smallgirlspr2", + "smartasset", + "smartbear", + "smarterdx", + "smarterdxprivate", + "smartling", + "smartlyio", + "smartrent", + "smartsheet", + "smartypantsvitamins", + "smavagmbh", + "smithrx", + "snapmobileinc", + "snorkelai", + "snowcompanies", + "snsone", + "soci", + "socialfinance", + "sociallabsa", + "socialscienceresearchcouncil", + "socket", + "solarisbank", + "soldejaneiro", + "soldejaneirointernship", + "solidpower", + "sollishealth", + "solmentalhealth", + "soloioinc", + "solutions", + "sonatus", + "sonderaustralia", + "sonicwall", + "sonobello", + "sonyinteractiveentertainmentglobal", + "sonymusicasiacareers", + "sonymusiccanada", + "sonymusiccareersafrica", + "sonymusiccareersfrance", + "sonymusicentertainment", + "sonypicturesanimation", + "sonypicturesimageworks", + "soraunion", + "sorcero", + "sothebys", + "soundagriculture", + "soundcloud71", + "sourcegraph91", + "sourcemeridian", + "southernpovertylawcenter", + "southwesttitleloans", + "sovrn", + "spacekinetic", + "spaceship", + "spacex", + "spacexglobal", + "sparetech", + "sparkadvisors", + "sparkfund", + "spauldingridge", + "spcareers", + "specialty1", + "specterops", + "spectrumvascular", + "speechify", + "spinnakersupport", + "splashfinancial", + "splice", + "splitero", + "sportandspinephysicaltherapy", + "spothopper", + "spotme", + "spotter", + "springboard", + "springboardmentors", + "springfertility", + "springhealth66", + "sprintersportses", + "spsnorthamerica", + "spycloud", + "spyretherapeutics", + "squishable", + "stablekernel", + "stackadapt", + "stackav", + "stackblitz", + "stackcommerce", + "stackline", + "standardmetrics", + "stanley1913-us", + "stannesbelfieldschool", + "star-catcher", + "starburst", + "starcloud", + "starfaceworld", + "starfishneuroscience", + "starrez", + "startale", + "startcampus", + "stateroadah", + "steadfasthealth", + "stemhealthcare", + "sterlingtonpllc", + "sti", + "stirlingpdf", + "stockx", + "stone", + "stonekite", + "stonepatrocina", + "storycannabis", + "strandtherapeutics", + "stratacareers", + "stratainformationgroup", + "strategichr", + "strategicprojectpartners", + "stratolaunch", + "stressfree", + "striiminc", + "strike", + "stripe", + "strivehealth", + "strivepharmacy", + "striveworks", + "strongpointpartners", + "stubhubinc", + "studiokraftonboard", + "studsinc", + "studycontractors", + "stylusmedicine", + "submittable", + "subsplash", + "successacademycharterschool", + "successkpiinc", + "sudstop", + "summer", + "summitonevanderbilt", + "summittherapeutics", + "sumofus", + "sumologic", + "sumup", + "sunnyside", + "sunrise", + "sunset", + "suntimes", + "supergoop", + "supernal", + "superset", + "supersod", + "supplyhouse", + "supportingstrategies", + "surefirecyber", + "survata", + "surveymonkey", + "sustainabletalent", + "sustainablewestchester", + "sustainment", + "svetness", + "swanloveland", + "swayable", + "swellmedia", + "swiftsolar", + "synack", + "synacksrt", + "synaptrixlabs", + "sypnewsitetest", + "syskahennessy", + "systemstechnologyresearch", + "tactilemedical", + "tailscale", + "takealotcom", + "takealotgroup", + "taketwo", + "talkdesk2", + "talkspace", + "talkspacepsychiatry", + "talkspacetherapist", + "tandemlaunch", + "tandemmoneylimited", + "tanium", + "tankww", + "tapestryenergy", + "taskrabbit", + "tastylive", + "tastytrade", + "tatari", + "taxbit", + "taxvalet", + "tazewellpikeanimalclinic", + "teachinglab", + "teads1", + "teague", + "teamlfg", + "teammate", + "teammobot", + "teampicnic", + "teamrubicon", + "tebra", + "techholding", + "techstars57", + "tecovas", + "tefron", + "tegnainc", + "tekion", + "tekmetric", + "telnyx54", + "temporaltechnologies", + "temus", + "tenableinc", + "teneolinkedin", + "tennesseetitleloansinc", + "tenon", + "tenstorrent", + "tenstorrentuniversity", + "tenstreet", + "teravision", + "terraclear", + "terranorbitalcorporation", + "terzo", + "tesseratherapeutics", + "testlio", + "testnisc", + "texasairsystems", + "texascartitleandpaydayloanservicesinc", + "texaschillersystemsasp", + "textus", + "thalamusgme", + "thanx", + "thatlot", + "thatsnomoonentertainment", + "thealleninstitute", + "thebaltimorebanner", + "thebrattlegroup", + "thechempetitivegroupllc", + "thedoulanetwork", + "thedutchie", + "theeconomistgroup", + "theeverycompany", + "thefarmersdog", + "thefloridapanthers", + "thefork", + "thegialliancemanagementllccompany", + "thehealthmanagementacademy", + "thehutgroup", + "theiconic", + "theknotworldwide", + "theloomisagency", + "themaritimeaquarium", + "thematherevanston", + "themichaeljfoxfoundation", + "themjcos", + "themotleyfool", + "themuseumofscience", + "thena", + "thenewyorktimes", + "thenuclearcompany", + "theoncologyinstitute", + "theorchard", + "theplaceforchildrenwithautism", + "theplanningshopus", + "thesciongroupllc", + "thesiscareers", + "thesocialhub", + "thetradedesk", + "thevirtussolution", + "thevitacococompany", + "theweathercompany", + "thinkacademymy", + "thinkacademyus", + "thinkingmachines", + "thinkmarkets", + "thirdlove", + "thirdwaveautomation", + "thomasdoor", + "thomasvillechildcare", + "thoropass", + "thoughtworksreferral", + "threatlocker", + "thrivemarket", + "thtbc", + "tia", + "tide", + "tigera", + "tigergraph", + "tines", + "tintai", + "tippingpointcommunity", + "tlatechinc", + "tmc", + "toast", + "toastmastersinternational", + "togetherai", + "tollbit", + "tomofunfurbo", + "tomorrow", + "tomorrowhealth", + "toogoodtogo", + "toojaysdeli", + "topsort", + "topsteptrader", + "torcrobotics", + "torq", + "toshibaglobalcommercesolutions", + "totalrecon", + "totusmedicines", + "towerresearchcapital", + "tpcengineeringholdingsllc", + "tpgcareers", + "tpreducationllc", + "trace3", + "traegergrills", + "transactlyconnect", + "transcendinc", + "transcendtherapeutics", + "transfergo", + "transmarketgroup", + "trase", + "traveledgenetwork", + "treasuryprime", + "treelinebiosciences", + "tribalscale", + "trident", + "trigild", + "trinityparktalent", + "tripadvisor", + "tripledotstudios", + "triplewhale", + "triumvirateenvironmental", + "trivelta", + "triviumpoint", + "trove", + "trovohealth", + "trueanomalyinc", + "truebill", + "truecaller", + "truemedia", + "trufflesecurity", + "trustautomation", + "trustbank", + "trustwill", + "truveta", + "tsg", + "ttcglobal", + "tubescience-labs", + "tubescience52", + "tubitv", + "tucows", + "tudorgroup", + "turbineone", + "turbotenant", + "turing", + "turnkeycareers", + "twilio", + "twinhealth", + "twistbioscience", + "twitch", + "twosixtechnologies", + "typeface", + "typeform", + "tysonmendesllp", + "uareai", + "uasi", + "uberfreight", + "ubiquiti", + "ubiquitygp", + "udacity", + "udemybedi", + "ujet", + "ultimagenomics", + "umaeducationinc", + "umistone", + "unanet", + "unbounce", + "undercontrolroboticsinc", + "underdogfantasy", + "understood", + "understoodcare", + "unframe", + "unisonhomeownershipinvestors", + "unispace", + "uniswapfoundation", + "unitedfirm", + "unitedmasterstranslation", + "unitedmedia", + "uniteus", + "unknownworlds", + "unlock", + "unlockhealth", + "unrealsnacks", + "unybrands", + "up", + "upbound", + "upboundext", + "updater", + "upgrade", + "upkeep", + "upriteconstruction", + "upshop", + "upstack", + "upstart", + "upstatementrecruiting", + "upstreamusa", + "upwork", + "urbansky", + "urbansportsclub", + "ursamajor", + "usconec", + "usenourish", + "utahtitleloansinc", + "vacasa", + "vacationinc", + "vaco", + "vailhealthprivate", + "valaratomics", + "valohealth", + "valpro", + "valtech", + "vanmetre", + "vannevarlabs", + "vantagescore", + "vardaspace", + "varicent", + "vast", + "vaticlabs", + "vaxcyte", + "vectara", + "veeamsoftware", + "vegaamericas", + "veir", + "velocityelectronics", + "venncity", + "ventureglobal", + "veocorporatecareers", + "veracyte", + "verainstituteofjustice", + "veranahealth", + "veratherapeuticsinc", + "vercel", + "verisign", + "veristainc", + "veritasvetpartners", + "verkada", + "verkada2", + "verramobility", + "versaterm", + "verse", + "versprite", + "verstela", + "verve", + "vesalius", + "vestmark", + "vestwell", + "veterinaryemergencygroupst", + "veterinaryemergencyservices", + "veterinarypracticepartners", + "vetevolve", + "vettoai", + "vgw", + "vgw-canada", + "vgw-eu", + "via", + "via99", + "viamrobotics", + "vianttechnology", + "vibesllc", + "vikingglobalinvestors", + "vipvermontinformationprocessing2", + "viralnation", + "virbiotechnologyinc", + "virginbetsa", + "virtru", + "virtu", + "viseai", + "visia", + "visiersolutionsinc", + "visitingmedia", + "visualconcepts", + "vitalvoicesglobalpartnership", + "vitoriainternational", + "vitta", + "vivcourtevents", + "vivvi", + "vixxo", + "vmax", + "vmlenterprisesolutions", + "vogliodigitalmarketing", + "volastratherapeutics", + "vonage", + "vooban", + "vorbiopharma", + "voxmedia", + "voyagertechnologiesinc", + "voyagertherapeutics", + "vpawashington", + "vsapartners", + "vscfiresecurityinc", + "vsco39", + "vtex", + "vulcanelements", + "vulncheck", + "vynamic", + "vynyl", + "wakam", + "waldensecurity", + "wallapop", + "walleyecapital-external-students", + "wallstreetprep", + "wargamingen", + "warp", + "wasabi", + "waterloocoop", + "watershed", + "waverlyadvisorsllc", + "waymark", + "waymo", + "weave", + "webershandwick", + "webflow", + "wedgewoodpharmacy", + "weedmaps77", + "weee", + "weinsteinproperties", + "weissassetmanagement", + "welbehealth", + "wellist", + "wellsaidlabs", + "wellthy-care-network", + "weploy", + "westcancercenter", + "westmonroe4", + "wettermarkkeith", + "wfclainc", + "whalarinc", + "wheely", + "whisperaero", + "whitewatermidstream", + "whogivesacrap", + "whop", + "wight", + "wikimedia", + "wildalaskancompany", + "wildcardcreativegroup", + "wildlifestudios", + "wilsonelser", + "wilsonelserattorneys", + "wimanagementllc", + "wing", + "wingspan", + "winhomeinspection", + "winnerscirclegroupoftexas1", + "wisconsinautotitleloansinc", + "wisetack", + "withcoverage", + "withmeinc", + "wizardcommerce", + "wizixtechnologygroupinc", + "wolt", + "wonderschool", + "wongdoody", + "woo", + "wooga", + "woolpert", + "workatbackbase", + "workato", + "workera", + "workhelix", + "workithealth", + "workleap", + "workleapfr", + "workoverseas", + "workrightnw", + "workshop", + "workstream", + "workwize", + "world4822stri986des", + "worldlabs", + "worldquant", + "wovencare", + "wpp", + "wppmedia", + "wrike", + "wundercapital", + "ww", + "wwclinic", + "xai", + "xairatherapeutics", + "xantium", + "xapo61", + "xdinizioengage", + "xealth", + "xebiaapac", + "xebiacee", + "xebiausa", + "xendit", + "xohealthinc", + "xometry", + "xometryeurope", + "xometryturkey", + "xpengmotors", + "xpinc", + "xsolisinc", + "xtxmarketstechnologies", + "yalochatinc", + "yerbamadre", + "yext", + "yipitdata", + "yipitdatajobs", + "ylopo", + "yoodliinc", + "youcom", + "yugabyte", + "yurtsai", + "zafinlabsamericasinc", + "zam", + "zambold", + "zarminalihealth", + "zeffy", + "zenbusiness", + "zencoder", + "zengrc", + "zennioptical", + "zenoti", + "zephyrhome", + "zerostudios", + "zerotothree", + "zetachain", + "zetaglobal", + "zinnia", + "zinnov", + "zipcolimited", + "ziprecruiter", + "ziro", + "zocalohealth", + "zocdoc", + "zone5technologies", + "zonecompanysoftwareconsultingllc", + "zoo", + "zora", + "zscaler", + "zubiad", + "zuora", + "zupinnovation", + "zyngacareers", + "zyngaearlycareers" + ] + } +} diff --git a/scraper-go/internal/cronjob/gupy_cursor.go b/scraper-go/internal/cronjob/gupy_cursor.go new file mode 100644 index 00000000..1f311fbc --- /dev/null +++ b/scraper-go/internal/cronjob/gupy_cursor.go @@ -0,0 +1,70 @@ +package cronjob + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/redis/go-redis/v9" +) + +const gupyCursorKey = "scraper:provider:gupy:checkpoint" + +type gupyCursor struct { + scheduler *Scheduler + lease *runlock.Lease +} +type gupyCursorState struct { + Manifest string `json:"manifest"` + Offset int `json:"offset"` +} + +func (c *gupyCursor) Load(ctx context.Context, manifest string, count int) (int, error) { + raw, err := c.scheduler.rdb.Get(ctx, gupyCursorKey).Bytes() + if err == redis.Nil { + return 0, nil + } + if err != nil { + return 0, err + } + var state gupyCursorState + if err = json.Unmarshal(raw, &state); err != nil { + return 0, err + } + if state.Manifest != manifest { + return 0, fmt.Errorf("gupy query manifest changed; explicit cursor migration required") + } + if state.Offset < 0 || state.Offset >= count { + return 0, fmt.Errorf("invalid gupy cursor") + } + return state.Offset, nil +} + +var advanceGupyCursor = redis.NewScript(` +if redis.call('GET',KEYS[1]) ~= ARGV[1] then return -1 end +local raw=redis.call('GET',KEYS[2]) +if raw then + local state=cjson.decode(raw) + if state.manifest ~= ARGV[2] or state.offset ~= tonumber(ARGV[3]) then return 0 end +elseif tonumber(ARGV[3]) ~= 0 then return 0 end +redis.call('SET',KEYS[2],ARGV[4]) +return 1 +`) + +func (c *gupyCursor) Advance(_ context.Context, manifest string, count, index int) error { + next, _ := json.Marshal(gupyCursorState{Manifest: manifest, Offset: (index + 1) % count}) + // The processing context permits a bounded checkpoint during collection drain, + // while still honoring shutdown and ownership loss. + written, err := advanceGupyCursor.Run(c.lease.Context(), c.scheduler.rdb, []string{runlock.LockKey, gupyCursorKey}, c.lease.Token(), manifest, index, string(next)).Int() + if err != nil { + return err + } + if written == -1 { + return runlock.ErrLost + } + if written != 1 { + return fmt.Errorf("gupy cursor conflict") + } + return nil +} diff --git a/scraper-go/internal/cronjob/gupy_cursor_test.go b/scraper-go/internal/cronjob/gupy_cursor_test.go new file mode 100644 index 00000000..b41d5b62 --- /dev/null +++ b/scraper-go/internal/cronjob/gupy_cursor_test.go @@ -0,0 +1,87 @@ +package cronjob + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/stretchr/testify/require" +) + +func TestGupyCursorSurvivesRestartAndFencesWrites(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + ctx := context.Background() + lease, err := s.runLock.Acquire(ctx, "cron") + require.NoError(t, err) + cursor := &gupyCursor{s, lease} + position, err := cursor.Load(ctx, "manifest", 200) + require.NoError(t, err) + require.Zero(t, position) + require.NoError(t, cursor.Advance(ctx, "manifest", 200, 0)) + require.NoError(t, lease.Release(ctx)) + lease, err = s.runLock.Acquire(ctx, "cron") + require.NoError(t, err) + defer lease.Release(ctx) + restarted := &gupyCursor{New(s.cfg, s.kwStore, s.jobStore, s.adapterList, client, s.runLock), lease} + position, err = restarted.Load(ctx, "manifest", 200) + require.NoError(t, err) + require.Equal(t, 1, position) + _, err = restarted.Load(ctx, "changed", 200) + require.Error(t, err) + require.NoError(t, client.Set(ctx, runlock.LockKey, "new-owner", time.Minute).Err()) + require.ErrorIs(t, restarted.Advance(ctx, "manifest", 200, 1), runlock.ErrLost) + position, err = restarted.Load(ctx, "manifest", 200) + require.NoError(t, err) + require.Equal(t, 1, position) +} + +type checkpointGupySource struct{ t *testing.T } + +func (s checkpointGupySource) SourceName() string { return "Gupy" } +func (s checkpointGupySource) Capabilities() ports.SourceCapabilities { + return ports.SourceCapabilities{Provider: ports.ProviderGupy, Mode: ports.DiscoveryBatch} +} +func (s checkpointGupySource) Search(context.Context, string, domain.ScrapeRequest) ([]domain.Job, error) { + return nil, errors.New("expected batch") +} +func (s checkpointGupySource) SearchBatch(ctx context.Context, _ []string, req domain.ScrapeRequest) ([]domain.Job, error) { + require.True(s.t, req.GupyWindow) + require.NotNil(s.t, req.QueryCursor) + require.NotEmpty(s.t, req.WindowID) + offset, err := req.QueryCursor.Load(ctx, "manifest", 200) + if err != nil { + return nil, err + } + if err = req.QueryCursor.Advance(ctx, "manifest", 200, offset); err != nil { + return nil, err + } + job := schedulerTestJob() + job.Source = "gupy" + return []domain.Job{job}, errors.New("query truncated") +} +func TestGupyWindowWiresCursorAndPersistsPartialJobs(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + s.adapterList = []ports.JobSource{checkpointGupySource{t}} + windows, err := config.ParseProviderWindows(config.ProviderSchedulePreset) + require.NoError(t, err) + s.cfg.ProviderWindows = windows + day := time.Now().Add(24 * time.Hour) + for _, index := range []int{1, 7} { + start, _ := windowBounds(day, windows[index]) + require.Error(t, s.dispatchProviderWindow(context.Background(), start)) + require.Equal(t, "partial", s.LastRun().Status) + require.Positive(t, s.LastRun().JobsSaved) + require.Positive(t, s.LastRun().JobsIndexed) + } + var state gupyCursorState + raw, err := client.Get(context.Background(), gupyCursorKey).Bytes() + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &state)) + require.Equal(t, 2, state.Offset) +} diff --git a/scraper-go/internal/cronjob/keyword_window_test.go b/scraper-go/internal/cronjob/keyword_window_test.go new file mode 100644 index 00000000..8ddf9255 --- /dev/null +++ b/scraper-go/internal/cronjob/keyword_window_test.go @@ -0,0 +1,141 @@ +package cronjob + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/adapters/adapterutil" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/keywords" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/stretchr/testify/require" +) + +type keywordWindowCall struct { + keywords []string + req domain.ScrapeRequest + deadline time.Time +} +type keywordWindowSource struct { + provider ports.ProviderID + calls []keywordWindowCall + search func(context.Context, []string, domain.ScrapeRequest) ([]domain.Job, error) +} + +func (a *keywordWindowSource) SourceName() string { return string(a.provider) } +func (a *keywordWindowSource) Capabilities() ports.SourceCapabilities { + return ports.SourceCapabilities{Provider: a.provider, Mode: ports.DiscoveryBatch} +} +func (a *keywordWindowSource) Search(context.Context, string, domain.ScrapeRequest) ([]domain.Job, error) { + return nil, errors.New("expected batch discovery") +} +func (a *keywordWindowSource) SearchBatch(ctx context.Context, kw []string, req domain.ScrapeRequest) ([]domain.Job, error) { + deadline, _ := ctx.Deadline() + a.calls = append(a.calls, keywordWindowCall{append([]string(nil), kw...), req, deadline}) + if a.search != nil { + return a.search(ctx, kw, req) + } + return nil, nil +} + +func TestPresetKeywordWindowsReceiveFullConfiguredSetAndKeepManualCompatibility(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + linkedin := &keywordWindowSource{provider: ports.ProviderLinkedIn} + adzuna := &keywordWindowSource{provider: ports.ProviderAdzuna} + s.adapterList = []ports.JobSource{linkedin, adzuna} + windows, err := config.ParseProviderWindows(config.ProviderSchedulePreset) + require.NoError(t, err) + s.cfg.ProviderWindows = windows + var configured []string + for i := 0; i < 37; i++ { + configured = append(configured, fmt.Sprintf("query-%02d", i)) + } + require.NoError(t, s.kwStore.Save(context.Background(), configured)) + expected := keywords.GenerateSearchKeywords(configured) + day := time.Now().In(mustProviderLocation(t)).AddDate(0, 0, 1) + for _, w := range windows { + if w.Provider != ports.ProviderLinkedIn && w.Provider != ports.ProviderAdzuna { + continue + } + start, _ := windowBounds(day, w) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start)) + target := linkedin + if w.Provider == ports.ProviderAdzuna { + target = adzuna + } + call := target.calls[len(target.calls)-1] + require.Equal(t, expected, call.keywords) + require.True(t, call.req.FullKeywordCoverage) + require.Equal(t, w.WindowID(), call.req.WindowID) + require.NotEmpty(t, call.req.RunID) + require.Equal(t, s.cfg.MaxConcurrency, call.req.MaxConcurrency) + require.WithinDuration(t, time.Now(), call.deadline, s.cfg.ScrapeTimeout) + } + require.Len(t, linkedin.calls, 3) + require.Len(t, adzuna.calls, 2) + runOneProvider(t, s, "linkedin") + require.False(t, linkedin.calls[3].req.FullKeywordCoverage) + require.Empty(t, linkedin.calls[3].req.WindowID) + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) +} + +func TestKeywordWindowPartialResultsPreservePersistenceAndRetryState(t *testing.T) { + for _, provider := range []ports.ProviderID{ports.ProviderLinkedIn, ports.ProviderAdzuna} { + t.Run(string(provider), func(t *testing.T) { + for _, kind := range []string{"timeout", "rate_limit"} { + t.Run(kind, func(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + s.cfg.ScrapeTimeout = 80 * time.Millisecond + source := &keywordWindowSource{provider: provider} + calls := 0 + source.search = func(ctx context.Context, kw []string, req domain.ScrapeRequest) ([]domain.Job, error) { + require.True(t, req.FullKeywordCoverage) + return adapterutil.SearchAllKeywords(ctx, kw, req, provider, func(ctx context.Context, _ string, _ domain.ScrapeRequest) ([]domain.Job, error) { + calls++ + if calls == 1 { + job := schedulerTestJob() + job.Source = string(provider) + return []domain.Job{job}, nil + } + if kind == "timeout" { + <-ctx.Done() + return nil, ctx.Err() + } + return nil, &ports.RateLimitError{Provider: provider, NotBefore: time.Now().Add(time.Hour)} + }) + } + s.adapterList = []ports.JobSource{source} + require.NoError(t, s.kwStore.Save(context.Background(), []string{"go", "java", "rust"})) + windows, err := config.ParseProviderWindows(config.ProviderSchedulePreset) + require.NoError(t, err) + index := 3 + if provider == ports.ProviderAdzuna { + index = 4 + } + w := windows[index] + require.Equal(t, string(provider)+"-1", w.WindowID()) + s.cfg.ProviderWindows = []config.ProviderWindow{w} + start, _ := windowBounds(time.Now().Add(24*time.Hour), w) + require.Error(t, s.dispatchProviderWindow(context.Background(), start)) + require.Equal(t, "partial", s.LastRun().Status) + require.Positive(t, s.LastRun().JobsSaved) + require.Positive(t, s.LastRun().JobsIndexed) + require.Equal(t, 2, calls) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start.Add(time.Second))) + require.Equal(t, 2, calls) + if kind == "rate_limit" { + require.NotEmpty(t, client.Get(context.Background(), providerRetryKey(provider)).Val()) + var limit *ports.RateLimitError + require.ErrorAs(t, s.RunProvidersNow(context.Background(), []string{string(provider)}), &limit) + } + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) + }) + } + }) + } +} diff --git a/scraper-go/internal/cronjob/operational_budget_test.go b/scraper-go/internal/cronjob/operational_budget_test.go new file mode 100644 index 00000000..342927bc --- /dev/null +++ b/scraper-go/internal/cronjob/operational_budget_test.go @@ -0,0 +1,126 @@ +package cronjob + +import ( + "context" + "errors" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/stretchr/testify/require" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestGlobalPauseAppliesAfterSuccessFailureAndManualDispatch(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "failure"}[fail], func(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + s.runLock, _ = runlock.New(runlock.NewValkeyStore(client), runlock.Config{TTL: 2 * time.Second, RenewInterval: 500 * time.Millisecond}) + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn}} + if fail { + s.BeforeRun = func(context.Context) error { return errors.New("preparation failed") } + } + started := time.Now() + runOneProvider(t, s, "linkedin") + require.Less(t, time.Since(started), time.Second) + require.NotEmpty(t, client.Get(context.Background(), runlock.NextAllowedKey).Val()) + var pause *runlock.CooldownError + require.ErrorAs(t, s.RunProvidersNow(context.Background(), []string{"linkedin"}), &pause) + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) + }) + } +} + +func TestPreparationConsumesCollectionBudgetAndPartialDrainKeepsLock(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + // The existing blocking fake returns a valid job when collection expires. + s.cfg.ScrapeTimeout = 120 * time.Millisecond + var preparationDeadline time.Time + s.BeforeRun = func(ctx context.Context) error { + preparationDeadline, _ = ctx.Deadline() + timer := time.NewTimer(70 * time.Millisecond) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + ctx, cancel := context.WithTimeout(context.Background(), 350*time.Millisecond) + defer cancel() + start := time.Now() + s.runCron(ctx) + require.Less(t, preparationDeadline.Sub(start), 150*time.Millisecond) + require.Equal(t, "partial", s.LastRun().Status) + require.Positive(t, s.LastRun().JobsSaved) + require.Positive(t, s.LastRun().JobsIndexed) + require.Less(t, time.Since(start), 350*time.Millisecond) + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) +} + +func TestUnavailableAdmissionDoesNotRetryWhenStorageReturns(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + var calls atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &calls}} + w := config.ProviderWindow{Provider: ports.ProviderLinkedIn, ID: "linkedin-1", Start: 60, End: 120} + s.cfg.ProviderWindows = []config.ProviderWindow{w} + start, _ := windowBounds(time.Now().Add(24*time.Hour), w) + s.rdb = nil + require.ErrorIs(t, s.dispatchProviderWindow(context.Background(), start), ErrLockUnavailable) + s.rdb = client + require.NoError(t, s.dispatchProviderWindow(context.Background(), start.Add(time.Second))) + require.Zero(t, calls.Load()) + require.Equal(t, "missed:admission_unavailable", client.Get(context.Background(), windowOccurrenceKey(start, w)).Val()) +} + +func TestFixedWindowToleranceAndMissedOccurrenceNeverCatchUp(t *testing.T) { + for _, late := range []time.Duration{0, ProviderStartTolerance, ProviderStartTolerance + time.Nanosecond} { + t.Run(late.String(), func(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + frozen := time.Now() + s.admissionClock = func() time.Time { return frozen } + var calls atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &calls}} + w := config.ProviderWindow{Provider: ports.ProviderLinkedIn, ID: "linkedin-1", Start: 60, End: 120, Timezone: config.ProviderScheduleTimezone} + s.cfg.ProviderWindows = []config.ProviderWindow{w} + day := time.Now().Add(24 * time.Hour) + start, _ := windowBounds(day, w) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start.Add(-time.Nanosecond))) + require.Zero(t, calls.Load()) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start.Add(late))) + if late <= ProviderStartTolerance { + require.EqualValues(t, 1, calls.Load()) + } else { + require.Zero(t, calls.Load()) + require.True(t, strings.HasPrefix(client.Get(context.Background(), windowOccurrenceKey(start, w)).Val(), "missed:")) + } + require.NoError(t, s.dispatchProviderWindow(context.Background(), start.Add(20*time.Minute))) + if late <= ProviderStartTolerance { + require.EqualValues(t, 1, calls.Load()) + } else { + require.Zero(t, calls.Load()) + } + }) + } +} + +func TestBusyWindowIsConsumedAndOnlyReturnsNextDay(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + var calls atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &calls}} + w := config.ProviderWindow{Provider: ports.ProviderLinkedIn, ID: "linkedin-1", Start: 60, End: 120} + s.cfg.ProviderWindows = []config.ProviderWindow{w} + start, _ := windowBounds(time.Now().Add(24*time.Hour), w) + held, err := s.runLock.Acquire(context.Background(), "admin_manual") + require.NoError(t, err) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start)) + require.NoError(t, held.Release(context.Background())) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start.Add(time.Second))) + require.Zero(t, calls.Load()) + require.Equal(t, "missed:lock_busy", client.Get(context.Background(), windowOccurrenceKey(start, w)).Val()) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start.AddDate(0, 0, 1))) + require.EqualValues(t, 1, calls.Load()) +} diff --git a/scraper-go/internal/cronjob/provider_admission_test.go b/scraper-go/internal/cronjob/provider_admission_test.go new file mode 100644 index 00000000..a1f0bd2b --- /dev/null +++ b/scraper-go/internal/cronjob/provider_admission_test.go @@ -0,0 +1,123 @@ +package cronjob + +import ( + "context" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +// Advance only after the run ID has been persisted: the global lease is already +// held and all admission coordination has finished. No real waiting is needed. +type admissionLatencyHook struct { + key string + advance func() +} + +func (h admissionLatencyHook) DialHook(next redis.DialHook) redis.DialHook { return next } +func (h admissionLatencyHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return next +} +func (h admissionLatencyHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + err := next(ctx, cmd) + args := cmd.Args() + if err == nil && cmd.Name() == "set" && len(args) > 2 && args[1] == h.key { + value, _ := args[2].(string) + if !strings.HasPrefix(value, "admitting:") && !strings.HasPrefix(value, "missed:") { + h.advance() + } + } + return err + } +} + +func TestProviderAdmissionRechecksClockAfterCoordination(t *testing.T) { + for _, latency := range []time.Duration{time.Second, 30 * time.Second, 30*time.Second + time.Nanosecond, 31 * time.Second} { + t.Run(latency.String(), func(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + var calls atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &calls}} + w := config.ProviderWindow{Provider: ports.ProviderLinkedIn, ID: "linkedin-1", Start: 60, End: 120, Timezone: config.ProviderScheduleTimezone} + s.cfg.ProviderWindows = []config.ProviderWindow{w} + start, _ := windowBounds(time.Now().AddDate(0, 0, 1), w) + var elapsed atomic.Int64 + base := time.Now() + s.admissionClock = func() time.Time { return base.Add(time.Duration(elapsed.Load())) } + key := windowOccurrenceKey(start, w) + client.AddHook(admissionLatencyHook{key: key, advance: func() { elapsed.Add(int64(latency)) }}) + + require.NoError(t, s.dispatchProviderWindow(context.Background(), start)) + require.Equal(t, int64(latency), elapsed.Load(), "latency must occur after lock acquisition") + require.Zero(t, client.Exists(context.Background(), runlock.LockKey, runlock.StateKey).Val()) + if latency <= ProviderStartTolerance { + require.EqualValues(t, 1, calls.Load()) + } else { + require.Zero(t, calls.Load()) + require.Nil(t, s.LastRun(), "rejected admission is not a provider run") + require.Equal(t, "missed:admission_late", client.Get(context.Background(), key).Val()) + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, s.Shutdown(shutdownCtx), "admission must release active execution accounting") + }) + } +} + +func TestLateAdmissionSurvivesRestartAndDoesNotConsumeNextWindow(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + var calls atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &calls}} + w := config.ProviderWindow{Provider: ports.ProviderLinkedIn, ID: "linkedin-1", Start: 60, End: 120} + next := config.ProviderWindow{Provider: ports.ProviderLinkedIn, ID: "linkedin-2", Start: 120, End: 180} + s.cfg.ProviderWindows = []config.ProviderWindow{w, next} + start, _ := windowBounds(time.Now().AddDate(0, 0, 1), w) + var elapsed atomic.Int64 + base := time.Now() + s.admissionClock = func() time.Time { return base.Add(time.Duration(elapsed.Load())) } + key := windowOccurrenceKey(start, w) + client.AddHook(admissionLatencyHook{key: key, advance: func() { elapsed.Add(int64(31 * time.Second)) }}) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start)) + require.Zero(t, calls.Load()) + require.Equal(t, "missed:admission_late", client.Get(context.Background(), key).Val()) + + restarted := New(s.cfg, s.kwStore, s.jobStore, s.adapterList, client, s.runLock) + restarted.admissionClock = func() time.Time { return base } + // Even a second replica with a clock still inside the tolerance cannot retry. + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), start.Add(time.Second))) + require.Zero(t, calls.Load()) + nextStart, _ := windowBounds(start, next) + require.Zero(t, client.Exists(context.Background(), windowOccurrenceKey(nextStart, next)).Val()) + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), nextStart)) + require.EqualValues(t, 1, calls.Load()) + require.Equal(t, "missed:admission_late", client.Get(context.Background(), key).Val()) +} + +func TestLateAdmissionPreservesGlobalPause(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + var calls atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &calls}} + var err error + s.runLock, err = runlock.New(runlock.NewValkeyStore(client), runlock.Config{TTL: 2 * time.Second, RenewInterval: 500 * time.Millisecond}) + require.NoError(t, err) + w := config.ProviderWindow{Provider: ports.ProviderLinkedIn, ID: "linkedin-1", Start: 60, End: 120} + s.cfg.ProviderWindows = []config.ProviderWindow{w} + start, _ := windowBounds(time.Now().AddDate(0, 0, 1), w) + base := time.Now() + var elapsed atomic.Int64 + s.admissionClock = func() time.Time { return base.Add(time.Duration(elapsed.Load())) } + client.AddHook(admissionLatencyHook{key: windowOccurrenceKey(start, w), advance: func() { elapsed.Add(int64(31 * time.Second)) }}) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start)) + require.Zero(t, calls.Load()) + require.Zero(t, client.Exists(context.Background(), runlock.LockKey, runlock.StateKey).Val()) + require.NotEmpty(t, client.Get(context.Background(), runlock.NextAllowedKey).Val()) + var pause *runlock.CooldownError + require.ErrorAs(t, s.RunProvidersNow(context.Background(), []string{"linkedin"}), &pause) +} diff --git a/scraper-go/internal/cronjob/provider_batch.go b/scraper-go/internal/cronjob/provider_batch.go new file mode 100644 index 00000000..6088476d --- /dev/null +++ b/scraper-go/internal/cronjob/provider_batch.go @@ -0,0 +1,88 @@ +package cronjob + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/pipeline" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/redis/go-redis/v9" +) + +const greenhouseCheckpointKey = "scraper:provider:greenhouse:checkpoint" + +type batchCheckpoint struct { + Manifest string `json:"manifest"` + Offset int `json:"offset"` +} + +var saveCheckpoint = redis.NewScript(` +if redis.call('GET',KEYS[1]) ~= ARGV[1] then return 0 end +redis.call('SET',KEYS[2],ARGV[2]) +return 1 +`) + +func (s *Scheduler) providerBatch(ctx context.Context, requested []string) ([]ports.JobSource, func(*runlock.Lease) error, error) { + sources, err := pipeline.SelectProviders(s.adapterList, requested) + if err != nil { + return nil, nil, err + } + if len(requested) != 1 { + return sources, nil, nil + } + provider, _ := ports.ParseProviderID(requested[0]) + if provider != ports.ProviderGreenhouse { + return sources, nil, nil + } + if s.rdb == nil { + return nil, nil, ErrLockUnavailable + } + // Sorting and hashing all source identities keeps resumed batches stable and + // restarts safely at zero when the configured company registry changes. + sources = append([]ports.JobSource(nil), sources...) + sort.SliceStable(sources, func(i, j int) bool { return sources[i].SourceName() < sources[j].SourceName() }) + h := sha256.New() + for _, source := range sources { + fmt.Fprintf(h, "%d:%s\n", len(source.SourceName()), source.SourceName()) + } + manifest := hex.EncodeToString(h.Sum(nil)) + state := batchCheckpoint{Manifest: manifest} + raw, err := s.rdb.Get(ctx, greenhouseCheckpointKey).Result() + if err != nil && err != redis.Nil { + return nil, nil, err + } + if err == nil { + if err := json.Unmarshal([]byte(raw), &state); err != nil { + return nil, nil, fmt.Errorf("invalid greenhouse checkpoint: %w", err) + } + } + if state.Manifest != manifest { + state = batchCheckpoint{Manifest: manifest} + } + if state.Offset < 0 || state.Offset >= len(sources) { + return nil, nil, fmt.Errorf("invalid greenhouse checkpoint offset") + } + size := s.cfg.GreenhouseBatchSize + if size <= 0 || size > 1000 { + size = 1000 + } + end := min(len(sources), state.Offset+size) + batch := sources[state.Offset:end] + next := batchCheckpoint{Manifest: manifest, Offset: end % len(sources)} + return batch, func(lease *runlock.Lease) error { + payload, _ := json.Marshal(next) + written, err := saveCheckpoint.Run(lease.Context(), s.rdb, []string{runlock.LockKey, greenhouseCheckpointKey}, lease.Token(), string(payload)).Int() + if err != nil { + return fmt.Errorf("greenhouse checkpoint: %w", err) + } + if written != 1 { + return fmt.Errorf("%w: checkpoint ownership changed", runlock.ErrLost) + } + return nil + }, nil +} diff --git a/scraper-go/internal/cronjob/provider_dispatch_test.go b/scraper-go/internal/cronjob/provider_dispatch_test.go new file mode 100644 index 00000000..8d166574 --- /dev/null +++ b/scraper-go/internal/cronjob/provider_dispatch_test.go @@ -0,0 +1,135 @@ +package cronjob + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/stretchr/testify/require" +) + +type dispatchSource struct { + err error + provider ports.ProviderID + name string + calls *atomic.Int32 + fail *atomic.Bool +} + +func (s dispatchSource) SourceName() string { return s.name } +func (s dispatchSource) Capabilities() ports.SourceCapabilities { + return ports.SourceCapabilities{Provider: s.provider, Mode: ports.DiscoveryKeyword} +} +func (s dispatchSource) Search(context.Context, string, domain.ScrapeRequest) ([]domain.Job, error) { + if s.calls != nil { + s.calls.Add(1) + } + if s.err != nil { + return nil, s.err + } + if s.fail != nil && s.fail.Load() { + return nil, errors.New("provider unavailable") + } + return nil, nil +} + +func runOneProvider(t *testing.T, s *Scheduler, provider string) { + t.Helper() + previous := s.LastRun() + previousID := "" + if previous != nil { + previousID = previous.RunID + } + require.NoError(t, s.RunProvidersNow(context.Background(), []string{provider})) + require.Eventually(t, func() bool { r := s.LastRun(); return r != nil && r.RunID != previousID }, time.Second, time.Millisecond) +} + +func TestManualProviderDispatchIsIsolatedAndValidated(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + var linkedin, gupy atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &linkedin}, dispatchSource{provider: ports.ProviderGupy, calls: &gupy}} + runOneProvider(t, s, "linkedin") + require.EqualValues(t, 1, linkedin.Load()) + require.Zero(t, gupy.Load()) + require.Error(t, s.RunProvidersNow(context.Background(), []string{"invalid"})) + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) +} + +func TestProviderWindowClaimSurvivesRestartAndWaitsForGlobalLock(t *testing.T) { + s, _, _ := newConcurrentTestScheduler(t) + var calls atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &calls}} + s.cfg.ProviderWindows = []config.ProviderWindow{{Provider: ports.ProviderLinkedIn, Start: 0, End: 1440}} + now := time.Now().UTC().AddDate(0, 0, 1) + now = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) + held, err := s.runLock.Acquire(context.Background(), "public_endpoint") + require.NoError(t, err) + require.NoError(t, s.dispatchProviderWindow(context.Background(), now)) + require.Zero(t, calls.Load()) + require.NoError(t, held.Release(context.Background())) + require.NoError(t, s.dispatchProviderWindow(context.Background(), now)) + require.Zero(t, calls.Load()) + now = now.AddDate(0, 0, 1) + require.NoError(t, s.dispatchProviderWindow(context.Background(), now)) + require.EqualValues(t, 1, calls.Load()) + restarted := New(s.cfg, s.kwStore, s.jobStore, s.adapterList, s.rdb, s.runLock) + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), now)) + require.EqualValues(t, 1, calls.Load()) + _, end := windowBounds(now, config.ProviderWindow{Start: 0, End: 1440}) + require.Equal(t, time.UTC, end.Location()) +} + +func TestGreenhouseBatchIsBoundedAndCheckpointIsFenced(t *testing.T) { + s, _, _ := newConcurrentTestScheduler(t) + s.adapterList = nil + for i := 0; i < 1005; i++ { + s.adapterList = append(s.adapterList, dispatchSource{provider: ports.ProviderGreenhouse, name: fmt.Sprintf("board-%04d", i)}) + } + batch, commit, err := s.providerBatch(context.Background(), []string{"greenhouse"}) + require.NoError(t, err) + require.Len(t, batch, 1000) + lease, err := s.runLock.Acquire(context.Background(), "admin_manual") + require.NoError(t, err) + require.NoError(t, commit(lease)) + require.NoError(t, lease.Release(context.Background())) + batch, commit, err = s.providerBatch(context.Background(), []string{"greenhouse"}) + require.NoError(t, err) + require.Len(t, batch, 5) + require.Equal(t, "board-1000", batch[0].SourceName()) + before := s.rdb.Get(context.Background(), greenhouseCheckpointKey).Val() + newLease, err := s.runLock.Acquire(context.Background(), "admin_manual") + require.NoError(t, err) + require.NoError(t, s.rdb.Set(context.Background(), runlock.LockKey, "replacement-owner", time.Minute).Err()) + require.ErrorIs(t, commit(newLease), runlock.ErrLost) + _ = newLease.Release(context.Background()) + require.Equal(t, before, s.rdb.Get(context.Background(), greenhouseCheckpointKey).Val()) + s.adapterList = append(s.adapterList, dispatchSource{provider: ports.ProviderGreenhouse, name: "board-new"}) + batch, _, err = s.providerBatch(context.Background(), []string{"greenhouse"}) + require.NoError(t, err) + require.Equal(t, "board-0000", batch[0].SourceName()) +} + +func TestGreenhouseCheckpointOnlyAdvancesAfterSuccessfulProcessing(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + s.cfg.GreenhouseBatchSize = 1 + var fail atomic.Bool + fail.Store(true) + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderGreenhouse, name: "a", fail: &fail}, dispatchSource{provider: ports.ProviderGreenhouse, name: "b"}} + runOneProvider(t, s, "greenhouse") + require.NotEqual(t, "success", s.LastRun().Status) + require.Zero(t, client.Exists(context.Background(), greenhouseCheckpointKey).Val()) + fail.Store(false) + runOneProvider(t, s, "greenhouse") + require.Equal(t, "success", s.LastRun().Status) + batch, _, err := s.providerBatch(context.Background(), []string{"greenhouse"}) + require.NoError(t, err) + require.Len(t, batch, 1) + require.Equal(t, "b", batch[0].SourceName()) +} diff --git a/scraper-go/internal/cronjob/provider_retry.go b/scraper-go/internal/cronjob/provider_retry.go new file mode 100644 index 00000000..0f48fe5c --- /dev/null +++ b/scraper-go/internal/cronjob/provider_retry.go @@ -0,0 +1,72 @@ +package cronjob + +import ( + "context" + "fmt" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/redis/go-redis/v9" +) + +func providerRetryKey(provider ports.ProviderID) string { + return "scraper:provider:" + string(provider) + ":retry_after" +} + +func (s *Scheduler) retryProviders(requested []string) []ports.ProviderID { + if len(requested) == 0 { + for _, source := range s.adapterList { + requested = append(requested, string(ports.CapabilitiesOf(source).Provider)) + } + } + var providers []ports.ProviderID + seen := map[ports.ProviderID]bool{} + for _, name := range requested { + if provider, known := ports.ParseProviderID(name); known && !seen[provider] { + seen[provider] = true + providers = append(providers, provider) + } + } + return providers +} + +func (s *Scheduler) checkProviderDelay(ctx context.Context, sources []string) error { + for _, provider := range s.retryProviders(sources) { + if s.rdb == nil { + return ErrLockUnavailable + } + value, err := s.rdb.Get(ctx, providerRetryKey(provider)).Result() + if err == redis.Nil { + continue + } + if err != nil { + return err + } + at, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return fmt.Errorf("invalid provider retry state: %w", err) + } + if time.Now().Before(at) { + return &ports.RateLimitError{Provider: provider, NotBefore: at} + } + } + return nil +} + +func (s *Scheduler) rememberRetryAfter(lease *runlock.Lease, sources []string, collectionError error) error { + for _, provider := range s.retryProviders(sources) { + at := ports.LatestRetryAfter(collectionError, provider) + if !at.After(time.Now()) { + continue + } + value, err := saveCheckpoint.Run(lease.Context(), s.rdb, []string{runlock.LockKey, providerRetryKey(provider)}, lease.Token(), at.UTC().Format(time.RFC3339Nano)).Int() + if err != nil { + return fmt.Errorf("provider retry state: %w", err) + } + if value != 1 { + return runlock.ErrLost + } + } + return nil +} diff --git a/scraper-go/internal/cronjob/provider_retry_test.go b/scraper-go/internal/cronjob/provider_retry_test.go new file mode 100644 index 00000000..416c31e2 --- /dev/null +++ b/scraper-go/internal/cronjob/provider_retry_test.go @@ -0,0 +1,34 @@ +package cronjob + +import ( + "context" + "errors" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" + "sync/atomic" + "testing" + "time" +) + +func TestProviderRetryAfterSurvivesRestartAndDoesNotConsumeWindow(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + var calls atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderGreenhouse, name: "a", calls: &calls, err: &ports.RateLimitError{Provider: ports.ProviderGreenhouse, NotBefore: time.Now().Add(time.Hour)}}} + runOneProvider(t, s, "greenhouse") + require.Equal(t, "rate_limit", s.LastRun().ErrorType) + require.NotEmpty(t, client.Get(context.Background(), providerRetryKey(ports.ProviderGreenhouse)).Val()) + restarted := New(s.cfg, s.kwStore, s.jobStore, s.adapterList, s.rdb, s.runLock) + var limited *ports.RateLimitError + require.True(t, errors.As(restarted.RunProvidersNow(context.Background(), []string{"greenhouse"}), &limited)) + require.Error(t, restarted.RunNow(context.Background())) // Global requests cannot bypass provider cooldowns. + restarted.cfg.ProviderWindows = []config.ProviderWindow{{Provider: ports.ProviderGreenhouse, Start: 0, End: 1440}} + now, _ := windowBounds(time.Now().Add(24*time.Hour), restarted.cfg.ProviderWindows[0]) + require.Error(t, restarted.dispatchProviderWindow(context.Background(), now)) + require.Len(t, client.Keys(context.Background(), "scraper:window:*").Val(), 1) + require.EqualValues(t, 1, calls.Load()) + require.NoError(t, client.Set(context.Background(), providerRetryKey(ports.ProviderGreenhouse), time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano), 0).Err()) + restarted.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderGreenhouse, name: "a", calls: &calls}} + runOneProvider(t, restarted, "greenhouse") + require.EqualValues(t, 2, calls.Load()) +} diff --git a/scraper-go/internal/cronjob/provider_schedule.go b/scraper-go/internal/cronjob/provider_schedule.go new file mode 100644 index 00000000..8a676614 --- /dev/null +++ b/scraper-go/internal/cronjob/provider_schedule.go @@ -0,0 +1,207 @@ +package cronjob + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/metrics" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/redis/go-redis/v9" +) + +// ProviderStartTolerance accommodates the existing 15-second polling cadence. +// It never permits retrying a consumed or impeded occurrence. +const ProviderStartTolerance = 30 * time.Second + +func minTime(a, b time.Time) time.Time { + if a.Before(b) { + return a + } + return b +} + +// Windows use the existing global lease. A persistent daily claim prevents +// repeated starts after completion, replica startup or a process restart. +func (s *Scheduler) startProviderWindows(ctx context.Context) { + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + defer metrics.SetNextRun(time.Time{}) + for { + if err := s.dispatchProviderWindow(ctx, time.Now().UTC()); err != nil { + slog.Warn("provider window not executed", "errorType", metrics.ErrorType(err)) + } + select { + case <-ctx.Done(): + return + case <-s.stop: + return + case <-ticker.C: + } + } +} + +func windowBounds(now time.Time, w config.ProviderWindow) (time.Time, time.Time) { + loc := w.Location() + now = now.In(loc) + return time.Date(now.Year(), now.Month(), now.Day(), w.Start/60, w.Start%60, 0, 0, loc), + time.Date(now.Year(), now.Month(), now.Day(), w.End/60, w.End%60, 0, 0, loc) +} + +func nextWindowStart(now time.Time, w config.ProviderWindow) time.Time { + start, _ := windowBounds(now, w) + if !start.After(now) { + start, _ = windowBounds(now.In(w.Location()).AddDate(0, 0, 1), w) + } + return start +} + +func windowOccurrenceKey(start time.Time, w config.ProviderWindow) string { + // Keep the existing persisted key for legacy configurations across upgrades. + if w.ID == "" && w.Location() == time.UTC { + return fmt.Sprintf("scraper:window:%s:%s:%d", start.Format("2006-01-02"), w.Provider, w.Start) + } + return fmt.Sprintf("scraper:window:%s:%s:%s", w.Location().String(), start.Format("2006-01-02"), w.WindowID()) +} + +func (s *Scheduler) dispatchProviderWindow(ctx context.Context, now time.Time) error { + clock := s.admissionClock + if clock == nil { + clock = time.Now + } + // Include coordination latency in the schedule timestamp. Monotonic elapsed + // time prevents wall-clock adjustments from reopening the admission period. + dispatchStarted := clock() + var next time.Time + for _, w := range s.cfg.ProviderWindows { + start := nextWindowStart(now, w) + if next.IsZero() || start.Before(next) { + next = start + } + } + metrics.SetNextRun(next) + for _, w := range s.cfg.ProviderWindows { + start, end := windowBounds(now, w) + if !now.Before(end) { + if err := s.omitProviderWindow(ctx, start, w, "window_expired"); err != nil { + return err + } + continue + } + if now.Before(start) || !now.Before(end) { + continue + } + if now.Sub(start) > ProviderStartTolerance { + if err := s.omitProviderWindow(ctx, start, w, "late_start"); err != nil { + return err + } + continue + } + key := windowOccurrenceKey(start, w) + attempt := "admitting:" + rand.Text() + s.mu.Lock() + previous := s.failedAdmissions[key] + s.mu.Unlock() + if previous != "" && s.rdb != nil { + value, readErr := s.rdb.Get(ctx, key).Result() + if readErr == redis.Nil { + return s.omitProviderWindow(ctx, start, w, "admission_unavailable") + } + if readErr != nil { + return readErr + } + if value == previous { + return s.rdb.SetXX(ctx, key, "missed:admission_unavailable", 48*time.Hour).Err() + } + return nil + } + rememberFailure := func() { + s.mu.Lock() + defer s.mu.Unlock() + if s.failedAdmissions == nil { + s.failedAdmissions = map[string]string{} + } + s.failedAdmissions[key] = attempt + } + if s.rdb == nil { + rememberFailure() + return ErrLockUnavailable + } + claimed, err := s.rdb.SetNX(ctx, key, attempt, 48*time.Hour).Result() + if err != nil { + rememberFailure() + } + if err != nil || !claimed { + return err + } + miss := func(reason string, cause error) error { + written, updateErr := s.rdb.SetXX(ctx, key, "missed:"+reason, 48*time.Hour).Result() + if written { + slog.Warn("provider window missed", "provider", w.Provider, "window_id", w.WindowID(), "local_date", start.Format("2006-01-02"), "reason", reason) + } + return errors.Join(cause, updateErr) + } + if err := s.checkProviderDelay(ctx, []string{string(w.Provider)}); err != nil { + return miss("provider_delay", err) + } + deadline := minTime(end, start.Add(runlock.MaxExecutionDuration)) + windowCtx, cancel := context.WithDeadline(ctx, deadline) + defer cancel() + lease, err := s.acquire(windowCtx, "cron") + if err != nil { + var pause *runlock.CooldownError + if errors.As(err, &pause) { + return miss("global_pause", nil) + } + if errors.Is(err, ErrAlreadyRunning) { + return miss("lock_busy", nil) + } + return miss("admission_failed", err) + } + if err := s.rdb.Set(windowCtx, key, lease.RunID(), 48*time.Hour).Err(); err != nil { + s.active.Done() + releaseCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) + defer stop() + _ = lease.Release(releaseCtx) + metrics.FinishRun(err, lease.RunID()) + return err + } + + defer s.active.Done() + // Decide admission while holding the global lease, after the last Redis + // operation. A slow claim or lock acquisition must never cause catch-up. + if now.Add(clock().Sub(dispatchStarted)).Sub(start) > ProviderStartTolerance { + missErr := miss("admission_late", nil) + releaseCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) + defer stop() + releaseErr := lease.Release(releaseCtx) + metrics.FinishRun(context.Canceled, lease.RunID()) + return errors.Join(missErr, releaseErr) + } + fields := []any{"provider", w.Provider, "window_id", w.WindowID(), "local_date", start.Format("2006-01-02"), "timezone", w.Location().String(), "window_end", end, "run_id", lease.RunID()} + slog.Info("provider window started", fields...) + err = s.runSelectedWindowWithLease(lease, []string{string(w.Provider)}, s.cfg.ScrapeTimeout, &w) + slog.Info("provider window finished", append(fields, "errorType", metrics.ErrorType(err))...) + return err + } + return nil +} + +func (s *Scheduler) omitProviderWindow(ctx context.Context, start time.Time, w config.ProviderWindow, reason string) error { + if s.rdb == nil { + return ErrLockUnavailable + } + written, err := s.rdb.SetNX(ctx, windowOccurrenceKey(start, w), "missed:"+reason, 48*time.Hour).Result() + if written { + slog.Warn("provider window omitted", "provider", w.Provider, "window_id", w.WindowID(), "local_date", start.Format("2006-01-02"), "reason", reason) + } + return err +} + +func (s *Scheduler) ProviderWindows() []config.ProviderWindow { + return append([]config.ProviderWindow(nil), s.cfg.ProviderWindows...) +} diff --git a/scraper-go/internal/cronjob/provider_schedule_p0_test.go b/scraper-go/internal/cronjob/provider_schedule_p0_test.go new file mode 100644 index 00000000..7134a0d9 --- /dev/null +++ b/scraper-go/internal/cronjob/provider_schedule_p0_test.go @@ -0,0 +1,149 @@ +package cronjob + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/stretchr/testify/require" +) + +func TestProviderWindowLocalDatesAndNextStart(t *testing.T) { + w := config.ProviderWindow{Provider: ports.ProviderGupy, ID: "gupy-night", Start: 23 * 60, End: 1440, Timezone: config.ProviderScheduleTimezone} + now := time.Date(2026, 11, 3, 2, 30, 0, 0, time.UTC) + start, end := windowBounds(now, w) + require.Equal(t, "2026-11-02", start.Format("2006-01-02")) + require.Equal(t, time.Date(2026, 11, 3, 2, 0, 0, 0, time.UTC), start.UTC()) + require.Equal(t, time.Date(2026, 11, 3, 3, 0, 0, 0, time.UTC), end.UTC()) + require.Equal(t, time.Date(2026, 11, 4, 2, 0, 0, 0, time.UTC), nextWindowStart(now, w).UTC()) + require.Contains(t, windowOccurrenceKey(start, w), "2026-11-02:gupy-night") + w.Start, w.End = 0, 60 + start, _ = windowBounds(end, w) + require.Equal(t, "2026-11-03", start.Format("2006-01-02")) + require.Equal(t, end.UTC(), start.UTC()) +} + +func TestProviderWindowUsesCivilDayAcrossTimezoneChanges(t *testing.T) { + w := config.ProviderWindow{Start: 0, End: 1440, Timezone: "America/New_York"} + now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC) + start, end := windowBounds(now, w) + require.Equal(t, 25*time.Hour, end.Sub(start)) + require.Equal(t, end, nextWindowStart(now, w)) +} + +func TestProviderWindowOccurrenceIsolationAndRestart(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + var linkedin, gupy atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &linkedin}, dispatchSource{provider: ports.ProviderGupy, calls: &gupy}} + // Use a future hour so the real lease deadline is valid even when tests + // start in the last seconds of a civil window. + now := time.Now().UTC().Truncate(time.Hour).Add(time.Hour) + local := now.In(mustProviderLocation(t)) + hour := local.Hour() + w := config.ProviderWindow{Provider: ports.ProviderLinkedIn, ID: "linkedin-1", Start: hour * 60, End: (hour + 1) * 60, Timezone: config.ProviderScheduleTimezone} + s.cfg.ProviderWindows = []config.ProviderWindow{w} + require.NoError(t, s.dispatchProviderWindow(context.Background(), now)) + require.EqualValues(t, 1, linkedin.Load()) + require.Zero(t, gupy.Load()) + restarted := New(s.cfg, s.kwStore, s.jobStore, s.adapterList, client, s.runLock) + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), now)) + require.EqualValues(t, 1, linkedin.Load()) + // A second same-provider window has its own daily occurrence. + w.ID = "linkedin-2" + w.Start, w.End = ((hour+1)%24)*60, ((hour+1)%24+1)*60 + restarted.cfg.ProviderWindows = []config.ProviderWindow{w} + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), now.Add(time.Hour))) + require.EqualValues(t, 2, linkedin.Load()) + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), now.Add(time.Hour))) + require.EqualValues(t, 2, linkedin.Load()) + // The following local day permits a new occurrence of that same window. + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), now.Add(25*time.Hour))) + require.EqualValues(t, 3, linkedin.Load()) + require.Zero(t, gupy.Load()) + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) +} + +func mustProviderLocation(t *testing.T) *time.Location { + t.Helper() + loc, err := time.LoadLocation(config.ProviderScheduleTimezone) + require.NoError(t, err) + return loc +} + +func TestProviderWindowLegacyOccurrenceKeyIsPreserved(t *testing.T) { + w := config.ProviderWindow{Provider: ports.ProviderLinkedIn, Start: 60, End: 120} + start, _ := windowBounds(time.Date(2026, 11, 2, 0, 0, 0, 0, time.UTC), w) + require.Equal(t, "scraper:window:2026-11-02:linkedin:60", windowOccurrenceKey(start, w)) +} + +func TestProviderScheduleDispatches21OccurrencesWithoutRepeatingAfterRestart(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + windows, err := config.ParseProviderWindows(config.ProviderSchedulePreset) + require.NoError(t, err) + s.cfg.ProviderWindows = windows + s.adapterList = nil + calls := map[ports.ProviderID]*atomic.Int32{} + for _, w := range windows { + if calls[w.Provider] == nil { + calls[w.Provider] = &atomic.Int32{} + if w.Provider == ports.ProviderGreenhouse { + s.adapterList = append(s.adapterList, fixedGroupTestRegistry(t, func(context.Context, string) ([]domain.Job, error) { + calls[ports.ProviderGreenhouse].Add(1) + return nil, nil + })...) + continue + } + s.adapterList = append(s.adapterList, dispatchSource{provider: w.Provider, name: string(w.Provider), calls: calls[w.Provider]}) + } + } + day := time.Now().In(mustProviderLocation(t)).AddDate(0, 0, 1) + for _, w := range windows { + start, _ := windowBounds(day, w) + require.NoError(t, s.dispatchProviderWindow(context.Background(), start), w.WindowID()) + } + require.Len(t, client.Keys(context.Background(), "scraper:window:*").Val(), 21) + expected := map[ports.ProviderID]int32{} + for _, w := range windows { + expected[w.Provider]++ + } + expected[ports.ProviderGreenhouse] = 3339 + for provider, count := range expected { + require.Equal(t, count, calls[provider].Load(), provider) + } + restarted := New(s.cfg, s.kwStore, s.jobStore, s.adapterList, client, s.runLock) + for _, w := range windows { + start, _ := windowBounds(day, w) + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), start)) + } + for provider, count := range expected { + require.Equal(t, count, calls[provider].Load(), provider) + } + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) +} + +func TestProviderWindowInterruptedClaimAndBusyLockDoNotStartProviders(t *testing.T) { + s, _, client := newConcurrentTestScheduler(t) + var calls atomic.Int32 + s.adapterList = []ports.JobSource{dispatchSource{provider: ports.ProviderLinkedIn, calls: &calls}} + w := config.ProviderWindow{Provider: ports.ProviderLinkedIn, ID: "linkedin-1", Start: 0, End: 1440, Timezone: config.ProviderScheduleTimezone} + s.cfg.ProviderWindows = []config.ProviderWindow{w} + now, _ := windowBounds(time.Now().UTC().Add(24*time.Hour), w) + held, err := s.runLock.Acquire(context.Background(), "admin_manual") + require.NoError(t, err) + require.NoError(t, s.dispatchProviderWindow(context.Background(), now)) + require.Zero(t, calls.Load()) + require.Len(t, client.Keys(context.Background(), "scraper:window:*").Val(), 1) + require.NoError(t, held.Release(context.Background())) + // Simulate a process that died after claiming but before completing work. + start, _ := windowBounds(now, w) + require.NoError(t, client.Set(context.Background(), windowOccurrenceKey(start, w), "interrupted-run", 48*time.Hour).Err()) + restarted := New(s.cfg, s.kwStore, s.jobStore, s.adapterList, client, s.runLock) + require.NoError(t, restarted.dispatchProviderWindow(context.Background(), now)) + require.Zero(t, calls.Load()) + require.Zero(t, client.Exists(context.Background(), runlock.LockKey).Val()) +} diff --git a/scraper-go/internal/domain/job.go b/scraper-go/internal/domain/job.go index 749a3624..00660a09 100644 --- a/scraper-go/internal/domain/job.go +++ b/scraper-go/internal/domain/job.go @@ -1,6 +1,15 @@ package domain -import "time" +import ( + "context" + "time" +) + +// QueryCursor checkpoints only queries admitted for an HTTP attempt. +type QueryCursor interface { + Load(context.Context, string, int) (int, error) + Advance(context.Context, string, int, int) error +} type Job struct { CatalogChange string `json:"-"` @@ -34,19 +43,24 @@ type Classification struct { } type ScrapeRequest struct { - Keywords []string `json:"keywords"` - SearchLocation string `json:"searchLocation"` - SearchGeoID string `json:"searchGeoId"` - SearchLanguage string `json:"searchLanguage"` - JobTypes string `json:"jobTypes"` - TimeFilter string `json:"timeFilter"` - RemoteOnly bool `json:"remoteOnly"` - Sources []string `json:"sources"` - ResultsPerPage int `json:"resultsPerPage"` - MaxPagesPerKeyword int `json:"maxPagesPerKeyword"` - WaitBetweenSearchesMs int `json:"waitBetweenSearchesMs"` - PageTimeoutMs int `json:"pageTimeoutMs"` - MaxConcurrency int `json:"maxConcurrency"` + GupyWindow bool `json:"-"` + QueryCursor QueryCursor `json:"-"` + FullKeywordCoverage bool `json:"-"` + WindowID string `json:"-"` + RunID string `json:"-"` + Keywords []string `json:"keywords"` + SearchLocation string `json:"searchLocation"` + SearchGeoID string `json:"searchGeoId"` + SearchLanguage string `json:"searchLanguage"` + JobTypes string `json:"jobTypes"` + TimeFilter string `json:"timeFilter"` + RemoteOnly bool `json:"remoteOnly"` + Sources []string `json:"sources"` + ResultsPerPage int `json:"resultsPerPage"` + MaxPagesPerKeyword int `json:"maxPagesPerKeyword"` + WaitBetweenSearchesMs int `json:"waitBetweenSearchesMs"` + PageTimeoutMs int `json:"pageTimeoutMs"` + MaxConcurrency int `json:"maxConcurrency"` } type ScrapeResponse struct { diff --git a/scraper-go/internal/jobindex/index.go b/scraper-go/internal/jobindex/index.go index d435f27e..f4004e68 100644 --- a/scraper-go/internal/jobindex/index.go +++ b/scraper-go/internal/jobindex/index.go @@ -105,35 +105,97 @@ func (m *Manager) Active(ctx context.Context) (string, error) { } return v, e } + +// ApplyStats describes one attempt. Confirmed is false after an ambiguous +// communication failure: the server may have applied writes without a reply. +type ApplyStats struct { + Received int + Changed int + FullApplied int + Removed int + Skipped int + Confirmed bool + PayloadBytes int + KeysTotal int + KeysMin int + KeysMax int + IndexKeys time.Duration + BuildMetadata time.Duration + Serialization time.Duration + RedisRoundTrip time.Duration + Total time.Duration + FailureStage string +} + func (m *Manager) Apply(ctx context.Context, jobs []domain.Job, keys func(domain.Job) []string) (int, error) { + result, err := m.ApplyDetailed(ctx, jobs, keys) + return result.Changed, err +} + +// ApplyDetailed adds observation only; Apply retains its existing contract. +func (m *Manager) ApplyDetailed(ctx context.Context, jobs []domain.Job, keys func(domain.Job) []string) (result ApplyStats, returnErr error) { started := time.Now() + result.Received = len(jobs) + defer func() { + result.Total = time.Since(started) + metrics.ObserveIndex(started, len(jobs), result.Changed, returnErr) + }() version, err := m.Active(ctx) if err != nil { - metrics.ObserveIndex(started, len(jobs), 0, err) - return 0, err + result.FailureStage = "active_lookup" + return result, err } - return m.ApplyVersion(ctx, version, jobs, keys, false) + returnErr = m.applyVersion(ctx, version, jobs, keys, false, &result) + return result, returnErr } -func (m *Manager) ApplyVersion(ctx context.Context, version string, jobs []domain.Job, keys func(domain.Job) []string, draft bool) (changed int, returnErr error) { + +func (m *Manager) ApplyVersion(ctx context.Context, version string, jobs []domain.Job, keys func(domain.Job) []string, draft bool) (int, error) { started := time.Now() - defer func() { metrics.ObserveIndex(started, len(jobs), changed, returnErr) }() + result := ApplyStats{Received: len(jobs)} + err := m.applyVersion(ctx, version, jobs, keys, draft, &result) + metrics.ObserveIndex(started, len(jobs), result.Changed, err) + return result.Changed, err +} + +func (m *Manager) applyVersion(ctx context.Context, version string, jobs []domain.Job, keys func(domain.Job) []string, draft bool, stats *ApplyStats) (returnErr error) { + stats.FailureStage = "validation" if len(jobs) > 2500 { - return 0, fmt.Errorf("index batch exceeds 2500") + return fmt.Errorf("index batch exceeds 2500") } if len(jobs) == 0 { - return 0, nil + stats.Confirmed = true + stats.FailureStage = "" + return nil } now := m.Now().Unix() plans := make([]Plan, 0, len(jobs)) seen := map[string]bool{} + buildAttempted := false + defer func() { + if buildAttempted { + var buildErr error + if stats.FailureStage == "build_metadata" { + buildErr = returnErr + } + metrics.ObserveStage("index_keys", stats.IndexKeys, nil) + metrics.ObserveStage("build_metadata", stats.BuildMetadata, buildErr) + } + }() for _, j := range jobs { if seen[j.ID] { - return 0, fmt.Errorf("duplicate ID in index batch") + return fmt.Errorf("duplicate ID in index batch") } seen[j.ID] = true - p, e := Build(j, keys(j)) + buildAttempted = true + keyStarted := time.Now() + indexKeys := keys(j) + stats.IndexKeys += time.Since(keyStarted) + stats.FailureStage = "build_metadata" + buildStarted := time.Now() + p, e := Build(j, indexKeys) if e != nil { - return 0, e + stats.BuildMetadata += time.Since(buildStarted) + return e } if p.Expires <= now { p.Keys = []string{} @@ -142,26 +204,60 @@ func (m *Manager) ApplyVersion(ctx context.Context, version string, jobs []domai meta := p meta.Document = "" encoded, e := json.Marshal(meta) + stats.BuildMetadata += time.Since(buildStarted) if e != nil { - return 0, e + return e } p.Metadata = string(encoded) plans = append(plans, p) + stats.KeysTotal += len(p.Keys) + if len(plans) == 1 || len(p.Keys) < stats.KeysMin { + stats.KeysMin = len(p.Keys) + } + if len(p.Keys) > stats.KeysMax { + stats.KeysMax = len(p.Keys) + } + stats.FailureStage = "validation" } + stats.FailureStage = "serialization" + serializationStarted := time.Now() raw, err := json.Marshal(plans) + stats.Serialization = time.Since(serializationStarted) + metrics.ObserveStage("serialization", stats.Serialization, err) if err != nil { - return 0, err + return err } + stats.PayloadBytes = len(raw) + metrics.IndexPayloadBytes.WithLabelValues().Observe(float64(len(raw))) + keyHistogram := metrics.IndexKeysPerJob.WithLabelValues() + for _, p := range plans { + keyHistogram.Observe(float64(len(p.Keys))) + } + stats.FailureStage = "redis_script_round_trip" scriptStarted := time.Now() - result, err := applyScript.Run(ctx, m.RDB, []string{ActiveKey, GenerationKey}, version, Prefix(version), string(raw), now, draft).Int() + result, err := applyScript.Run(ctx, m.RDB, []string{ActiveKey, GenerationKey}, version, Prefix(version), string(raw), now, draft).Int64Slice() + stats.RedisRoundTrip = time.Since(scriptStarted) + metrics.ObserveStage("redis_script_round_trip", stats.RedisRoundTrip, err) if err != nil { - return 0, fmt.Errorf("atomic index batch: %w", err) + return fmt.Errorf("atomic index batch: %w", err) } - - if !draft && result > 0 { - metrics.CacheInvalidationDuration.WithLabelValues("invalidate").Observe(time.Since(scriptStarted).Seconds()) + stats.Changed = int(result[0]) + stats.FullApplied = int(result[1]) + stats.Removed = int(result[2]) + stats.Skipped = int(result[3]) + stats.Confirmed = true + stats.FailureStage = "" + mode := "live" + if draft { + mode = "draft" + } + metrics.IndexApplicationJobs.WithLabelValues("full_applied", mode).Add(float64(stats.FullApplied)) + metrics.IndexApplicationJobs.WithLabelValues("removed", mode).Add(float64(stats.Removed)) + metrics.IndexApplicationJobs.WithLabelValues("skipped", mode).Add(float64(stats.Skipped)) + if !draft && stats.Changed > 0 { + metrics.CacheInvalidationDuration.WithLabelValues("invalidate").Observe(stats.RedisRoundTrip.Seconds()) } - return result, nil + return nil } // Preflight validates every key type before any mutation: Redis Lua errors do @@ -200,7 +296,7 @@ for i,p in ipairs(plans) do for _,k in ipairs(p.keys) do typed(prefix..k,'set');if not draft then typed('scraper:jobs:'..k,'set') end end if not draft then typed('scraper:job:'..p.id,'string');typed('scraper:jobs:index-membership:'..p.id,'string');typed('scraper:job:'..p.id..':idx','set') end end -local changed=0;local invalidations={} +local changed=0;local fullApplied=0;local removed=0;local invalidations={} for i,p in ipairs(plans) do if tonumber(old[i].revision)0 or redis.call('SISMEMBER',prefix..'index',p.id)==1)) then for _,k in ipairs(old[i].keys) do @@ -208,6 +304,7 @@ for i,p in ipairs(plans) do if not draft then redis.call('SREM','scraper:jobs:'..k,p.id) end end local alive=p.expiresAt>now + if alive then fullApplied=fullApplied+1 else removed=removed+1 end if alive then for _,k in ipairs(p.keys) do redis.call('SADD',prefix..k,p.id);redis.call('SADD',prefix..'keys',prefix..k) @@ -245,5 +342,5 @@ for i,p in ipairs(plans) do end end if not draft and changed>0 then redis.call('SET',KEYS[1],version);redis.call('INCR',KEYS[2]);if telemetryOk then for reason,_ in pairs(invalidations) do redis.call('HINCRBY','scraper:observability:maintenance-metrics','cache:'..reason,1) end end;local old=redis.call('GET','scraper:jobs:taxonomy-version');local tax=plans[1].taxonomyVersion;if telemetryOk and old and old~=tax then redis.call('HINCRBY','scraper:observability:maintenance-metrics','cache:taxonomy_changed',1) end;redis.call('SET','scraper:jobs:taxonomy-version',tax) end -return changed +return {changed,fullApplied,removed,#plans-changed} `) diff --git a/scraper-go/internal/jobindex/index_test.go b/scraper-go/internal/jobindex/index_test.go index cefa85a1..81824938 100644 --- a/scraper-go/internal/jobindex/index_test.go +++ b/scraper-go/internal/jobindex/index_test.go @@ -230,3 +230,88 @@ func TestIndexTelemetryIncludesActiveNamespaceLookupFailure(t *testing.T) { require.Error(t, err) require.Equal(t, before+1, testutil.ToFloat64(metrics.IndexBatches.WithLabelValues("failed"))) } + +type payloadHook struct { + redis.Hook + payload string +} + +func (h *payloadHook) DialHook(next redis.DialHook) redis.DialHook { return next } +func (h *payloadHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return next +} +func (h *payloadHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + if cmd.Name() == "eval" || cmd.Name() == "evalsha" { + h.payload = cmd.Args()[7].(string) + } + return next(ctx, cmd) + } +} + +func TestDetailedMixedResultsAndPayload(t *testing.T) { + m, client := setup(t) + ctx := context.Background() + seed := []domain.Job{committed("equal", "backend", nil, 2), committed("stale", "backend", nil, 3), committed("expired", "backend", nil, 4)} + _, err := m.Apply(ctx, seed, noKeys) + require.NoError(t, err) + jobs := []domain.Job{committed("new", "backend", nil, 5), seed[0], committed("stale", "frontend", nil, 1), seed[2]} + jobs[3].CatalogExpiresAt = m.Now() + hook := &payloadHook{} + client.AddHook(hook) + result, err := m.ApplyDetailed(ctx, jobs, func(domain.Job) []string { + return []string{"scraper:jobs:keyword:go", "scraper:jobs:keyword:go"} + }) + require.NoError(t, err) + require.True(t, result.Confirmed) + require.Equal(t, 4, result.Received) + require.Equal(t, 2, result.Changed) + require.Equal(t, 1, result.FullApplied) + require.Equal(t, 1, result.Removed) + require.Equal(t, 2, result.Skipped) + require.Equal(t, len(hook.payload), result.PayloadBytes) + require.Equal(t, 9, result.KeysTotal) + require.Equal(t, 0, result.KeysMin) + require.Equal(t, 3, result.KeysMax) + require.Positive(t, result.IndexKeys) + require.Positive(t, result.BuildMetadata) + require.Positive(t, result.Serialization) + require.Positive(t, result.RedisRoundTrip) + require.GreaterOrEqual(t, result.Total, result.IndexKeys+result.BuildMetadata+result.Serialization+result.RedisRoundTrip) + require.True(t, client.SIsMember(ctx, Prefix(Bootstrap)+"family:primary:backend", "stale").Val()) + require.False(t, client.SIsMember(ctx, Prefix(Bootstrap)+"index", "expired").Val()) + require.Equal(t, "2", client.Get(ctx, GenerationKey).Val()) +} + +func TestDetailedFailuresAndRetry(t *testing.T) { + m, client := setup(t) + ctx := context.Background() + j := committed("a", "backend", nil, 1) + result, err := m.ApplyDetailed(ctx, []domain.Job{j}, func(domain.Job) []string { return []string{"invalid:key"} }) + require.Error(t, err) + require.False(t, result.Confirmed) + require.Equal(t, "build_metadata", result.FailureStage) + require.Zero(t, result.PayloadBytes) + require.Zero(t, client.Exists(ctx, GenerationKey).Val()) + _, err = m.Apply(ctx, []domain.Job{j}, noKeys) + require.NoError(t, err) + before := client.Get(ctx, GenerationKey).Val() + result, err = m.ApplyDetailed(ctx, []domain.Job{j}, noKeys) + require.NoError(t, err) + require.True(t, result.Confirmed) + require.Zero(t, result.FullApplied) + require.Equal(t, 1, result.Skipped) + require.Equal(t, before, client.Get(ctx, GenerationKey).Val()) + require.NoError(t, client.Set(ctx, Prefix(Bootstrap)+"family:primary:frontend", "wrongtype", 0).Err()) + result, err = m.ApplyDetailed(ctx, []domain.Job{committed("b", "frontend", nil, 2)}, noKeys) + require.Error(t, err) + require.Equal(t, "redis_script_round_trip", result.FailureStage) + require.False(t, result.Confirmed) + require.Zero(t, result.Changed) + require.NoError(t, client.Close()) + result, err = m.ApplyDetailed(ctx, []domain.Job{j}, noKeys) + require.Error(t, err) + require.Equal(t, "active_lookup", result.FailureStage) + require.Positive(t, result.Total) + require.False(t, result.Confirmed) +} diff --git a/scraper-go/internal/jobstore/jobstore.go b/scraper-go/internal/jobstore/jobstore.go index c42906a0..117ab1cd 100644 --- a/scraper-go/internal/jobstore/jobstore.go +++ b/scraper-go/internal/jobstore/jobstore.go @@ -39,14 +39,19 @@ func New(rdb *redis.Client) *Store { } // SaveResult is the outcome of one persist batch. -// Conflict strategy: existing IDs are upserted in a single MULTI/EXEC. +// Durable persistence distinguishes content/lifecycle changes from freshness; +// legacy persistence upserts existing IDs in a single MULTI/EXEC. // Allowed updates: description, URL, salary, postedAt, modality, location, // classification, and merged sources/keywords. The stable ID is preserved. type SaveResult struct { - Inserted int - Updated int - Invalid int - Persisted []domain.Job + Inserted int + Updated int // Legacy total: real updates + refreshes + reactivations. + RealUpdated int + Refreshed int + Reactivated int + ContentChangeDetectionAvailable bool + Invalid int + Persisted []domain.Job } // SaveBatch returns only committed durable rows; the legacy adapter remains for migration/tests. diff --git a/scraper-go/internal/metrics/operational.go b/scraper-go/internal/metrics/operational.go index 62ac54aa..fe5a802c 100644 --- a/scraper-go/internal/metrics/operational.go +++ b/scraper-go/internal/metrics/operational.go @@ -5,6 +5,7 @@ import ( "errors" "net" "strings" + "sync/atomic" "time" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" @@ -87,6 +88,10 @@ func ErrorType(err error) string { if errors.Is(err, context.Canceled) { return "canceled" } + var limit *ports.RateLimitError + if errors.As(err, &limit) { + return "rate_limit" + } var e net.Error if errors.As(err, &e) { if e.Timeout() { @@ -204,3 +209,51 @@ func PersistenceErrorReason(err error) string { } return ErrorType(err) } + +// These observations measure elapsed client time, not server-only Lua time. +// Stages are fixed at call sites; never pass provider/user text as a stage. +var StageDuration = histogram("candidate_scraper_stage_duration_seconds", []float64{.001, .005, .01, .05, .1, .25, .5, 1, 2, 5, 10, 15, 20, 30, 60, 120, 600}, "stage", "status") +var ChannelSendWait = histogram("candidate_scraper_channel_send_wait_seconds", durationBuckets, "queue") +var IndexPayloadBytes = histogram("candidate_scraper_index_payload_bytes", []float64{1024, 16384, 65536, 262144, 1048576, 4194304, 16777216}) +var IndexKeysPerJob = histogram("candidate_scraper_index_keys_per_job", []float64{0, 4, 8, 16, 32, 64, 128, 256, 512}) +var IndexApplicationJobs = counter("candidate_scraper_index_application_jobs_total", "result", "mode") +var IndexRetries = counter("candidate_scraper_index_retries_total", "reason") +var CatalogCommittedJobs = counter("candidate_scraper_catalog_committed_jobs_total", "result") + +// Unsupported measurements are explicitly unavailable, rather than reported +// as zero observed jobs. Enabled after the durable catalog confirms detection. +var contentDetectionAvailable atomic.Bool +var unavailableMeasurements = []string{"real_update", "refreshed"} + +func init() { + for _, measurement := range unavailableMeasurements { + promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "candidate_scraper_measurement_available", Help: "Whether an outcome or timing is measured by this implementation", + ConstLabels: prometheus.Labels{"kind": measurement}, + }, func() float64 { + if contentDetectionAvailable.Load() { + return 1 + } + return 0 + }) + } +} + +func ObserveStage(stage string, elapsed time.Duration, err error) { + StageDuration.WithLabelValues(stage, BatchStatus(err)).Observe(elapsed.Seconds()) +} + +func ObserveCommittedJobs(created, existingUpsert int) { + CatalogCommittedJobs.WithLabelValues("created").Add(float64(created)) + CatalogCommittedJobs.WithLabelValues("existing_upsert").Add(float64(existingUpsert)) +} + +// ObserveCommittedContent is called only for committed, detected catalog outcomes. +// The existing_upsert aggregate is emitted only without detection: outcome +// series are exclusive, so summing result labels never double-counts commits. +func ObserveCommittedContent(realUpdated, refreshed, reactivated int) { + contentDetectionAvailable.Store(true) + CatalogCommittedJobs.WithLabelValues("real_updated").Add(float64(realUpdated)) + CatalogCommittedJobs.WithLabelValues("refreshed").Add(float64(refreshed)) + CatalogCommittedJobs.WithLabelValues("reactivated").Add(float64(reactivated)) +} diff --git a/scraper-go/internal/metrics/operational_test.go b/scraper-go/internal/metrics/operational_test.go index 5f2811fc..63561e27 100644 --- a/scraper-go/internal/metrics/operational_test.go +++ b/scraper-go/internal/metrics/operational_test.go @@ -12,6 +12,7 @@ import ( "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" ) func TestExecutionOutcomesAndState(t *testing.T) { @@ -52,7 +53,7 @@ func TestConcurrencyQueuesAndLock(t *testing.T) { Configure(12, 8, nil, 0, func(ports.ProviderID) int { return 2 }) Active("linkedin", 1) Waiting("linkedin", 1) - BindQueue("collection", func() Queue { return Queue{2, 4} }) + BindQueue("collection", func() Queue { return Queue{Depth: 2, Capacity: 4} }) s := Current() if s.Concurrency.Configured != 12 || s.Concurrency.Effective != 8 || s.Concurrency.Active != 1 || s.Concurrency.Waiting != 1 || s.Queues["collection"].Depth != 2 { t.Fatalf("wrong concurrency %+v", s) @@ -127,9 +128,9 @@ func TestClassificationRejectionsAndBoundedTitles(t *testing.T) { func TestMetricLabelCardinalityPolicy(t *testing.T) { // Exercise arbitrary IDs/text first, then inspect actual registered series. RecordProvider("url/user-secret", "keyword", 2, 0, time.Now(), errors.New("job-id secret error")) - allowedNames := map[string]bool{"adapter": true, "provider": true, "source": true, "status": true, "result": true, "mode": true, "family": true, "reason_code": true, "error_type": true, "reason": true, "kind": true, "queue": true, "operation": true} + allowedNames := map[string]bool{"adapter": true, "provider": true, "source": true, "status": true, "result": true, "mode": true, "family": true, "reason_code": true, "error_type": true, "reason": true, "kind": true, "queue": true, "operation": true, "stage": true} allowedValues := map[string]bool{} - for _, s := range strings.Fields("linkedin adzuna themuse gupy inhire jooble greenhouse lever unknown manual cron success failed canceled skipped timeout approved rejected other invalid collected valid duplicate persisted indexed keyword batch catalog backend frontend fullstack mobile data devops platform qa security product product_design software leadership no_family_recognized negative_title insufficient_title_evidence missing_required_field classification_error network rate_limit authentication parsing validation persistence indexing inserted updated no_family_recognized collection classification tasksTotal tasksCompleted tasksCanceled providersTotal providersCompleted adaptersTotal adaptersProcessed keywordsTotal keywordsProcessed batchesCompleted missing stale membership primary related any documents counts job_created job_updated job_removed job_reclassified index_rebuilt taxonomy_changed invalidate conflict permanent consistent divergent rebuild reconcile backfill reclassify expire rollback") { + for _, s := range strings.Fields("linkedin adzuna themuse gupy inhire jooble greenhouse lever unknown manual cron success failed canceled skipped timeout approved rejected other invalid collected valid duplicate persisted indexed keyword batch catalog backend frontend fullstack mobile data devops platform qa security product product_design software leadership no_family_recognized negative_title insufficient_title_evidence missing_required_field classification_error network rate_limit authentication parsing validation persistence indexing inserted updated no_family_recognized collection classification tasksTotal tasksCompleted tasksCanceled providersTotal providersCompleted adaptersTotal adaptersProcessed keywordsTotal keywordsProcessed batchesCompleted missing stale membership primary related any documents counts job_created job_updated job_removed job_reclassified index_rebuilt taxonomy_changed invalidate conflict permanent consistent divergent rebuild reconcile backfill reclassify expire rollback active_lookup index_keys build_metadata serialization redis_script_round_trip mark_indexed full_applied removed created real_updated reactivated existing_upsert results incoming real_update refreshed live draft") { allowedValues[s] = true } all, err := prometheus.DefaultGatherer.Gather() @@ -193,3 +194,41 @@ func TestRuntimeMetricsAreReused(t *testing.T) { } } } + +func TestUnsupportedMeasurementsAreUnavailable(t *testing.T) { + families, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + observed := map[string]float64{} + for _, f := range families { + if f.GetName() != "candidate_scraper_measurement_available" { + continue + } + for _, m := range f.Metric { + observed[m.Label[0].GetValue()] = m.GetGauge().GetValue() + } + } + require.Len(t, observed, len(unavailableMeasurements)) + for _, kind := range unavailableMeasurements { + require.Contains(t, observed, kind) + require.Zero(t, observed[kind]) + } +} + +func TestCommittedContentEnablesAvailabilityAndCounters(t *testing.T) { + previous := contentDetectionAvailable.Load() + t.Cleanup(func() { contentDetectionAvailable.Store(previous) }) + contentDetectionAvailable.Store(false) + before := testutil.ToFloat64(CatalogCommittedJobs.WithLabelValues("refreshed")) + ObserveCommittedContent(1, 2, 3) + require.Equal(t, before+2, testutil.ToFloat64(CatalogCommittedJobs.WithLabelValues("refreshed"))) + require.True(t, contentDetectionAvailable.Load()) + families, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + for _, family := range families { + if family.GetName() == "candidate_scraper_measurement_available" { + for _, measurement := range family.Metric { + require.Equal(t, float64(1), measurement.GetGauge().GetValue()) + } + } + } +} diff --git a/scraper-go/internal/metrics/partial_run_test.go b/scraper-go/internal/metrics/partial_run_test.go new file mode 100644 index 00000000..8ab7d505 --- /dev/null +++ b/scraper-go/internal/metrics/partial_run_test.go @@ -0,0 +1,24 @@ +package metrics + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPartialRunReportsSavedWorkWithoutSuccess(t *testing.T) { + StartRun("cron", time.Now(), "partial-test") + BeginRunProcessing("partial-test", true) + RecordRunPersistence("partial-test", 3, 2, 1, 3, nil) + FinishRun(context.DeadlineExceeded, "partial-test") + s := Current() + require.True(t, s.Execution.Partial) + require.Equal(t, "failed", s.Execution.Status) + require.Equal(t, "timeout", *s.Execution.LastStatus) + require.Equal(t, 2, s.Persistence.Inserted) + StartRun("cron", time.Now(), "next-test") + require.False(t, Current().Execution.Partial) + FinishRun(nil, "next-test") +} diff --git a/scraper-go/internal/metrics/state.go b/scraper-go/internal/metrics/state.go index 08b88b0b..3fcd9f5d 100644 --- a/scraper-go/internal/metrics/state.go +++ b/scraper-go/internal/metrics/state.go @@ -15,6 +15,7 @@ import ( ) type Execution struct { + Partial bool `json:"partial"` RunID *string `json:"runId"` Status string `json:"status"` Source *string `json:"source"` @@ -41,6 +42,7 @@ type Lock struct { type Queue struct { Depth int `json:"depth"` Capacity int `json:"capacity"` + InFlight int `json:"inFlight"` } type RejectedTitle struct { Title string `json:"title"` @@ -142,12 +144,18 @@ func init() { promauto.NewGaugeFunc(prometheus.GaugeOpts{Name: item.name, Help: item.name}, item.get) } for _, q := range queueNames { - for _, kind := range []string{"depth", "capacity"} { + for _, kind := range []string{"depth", "capacity", "in_flight"} { + if q != "indexing" && kind != "depth" && kind != "capacity" { + continue + } promauto.NewGaugeFunc(prometheus.GaugeOpts{Name: "candidate_scraper_queue_" + kind, Help: "Actual bounded queue " + kind, ConstLabels: prometheus.Labels{"queue": q}}, func() float64 { v := Current().Queues[q] if kind == "capacity" { return float64(v.Capacity) } + if kind == "in_flight" { + return float64(v.InFlight) + } return float64(v.Depth) }) } @@ -182,6 +190,7 @@ func StartRun(source string, started time.Time, runID ...string) { state.execution.RunID = &id } state.execution.FinishedAt = nil + state.execution.Partial = false state.classification = nil state.persistence = nil state.providers = nil @@ -221,6 +230,7 @@ func FinishRun(err error, runID ...string) { state.execution.FinishedAt = &now state.execution.LastDurationSeconds = duration state.execution.LastStatus = &status + state.execution.Partial = err != nil && state.persistence != nil && (state.persistence.Inserted+state.persistence.Updated) > 0 if err == nil { state.execution.Status = "completed" } else { diff --git a/scraper-go/internal/pipeline/absolute_deadline_test.go b/scraper-go/internal/pipeline/absolute_deadline_test.go new file mode 100644 index 00000000..e35f1100 --- /dev/null +++ b/scraper-go/internal/pipeline/absolute_deadline_test.go @@ -0,0 +1,34 @@ +package pipeline + +import ( + "context" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + "sync/atomic" + "testing" + "time" +) + +func TestAbsoluteDeadlineIncludesFinalizationAndJoinsWorkers(t *testing.T) { + defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + var saved atomic.Int32 + cfg := recoveryConfig(&saved) + cfg.ClassificationBatchSize = 1 + cfg.PersistBatchSize = 1 + cfg.IndexBatchSize = 1 + start := time.Now() + deadline := start.Add(150 * time.Millisecond) + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + cfg.CollectionDeadline = start.Add(70 * time.Millisecond) + cfg.Index = func(ctx context.Context, _ []domain.Job) error { <-ctx.Done(); return ctx.Err() } + source := schedulerTestAdapter{provider: ports.ProviderGupy, search: func(context.Context, string) ([]domain.Job, error) { return []domain.Job{inScopeJob(1)}, nil }} + _, stats, err := runWithConcurrency(ctx, []ports.JobSource{source}, domain.ScrapeRequest{Keywords: []string{"go"}, MaxConcurrency: 1}, 1, nil, cfg) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.EqualValues(t, 1, saved.Load()) + require.Equal(t, 1, stats.Saved()) + require.Zero(t, stats.Indexed) + require.Less(t, time.Since(start), time.Second) +} diff --git a/scraper-go/internal/pipeline/cache_key.go b/scraper-go/internal/pipeline/cache_key.go index 407654a6..9babfab1 100644 --- a/scraper-go/internal/pipeline/cache_key.go +++ b/scraper-go/internal/pipeline/cache_key.go @@ -13,9 +13,15 @@ func BuildCacheKey(config SearchConfig) string { sort.Strings(normalizedKeywords) sources := normalizeCacheValues(config.Sources) + cacheNamespace := "jobs" + if len(config.Sources) > 0 { + // Earlier versions accepted sources without filtering adapters. Do not + // reuse those results for a now-enforced provider selection. + cacheNamespace = "jobs:providers:v1" + } return strings.Join([]string{ - "jobs", + cacheNamespace, strings.Join(normalizedKeywords, ","), normalizeCacheValue(config.SearchLocation), normalizeCacheValue(config.SearchGeoID), diff --git a/scraper-go/internal/pipeline/catalog_integration_test.go b/scraper-go/internal/pipeline/catalog_integration_test.go index 6ffd5c2a..d3cd1f9d 100644 --- a/scraper-go/internal/pipeline/catalog_integration_test.go +++ b/scraper-go/internal/pipeline/catalog_integration_test.go @@ -54,6 +54,16 @@ func TestDurablePipelineOnlyIndexesAfterConfirmedCommit(t *testing.T) { require.Len(t, jobs, 2) require.Equal(t, 2, stats.Indexed) require.Equal(t, 2, stats.Saved()) + // Identical recollection uses detected refresh and full projection fallback. + refreshed, refreshStats, e := processIncomingJobs(ctx, feedJobs(inScopeJob(1), inScopeJob(2)), cfg) + require.NoError(t, e) + require.Equal(t, 2, refreshStats.Refreshed) + require.Zero(t, refreshStats.RealUpdated) + require.True(t, refreshStats.ContentChangeDetectionAvailable) + for i := range jobs { + require.Greater(t, refreshed[i].CatalogRevision, jobs[i].CatalogRevision) + } + jobs = refreshed for _, j := range jobs { var revision, indexed int64 require.NoError(t, db.QueryRow("SELECT revision,indexed_revision FROM job_catalog WHERE id=$1", j.ID).Scan(&revision, &indexed)) diff --git a/scraper-go/internal/pipeline/collection_recovery_test.go b/scraper-go/internal/pipeline/collection_recovery_test.go new file mode 100644 index 00000000..c8663daf --- /dev/null +++ b/scraper-go/internal/pipeline/collection_recovery_test.go @@ -0,0 +1,156 @@ +package pipeline + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/jobstore" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" +) + +func recoveryConfig(saved *atomic.Int32) processConfig { + return processConfig{ + ClassificationBatchSize: 10, PersistBatchSize: 10, IndexBatchSize: 10, + Persist: func(ctx context.Context, jobs []domain.Job) (jobstore.SaveResult, error) { + if err := ctx.Err(); err != nil { + return jobstore.SaveResult{}, err + } + saved.Add(int32(len(jobs))) + return jobstore.SaveResult{Inserted: len(jobs), Persisted: jobs}, nil + }, + Index: func(ctx context.Context, _ []domain.Job) error { return ctx.Err() }, + } +} + +func TestCollectionRecoveryKeepsJobsReturnedWithAdapterError(t *testing.T) { + var saved atomic.Int32 + sources := []ports.JobSource{ + schedulerTestAdapter{provider: ports.ProviderGupy, search: func(context.Context, string) ([]domain.Job, error) { + return []domain.Job{inScopeJob(1)}, errors.New("later page failed") + }}, + schedulerTestAdapter{provider: ports.ProviderAdzuna, search: func(context.Context, string) ([]domain.Job, error) { + return []domain.Job{inScopeJob(2)}, nil + }}, + } + jobs, stats, err := runWithConcurrency(context.Background(), sources, domain.ScrapeRequest{Keywords: []string{"go"}, MaxConcurrency: 2}, 1, nil, recoveryConfig(&saved)) + require.NoError(t, err) // Adapter errors remain isolated; their metrics retain the failure. + require.Len(t, jobs, 2) + require.EqualValues(t, 2, saved.Load()) + require.Equal(t, 2, stats.Indexed) +} + +func TestCollectionRecoveryDrainsAfterTimeoutWithoutLosingCommittedJobs(t *testing.T) { + defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + var saved atomic.Int32 + committed := make(chan struct{}) + cfg := recoveryConfig(&saved) + cfg.ClassificationBatchSize, cfg.IndexBatchSize = 1, 1 + cfg.CollectionTimeout, cfg.DrainTimeout = 100*time.Millisecond, time.Second + persist := cfg.Persist + cfg.Persist = func(ctx context.Context, jobs []domain.Job) (jobstore.SaveResult, error) { + r, err := persist(ctx, jobs) + if saved.Load() == 1 { + close(committed) + } + return r, err + } + sources := []ports.JobSource{ + schedulerTestAdapter{provider: ports.ProviderGupy, search: func(context.Context, string) ([]domain.Job, error) { return []domain.Job{inScopeJob(1)}, nil }}, + schedulerTestAdapter{provider: ports.ProviderAdzuna, search: func(ctx context.Context, _ string) ([]domain.Job, error) { + <-committed + <-ctx.Done() + return []domain.Job{inScopeJob(2)}, ctx.Err() + }}, + } + jobs, stats, err := runWithConcurrency(context.Background(), sources, domain.ScrapeRequest{Keywords: []string{"go"}, MaxConcurrency: 2}, 1, nil, cfg) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Len(t, jobs, 2) + require.EqualValues(t, 2, saved.Load()) + require.Equal(t, 2, stats.Saved()) + require.Equal(t, 2, stats.Indexed) +} + +func TestCollectionRecoveryFlushesSmallPartialBatchAfterTimeout(t *testing.T) { + var saved atomic.Int32 + cfg := recoveryConfig(&saved) + cfg.CollectionTimeout, cfg.DrainTimeout = 10*time.Millisecond, time.Second + source := schedulerTestAdapter{provider: ports.ProviderGupy, search: func(ctx context.Context, _ string) ([]domain.Job, error) { + <-ctx.Done() + return []domain.Job{inScopeJob(1)}, ctx.Err() + }} + _, stats, err := runWithConcurrency(context.Background(), []ports.JobSource{source}, domain.ScrapeRequest{Keywords: []string{"go"}, MaxConcurrency: 1}, 1, nil, cfg) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.EqualValues(t, 1, saved.Load()) + require.Equal(t, 1, stats.Indexed) +} + +func TestCollectionRecoveryProcessingFailureNeverReportsSuccess(t *testing.T) { + for _, stage := range []string{"persist", "index"} { + t.Run(stage, func(t *testing.T) { + var saved atomic.Int32 + cfg := recoveryConfig(&saved) + failure := errors.New(stage + " failed") + if stage == "persist" { + cfg.Persist = func(context.Context, []domain.Job) (jobstore.SaveResult, error) { + return jobstore.SaveResult{}, failure + } + } else { + cfg.Index = func(context.Context, []domain.Job) error { return failure } + } + source := schedulerTestAdapter{provider: ports.ProviderGupy, search: func(context.Context, string) ([]domain.Job, error) { return []domain.Job{inScopeJob(1)}, nil }} + _, stats, err := runWithConcurrency(context.Background(), []ports.JobSource{source}, domain.ScrapeRequest{Keywords: []string{"go"}, MaxConcurrency: 1}, 1, nil, cfg) + require.ErrorIs(t, err, failure) + require.Zero(t, stats.Indexed) + if stage == "persist" { + require.Zero(t, stats.Saved()) + } else { + require.Equal(t, 1, stats.Saved()) + } + }) + } +} + +func TestCollectionRecoveryIndexFailureDoesNotAdvanceDurableCheckpoint(t *testing.T) { + catalog := &observedCatalog{} + cfg := defaultProcessConfig() + cfg.Store = jobstore.NewDurable(nil, catalog) + cfg.Index = func(context.Context, []domain.Job) error { return errors.New("projection unavailable") } + _, stats, err := processIncomingJobs(context.Background(), feedJobs(committedIndexJob()), cfg) + require.Error(t, err) + require.Equal(t, 1, stats.Saved()) + require.Zero(t, catalog.marks) +} + +func TestCollectionRecoveryBoundsDrainAndRespectsShutdown(t *testing.T) { + defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + for _, shutdown := range []bool{false, true} { + parent, cancel := context.WithCancel(context.Background()) + var saved atomic.Int32 + cfg := recoveryConfig(&saved) + cfg.CollectionTimeout, cfg.DrainTimeout = 10*time.Millisecond, 20*time.Millisecond + cfg.Persist = func(ctx context.Context, _ []domain.Job) (jobstore.SaveResult, error) { + if shutdown { + cancel() + } + <-ctx.Done() + return jobstore.SaveResult{}, ctx.Err() + } + source := schedulerTestAdapter{provider: ports.ProviderGupy, search: func(context.Context, string) ([]domain.Job, error) { return []domain.Job{inScopeJob(1)}, nil }} + started := time.Now() + _, _, err := runWithConcurrency(parent, []ports.JobSource{source}, domain.ScrapeRequest{Keywords: []string{"go"}, MaxConcurrency: 1}, 1, nil, cfg) + cancel() + require.Error(t, err) + require.Less(t, time.Since(started), time.Second) + if shutdown { + require.ErrorIs(t, err, context.Canceled) + } else { + require.ErrorIs(t, err, context.DeadlineExceeded) + } + } +} diff --git a/scraper-go/internal/pipeline/index.go b/scraper-go/internal/pipeline/index.go index 618e4498..af411ee5 100644 --- a/scraper-go/internal/pipeline/index.go +++ b/scraper-go/internal/pipeline/index.go @@ -92,6 +92,7 @@ func indexJobsInValkeyBatched( session = newIndexSession("") } + prepared := prepareIndexKeywords(keywords) commands := 0 var err error for start := 0; start < len(jobs); start += batchSize { @@ -100,7 +101,7 @@ func indexJobsInValkeyBatched( break } end := min(start+batchSize, len(jobs)) - chunkCommands, chunkErr := indexJobChunk(ctx, rdb, jobs[start:end], keywords, session) + chunkCommands, chunkErr := indexJobChunk(ctx, rdb, jobs[start:end], prepared, session) commands += chunkCommands if chunkErr != nil { err = chunkErr @@ -126,7 +127,7 @@ func indexJobChunk( ctx context.Context, rdb *redis.Client, jobs []domain.Job, - keywords []string, + prepared *preparedIndexKeywords, session *indexSession, ) (int, error) { if session == nil { @@ -157,7 +158,7 @@ func indexJobChunk( ids = append(ids, id) additions[globalIndexKey] = append(additions[globalIndexKey], id) - newKeys := invertedIndexKeys(job, keywords) + newKeys := prepared.keys(job) newSet := make(map[string]struct{}, len(newKeys)) for _, key := range newKeys { newSet[key] = struct{}{} @@ -405,29 +406,92 @@ func (s *indexSession) Discard(ctx context.Context, rdb *redis.Client) error { }) } -func invertedIndexKeys(job domain.Job, keywords []string) []string { - keys := make([]string, 0) - searchText := keywordSearchText(job) +// preparedIndexKeywords owns an immutable snapshot of the execution's keywords. +// No job-specific results or mutable caches are shared between callers. +type preparedIndexKeywords struct { + plans []indexKeywordPlan +} + +type indexKeywordPlan struct { + terms []string + aliases []string + subterms []indexSubterm +} + +type indexSubterm struct { + term string + key string +} + +func prepareIndexKeywords(keywords []string) *preparedIndexKeywords { + prepared := &preparedIndexKeywords{plans: make([]indexKeywordPlan, 0, len(keywords))} for _, kw := range keywords { - sanitizedKw := strings.ToLower(strings.TrimSpace(kw)) - if sanitizedKw == "" { + sanitized := strings.ToLower(strings.TrimSpace(kw)) + if sanitized == "" { continue } - if keywordMatches(searchText, sanitizedKw) { - for _, alias := range keywordIndexAliases(sanitizedKw) { - keys = append(keys, fmt.Sprintf("scraper:jobs:keyword:%s", alias)) + normalized := normalizeIndexValue(sanitized) + rawTerms := strings.Fields(normalized) + plan := indexKeywordPlan{terms: make([]string, 0, len(rawTerms))} + for i := 0; i < len(rawTerms); i++ { + if rawTerms[i] == "node" && i+1 < len(rawTerms) && rawTerms[i+1] == "js" { + plan.terms = append(plan.terms, "node js") + i++ + } else { + plan.terms = append(plan.terms, rawTerms[i]) } } - for _, term := range keywordSubTerms(sanitizedKw) { - if term == "" { - continue + aliasValues := []string{strings.Join(strings.Fields(strings.ReplaceAll(sanitized, "/", " ")), " "), normalized} + if strings.Contains(normalized, "node js") { + aliasValues = append(aliasValues, strings.ReplaceAll(normalized, "node js", "nodejs"), "nodejs", "node") + } + for _, alias := range uniqueStrings(aliasValues) { + plan.aliases = append(plan.aliases, "scraper:jobs:keyword:"+alias) + } + subterms := rawTerms + if len(rawTerms) >= 2 && rawTerms[0] == "node" && rawTerms[1] == "js" { + subterms = append(subterms, "nodejs") + } + for _, term := range uniqueStrings(subterms) { + plan.subterms = append(plan.subterms, indexSubterm{term: term, key: "scraper:jobs:keyword:" + term}) + } + prepared.plans = append(prepared.plans, plan) + } + return prepared +} + +func invertedIndexKeys(job domain.Job, keywords []string) []string { + return prepareIndexKeywords(keywords).keys(job) +} + +func (prepared *preparedIndexKeywords) keys(job domain.Job) []string { + searchText := keywordSearchText(job) + // normalizeIndexValue joins words using literal spaces. Token membership is + // exactly the old padded-token search; phrases retain substring semantics. + tokens := make(map[string]bool) + for _, token := range strings.Fields(searchText) { + tokens[token] = true + } + nodeJS := strings.Contains(searchText, "node js") || tokens["nodejs"] + keys := make([]string, 0, 32) + for _, plan := range prepared.plans { + matches := len(plan.terms) > 0 + for _, term := range plan.terms { + if (term == "node js" && !nodeJS) || (term != "node js" && !tokens[term]) { + matches = false + break } - if containsTokenOrPhrase(searchText, term) { - keys = append(keys, fmt.Sprintf("scraper:jobs:keyword:%s", term)) + } + if matches { + keys = append(keys, plan.aliases...) + } + for _, subterm := range plan.subterms { + if tokens[subterm.term] { + keys = append(keys, subterm.key) } } } - keys = append(keys, structuredIndexKeys(job)...) + keys = append(keys, structuredIndexKeysFromText(job, searchText)...) keys = append(keys, classificationIndexKeys(job)...) return uniqueIndexMembers(keys) } diff --git a/scraper-go/internal/pipeline/normalize_test.go b/scraper-go/internal/pipeline/normalize_test.go index b8ab3812..138783d1 100644 --- a/scraper-go/internal/pipeline/normalize_test.go +++ b/scraper-go/internal/pipeline/normalize_test.go @@ -1,7 +1,21 @@ package pipeline import ( + "encoding/json" + "fmt" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/keywords" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/taxonomy" + "github.com/stretchr/testify/require" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" + "math/rand" + "os" + "reflect" + "sort" + "strings" + "sync" "testing" + "unicode" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" "github.com/stretchr/testify/assert" @@ -32,3 +46,539 @@ func TestJobIsInvalidWithoutStableIdentity(t *testing.T) { assert.True(t, jobIsInvalid(domain.Job{Title: "Só título"})) assert.False(t, jobIsInvalid(domain.Job{Title: "Dev Go", Company: "Acme", Location: "Brasil"})) } + +// Frozen projection from 5a80b8c. Test-only oracle; never calls the optimized matcher. +func referenceInvertedIndexKeys(job domain.Job, keywords []string) []string { + keys := make([]string, 0) + searchText := referenceKeywordSearchText(job) + for _, kw := range keywords { + sanitizedKw := strings.ToLower(strings.TrimSpace(kw)) + if sanitizedKw == "" { + continue + } + if referenceKeywordMatches(searchText, sanitizedKw) { + for _, alias := range referenceKeywordIndexAliases(sanitizedKw) { + keys = append(keys, fmt.Sprintf("scraper:jobs:keyword:%s", alias)) + } + } + for _, term := range referenceKeywordSubTerms(sanitizedKw) { + if term == "" { + continue + } + if referenceContainsTokenOrPhrase(searchText, term) { + keys = append(keys, fmt.Sprintf("scraper:jobs:keyword:%s", term)) + } + } + } + keys = append(keys, referenceStructuredIndexKeys(job)...) + keys = append(keys, referenceClassificationIndexKeys(job)...) + return referenceUniqueIndexMembers(keys) +} + +func referenceUniqueIndexMembers(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + if value == "" { + continue + } + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + return out +} + +func referenceClassificationIndexKeys(job domain.Job) []string { + if job.Classification == nil { + return nil + } + + classification := job.Classification + if !classification.InScope { + return nil + } + + values := make([]string, 0, 1+len(classification.RelatedFamilies)+len(classification.Technologies)) + + primary := classification.PrimaryFamily + if taxonomy.IsPublic(primary) { + values = append(values, "scraper:jobs:family:"+primary, "scraper:jobs:family:primary:"+primary, "scraper:jobs:keyword:"+referenceNormalizeIndexValue(primary)) + } + for _, family := range taxonomy.Related(primary, classification.RelatedFamilies) { + values = append(values, "scraper:jobs:family:"+family, "scraper:jobs:family:related:"+family, "scraper:jobs:keyword:"+referenceNormalizeIndexValue(family)) + } + for _, technology := range classification.Technologies { + normalized := referenceNormalizeIndexValue(technology) + if normalized != "" { + values = append(values, + fmt.Sprintf("scraper:jobs:technology:%s", normalized), + fmt.Sprintf("scraper:jobs:keyword:%s", normalized), + ) + } + } + if classification.Seniority != "" { + normalized := referenceNormalizeIndexValue(classification.Seniority) + if normalized != "" { + values = append(values, fmt.Sprintf("scraper:jobs:seniority:%s", normalized)) + } + } + + return referenceUniqueStrings(values) +} + +func referenceStructuredIndexKeys(job domain.Job) []string { + values := map[string]string{ + "level": referenceInferLevel(job), + "model": referenceInferWorkModel(job), + "contract": referenceInferContract(job), + } + + for key, value := range referenceInferLocation(job.Location) { + values[key] = value + } + + keys := make([]string, 0, len(values)) + for kind, value := range values { + normalized := referenceNormalizeIndexValue(value) + if normalized == "" || normalized == "todos" || normalized == "all" { + continue + } + keys = append(keys, fmt.Sprintf("scraper:jobs:%s:%s", kind, normalized)) + } + + return keys +} + +func referenceNormalizeIndexValue(value string) string { + t := transform.Chain(norm.NFD, transform.RemoveFunc(func(r rune) bool { + return unicode.Is(unicode.Mn, r) + }), norm.NFC) + + result, _, _ := transform.String(t, value) + result = strings.ReplaceAll(result, "/", " ") + + var b strings.Builder + for _, r := range strings.ToLower(result) { + if unicode.IsLetter(r) || unicode.IsNumber(r) { + b.WriteRune(r) + } else { + b.WriteRune(' ') + } + } + + return strings.Join(strings.Fields(b.String()), " ") +} + +func referenceKeywordSearchText(job domain.Job) string { + return referenceNormalizeIndexValue(strings.Join([]string{ + job.Title, + job.Company, + job.Location, + job.Modality, + job.Description, + }, " ")) +} + +func referenceSearchableJobText(job domain.Job) string { + return referenceNormalizeIndexValue(strings.Join([]string{ + job.Title, + job.Company, + job.Location, + job.Modality, + job.Description, + }, " ")) +} + +func referenceKeywordSubTerms(keyword string) []string { + terms := strings.Fields(referenceNormalizeIndexValue(keyword)) + if len(terms) >= 2 && terms[0] == "node" && terms[1] == "js" { + terms = append(terms, "nodejs") + } + return referenceUniqueStrings(terms) +} + +func referenceKeywordIndexAliases(keyword string) []string { + sanitized := strings.Join(strings.Fields(strings.ReplaceAll(strings.ToLower(keyword), "/", " ")), " ") + normalized := referenceNormalizeIndexValue(keyword) + + aliases := []string{sanitized, normalized} + if strings.Contains(normalized, "node js") { + aliases = append(aliases, strings.ReplaceAll(normalized, "node js", "nodejs"), "nodejs", "node") + } + + return referenceUniqueStrings(aliases) +} + +func referenceKeywordMatches(searchText, keyword string) bool { + terms := strings.Fields(referenceNormalizeIndexValue(keyword)) + if len(terms) == 0 { + return false + } + + for i := 0; i < len(terms); i++ { + term := terms[i] + if term == "node" && i+1 < len(terms) && terms[i+1] == "js" { + if !referenceContainsTokenOrPhrase(searchText, "node js") && !referenceContainsTokenOrPhrase(searchText, "nodejs") { + return false + } + i++ + continue + } + if !referenceContainsTokenOrPhrase(searchText, term) { + return false + } + } + + return true +} + +func referenceUniqueStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + result := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} + +func referenceInferLevel(job domain.Job) string { + text := referenceSearchableJobText(job) + + if referenceContainsAny(text, "estagio", "intern", "trainee") { + return "estagio" + } + if referenceContainsAny(text, "senior", "sr", "especialista", "lead", "principal", "staff") { + return "senior" + } + if referenceContainsAny(text, "junior", "jr", "entry level", "assistente") { + return "junior" + } + + return "pleno" +} + +func referenceInferWorkModel(job domain.Job) string { + text := referenceSearchableJobText(job) + + if referenceContainsAny(text, "hibrido", "hybrid") { + return "hibrido" + } + if referenceContainsAny(text, + "remoto", + "remote", + "home office", + "teletrabalho", + "anywhere", + "worldwide", + "global", + ) { + return "remoto" + } + if referenceContainsAny(text, + "presencial", + "onsite", + "on site", + "on-site", + "in office", + "escritorio", + ) { + return "presencial" + } + + return "presencial" +} + +func referenceInferContract(job domain.Job) string { + text := referenceSearchableJobText(job) + + if referenceContainsAny(text, "cooperado", "cooperativa") { + return "cooperado" + } + if referenceContainsAny(text, "pj", "pessoa juridica", "contractor", "freelance", "freela") { + return "pj" + } + if referenceContainsAny(text, "clt", "full time", "full-time", "efetivo", "permanent") { + return "clt" + } + + return "" +} + +func referenceInferLocation(location string) map[string]string { + text := referenceNormalizeIndexValue(location) + values := make(map[string]string) + if text == "" { + return values + } + + type countryRule struct { + country string + continent string + aliases []string + } + + rules := []countryRule{ + {"Brasil", "América do Sul", []string{"brasil", "brazil", "br"}}, + {"Estados Unidos", "América do Norte", []string{"estados unidos", "united states", "usa", "us", "eua"}}, + {"Canadá", "América do Norte", []string{"canada", "canadá"}}, + {"México", "América do Norte", []string{"mexico", "méxico"}}, + {"Argentina", "América do Sul", []string{"argentina"}}, + {"Chile", "América do Sul", []string{"chile"}}, + {"Colômbia", "América do Sul", []string{"colombia", "colômbia"}}, + {"Portugal", "Europa", []string{"portugal"}}, + {"Espanha", "Europa", []string{"spain", "espanha", "espana", "españa"}}, + {"Reino Unido", "Europa", []string{"reino unido", "united kingdom", "uk", "england", "london"}}, + {"França", "Europa", []string{"france", "frança", "franca"}}, + {"Alemanha", "Europa", []string{"germany", "alemanha", "deutschland"}}, + {"Países Baixos", "Europa", []string{"netherlands", "paises baixos", "holanda"}}, + {"Índia", "Ásia", []string{"india", "índia"}}, + {"Singapura", "Ásia", []string{"singapore", "singapura"}}, + {"Austrália", "Oceania", []string{"australia", "austrália"}}, + {"Nova Zelândia", "Oceania", []string{"new zealand", "nova zelandia", "nova zelândia"}}, + {"África do Sul", "África", []string{"south africa", "africa do sul", "áfrica do sul"}}, + } + + for _, rule := range rules { + for _, alias := range rule.aliases { + if referenceContainsTokenOrPhrase(text, referenceNormalizeIndexValue(alias)) { + values["country"] = rule.country + values["location"] = rule.country + values["continent"] = rule.continent + break + } + } + if values["country"] != "" { + break + } + } + + if values["continent"] == "" && referenceContainsAny(text, "remote", "remoto", "global", "worldwide") { + values["continent"] = "Global / Remoto" + values["location"] = "Global / Remoto" + } + + if state := referenceInferBrazilianState(text); state != "" { + values["state"] = state + if values["country"] == "" { + values["country"] = "Brasil" + values["location"] = "Brasil" + values["continent"] = "América do Sul" + } + } + if city := referenceInferKnownCity(text); city != "" { + values["city"] = city + } + + return values +} + +func referenceInferBrazilianState(text string) string { + states := map[string][]string{ + "SP": {"sp", "sao paulo"}, + "RJ": {"rj", "rio de janeiro"}, + "MG": {"mg", "minas gerais", "belo horizonte"}, + "PR": {"pr", "parana", "curitiba"}, + "SC": {"sc", "santa catarina", "florianopolis"}, + "RS": {"rs", "rio grande do sul", "porto alegre"}, + "BA": {"ba", "bahia", "salvador"}, + "PE": {"pe", "pernambuco", "recife"}, + "CE": {"ce", "ceara", "fortaleza"}, + "DF": {"df", "distrito federal", "brasilia"}, + } + + for state, aliases := range states { + for _, alias := range aliases { + if referenceContainsTokenOrPhrase(text, referenceNormalizeIndexValue(alias)) { + return state + } + } + } + + return "" +} + +func referenceInferKnownCity(text string) string { + cities := []string{ + "sao paulo", + "rio de janeiro", + "belo horizonte", + "curitiba", + "joinville", + "florianopolis", + "porto alegre", + "salvador", + "recife", + "fortaleza", + "brasilia", + "lisboa", + "porto", + "madrid", + "barcelona", + "london", + "paris", + "berlin", + "amsterdam", + "toronto", + "vancouver", + "new york", + "san francisco", + "singapore", + "sydney", + } + + for _, city := range cities { + if referenceContainsTokenOrPhrase(text, city) { + return city + } + } + + return "" +} + +func referenceContainsAny(text string, needles ...string) bool { + for _, needle := range needles { + if referenceContainsTokenOrPhrase(text, referenceNormalizeIndexValue(needle)) { + return true + } + } + return false +} + +func referenceContainsTokenOrPhrase(text string, needle string) bool { + if needle == "" { + return false + } + if strings.Contains(needle, " ") { + return strings.Contains(text, needle) + } + + return strings.Contains(" "+text+" ", " "+needle+" ") +} + +func realIndexKeywords(t testing.TB) []string { + t.Helper() + raw, err := os.ReadFile("../keywords/keywords.json") + require.NoError(t, err) + var seed struct { + Keywords []string `json:"KEYWORDS"` + } + require.NoError(t, json.Unmarshal(raw, &seed)) + result := keywords.GenerateSearchKeywords(seed.Keywords) + require.Len(t, result, 445) + return result +} + +func sortedIndexKeys(keys []string) []string { + sort.Strings(keys) + return keys +} + +func requireIndexEquivalence(t *testing.T, job domain.Job, keywords []string) { + t.Helper() + want := sortedIndexKeys(referenceInvertedIndexKeys(job, keywords)) + require.Equal(t, want, sortedIndexKeys(IndexKeys(job, keywords))) + require.Equal(t, want, sortedIndexKeys(prepareIndexKeywords(keywords).keys(job))) +} + +func TestIndexKeysEquivalentToCommittedProjection(t *testing.T) { + queries := []string{"go", "Go", "backend engineer", "engineer backend", "node.js", "node js engineer", "engineer node js", "nodejs", "C#", ".NET", "UX/UI", "operações de produto", "", " ", "---", "react", "React", "a/b", "a b", "entry level"} + texts := []string{ + "", "go", "golang", "django gopher ongoing", "go backend distant engineer", + "ENGINEER GO BACKEND", "Node.js backend engineer", "nodejs engineer", + "anode json engineer", "node distant js engineer", "engineer nodejs", + "C# .NET React", "UX/UI designer", "OPERAÇÕES DE PRODUTO", "operações de produto", + "go\tbackend\nengineer", "go backend engineer", "go-react/node.js", + "Senior remote full-time", "Júnior híbrido PJ", "Estágio presencial CLT", + "Staff hybrid contractor", "Intern home office cooperativa", "entry level on-site", + strings.Repeat("Golang go APIs backend engineer React Node.js C# .NET Brasil. ", 300), + } + for i, text := range texts { + t.Run(fmt.Sprint(i), func(t *testing.T) { + job := domain.Job{Title: text, Company: "Acme", Location: "São Paulo, SP, Brasil", Description: text, Modality: text, + Classification: &domain.Classification{InScope: true, PrimaryFamily: "backend", RelatedFamilies: []string{"platform", "backend", "platform"}, Technologies: []string{"Go", "C#", ".NET", "Node.js"}, Seniority: "sênior"}} + requireIndexEquivalence(t, job, queries) + requireIndexEquivalence(t, job, realIndexKeywords(t)) + job.Classification = nil + requireIndexEquivalence(t, job, nil) + job.Classification = &domain.Classification{InScope: false, PrimaryFamily: "other"} + requireIndexEquivalence(t, job, queries) + }) + } + for _, location := range []string{"", "Remote worldwide", "Lisboa Portugal", "London UK", "Toronto Canada", "Rio de Janeiro RJ", "Curitiba PR", "Belo Horizonte MG", "New York USA", "Sydney Australia"} { + requireIndexEquivalence(t, domain.Job{Title: "Backend Engineer", Location: location}, queries) + } +} + +func TestContainsTokenOrPhraseMatchesCommittedBoundaries(t *testing.T) { + texts := []string{"", "go", "gopher go", "go gopher", "go-go", "go\tgo", "go\ngo", " gopher ", "node js", "anode json", "égo", "é go é", "go go go", " go "} + needles := []string{"", "go", "gopher", "node js", "on site", "go go", "go\tgo", "é", "verylongneedle"} + for _, text := range texts { + for _, needle := range needles { + require.Equal(t, referenceContainsTokenOrPhrase(text, needle), containsTokenOrPhrase(text, needle), "%q / %q", text, needle) + } + } +} + +func TestIndexKeysGeneratedEquivalenceAndImmutableReuse(t *testing.T) { + rng := rand.New(rand.NewSource(178)) + words := []string{"go", "Golang", "node", "js", "nodejs", "C#", ".NET", "sênior", "remote", "backend", "engineer", "React", "a/b", "operações", "UX/UI", "python", "gopher"} + queries := realIndexKeywords(t) + prepared := prepareIndexKeywords(queries) + saved := append([]string(nil), queries...) + for i := 0; i < 150; i++ { + parts := make([]string, 12) + for j := range parts { + parts[j] = words[rng.Intn(len(words))] + } + job := domain.Job{Title: parts[0], Company: parts[1], Description: strings.Join(parts, " "), Location: "Lisboa Portugal"} + want := sortedIndexKeys(referenceInvertedIndexKeys(job, queries)) + require.Equal(t, want, sortedIndexKeys(prepared.keys(job))) + require.Equal(t, want, sortedIndexKeys(IndexKeys(job, queries))) + arbitrary := make([]string, 12) + for n := range arbitrary { + arbitrary[n] = words[rng.Intn(len(words))] + " / " + words[rng.Intn(len(words))] + } + requireIndexEquivalence(t, job, arbitrary) + } + require.Equal(t, saved, queries) + // The compiled snapshot must survive changes to the caller's slice. + job := domain.Job{Title: "Backend engineer Go", Location: "Brasil"} + want := sortedIndexKeys(prepared.keys(job)) + queries[0] = "mutated keyword" + require.Equal(t, want, sortedIndexKeys(prepared.keys(job))) + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for n := 0; n < 20; n++ { + if !reflect.DeepEqual(want, sortedIndexKeys(prepared.keys(job))) { + t.Error("shared matcher changed") + } + } + }() + } + wg.Wait() +} + +func FuzzContainsTokenOrPhraseEquivalent(f *testing.F) { + for _, pair := range [][2]string{{"go gopher", "go"}, {"anode json", "node js"}, {" go\tgo ", "go"}, {"é Go", "Go"}, {"", ""}} { + f.Add(pair[0], pair[1]) + } + f.Fuzz(func(t *testing.T, text, needle string) { + if containsTokenOrPhrase(text, needle) != referenceContainsTokenOrPhrase(text, needle) { + t.Fatalf("boundary mismatch: %q / %q", text, needle) + } + }) +} diff --git a/scraper-go/internal/pipeline/pav178_integration_test.go b/scraper-go/internal/pipeline/pav178_integration_test.go new file mode 100644 index 00000000..cd098efd --- /dev/null +++ b/scraper-go/internal/pipeline/pav178_integration_test.go @@ -0,0 +1,71 @@ +package pipeline + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/catalog" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/jobindex" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/jobstore" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func TestPAV178PartialCollectionCommitsAndIndexesOnRealDisposableServices(t *testing.T) { + dsn, endpoint := os.Getenv("PAV125_TEST_DATABASE_URL"), os.Getenv("PAV125_TEST_VALKEY_URL") + if dsn == "" || endpoint == "" { + t.Skip("isolated PostgreSQL and Valkey required") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + root, err := sql.Open("postgres", dsn) + require.NoError(t, err) + defer root.Close() + schema := fmt.Sprintf("pav178_%d", time.Now().UnixNano()) + _, err = root.ExecContext(ctx, "CREATE SCHEMA "+schema) + require.NoError(t, err) + defer root.Exec("DROP SCHEMA " + schema + " CASCADE") + db, err := sql.Open("postgres", dsn+"&search_path="+schema) + require.NoError(t, err) + defer db.Close() + migration, err := os.ReadFile("../../../backend/drizzle/0015_job_catalog.sql") + require.NoError(t, err) + _, err = db.ExecContext(ctx, string(migration)) + require.NoError(t, err) + options, err := redis.ParseURL(endpoint) + require.NoError(t, err) + rdb := redis.NewClient(options) + defer rdb.Close() + catalogStore := catalog.New(db, catalog.DefaultLifetime) + sources := []ports.JobSource{ + schedulerTestAdapter{provider: ports.ProviderGupy, search: func(context.Context, string) ([]domain.Job, error) { return []domain.Job{inScopeJob(701)}, nil }}, + schedulerTestAdapter{provider: ports.ProviderAdzuna, search: func(ctx context.Context, _ string) ([]domain.Job, error) { + <-ctx.Done() + return []domain.Job{inScopeJob(702)}, ctx.Err() + }}, + } + jobs, stats, err := ScrapeAllSources(ctx, SearchConfig{RunID: "pav178-real-partial", Keywords: []string{"go"}, CollectionTimeout: 150 * time.Millisecond, MaxConcurrency: 2, ProviderMaxConcurrency: 1, Store: jobstore.NewDurable(rdb, catalogStore), ClassificationBatchSize: 1, PersistBatchSize: 1, IndexBatchSize: 1}, sources, rdb) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Len(t, jobs, 2) + require.Equal(t, 2, stats.Saved()) + require.Equal(t, 2, stats.Indexed) + version, err := jobindex.New(rdb).Active(ctx) + require.NoError(t, err) + for _, job := range jobs { + var revision, indexed int64 + require.NoError(t, db.QueryRowContext(ctx, "SELECT revision,indexed_revision FROM job_catalog WHERE id=$1", job.ID).Scan(&revision, &indexed)) + require.Equal(t, revision, indexed) + raw, err := rdb.Get(ctx, jobindex.Prefix(version)+"job:"+job.ID).Bytes() + require.NoError(t, err) + var document domain.Job + require.NoError(t, json.Unmarshal(raw, &document)) + require.Equal(t, job.ID, document.ID) + } +} diff --git a/scraper-go/internal/pipeline/pipeline.go b/scraper-go/internal/pipeline/pipeline.go index 557780fe..0a0eda4c 100644 --- a/scraper-go/internal/pipeline/pipeline.go +++ b/scraper-go/internal/pipeline/pipeline.go @@ -2,6 +2,7 @@ package pipeline import ( "context" + "errors" "fmt" "log/slog" "strings" @@ -34,6 +35,10 @@ type adapterTask struct { } func Run(ctx context.Context, adapterList []ports.JobSource, req domain.ScrapeRequest) ([]domain.Job, error) { + adapterList, err := selectProviders(adapterList, req.Sources) + if err != nil { + return nil, err + } jobs, _, err := runWithConcurrency(ctx, adapterList, req, 2, nil, defaultProcessConfig()) return jobs, err } @@ -47,6 +52,35 @@ func runWithConcurrency( processCfg processConfig, ) ([]domain.Job, ProcessStats, error) { pipelineStart := time.Now() + req.RunID = processCfg.RunID + if req.RunID == "" { + req.RunID = pipelineStart.Format(time.RFC3339Nano) + } + // Collection expiry must not cancel already returned jobs. Both contexts + // still inherit shutdown and lock loss; draining has a hard upper bound. + processingCtx := ctx + collectionCtx := ctx + if !processCfg.CollectionDeadline.IsZero() { + drain := processCfg.DrainTimeout + if drain <= 0 { + drain = 30 * time.Second + } + var cancelProcessing, cancelCollection context.CancelFunc + processingCtx, cancelProcessing = context.WithDeadline(ctx, processCfg.CollectionDeadline.Add(drain)) + defer cancelProcessing() + collectionCtx, cancelCollection = context.WithDeadline(processingCtx, processCfg.CollectionDeadline) + defer cancelCollection() + } else if processCfg.CollectionTimeout > 0 { + drain := processCfg.DrainTimeout + if drain <= 0 { + drain = 30 * time.Second + } + var cancelProcessing, cancelCollection context.CancelFunc + processingCtx, cancelProcessing = context.WithTimeout(ctx, processCfg.CollectionTimeout+drain) + defer cancelProcessing() + collectionCtx, cancelCollection = context.WithTimeout(processingCtx, processCfg.CollectionTimeout) + defer cancelCollection() + } maxConcurrency := req.MaxConcurrency if maxConcurrency <= 0 { @@ -79,12 +113,12 @@ func runWithConcurrency( tasksWg.Add(1 + maxConcurrency) go func() { defer tasksWg.Done() - produceTasks(ctx, tasks, adapterList, req.Keywords, runStats) + produceTasks(collectionCtx, tasks, adapterList, req.Keywords, runStats) }() for range maxConcurrency { go func() { defer tasksWg.Done() - runWorker(ctx, tasks, results, budget, runStats, req, processCfg.RunID) + runWorker(collectionCtx, processingCtx, tasks, results, budget, runStats, req, processCfg.RunID) }() } @@ -103,7 +137,7 @@ func runWithConcurrency( processWg.Add(1) go func() { defer processWg.Done() - processed, processStats, processErr = processIncomingJobs(ctx, incoming, processCfg) + processed, processStats, processErr = processIncomingJobs(processingCtx, incoming, processCfg) }() var closeIncoming sync.Once @@ -112,22 +146,31 @@ func runWithConcurrency( } defer closeIncomingFn() + var incomingSendWait time.Duration + collectionFailed := 0 + var collectionError error + defer func() { metrics.ChannelSendWait.WithLabelValues("incoming").Observe(incomingSendWait.Seconds()) }() for r := range results { if r.err != nil { - continue + collectionFailed++ + collectionError = errors.Join(collectionError, r.err) } for _, job := range r.jobs { - if context.Cause(ctx) != nil { + if context.Cause(processingCtx) != nil { break } + sendStarted := time.Now() select { - case <-ctx.Done(): + case <-processingCtx.Done(): case incoming <- job: } + incomingSendWait += time.Since(sendStarted) } } closeIncomingFn() processWg.Wait() + processStats.CollectionFailed = collectionFailed + processStats.CollectionError = collectionError gatherWg.Wait() logProviderRunStats(runStats) if processErr != nil { @@ -136,6 +179,9 @@ func runWithConcurrency( if cause := context.Cause(ctx); cause != nil { return processed, processStats, cause } + if cause := context.Cause(collectionCtx); cause != nil { + return processed, processStats, cause + } metrics.PipelineRunDuration.Observe(time.Since(pipelineStart).Seconds()) metrics.PipelineJobsTotal.Observe(float64(len(processed))) @@ -165,6 +211,8 @@ func produceTasks( runStats *providerRunStats, ) { defer close(queue) + var taskSendWait time.Duration + defer func() { metrics.ChannelSendWait.WithLabelValues("collection").Observe(taskSendWait.Seconds()) }() providerIndexes := make(map[ports.ProviderID]int) cursors := make([]providerTaskCursor, 0, len(adapterList)) @@ -197,13 +245,16 @@ func produceTasks( return } metrics.Waiting(string(task.provider), 1) + sendStarted := time.Now() select { case queue <- task: runStats.recordProduced(task) case <-ctx.Done(): metrics.Waiting(string(task.provider), -1) + taskSendWait += time.Since(sendStarted) return } + taskSendWait += time.Since(sendStarted) } if !produced { return @@ -298,6 +349,7 @@ func validateSources(adapterList []ports.JobSource) error { func runWorker( ctx context.Context, + processingCtx context.Context, tasks <-chan adapterTask, results chan<- result, budget *concurrencyBudget, @@ -320,13 +372,14 @@ func runWorker( if context.Cause(ctx) != nil { return } - runScheduledTask(ctx, task, results, budget, runStats, req, runID) + runScheduledTask(ctx, processingCtx, task, results, budget, runStats, req, runID) } } } func runScheduledTask( ctx context.Context, + processingCtx context.Context, task adapterTask, results chan<- result, budget *concurrencyBudget, @@ -361,10 +414,12 @@ func runScheduledTask( metrics.JobsFoundTotal.WithLabelValues(source).Add(float64(len(jobs))) } + sendStarted := time.Now() select { case results <- result{jobs: jobs, err: err}: - case <-ctx.Done(): + case <-processingCtx.Done(): } + metrics.ChannelSendWait.WithLabelValues("results").Observe(time.Since(sendStarted).Seconds()) } func logProviderRunStats(runStats *providerRunStats) { @@ -463,10 +518,14 @@ func indexedClassificationFamily(family string) bool { } func structuredIndexKeys(job domain.Job) []string { + return structuredIndexKeysFromText(job, searchableJobText(job)) +} + +func structuredIndexKeysFromText(job domain.Job, text string) []string { values := map[string]string{ - "level": inferLevel(job), - "model": inferWorkModel(job), - "contract": inferContract(job), + "level": inferLevelFromText(text), + "model": inferWorkModelFromText(text), + "contract": inferContractFromText(text), } for key, value := range inferLocation(job.Location) { @@ -525,49 +584,6 @@ func searchableJobText(job domain.Job) string { }, " ")) } -func keywordSubTerms(keyword string) []string { - terms := strings.Fields(normalizeIndexValue(keyword)) - if len(terms) >= 2 && terms[0] == "node" && terms[1] == "js" { - terms = append(terms, "nodejs") - } - return uniqueStrings(terms) -} - -func keywordIndexAliases(keyword string) []string { - sanitized := strings.Join(strings.Fields(strings.ReplaceAll(strings.ToLower(keyword), "/", " ")), " ") - normalized := normalizeIndexValue(keyword) - - aliases := []string{sanitized, normalized} - if strings.Contains(normalized, "node js") { - aliases = append(aliases, strings.ReplaceAll(normalized, "node js", "nodejs"), "nodejs", "node") - } - - return uniqueStrings(aliases) -} - -func keywordMatches(searchText, keyword string) bool { - terms := strings.Fields(normalizeIndexValue(keyword)) - if len(terms) == 0 { - return false - } - - for i := 0; i < len(terms); i++ { - term := terms[i] - if term == "node" && i+1 < len(terms) && terms[i+1] == "js" { - if !containsTokenOrPhrase(searchText, "node js") && !containsTokenOrPhrase(searchText, "nodejs") { - return false - } - i++ - continue - } - if !containsTokenOrPhrase(searchText, term) { - return false - } - } - - return true -} - func uniqueStrings(values []string) []string { seen := make(map[string]struct{}, len(values)) result := make([]string, 0, len(values)) @@ -586,8 +602,10 @@ func uniqueStrings(values []string) []string { } func inferLevel(job domain.Job) string { - text := searchableJobText(job) + return inferLevelFromText(searchableJobText(job)) +} +func inferLevelFromText(text string) string { if containsAny(text, "estagio", "intern", "trainee") { return "estagio" } @@ -602,8 +620,10 @@ func inferLevel(job domain.Job) string { } func inferWorkModel(job domain.Job) string { - text := searchableJobText(job) + return inferWorkModelFromText(searchableJobText(job)) +} +func inferWorkModelFromText(text string) string { if containsAny(text, "hibrido", "hybrid") { return "hibrido" } @@ -633,8 +653,10 @@ func inferWorkModel(job domain.Job) string { } func inferContract(job domain.Job) string { - text := searchableJobText(job) + return inferContractFromText(searchableJobText(job)) +} +func inferContractFromText(text string) string { if containsAny(text, "cooperado", "cooperativa") { return "cooperado" } @@ -796,5 +818,18 @@ func containsTokenOrPhrase(text string, needle string) bool { return strings.Contains(text, needle) } - return strings.Contains(" "+text+" ", " "+needle+" ") + // Check the same literal-space boundaries without copying the haystack. + for offset := 0; offset <= len(text)-len(needle); { + relative := strings.Index(text[offset:], needle) + if relative < 0 { + return false + } + start := offset + relative + end := start + len(needle) + if (start == 0 || text[start-1] == ' ') && (end == len(text) || text[end] == ' ') { + return true + } + offset = start + 1 + } + return false } diff --git a/scraper-go/internal/pipeline/process.go b/scraper-go/internal/pipeline/process.go index fdc4b18b..7c04b7fd 100644 --- a/scraper-go/internal/pipeline/process.go +++ b/scraper-go/internal/pipeline/process.go @@ -20,8 +20,12 @@ type persistBatchFn func(ctx context.Context, jobs []domain.Job) (jobstore.SaveR type indexBatchFn func(ctx context.Context, jobs []domain.Job) error type processConfig struct { + CollectionTimeout time.Duration + CollectionDeadline time.Time + DrainTimeout time.Duration RunID string Keywords []string + indexKeywords *preparedIndexKeywords ClassificationBatchSize int PersistBatchSize int IndexBatchSize int @@ -33,17 +37,23 @@ type processConfig struct { } type ProcessStats struct { - Received int - Valid int - Invalid int - Duplicates int - Classified int - Approved int - Rejected int - Inserted int - Updated int - Indexed int - Failed int + CollectionFailed int + CollectionError error + Received int + Valid int + Invalid int + Duplicates int + Classified int + Approved int + Rejected int + Inserted int + Updated int + RealUpdated int + Refreshed int + Reactivated int + ContentChangeDetectionAvailable bool + Indexed int // Successful input jobs, including idempotent skips; retained for compatibility. + Failed int } func (s ProcessStats) Saved() int { @@ -76,6 +86,10 @@ func processIncomingJobs( cfg.IndexSession = newIndexSession(cfg.RunID) } + if cfg.Index == nil && cfg.RDB != nil && cfg.Store.Durable() { + cfg.indexKeywords = prepareIndexKeywords(cfg.Keywords) + } + // seen holds only the current classification window. flushed maps identity // keys to persisted IDs so cross-batch duplicates can merge with the // document already written in this run. @@ -84,16 +98,16 @@ func processIncomingJobs( persistedByID := make(map[string]domain.Job) pending := make([]*domain.Job, 0, cfg.ClassificationBatchSize) indexBuf := make([]domain.Job, 0, cfg.IndexBatchSize) - var persistenceDepth, indexDepth, persistenceCapacity, indexCapacity atomic.Int64 + var persistenceDepth, indexDepth, persistenceCapacity, indexCapacity, indexInFlight atomic.Int64 persistenceCapacity.Store(int64(cfg.ClassificationBatchSize)) indexCapacity.Store(int64(cap(indexBuf))) metrics.BindQueue("persistence", func() metrics.Queue { return metrics.Queue{Depth: int(persistenceDepth.Load()), Capacity: int(persistenceCapacity.Load())} }) metrics.BindQueue("indexing", func() metrics.Queue { - return metrics.Queue{Depth: int(indexDepth.Load()), Capacity: int(indexCapacity.Load())} + return metrics.Queue{Depth: int(indexDepth.Load()), Capacity: int(indexCapacity.Load()), InFlight: int(indexInFlight.Load())} }) - defer func() { persistenceDepth.Store(0); indexDepth.Store(0) }() + defer func() { persistenceDepth.Store(0); indexDepth.Store(0); indexInFlight.Store(0) }() approved := make([]domain.Job, 0) metrics.BeginRunProcessing(cfg.RunID, cfg.Persist != nil || cfg.Store != nil) stats := ProcessStats{} @@ -124,7 +138,10 @@ func processIncomingJobs( indexBuf = append([]domain.Job(nil), indexBuf[n:]...) indexDepth.Store(int64(len(indexBuf))) indexCapacity.Store(int64(cap(indexBuf))) - if err := indexPersistedChunk(ctx, cfg, batchNo, chunk, &stats); err != nil { + indexInFlight.Store(int64(len(chunk))) + err := indexPersistedChunk(ctx, cfg, batchNo, chunk, &stats) + indexInFlight.Store(0) + if err != nil { return err } } @@ -315,9 +332,11 @@ func processIncomingJobs( "approved", stats.Approved, "rejected", stats.Rejected, "inserted", stats.Inserted, - "updated", stats.Updated, + "existing_upsert", stats.Updated, + "created", stats.Inserted, "real_updated", stats.RealUpdated, "refreshed", stats.Refreshed, "reactivated", stats.Reactivated, + "content_change_detection_available", stats.ContentChangeDetectionAvailable, "saved", stats.Saved(), - "indexed", stats.Indexed, + "index_completed_inputs", stats.Indexed, "failed", stats.Failed, ) return approved, stats, nil @@ -412,6 +431,7 @@ func processStageBatch( metrics.Stage("classification") defer metrics.AddProgress("batchesCompleted", 1) metrics.RunClassificationReceived(cfg.RunID, len(jobs)) + validationStarted := time.Now() valid := make([]domain.Job, 0, len(jobs)) invalid := 0 for _, job := range jobs { @@ -427,6 +447,8 @@ func processStageBatch( valid = append(valid, job) } + validationDuration := time.Since(validationStarted) + classificationStarted := time.Now() approved := make([]domain.Job, 0, len(valid)) rejected := 0 for _, job := range valid { @@ -443,16 +465,23 @@ func processStageBatch( stats.Approved++ } + classificationDuration := time.Since(classificationStarted) + var persistenceDuration time.Duration persisted := approved + before := *stats + batchDetection := false batchInserted := 0 batchUpdated := 0 var persistErr error if cfg.Persist != nil || cfg.Store != nil { var written []domain.Job + persistenceStarted := time.Now() written, batchInserted, batchUpdated, persistErr = persistJobs(ctx, cfg, batchNo, approved, stats) + persistenceDuration = time.Since(persistenceStarted) stats.Inserted += batchInserted stats.Updated += batchUpdated persisted = written + batchDetection = len(written) > 0 && stats.ContentChangeDetectionAvailable } else { for i := range persisted { if persisted[i].ID == "" { @@ -473,7 +502,13 @@ func processStageBatch( "approved", len(approved), "rejected", rejected, "inserted", batchInserted, - "updated", batchUpdated, + "existing_upsert", batchUpdated, + "created", batchInserted, "real_updated", stats.RealUpdated-before.RealUpdated, + "refreshed", stats.Refreshed-before.Refreshed, "reactivated", stats.Reactivated-before.Reactivated, + "content_change_detection_available", batchDetection, + "validation_duration", validationDuration, + "classification_duration", classificationDuration, + "persistence_duration", persistenceDuration, "duration", time.Since(started).Round(time.Millisecond), ) return persisted, persistErr @@ -516,6 +551,20 @@ func persistJobs( ) return written, inserted, updated, err } + if cfg.Persist == nil && cfg.Store.Durable() { + legacyExisting := result.Updated + if result.ContentChangeDetectionAvailable { + legacyExisting = 0 + } + metrics.ObserveCommittedJobs(result.Inserted, legacyExisting) + } + if result.ContentChangeDetectionAvailable { + stats.ContentChangeDetectionAvailable = true + stats.RealUpdated += result.RealUpdated + stats.Refreshed += result.Refreshed + stats.Reactivated += result.Reactivated + metrics.ObserveCommittedContent(result.RealUpdated, result.Refreshed, result.Reactivated) + } inserted += result.Inserted updated += result.Updated for _, j := range result.Persisted { @@ -546,18 +595,82 @@ func indexPersistedChunk( var err error commands := 0 + indexPath := "callback" + var observed jobindex.ApplyStats + var indexKeysDuration, buildDuration, serializationDuration, redisDuration, markDuration time.Duration + attempts := 0 + priorUnconfirmed := false + checkpointStatus := "not_attempted" + defer func() { + elapsed := time.Since(started) + attrs := []any{"run_id", cfg.RunID, "stage", "index", "batch", batchNo, "index_path", indexPath, + "received", len(jobs), "duration", elapsed.Round(time.Millisecond), "status", metrics.BatchStatus(err), + "checkpoint_status", checkpointStatus, "mark_indexed_duration", markDuration, + "content_change_detection_available", stats.ContentChangeDetectionAvailable, "refresh_projection_available", false} + if indexPath == "durable" { + refreshFallback := 0 + for _, job := range jobs { + if job.CatalogChange == "job_refreshed" { + refreshFallback++ + } + } + attrs = append(attrs, "projection_mode", "full", "refresh_projection", 0, + "refresh_fallback_full", refreshFallback, "refresh_fallback_reason", "light_refresh_not_implemented", + "result_confirmed", observed.Confirmed, "full_applied", observed.FullApplied, + "removed", observed.Removed, "skipped", observed.Skipped, "attempts", attempts, "retries", max(0, attempts-1), + "prior_attempt_unconfirmed", priorUnconfirmed, "failure_stage", observed.FailureStage, + "index_keys_duration", indexKeysDuration, + "build_metadata_duration", buildDuration, "serialization_duration", serializationDuration, + "redis_script_round_trip_duration", redisDuration, "batch_bytes", observed.PayloadBytes, + "keys_total", observed.KeysTotal, "keys_min", observed.KeysMin, "keys_max", observed.KeysMax) + } else if indexPath == "legacy" { + attrs = append(attrs, "legacy_commands", commands) + } + if err != nil { + slog.Error("scraper stage batch", append(attrs, "error", err)...) + } else { + slog.Info("scraper stage batch", attrs...) + } + }() metrics.Stage("indexing") if cfg.Index != nil { err = cfg.Index(ctx, jobs) } else if cfg.Store.Durable() { + indexPath = "durable" + prepared := cfg.indexKeywords + if prepared == nil { + prepared = prepareIndexKeywords(cfg.Keywords) + } + var previousErr error err = jobstore.RetryTransient(ctx, func() error { - _, e := jobindex.New(cfg.RDB).Apply(ctx, jobs, func(job domain.Job) []string { return invertedIndexKeys(job, cfg.Keywords) }) + if previousErr != nil { + metrics.IndexRetries.WithLabelValues(metrics.ErrorType(previousErr)).Inc() + } + attempts++ + var e error + observed, e = jobindex.New(cfg.RDB).ApplyDetailed(ctx, jobs, prepared.keys) + indexKeysDuration += observed.IndexKeys + buildDuration += observed.BuildMetadata + serializationDuration += observed.Serialization + redisDuration += observed.RedisRoundTrip + if e != nil && observed.FailureStage == "redis_script_round_trip" { + priorUnconfirmed = true + } + previousErr = e return e }) if err == nil { + markStarted := time.Now() err = cfg.Store.MarkIndexed(ctx, jobs) + markDuration = time.Since(markStarted) + metrics.ObserveStage("mark_indexed", markDuration, err) + checkpointStatus = metrics.BatchStatus(err) + if err != nil { + observed.FailureStage = "mark_indexed" + } } } else { + indexPath = "legacy" commands, err = indexJobsInValkeyBatched(ctx, cfg.RDB, jobs, cfg.Keywords, cfg.IndexBatchSize, cfg.IndexSession) } if err != nil { @@ -581,6 +694,7 @@ func indexPersistedChunk( ) return err } + err = nil slog.Warn("scraper index reconciled after failure", "run_id", cfg.RunID, "stage", "index_reconcile", @@ -592,14 +706,7 @@ func indexPersistedChunk( metrics.JobResult(j.Source, "indexed", 1) } stats.Indexed += len(jobs) - slog.Info("scraper stage batch", - "run_id", cfg.RunID, - "stage", "index", - "batch", batchNo, - "indexed", len(jobs), - "commands", commands, - "duration", time.Since(started).Round(time.Millisecond), - ) + return nil } diff --git a/scraper-go/internal/pipeline/process_bench_test.go b/scraper-go/internal/pipeline/process_bench_test.go index 8903fda0..9635f965 100644 --- a/scraper-go/internal/pipeline/process_bench_test.go +++ b/scraper-go/internal/pipeline/process_bench_test.go @@ -3,9 +3,14 @@ package pipeline import ( "context" "fmt" + "os" + "reflect" + "strings" "testing" + "time" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/jobindex" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/jobstore" "github.com/alicebob/miniredis/v2" "github.com/redis/go-redis/v9" @@ -122,3 +127,89 @@ func BenchmarkProcessIncomingBatched(b *testing.B) { } } } + +// Requires an explicitly supplied disposable server: this benchmark writes index +// data. It never reads the application Redis URL or starts external processes. +func BenchmarkDurableIndexApply(b *testing.B) { + socket := os.Getenv("PAV178_BENCH_REDIS_SOCKET") + if socket == "" { + b.Skip("PAV178_BENCH_REDIS_SOCKET must identify an isolated disposable server") + } + rdb := redis.NewClient(&redis.Options{Network: "unix", Addr: socket, MaxRetries: -1}) + b.Cleanup(func() { rdb.Close() }) + manager := jobindex.New(rdb) + jobs := benchJobs(250) + description := strings.Repeat("Golang go APIs microservices postgresql redis kafka backend engineer remote Brasil. ", 52)[:4096] + expires := time.Now().Add(time.Hour) + for i := range jobs { + jobs[i].ID = fmt.Sprintf("bench-%d-%d-%d", os.Getpid(), expires.UnixNano(), i) + jobs[i].Description = description + jobs[i].CatalogRevision = 1 + jobs[i].CatalogExpiresAt = expires + jobs[i].Classification = &domain.Classification{PrimaryFamily: "backend", InScope: true} + } + keywords := make([]string, 445) + for i := range keywords { + keywords[i] = fmt.Sprintf("backend engineer technology%d", i) + } + keywords[0] = "go" + keys := func(j domain.Job) []string { return IndexKeys(j, keywords) } + if _, err := manager.Apply(context.Background(), jobs, keys); err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := range jobs { + jobs[i].CatalogRevision++ + } + if _, err := manager.Apply(context.Background(), jobs, keys); err != nil { + b.Fatal(err) + } + } +} + +// Local projection only: no datastore or provider I/O. Every ns/op is a batch +// of 250 jobs; compilation is included in isolated and excluded in prepared. +func BenchmarkIndexKeys(b *testing.B) { + keywords := realIndexKeywords(b) + for _, size := range []int{4096, 16384} { + jobs := benchJobs(250) + text := strings.Repeat("Golang go APIs microservices postgresql redis kafka backend engineer remote Brasil Node.js React C# .NET. ", size/50+1)[:size] + for i := range jobs { + jobs[i].Description = text + jobs[i].Classification = &domain.Classification{PrimaryFamily: "backend", InScope: true, Technologies: []string{"Go", "Node.js"}, Seniority: "senior"} + } + prepared := prepareIndexKeywords(keywords) + totalKeys := 0 + // Verify every job against the frozen oracle outside timed regions. + for _, job := range jobs { + expected := sortedIndexKeys(referenceInvertedIndexKeys(job, keywords)) + if !reflect.DeepEqual(expected, sortedIndexKeys(IndexKeys(job, keywords))) || !reflect.DeepEqual(expected, sortedIndexKeys(prepared.keys(job))) { + b.Fatal("projection differs from committed reference") + } + totalKeys += len(expected) + } + for _, mode := range []string{"reference", "isolated", "prepared"} { + b.Run(fmt.Sprintf("%dKiB/%s", size/1024, mode), func(b *testing.B) { + project := prepared.keys + if mode == "reference" { + project = func(j domain.Job) []string { return referenceInvertedIndexKeys(j, keywords) } + } + if mode == "isolated" { + project = func(j domain.Job) []string { return IndexKeys(j, keywords) } + } + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + for _, job := range jobs { + project(job) + } + } + b.StopTimer() + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/float64(len(jobs)), "ns/job") + b.ReportMetric(float64(totalKeys), "keys/batch") + }) + } + } +} diff --git a/scraper-go/internal/pipeline/process_test.go b/scraper-go/internal/pipeline/process_test.go index 1916f5ca..4314319b 100644 --- a/scraper-go/internal/pipeline/process_test.go +++ b/scraper-go/internal/pipeline/process_test.go @@ -1,9 +1,13 @@ package pipeline import ( + "bytes" "context" + "encoding/json" "errors" "fmt" + "log/slog" + "strings" "sync" "sync/atomic" "testing" @@ -15,6 +19,7 @@ import ( "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/metrics" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" "github.com/alicebob/miniredis/v2" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -721,3 +726,321 @@ func TestRunSnapshotProviderTracksRealScheduledTask(t *testing.T) { assert.Equal(t, "success", *providers[0].LastTaskStatus) assert.Zero(t, providers[0].Active) } + +type observedCatalog struct { + jobstore.Catalog + saveErr error + markErr error + marks int +} + +func (c *observedCatalog) SaveBatch(_ context.Context, jobs []domain.Job) (jobstore.SaveResult, error) { + if c.saveErr != nil { + return jobstore.SaveResult{}, c.saveErr + } + return jobstore.SaveResult{Inserted: len(jobs), Persisted: jobs}, nil +} +func (c *observedCatalog) MarkIndexed(context.Context, []domain.Job) error { + c.marks++ + return c.markErr +} + +func committedIndexJob() domain.Job { + j := inScopeJob(1) + j.ID = jobstore.StableID(&j) + j.CatalogRevision = 1 + j.CatalogExpiresAt = time.Now().Add(time.Hour) + j.Classification = &domain.Classification{PrimaryFamily: "backend", InScope: true} + return j +} + +func captureIndexLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var output bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + return &output +} + +func lastIndexLog(t *testing.T, output *bytes.Buffer) map[string]any { + t.Helper() + var result map[string]any + for _, line := range strings.Split(strings.TrimSpace(output.String()), "\n") { + var record map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &record)) + if record["msg"] == "scraper stage batch" && record["stage"] == "index" { + result = record + } + } + require.NotNil(t, result) + return result +} + +func TestDurableLogSeparatesApplyAndCheckpoint(t *testing.T) { + for _, checkpointErr := range []error{nil, errors.New("checkpoint failure")} { + t.Run(metrics.BatchStatus(checkpointErr), func(t *testing.T) { + output := captureIndexLogs(t) + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { rdb.Close() }) + catalog := &observedCatalog{markErr: checkpointErr} + cfg := processConfig{Store: jobstore.NewDurable(rdb, catalog), RDB: rdb, Keywords: []string{"go"}} + stats := ProcessStats{} + err := indexPersistedChunk(context.Background(), cfg, 1, []domain.Job{committedIndexJob()}, &stats) + require.ErrorIs(t, err, checkpointErr) + record := lastIndexLog(t, output) + require.NotContains(t, record, "commands") + require.NotContains(t, record, "indexed") + require.Equal(t, "durable", record["index_path"]) + require.Equal(t, true, record["result_confirmed"]) + require.Equal(t, float64(1), record["full_applied"]) + require.Equal(t, metrics.BatchStatus(checkpointErr), record["checkpoint_status"]) + require.Equal(t, false, record["content_change_detection_available"]) + require.Equal(t, 1, catalog.marks) + if checkpointErr == nil { + require.NoError(t, indexPersistedChunk(context.Background(), cfg, 2, []domain.Job{committedIndexJob()}, &stats)) + record = lastIndexLog(t, output) + require.Equal(t, float64(0), record["full_applied"]) + require.Equal(t, float64(1), record["skipped"]) + require.Equal(t, 2, stats.Indexed) // Compatibility: successful inputs, not mutations. + } + }) + } +} + +func TestIndexInFlightSurvivesEmptyBuffer(t *testing.T) { + for _, terminalErr := range []error{nil, errors.New("index failure"), context.Canceled} { + t.Run(metrics.BatchStatus(terminalErr), func(t *testing.T) { + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan error, 1) + var releaseOnce sync.Once + joined := false + t.Cleanup(func() { + releaseOnce.Do(func() { close(release) }) + if !joined { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("index goroutine did not exit") + } + } + }) + cfg := processConfig{ClassificationBatchSize: 1, PersistBatchSize: 1, IndexBatchSize: 1, + Index: func(context.Context, []domain.Job) error { close(entered); <-release; return terminalErr }} + go func() { + _, _, err := processIncomingJobs(context.Background(), feedJobs(inScopeJob(1)), cfg) + done <- err + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("index not entered") + } + q := metrics.Current().Queues["indexing"] + require.Equal(t, 0, q.Depth) + require.Equal(t, 1, q.InFlight) + releaseOnce.Do(func() { close(release) }) + gotErr := <-done + joined = true + require.ErrorIs(t, gotErr, terminalErr) + q = metrics.Current().Queues["indexing"] + require.Zero(t, q.InFlight) + metrics.ClearQueues() + }) + } +} + +type lostReplyHook struct { + redis.Hook + lost bool +} + +func (h *lostReplyHook) DialHook(next redis.DialHook) redis.DialHook { return next } +func (h *lostReplyHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return next +} +func (h *lostReplyHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + err := next(ctx, cmd) + if err == nil && cmd.Name() == "eval" && !h.lost { + h.lost = true + return errors.New("network lost reply") + } + return err + } +} + +func TestRetryLogsUnconfirmedAttemptAndIdempotentSkip(t *testing.T) { + output := captureIndexLogs(t) + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + defer rdb.Close() + rdb.AddHook(&lostReplyHook{}) + catalog := &observedCatalog{} + cfg := processConfig{Store: jobstore.NewDurable(rdb, catalog), RDB: rdb} + before := testutil.ToFloat64(metrics.IndexRetries.WithLabelValues("network")) + stats := ProcessStats{} + require.NoError(t, indexPersistedChunk(context.Background(), cfg, 1, []domain.Job{committedIndexJob()}, &stats)) + record := lastIndexLog(t, output) + require.Equal(t, float64(2), record["attempts"]) + require.Equal(t, float64(1), record["retries"]) + require.Equal(t, true, record["prior_attempt_unconfirmed"]) + require.Equal(t, float64(0), record["full_applied"]) + require.Equal(t, float64(1), record["skipped"]) + require.Equal(t, before+1, testutil.ToFloat64(metrics.IndexRetries.WithLabelValues("network"))) + require.Equal(t, 1, catalog.marks) +} + +func TestFailedPersistDoesNotReportCommittedJobs(t *testing.T) { + before := testutil.ToFloat64(metrics.CatalogCommittedJobs.WithLabelValues("created")) + catalog := &observedCatalog{saveErr: errors.New("invalid commit")} + cfg := processConfig{Store: jobstore.NewDurable(nil, catalog), PersistBatchSize: 1} + stats := ProcessStats{} + _, _, _, err := persistJobs(context.Background(), cfg, 1, []domain.Job{committedIndexJob()}, &stats) + require.Error(t, err) + require.Equal(t, before, testutil.ToFloat64(metrics.CatalogCommittedJobs.WithLabelValues("created"))) + require.Zero(t, catalog.marks) +} + +func TestDurablePipelineUsesPreparedKeywordSnapshot(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { rdb.Close() }) + queries := []string{"initialtoken"} + first, second := inScopeJob(101), inScopeJob(102) + first.Description += " initialtoken" + second.Description += " initialtoken" + saves := 0 + cfg := processConfig{Keywords: queries, RDB: rdb, Store: jobstore.NewDurable(rdb, &observedCatalog{}), ClassificationBatchSize: 1, PersistBatchSize: 1, IndexBatchSize: 1, + Persist: func(_ context.Context, jobs []domain.Job) (jobstore.SaveResult, error) { + saves++ + // Simulates a caller changing its source slice after execution preparation. + // Synchronous mutation avoids races; the matcher must own its snapshot. + queries[0] = "replacementtoken" + persisted := append([]domain.Job(nil), jobs...) + for i := range persisted { + persisted[i].ID = jobstore.StableID(&persisted[i]) + persisted[i].CatalogRevision = int64(saves) + persisted[i].CatalogExpiresAt = time.Now().Add(time.Hour) + } + return jobstore.SaveResult{Inserted: len(persisted), Persisted: persisted}, nil + }} + _, stats, err := processIncomingJobs(context.Background(), feedJobs(first, second), cfg) + require.NoError(t, err) + require.Equal(t, 2, stats.Indexed) + require.Equal(t, int64(2), rdb.SCard(context.Background(), "scraper:jobs:keyword:initialtoken").Val()) + require.Zero(t, rdb.SCard(context.Background(), "scraper:jobs:keyword:replacementtoken").Val()) +} + +func TestClassifyPersistLogSeparatesStageDurations(t *testing.T) { + for _, scenario := range []string{"success", "failure", "no_store"} { + t.Run(scenario, func(t *testing.T) { + output := captureIndexLogs(t) + cfg := processConfig{PersistBatchSize: 100} + var expectedErr error + if scenario == "failure" { + expectedErr = errors.New("persist failure") + } + if scenario != "no_store" { + cfg.Persist = func(_ context.Context, jobs []domain.Job) (jobstore.SaveResult, error) { + time.Sleep(10 * time.Millisecond) + return jobstore.SaveResult{Inserted: len(jobs), Persisted: jobs}, expectedErr + } + } + stats := ProcessStats{} + _, err := processStageBatch(context.Background(), cfg, 1, []domain.Job{ + inScopeJob(1), {}, {Title: "Graphic Designer", Company: "Acme"}, + }, &stats, 0) + require.ErrorIs(t, err, expectedErr) + var record map[string]any + for _, line := range strings.Split(strings.TrimSpace(output.String()), "\n") { + var candidate map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &candidate)) + if candidate["stage"] == "classify_persist" { + record = candidate + } + } + require.NotNil(t, record) + validation := record["validation_duration"].(float64) + classification := record["classification_duration"].(float64) + persistence := record["persistence_duration"].(float64) + require.Positive(t, validation) + require.Positive(t, classification) + if scenario == "no_store" { + require.Zero(t, persistence) + } else { + require.GreaterOrEqual(t, persistence, float64(10*time.Millisecond)) + } + // Total remains rounded to milliseconds; component durations are exact. + require.GreaterOrEqual(t, record["duration"].(float64)+float64(time.Millisecond), validation+classification+persistence) + require.Equal(t, float64(1), record["invalid"]) + require.Equal(t, float64(1), record["rejected"]) + }) + } +} + +func TestRefreshedProjectionUsesConfirmedFullFallback(t *testing.T) { + for _, failure := range []string{"none", "checkpoint", "valkey", "lost_reply"} { + t.Run(failure, func(t *testing.T) { + output := captureIndexLogs(t) + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { rdb.Close() }) + catalog := &observedCatalog{} + if failure == "checkpoint" { + catalog.markErr = errors.New("checkpoint failed") + } + if failure == "valkey" { + require.NoError(t, rdb.Set(context.Background(), "scraper:jobs:index", "wrong type", 0).Err()) + } + if failure == "lost_reply" { + rdb.AddHook(&lostReplyHook{}) + } + j := committedIndexJob() + j.CatalogChange = "job_refreshed" + j.CatalogRevision = 2 + cfg := processConfig{Store: jobstore.NewDurable(rdb, catalog), RDB: rdb} + stats := ProcessStats{ContentChangeDetectionAvailable: true} + err := indexPersistedChunk(context.Background(), cfg, 1, []domain.Job{j}, &stats) + record := lastIndexLog(t, output) + require.Equal(t, "full", record["projection_mode"]) + require.Equal(t, float64(1), record["refresh_fallback_full"]) + require.Equal(t, float64(0), record["refresh_projection"]) + if failure == "valkey" { + require.Error(t, err) + require.Zero(t, catalog.marks) + require.Equal(t, "not_attempted", record["checkpoint_status"]) + } else { + require.Equal(t, 1, catalog.marks) + if failure == "checkpoint" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + } + }) + } +} + +func TestPersistDetectedOutcomesAreCountedOnlyAfterSuccess(t *testing.T) { + cfg := processConfig{PersistBatchSize: 4, Persist: func(context.Context, []domain.Job) (jobstore.SaveResult, error) { + return jobstore.SaveResult{Inserted: 1, Updated: 3, RealUpdated: 1, Refreshed: 1, Reactivated: 1, ContentChangeDetectionAvailable: true, Persisted: []domain.Job{committedIndexJob()}}, nil + }} + stats := ProcessStats{} + _, created, existing, err := persistJobs(context.Background(), cfg, 1, []domain.Job{committedIndexJob()}, &stats) + require.NoError(t, err) + require.Equal(t, 1, created) + require.Equal(t, 3, existing) + require.Equal(t, 1, stats.RealUpdated) + require.Equal(t, 1, stats.Refreshed) + require.Equal(t, 1, stats.Reactivated) + require.True(t, stats.ContentChangeDetectionAvailable) + cfg.Persist = func(context.Context, []domain.Job) (jobstore.SaveResult, error) { + return jobstore.SaveResult{Refreshed: 1, ContentChangeDetectionAvailable: true}, errors.New("commit failed") + } + stats = ProcessStats{} + _, _, _, err = persistJobs(context.Background(), cfg, 1, []domain.Job{committedIndexJob()}, &stats) + require.Error(t, err) + require.Zero(t, stats.Refreshed) + require.False(t, stats.ContentChangeDetectionAvailable) +} diff --git a/scraper-go/internal/pipeline/provider_selection.go b/scraper-go/internal/pipeline/provider_selection.go new file mode 100644 index 00000000..1ae6d48d --- /dev/null +++ b/scraper-go/internal/pipeline/provider_selection.go @@ -0,0 +1,47 @@ +package pipeline + +import ( + "errors" + "fmt" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" +) + +// ErrInvalidProviderSelection includes unknown IDs and providers absent from the +// configured registry. An explicit selection must never fall back to all sources. +var ErrInvalidProviderSelection = errors.New("invalid provider selection") + +// SelectProviders validates an explicit request against the startup registry. +// Callers use the same selection rules for scheduling and manual execution. +func SelectProviders(sources []ports.JobSource, requested []string) ([]ports.JobSource, error) { + return selectProviders(sources, requested) +} + +func selectProviders(sources []ports.JobSource, requested []string) ([]ports.JobSource, error) { + if len(requested) == 0 { + return sources, nil + } + selected := make(map[ports.ProviderID]bool, len(requested)) + for _, value := range requested { + provider, known := ports.ParseProviderID(value) + if !known { + return nil, fmt.Errorf("%w: unknown provider %q", ErrInvalidProviderSelection, value) + } + selected[provider] = false + } + filtered := make([]ports.JobSource, 0, len(sources)) + for _, source := range sources { + provider := ports.CapabilitiesOf(source).Provider + if _, wanted := selected[provider]; wanted { + filtered = append(filtered, source) + selected[provider] = true + } + } + for _, value := range requested { + provider, _ := ports.ParseProviderID(value) + if !selected[provider] { + return nil, fmt.Errorf("%w: provider %q is not configured", ErrInvalidProviderSelection, provider) + } + } + return filtered, nil +} diff --git a/scraper-go/internal/pipeline/provider_selection_test.go b/scraper-go/internal/pipeline/provider_selection_test.go new file mode 100644 index 00000000..1554160e --- /dev/null +++ b/scraper-go/internal/pipeline/provider_selection_test.go @@ -0,0 +1,145 @@ +package pipeline + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/cache" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/ports" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/runlock" + "github.com/stretchr/testify/require" +) + +func selectionSources() ([]ports.JobSource, []*atomic.Int32) { + ids := []ports.ProviderID{ports.ProviderLinkedIn, ports.ProviderAdzuna, ports.ProviderGupy, + ports.ProviderInHire, ports.ProviderGreenhouse, ports.ProviderLever, ports.ProviderJooble, ports.ProviderTheMuse} + sources := make([]ports.JobSource, 0, len(ids)) + calls := make([]*atomic.Int32, 0, len(ids)) + for _, id := range ids { + count := new(atomic.Int32) + calls = append(calls, count) + sources = append(sources, schedulerTestAdapter{provider: id, search: func(context.Context, string) ([]domain.Job, error) { + count.Add(1) + return nil, nil + }}) + } + return sources, calls +} + +func TestProviderSelectionRunsEachProviderInIsolation(t *testing.T) { + sources, _ := selectionSources() + for i, source := range sources { + t.Run(string(ports.CapabilitiesOf(source).Provider), func(t *testing.T) { + adapters, calls := selectionSources() + _, err := Run(context.Background(), adapters, domain.ScrapeRequest{ + Keywords: []string{"go"}, Sources: []string{string(ports.CapabilitiesOf(source).Provider)}, MaxConcurrency: 4, + }) + require.NoError(t, err) + for j, count := range calls { + want := int32(0) + if i == j { + want = 1 + } + require.Equal(t, want, count.Load()) + } + }) + } +} + +func TestProviderSelectionWithoutSourcesPreservesAllAdapters(t *testing.T) { + for _, requested := range [][]string{nil, {}} { + sources, calls := selectionSources() + _, err := Run(context.Background(), sources, domain.ScrapeRequest{Keywords: []string{"go"}, Sources: requested, MaxConcurrency: 4}) + require.NoError(t, err) + for _, count := range calls { + require.EqualValues(t, 1, count.Load()) + } + } +} + +func TestProviderSelectionRejectsUnknownEmptyAndUnconfiguredBeforeExecution(t *testing.T) { + for _, requested := range [][]string{{"invalid"}, {""}, {" "}, {"linkedin", "invalid"}, {"gupy"}} { + sources, calls := selectionSources() + _, _, err := ScrapeAllSources(context.Background(), SearchConfig{Keywords: []string{"go"}, Sources: requested, MaxConcurrency: 4}, sources[:1], nil) + require.ErrorIs(t, err, ErrInvalidProviderSelection) + for _, count := range calls { + require.Zero(t, count.Load()) + } + } +} + +func TestProviderSelectionUsesCapabilitiesAndKeepsAllProviderInstances(t *testing.T) { + sources := []ports.JobSource{ + schedulerTestAdapter{name: "company-one", provider: ports.ProviderGreenhouse}, + schedulerTestAdapter{name: "linkedin", provider: ports.ProviderGupy}, + schedulerTestAdapter{name: "company-two", provider: ports.ProviderGreenhouse}, + } + selected, err := selectProviders(sources, []string{" GreenHouse ", "greenhouse"}) + require.NoError(t, err) + require.Equal(t, []ports.JobSource{sources[0], sources[2]}, selected) + _, err = selectProviders(sources, []string{"linkedin"}) + require.ErrorIs(t, err, ErrInvalidProviderSelection) + selected, err = selectProviders(sources, []string{"gupy", "greenhouse"}) + require.NoError(t, err) + require.Equal(t, sources, selected) +} + +func TestProviderSelectionPrecedesCacheAndCadence(t *testing.T) { + manager, client, _ := newSearchLockManager(t) + sources, calls := selectionSources() + c := cache.NewMemoryCache() + cfg := SearchConfig{Keywords: []string{"go"}, Sources: []string{"gupy"}, MaxConcurrency: 4} + require.NoError(t, c.Set(context.Background(), BuildCacheKey(cfg), SearchResult{}, time.Minute)) + _, err := SearchJobs(context.Background(), c, cfg, sources[:1], time.Minute, client, manager, "public_endpoint") + require.ErrorIs(t, err, ErrInvalidProviderSelection) + cfg.Sources = []string{"linkedin"} + _, _, err = ScrapeAllSources(context.Background(), cfg, sources, client) + require.NoError(t, err) + require.EqualValues(t, 1, calls[0].Load()) + for _, count := range calls[1:] { + require.Zero(t, count.Load()) + } + require.Zero(t, client.Exists(context.Background(), joobleRunGateKey).Val()) +} + +func TestProviderSelectionPreservesSharedRunLock(t *testing.T) { + manager, client, _ := newSearchLockManager(t) + sources, calls := selectionSources() + held, err := manager.Acquire(context.Background(), "cron") + require.NoError(t, err) + t.Cleanup(func() { _ = held.Release(context.Background()) }) + _, err = SearchJobs(context.Background(), cache.NewMemoryCache(), SearchConfig{ + Keywords: []string{"go"}, Sources: []string{"linkedin"}, MaxConcurrency: 4, + }, sources, time.Minute, client, manager, "public_endpoint") + require.ErrorIs(t, err, runlock.ErrAlreadyHeld) + for _, count := range calls { + require.Zero(t, count.Load()) + } +} + +func TestProviderSelectionPreservesJoobleCadence(t *testing.T) { + _, client, _ := newSearchLockManager(t) + sources, calls := selectionSources() + cfg := SearchConfig{Keywords: []string{"go"}, Sources: []string{"jooble"}, MaxConcurrency: 4} + for range 2 { + _, _, err := ScrapeAllSources(context.Background(), cfg, sources, client) + require.NoError(t, err) + } + require.EqualValues(t, 1, calls[6].Load()) + for i, count := range calls { + if i != 6 { + require.Zero(t, count.Load()) + } + } +} + +func TestProviderSelectionSeparatesLegacyCacheResults(t *testing.T) { + cfg := SearchConfig{Keywords: []string{"go"}, MaxConcurrency: 4} + require.Contains(t, BuildCacheKey(cfg), "jobs:") + require.NotContains(t, BuildCacheKey(cfg), "providers:v1") + cfg.Sources = []string{"linkedin"} + require.Contains(t, BuildCacheKey(cfg), "jobs:providers:v1:") +} diff --git a/scraper-go/internal/pipeline/scrape.go b/scraper-go/internal/pipeline/scrape.go index c6f8deb1..4d9bc655 100644 --- a/scraper-go/internal/pipeline/scrape.go +++ b/scraper-go/internal/pipeline/scrape.go @@ -6,6 +6,7 @@ import ( "log/slog" "sort" "strings" + "time" cfgpkg "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/config" "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/domain" @@ -16,6 +17,12 @@ import ( ) type SearchConfig struct { + GupyWindow bool `json:"-"` + QueryCursor domain.QueryCursor `json:"-"` + FullKeywordCoverage bool `json:"-"` + WindowID string `json:"-"` + CollectionTimeout time.Duration `json:"-"` + CollectionDeadline time.Time `json:"-"` Store *jobstore.Store `json:"-"` Keywords []string `json:"keywords"` SearchLocation string `json:"searchLocation"` @@ -38,6 +45,16 @@ type SearchConfig struct { IndexBatchSize int `json:"-"` } +type collectionFailure struct { + count int + cause error +} + +func (e collectionFailure) Error() string { + return fmt.Sprintf("collection: %d task(s) failed", e.count) +} +func (e collectionFailure) Unwrap() error { return e.cause } + func normalizeSearchConfig(config SearchConfig) SearchConfig { config.Keywords = keywords.GenerateSearchKeywords(config.Keywords) if config.ProviderMaxConcurrency <= 0 { @@ -52,6 +69,10 @@ func ScrapeAllSources( adapterList []ports.JobSource, rdb *redis.Client, ) ([]domain.Job, ProcessStats, error) { + adapterList, err := selectProviders(adapterList, config.Sources) + if err != nil { + return nil, ProcessStats{}, err + } config = normalizeSearchConfig(config) if config.Store != nil && config.Store.Durable() { release, err := config.Store.ProcessingLease(ctx) @@ -68,6 +89,8 @@ func ScrapeAllSources( ) processCfg := processConfig{ + CollectionTimeout: config.CollectionTimeout, + CollectionDeadline: config.CollectionDeadline, RunID: config.RunID, Keywords: config.Keywords, ClassificationBatchSize: config.ClassificationBatchSize, @@ -94,6 +117,10 @@ func ScrapeAllSources( adapterList = filterAdaptersByCadence(ctx, rdb, adapterList) req := domain.ScrapeRequest{ + GupyWindow: config.GupyWindow, + QueryCursor: config.QueryCursor, + FullKeywordCoverage: config.FullKeywordCoverage, + WindowID: config.WindowID, Keywords: config.Keywords, SearchLocation: config.SearchLocation, SearchGeoID: config.SearchGeoID, @@ -127,6 +154,9 @@ func ScrapeAllSources( if err != nil { return jobs, stats, err } + if stats.CollectionFailed > 0 { + return jobs, stats, collectionFailure{stats.CollectionFailed, stats.CollectionError} + } slog.Info("scrape finished", "total_jobs", len(jobs), diff --git a/scraper-go/internal/pipeline/search.go b/scraper-go/internal/pipeline/search.go index c8a5988b..11c89802 100644 --- a/scraper-go/internal/pipeline/search.go +++ b/scraper-go/internal/pipeline/search.go @@ -34,6 +34,10 @@ func SearchJobs( runLock *runlock.Manager, source string, ) (SearchResult, error) { + adapterList, err := selectProviders(adapterList, config.Sources) + if err != nil { + return SearchResult{}, err + } config = normalizeSearchConfig(config) cacheKey := BuildCacheKey(config) @@ -70,6 +74,7 @@ func SearchJobs( runCtx := lease.Context() config.RunID = lease.RunID() + config.CollectionDeadline = lease.CollectionDeadline(config.CollectionTimeout) jobs, _, err := ScrapeAllSources(runCtx, config, adapterList, rdb) if err != nil { return SearchResult{}, fmt.Errorf("pipeline.SearchJobs: scrape: %w", err) diff --git a/scraper-go/internal/pipeline/search_lock_test.go b/scraper-go/internal/pipeline/search_lock_test.go index e1d07c4f..72bf51b8 100644 --- a/scraper-go/internal/pipeline/search_lock_test.go +++ b/scraper-go/internal/pipeline/search_lock_test.go @@ -133,7 +133,7 @@ func TestSearchJobsCancelsWithoutIndexingAfterConfirmedLockLoss(t *testing.T) { server := miniredis.RunT(t) client := redis.NewClient(&redis.Options{Addr: server.Addr()}) t.Cleanup(func() { _ = client.Close() }) - manager, err := runlock.New(runlock.NewValkeyStore(client), runlock.Config{ + manager, err := runlock.New(runlock.NewValkeyStore(client, 0), runlock.Config{ TTL: 200 * time.Millisecond, RenewInterval: 25 * time.Millisecond, }) @@ -203,7 +203,7 @@ func newSearchLockManager( client := redis.NewClient(&redis.Options{Addr: server.Addr()}) t.Cleanup(func() { _ = client.Close() }) - manager, err := runlock.New(runlock.NewValkeyStore(client), runlock.Config{ + manager, err := runlock.New(runlock.NewValkeyStore(client, 0), runlock.Config{ TTL: 2 * time.Second, RenewInterval: 500 * time.Millisecond, }) diff --git a/scraper-go/internal/ports/job_source.go b/scraper-go/internal/ports/job_source.go index 11c41f4a..104880a9 100644 --- a/scraper-go/internal/ports/job_source.go +++ b/scraper-go/internal/ports/job_source.go @@ -56,6 +56,13 @@ type JobSource interface { Search(ctx context.Context, keyword string, req domain.ScrapeRequest) ([]domain.Job, error) } +// StableJobSource identifies a board independently of its display name. +// Providers with fixed scheduling groups must implement this optional contract. +type StableJobSource interface { + JobSource + SourceID() string +} + type CapabilityJobSource interface { JobSource Capabilities() SourceCapabilities diff --git a/scraper-go/internal/ports/rate_limit.go b/scraper-go/internal/ports/rate_limit.go new file mode 100644 index 00000000..66f2fd05 --- /dev/null +++ b/scraper-go/internal/ports/rate_limit.go @@ -0,0 +1,38 @@ +package ports + +import ( + "fmt" + "time" +) + +type RateLimitError struct { + Provider ProviderID + NotBefore time.Time +} + +func (e *RateLimitError) Error() string { + return fmt.Sprintf("%s: rate limit (429), retry after %s", e.Provider, e.NotBefore.UTC().Format(time.RFC3339)) +} + +// LatestRetryAfter also handles errors.Join from multiple failed collection tasks. +func LatestRetryAfter(err error, provider ProviderID) time.Time { + if err == nil { + return time.Time{} + } + var latest time.Time + if limit, ok := err.(*RateLimitError); ok && limit.Provider == provider { + latest = limit.NotBefore + } + var causes []error + if group, ok := err.(interface{ Unwrap() []error }); ok { + causes = group.Unwrap() + } else if wrapped, ok := err.(interface{ Unwrap() error }); ok { + causes = []error{wrapped.Unwrap()} + } + for _, cause := range causes { + if next := LatestRetryAfter(cause, provider); next.After(latest) { + latest = next + } + } + return latest +} diff --git a/scraper-go/internal/ports/rate_limit_test.go b/scraper-go/internal/ports/rate_limit_test.go new file mode 100644 index 00000000..6ef6f500 --- /dev/null +++ b/scraper-go/internal/ports/rate_limit_test.go @@ -0,0 +1,16 @@ +package ports + +import ( + "errors" + "fmt" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestLatestRetryAfterPreservesLongestProviderDeadline(t *testing.T) { + now := time.Now() + err := errors.Join(&RateLimitError{ProviderGreenhouse, now.Add(time.Minute)}, fmt.Errorf("wrapped: %w", &RateLimitError{ProviderGreenhouse, now.Add(time.Hour)}), &RateLimitError{ProviderLever, now.Add(24 * time.Hour)}) + require.Equal(t, now.Add(time.Hour), LatestRetryAfter(err, ProviderGreenhouse)) + require.True(t, LatestRetryAfter(err, ProviderGupy).IsZero()) +} diff --git a/scraper-go/internal/runlock/budget.go b/scraper-go/internal/runlock/budget.go new file mode 100644 index 00000000..e800cf5c --- /dev/null +++ b/scraper-go/internal/runlock/budget.go @@ -0,0 +1,41 @@ +package runlock + +import ( + "context" + "fmt" + "time" +) + +const MaxExecutionDuration = 55 * time.Minute +const MinimumInterval = 5 * time.Minute + +type CooldownError struct{ NextAllowedAt time.Time } + +func (e *CooldownError) Error() string { + return fmt.Sprintf("scraper global pause until %s", e.NextAllowedAt.UTC().Format(time.RFC3339Nano)) +} + +// Keep the existing HTTP 409 admission contract; callers can distinguish pause. +func (e *CooldownError) Unwrap() error { return ErrAlreadyHeld } + +func executionDeadlines(ctx context.Context, start time.Time) (total, processing, collection time.Time) { + total = start.Add(MaxExecutionDuration) + if parent, ok := ctx.Deadline(); ok && parent.Before(total) { + total = parent + } + remaining := max(time.Duration(0), total.Sub(start)) + release := min(5*time.Second, remaining/20) + drain := min(30*time.Second, (remaining-release)/10) + processing = total.Add(-release) + collection = processing.Add(-drain) + return +} + +func (l *Lease) Deadline() time.Time { return l.deadline } +func (l *Lease) CollectionDeadline(timeout time.Duration) time.Time { + deadline := l.collectionDeadline + if timeout > 0 && l.startedAt.Add(timeout).Before(deadline) { + deadline = l.startedAt.Add(timeout) + } + return deadline +} diff --git a/scraper-go/internal/runlock/budget_test.go b/scraper-go/internal/runlock/budget_test.go new file mode 100644 index 00000000..0d882e44 --- /dev/null +++ b/scraper-go/internal/runlock/budget_test.go @@ -0,0 +1,95 @@ +package runlock + +import ( + "context" + "github.com/stretchr/testify/require" + "strconv" + "testing" + "time" +) + +func TestExecutionAbsoluteBudgetIncludesDrainAndRelease(t *testing.T) { + start := time.Now() + total, processing, collection := executionDeadlines(context.Background(), start) + require.Equal(t, 55*time.Minute, total.Sub(start)) + require.Equal(t, 5*time.Second, total.Sub(processing)) + require.Equal(t, 30*time.Second, processing.Sub(collection)) + parent, cancel := context.WithDeadline(context.Background(), start.Add(time.Second)) + defer cancel() + total, processing, collection = executionDeadlines(parent, start) + require.Equal(t, start.Add(time.Second), total) + require.True(t, collection.Before(processing)) + require.True(t, processing.Before(total)) +} + +func TestGlobalPausePersistsAcrossRestartAndSerializesReplicas(t *testing.T) { + m, client, server := newTestManager(t, 2*time.Second, 500*time.Millisecond) + m.store = NewValkeyStore(client) + now := time.Now().UTC().Truncate(time.Millisecond) + server.SetTime(now) + lease, err := m.Acquire(context.Background(), "cron") + require.NoError(t, err) + require.NoError(t, lease.Release(context.Background())) + stamp, err := strconv.ParseInt(client.Get(context.Background(), NextAllowedKey).Val(), 10, 64) + require.NoError(t, err) + require.Equal(t, now.Add(MinimumInterval), time.UnixMilli(stamp).UTC()) + restarted, err := New(NewValkeyStore(client), m.cfg) + require.NoError(t, err) + server.SetTime(now.Add(MinimumInterval - time.Millisecond)) + _, err = restarted.Acquire(context.Background(), "admin_manual") + var pause *CooldownError + require.ErrorAs(t, err, &pause) + server.SetTime(now.Add(MinimumInterval)) + type answer struct { + lease *Lease + err error + } + out := make(chan answer, 2) + for _, replica := range []*Manager{m, restarted} { + go func(m *Manager) { l, e := m.Acquire(context.Background(), "cron"); out <- answer{l, e} }(replica) + } + wins := 0 + for range 2 { + a := <-out + if a.err == nil { + wins++ + defer a.lease.Release(context.Background()) + } else { + require.ErrorIs(t, a.err, ErrAlreadyHeld) + } + } + require.Equal(t, 1, wins) +} + +func TestCrashRetainsPauseBeyondLeaseExpiration(t *testing.T) { + m, client, server := newTestManager(t, 2*time.Second, 500*time.Millisecond) + m.store = NewValkeyStore(client) + now := time.Now().UTC() + server.SetTime(now) + lease, err := m.Acquire(context.Background(), "cron") + require.NoError(t, err) + lease.cancelRenewal() + lease.stopOnce.Do(func() { close(lease.stopRenewal) }) + <-lease.renewalDone + server.FastForward(3 * time.Second) + server.SetTime(now.Add(3 * time.Second)) + _, err = m.Acquire(context.Background(), "admin_manual") + var pause *CooldownError + require.ErrorAs(t, err, &pause) + lease.cancel(nil) + lease.cancelDeadline() +} + +func TestCanceledExecutionRenewsUntilWorkersJoinAndRelease(t *testing.T) { + m, client, server := newTestManager(t, 100*time.Millisecond, 10*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + lease, err := m.Acquire(ctx, "cron") + require.NoError(t, err) + defer lease.Release(context.Background()) + cancel() + require.ErrorIs(t, context.Cause(lease.Context()), context.Canceled) + server.FastForward(90 * time.Millisecond) + require.Eventually(t, func() bool { return client.PTTL(context.Background(), LockKey).Val() > 90*time.Millisecond }, time.Second, 5*time.Millisecond) + _, err = m.Acquire(context.Background(), "admin_manual") + require.ErrorIs(t, err, ErrAlreadyHeld) +} diff --git a/scraper-go/internal/runlock/runlock.go b/scraper-go/internal/runlock/runlock.go index 00218a38..bb17d688 100644 --- a/scraper-go/internal/runlock/runlock.go +++ b/scraper-go/internal/runlock/runlock.go @@ -65,6 +65,8 @@ func New(store Store, cfg Config) (*Manager, error) { } func (m *Manager) Acquire(ctx context.Context, source string) (*Lease, error) { + startedAt := m.now().UTC() + deadline, processingDeadline, collectionDeadline := executionDeadlines(ctx, startedAt) token, err := newToken() if err != nil { return nil, fmt.Errorf("runlock: generate token: %w", err) @@ -74,18 +76,25 @@ func (m *Manager) Acquire(ctx context.Context, source string) (*Lease, error) { return nil, fmt.Errorf("runlock: generate run id: %w", err) } - acquired, err := m.store.TryAcquire(ctx, token, m.cfg.TTL) + deadlineCtx, cancelDeadline := context.WithDeadline(ctx, processingDeadline) + acquired, err := m.store.TryAcquire(deadlineCtx, token, m.cfg.TTL) if err != nil { + cancelDeadline() + var cooldown *CooldownError + if errors.As(err, &cooldown) { + metrics.Attempt(source, "skipped") + return nil, err + } metrics.Attempt(source, "failed") return nil, fmt.Errorf("%w: acquire: %v", ErrUnavailable, err) } if !acquired { + cancelDeadline() metrics.LockConflicts.WithLabelValues(metrics.Source(source)).Inc() metrics.Attempt(source, "skipped") return nil, ErrAlreadyHeld } - startedAt := m.now().UTC() metrics.StartRun(source, startedAt, runID) metrics.LockAcquired(m.cfg.TTL, runID) state := State{ @@ -94,7 +103,7 @@ func (m *Manager) Acquire(ctx context.Context, source string) (*Lease, error) { StartedAt: startedAt, LockExpiresAt: startedAt.Add(m.cfg.TTL), } - if written, stateErr := m.store.WriteState(ctx, token, state, m.cfg.TTL); stateErr != nil || !written { + if written, stateErr := m.store.WriteState(deadlineCtx, token, state, m.cfg.TTL); stateErr != nil || !written { slog.Warn("scraper run state unavailable", "source", source, "run_id", runID, @@ -102,13 +111,18 @@ func (m *Manager) Acquire(ctx context.Context, source string) (*Lease, error) { ) } - leaseCtx, cancel := context.WithCancelCause(ctx) + leaseCtx, cancel := context.WithCancelCause(deadlineCtx) + // Keep ownership while canceled workers join. This context only coordinates + // renewal; every Redis RPC is bounded and Release cancels in-flight renewal. + renewCtx, cancelRenewal := context.WithCancel(context.WithoutCancel(ctx)) lease := &Lease{ - manager: m, - token: token, - runID: runID, - source: source, - startedAt: startedAt, + manager: m, + token: token, + runID: runID, + source: source, + startedAt: startedAt, + deadline: deadline, collectionDeadline: collectionDeadline, + cancelDeadline: cancelDeadline, renewCtx: renewCtx, cancelRenewal: cancelRenewal, ctx: leaseCtx, cancel: cancel, stopRenewal: make(chan struct{}), @@ -125,11 +139,16 @@ func (m *Manager) Acquire(ctx context.Context, source string) (*Lease, error) { } type Lease struct { - manager *Manager - token string - runID string - source string - startedAt time.Time + manager *Manager + token string + runID string + source string + startedAt time.Time + deadline time.Time + collectionDeadline time.Time + cancelDeadline context.CancelFunc + renewCtx context.Context + cancelRenewal context.CancelFunc ctx context.Context cancel context.CancelCauseFunc @@ -166,12 +185,16 @@ func (l *Lease) Release(ctx context.Context) error { l.releaseOnce.Do(func() { defer metrics.LockReleased(l.runID) l.cancel(nil) + l.cancelDeadline() + l.cancelRenewal() l.stopOnce.Do(func() { close(l.stopRenewal) }) <-l.renewalDone - released, err := l.manager.store.Release(ctx, l.token, l.runID) + releaseCtx, cancel := context.WithDeadline(ctx, l.deadline) + defer cancel() + released, err := l.manager.store.Release(releaseCtx, l.token, l.runID) switch { case err != nil: l.releaseErr = fmt.Errorf("%w: release: %v", ErrUnavailable, err) @@ -204,14 +227,15 @@ func (l *Lease) renew() { safetyTimer := time.NewTimer(l.manager.cfg.TTL - l.manager.cfg.RenewInterval) defer safetyTimer.Stop() + executionDone := l.ctx.Done() for { select { case <-l.stopRenewal: return - case <-l.ctx.Done(): + case <-executionDone: metrics.Canceling(l.runID) - return + executionDone = nil // Keep the lock until workers have joined and Release runs. case <-safetyTimer.C: cause := fmt.Errorf("%w: renewal could not be confirmed before safety deadline", ErrLost) slog.Error("scraper run lock lost", @@ -226,7 +250,7 @@ func (l *Lease) renew() { return case <-ticker.C: state := l.State() - renewCtx, cancelRenew := context.WithTimeout(l.ctx, l.renewTimeout()) + renewCtx, cancelRenew := context.WithTimeout(l.renewCtx, l.renewTimeout()) renewed, err := l.manager.store.Renew( renewCtx, l.token, @@ -235,7 +259,7 @@ func (l *Lease) renew() { ) cancelRenew() if err != nil { - if l.ctx.Err() != nil { + if l.renewCtx.Err() != nil { return } metrics.LockRenewFailures.Inc() diff --git a/scraper-go/internal/runlock/runlock_test.go b/scraper-go/internal/runlock/runlock_test.go index 7a49aa5d..011c82b8 100644 --- a/scraper-go/internal/runlock/runlock_test.go +++ b/scraper-go/internal/runlock/runlock_test.go @@ -134,6 +134,8 @@ func TestLockExpiresAfterOwnerCrashes(t *testing.T) { lease, err := manager.Acquire(ctx, "cron") require.NoError(t, err) cancel() + lease.cancelRenewal() + lease.stopOnce.Do(func() { close(lease.stopRenewal) }) select { case <-lease.renewalDone: @@ -315,7 +317,7 @@ func newTestManager( _ = client.Close() }) - manager, err := New(NewValkeyStore(client), Config{ + manager, err := New(NewValkeyStore(client, 0), Config{ TTL: ttl, RenewInterval: renewInterval, }) diff --git a/scraper-go/internal/runlock/valkey.go b/scraper-go/internal/runlock/valkey.go index a143946f..1cf21237 100644 --- a/scraper-go/internal/runlock/valkey.go +++ b/scraper-go/internal/runlock/valkey.go @@ -9,10 +9,22 @@ import ( ) const ( - LockKey = "scraper:run:lock" - StateKey = "scraper:run:state" + LockKey = "scraper:run:lock" + StateKey = "scraper:run:state" + NextAllowedKey = "scraper:run:next_allowed_at" ) +var acquireScript = redis.NewScript(` +if redis.call('EXISTS',KEYS[1]) == 1 then return {0,0} end +local clock = redis.call('TIME') +local now = tonumber(clock[1])*1000 + math.floor(tonumber(clock[2])/1000) +local allowed = tonumber(redis.call('GET',KEYS[2]) or '0') +if allowed > now then return {-1,allowed} end +redis.call('SET',KEYS[1],ARGV[1],'PX',ARGV[2]) +if tonumber(ARGV[3]) > 0 then redis.call('SET',KEYS[2],now+tonumber(ARGV[2])+tonumber(ARGV[3])) end +return {1,0} +`) + var writeStateScript = redis.NewScript(` if redis.call("GET", KEYS[1]) ~= ARGV[1] then return 0 @@ -32,6 +44,11 @@ if redis.call("GET", KEYS[1]) ~= ARGV[1] then return 0 end redis.call("PEXPIRE", KEYS[1], ARGV[6]) +if tonumber(ARGV[7]) > 0 then + local clock = redis.call('TIME') + local now = tonumber(clock[1])*1000 + math.floor(tonumber(clock[2])/1000) + redis.call('SET',KEYS[3],now+tonumber(ARGV[6])+tonumber(ARGV[7])) +end if redis.call("HGET", KEYS[2], "runId") ~= ARGV[2] then redis.call("DEL", KEYS[2]) end @@ -48,6 +65,11 @@ var releaseScript = redis.NewScript(` if redis.call("GET", KEYS[1]) ~= ARGV[1] then return 0 end +if tonumber(ARGV[3]) > 0 then + local clock = redis.call('TIME') + local now = tonumber(clock[1])*1000 + math.floor(tonumber(clock[2])/1000) + redis.call('SET',KEYS[3],now+tonumber(ARGV[3])) +end redis.call("DEL", KEYS[1]) if redis.call("HGET", KEYS[2], "runId") == ARGV[2] then redis.call("DEL", KEYS[2]) @@ -64,21 +86,28 @@ type Store interface { type ValkeyStore struct { client *redis.Client + pause time.Duration } -func NewValkeyStore(client *redis.Client) *ValkeyStore { - return &ValkeyStore{client: client} +// Production always uses the five-minute default. An explicit override supports +// isolated legacy test fixtures; there is no runtime/env option to disable it. +func NewValkeyStore(client *redis.Client, testPause ...time.Duration) *ValkeyStore { + pause := MinimumInterval + if len(testPause) > 0 { + pause = testPause[0] + } + return &ValkeyStore{client: client, pause: pause} } func (s *ValkeyStore) TryAcquire(ctx context.Context, token string, ttl time.Duration) (bool, error) { - result, err := s.client.Do(ctx, acquireCommand(token, ttl)...).Text() - if err == redis.Nil { - return false, nil - } + result, err := acquireScript.Run(ctx, s.client, []string{LockKey, NextAllowedKey}, token, ttl.Milliseconds(), s.pause.Milliseconds()).Int64Slice() if err != nil { return false, err } - return result == "OK", nil + if result[0] == -1 { + return false, &CooldownError{NextAllowedAt: time.UnixMilli(result[1])} + } + return result[0] == 1, nil } func acquireCommand(token string, ttl time.Duration) []any { @@ -114,13 +143,14 @@ func (s *ValkeyStore) Renew( result, err := renewScript.Run( ctx, s.client, - []string{LockKey, StateKey}, + []string{LockKey, StateKey, NextAllowedKey}, token, state.RunID, state.Source, state.StartedAt.UTC().Format(time.RFC3339Nano), state.LockExpiresAt.UTC().Format(time.RFC3339Nano), strconv.FormatInt(ttl.Milliseconds(), 10), + s.pause.Milliseconds(), ).Int64() return result == 1, err } @@ -129,9 +159,10 @@ func (s *ValkeyStore) Release(ctx context.Context, token, runID string) (bool, e result, err := releaseScript.Run( ctx, s.client, - []string{LockKey, StateKey}, + []string{LockKey, StateKey, NextAllowedKey}, token, runID, + s.pause.Milliseconds(), ).Int64() return result == 1, err }