PAV-178: [Jobs Processor] Otimizar agendamento e distribuição de carga dos providers

backend/package.json

@@ -34,41 +34,41 @@
   "author": "Projeto Open Source - Bene Tesla Dev",
   "license": "ISC",
   "dependencies": {
-    "@react-email/render": "^2.1.0",
+    "@react-email/render": "^2.2.0",
     "argon2": "^0.45.1",
-    "axios": "^1.20.0",
-    "bullmq": "^6.3.11",
+    "axios": "^1.21.0",
+    "bullmq": "^6.5.1",
     "cheerio": "^1.2.0",
     "cors": "^2.8.6",
-    "dotenv": "^17.4.2",
+    "dotenv": "^18.0.9",
     "drizzle-orm": "^0.45.4",
     "express": "^5.2.1",
     "ioredis": "^5.11.1",
     "iron-session": "^8.0.4",
     "pdfkit": "^0.20.2",
     "prom-client": "^15.1.3",
-    "react": "^19.2.8",
-    "react-dom": "^19.2.8",
+    "react": "^19.3.1",
+    "react-dom": "^19.3.1",
     "redis": "^6.3.0",
-    "resend": "^6.18.0",
+    "resend": "^6.35.0",
     "swagger-jsdoc": "^6.3.0",
     "swagger-ui-express": "^5.0.1",
     "xlsx": "^0.18.5",
-    "zod": "^4.6.5"
+    "zod": "^4.6.7"
   },
   "engines": {
     "node": ">=22.0.0"
   },
   "devDependencies": {
-    "@types/node": "^26.6.4",
+    "@types/node": "^26.6.6",
     "@types/pg": "^8.20.0",
     "@types/react": "^19.2.17",
     "@types/supertest": "^7.2.0",
-    "@vitest/coverage-v8": "5.0.3",
+    "@vitest/coverage-v8": "5.0.5",
     "drizzle-kit": "^0.31.10",
     "supertest": "^7.2.2",
     "tsx": "^4.22.4",
     "typescript": "~5.9.3",
-    "vitest": "5.0.3"
+    "vitest": "5.0.5"
   }
 }

backend/src/modules/admin/observability/observability.types.ts

@@ -108,6 +108,7 @@ export const ProcessorSnapshotSchema = z.object({
   status: z.enum(["ok", "partial"]),
   timestamp: z.string().datetime(),
   execution: z.object({
+    partial: z.boolean().optional(),
     runId: z.string().min(1).max(128).nullable().optional().default(null),
     status: z.enum(["idle", "running", "canceling", "failed", "completed"]),
     source: z.enum(["manual", "cron"]).nullable(),

backend/src/modules/admin/scrapers/scraperClient.ts

@@ -1,4 +1,4 @@
-import { JobsCountResultSchema } from "./scrapers.types";
+import { JobsCountResultSchema, ScraperStatusSchema } from "./scrapers.types";
 import { config } from "../../../config";
 import type {
   GetJobsResult,
@@ -10,6 +10,15 @@ import type {
 
 const TIMEOUT_MS = 5000;
 
+export class ScraperInvalidSelectionError extends Error {
+  readonly code = "INVALID_PROVIDER_SELECTION";
+}
+
+export class ScraperRateLimitedError extends Error {
+  readonly code = "PROVIDER_RATE_LIMITED";
+  constructor(message: string, readonly retryAfter: string | null) { super(message); }
+}
+
 /** Lançado quando o Go scraper responde 409 (já em execução) */
 export class ScraperAlreadyRunningError extends Error {
   readonly code = "SCRAPER_ALREADY_RUNNING";
@@ -41,6 +50,14 @@ async function request<T>(path: string, init?: RequestInit): Promise<T> {
     const body = await response.json().catch(() => null);
     throw new ScraperAlreadyRunningError(body?.message);
   }
+	if (response.status === 400) {
+		const body = await response.json().catch(() => null);
+		throw new ScraperInvalidSelectionError(body?.message ?? "Seleção de provedor inválida.");
+	}
+  if (response.status === 429) {
+    const body = await response.json().catch(() => null);
+    throw new ScraperRateLimitedError(body?.message ?? "Provedor aguardando Retry-After.", response.headers?.get("Retry-After") ?? null);
+  }
 
   if (response.status === 503) {
     const body = await response.json().catch(() => null);
@@ -55,12 +72,15 @@ async function request<T>(path: string, init?: RequestInit): Promise<T> {
 }
 
 export const scraperClient = {
-  triggerScrape(): Promise<TriggerScrapeResult> {
-    return request<TriggerScrapeResult>("/admin/scrape", { method: "POST" });
+  triggerScrape(sources?: string[]): Promise<TriggerScrapeResult> {
+    return request<TriggerScrapeResult>("/admin/scrape", {
+      method: "POST",
+      ...(sources !== undefined ? { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sources }) } : {}),
+    });
   },
 
-  getStatus(): Promise<ScraperStatus> {
-    return request<ScraperStatus>("/admin/scrape/status");
+  async getStatus(): Promise<ScraperStatus> {
+    return ScraperStatusSchema.parse(await request<unknown>("/admin/scrape/status"));
   },
 
   getJobs(limit?: number): Promise<GetJobsResult> {

backend/src/modules/admin/scrapers/scrapers.controller.ts

@@ -3,8 +3,11 @@ import type { AuditService } from "../audit/audit.service";
 import {
   ScraperAlreadyRunningError,
   ScraperRunLockUnavailableError,
+  ScraperInvalidSelectionError,
+  ScraperRateLimitedError,
 } from "./scraperClient";
 import { ScrapersService } from "./scrapers.service";
+import { TriggerScrapeRequestSchema } from "./scrapers.types";
 
 export class ScrapersController {
   constructor(
@@ -13,13 +16,20 @@ export class ScrapersController {
   ) {}
 
   async trigger(req: Request, res: Response): Promise<void> {
+    const selection = TriggerScrapeRequestSchema.safeParse(req.body ?? {});
+    if (!selection.success) { res.status(400).json({ ok: false, code: "INVALID_PROVIDER_SELECTION" }); return; }
     try {
-      const result = await this.scrapersService.triggerScrape();
+      const result = await this.scrapersService.triggerScrape(selection.data.sources);
 
       this.auditService.fromRequest(req, "scrapers.trigger");
 
       res.status(202).json(result);
     } catch (error) {
+      if (error instanceof ScraperRateLimitedError) {
+        if (error.retryAfter) res.setHeader("Retry-After", error.retryAfter);
+        res.status(429).json({ ok: false, code: error.code, message: error.message }); return;
+      }
+      if (error instanceof ScraperInvalidSelectionError) { res.status(400).json({ ok: false, code: error.code, message: error.message }); return; }
       if (error instanceof ScraperAlreadyRunningError) {
         res.status(409).json({
           ok: false,
@@ -52,6 +62,11 @@ export class ScrapersController {
 
       res.status(202).json({ ...result, scraper: scraperName });
     } catch (error) {
+      if (error instanceof ScraperRateLimitedError) {
+        if (error.retryAfter) res.setHeader("Retry-After", error.retryAfter);
+        res.status(429).json({ ok: false, code: error.code, message: error.message }); return;
+      }
+      if (error instanceof ScraperInvalidSelectionError) { res.status(400).json({ ok: false, code: error.code, message: error.message }); return; }
       if (error instanceof ScraperAlreadyRunningError) {
         res.status(409).json({
           ok: false,

backend/src/modules/admin/scrapers/scrapers.service.ts

@@ -1,6 +1,8 @@
 import {
   ScraperAlreadyRunningError,
   ScraperRunLockUnavailableError,
+  ScraperInvalidSelectionError,
+  ScraperRateLimitedError,
   scraperClient,
 } from "./scraperClient";
 import type {
@@ -11,15 +13,16 @@ import type {
   ScraperStatus,
   TriggerScrapeResult,
 } from "./scrapers.types";
+import { TriggerScrapeRequestSchema } from "./scrapers.types";
 
 export class ScrapersService {
-  async triggerScrape(): Promise<TriggerScrapeResult> {
+  async triggerScrape(sources?: string[]): Promise<TriggerScrapeResult> {
     try {
-      return await scraperClient.triggerScrape();
+      return await scraperClient.triggerScrape(sources);
     } catch (error) {
       if (
         error instanceof ScraperAlreadyRunningError ||
-        error instanceof ScraperRunLockUnavailableError
+        error instanceof ScraperRunLockUnavailableError || error instanceof ScraperInvalidSelectionError || error instanceof ScraperRateLimitedError
       ) {
         // repropaga para o controller decidir o status HTTP (409)
         throw error;
@@ -31,7 +34,9 @@ export class ScrapersService {
   async triggerScraper(name: string): Promise<TriggerScrapeResult> {
     const normalized = name.trim().toLowerCase();
     if (normalized !== "go-scraper") {
-      throw new Error(`scraper desconhecido: ${name}`);
+      const selection = TriggerScrapeRequestSchema.safeParse({ sources: [normalized] });
+      if (!selection.success) throw new Error(`scraper desconhecido: ${name}`);
+      return this.triggerScrape(selection.data.sources);
     }
 
     return this.triggerScrape();

backend/src/modules/admin/scrapers/scrapers.types.ts

@@ -25,12 +25,18 @@ export const TriggerScrapeResultSchema = z.object({
 });
 export type TriggerScrapeResult = z.infer<typeof TriggerScrapeResultSchema>;
 
+export const TriggerScrapeRequestSchema = z.object({
+  sources: z.array(z.enum(["linkedin", "adzuna", "gupy", "inhire", "greenhouse", "lever", "jooble", "themuse"])).max(8).optional(),
+}).strict();
+
 export const TriggerScrapeErrorSchema = z.object({
   ok: z.literal(false),
   code: z.enum([
     "SCRAPER_ALREADY_RUNNING",
     "SCRAPER_RUN_LOCK_UNAVAILABLE",
     "SCRAPER_RUN_LOCK_LOST",
+    "INVALID_PROVIDER_SELECTION",
+    "PROVIDER_RATE_LIMITED",
   ]),
   message: z.string(),
 });
@@ -40,8 +46,27 @@ export type TriggerScrapeError = z.infer<typeof TriggerScrapeErrorSchema>;
 export const ScraperStatusSchema = z.object({
   name: z.string().optional(),
   running: z.boolean(),
-  lastRunAt: z.string().optional(),
+  lastRunAt: z.string().nullable().optional(),
   jobsCollected: z.number().int().min(0).optional(),
+  lastRun: z.object({
+    runId: z.string(),
+    providers: z.array(z.string()).nullable().optional(),
+    status: z.enum(["success", "partial", "failed", "timeout", "canceled"]),
+    finishedAt: z.string().datetime(),
+    jobsSaved: z.number().int().nonnegative(),
+    jobsIndexed: z.number().int().nonnegative(),
+    errorType: z.string(),
+  }).nullable().optional(),
+  providerWindows: z.array(z.union([
+    z.object({
+      provider: z.string(), id: z.string().optional(), timezone: z.literal("UTC").optional(),
+      startMinuteUTC: z.number().int().min(0).max(1439), endMinuteUTC: z.number().int().min(1).max(1440),
+    }),
+    z.object({
+      provider: z.string(), id: z.string(), timezone: z.string().min(1),
+      startMinuteLocal: z.number().int().min(0).max(1439), endMinuteLocal: z.number().int().min(1).max(1440),
+    }),
+  ])).nullable().optional(),
 });
 export type ScraperStatus = z.infer<typeof ScraperStatusSchema>;
 

backend/src/scripts/seed.ts

@@ -1,4 +1,5 @@
 import * as argon2 from "argon2";
+import { assertLocalSeedAllowed } from "./seedPolicy";
 import "dotenv/config";
 import { and, eq } from "drizzle-orm";
 import { db, pool } from "../db/client";
@@ -237,6 +238,7 @@ async function upsertApplicationEvent(
 
 
 async function main() {
+  assertLocalSeedAllowed();
   console.log("Iniciando seed de desenvolvimento local...");
 
   const devUserSpec = SEED_USERS.find((spec) => spec.role === "user")!;

backend/src/scripts/seedCatalogJobs.ts

@@ -1,4 +1,5 @@
 import type { ProfessionalFamily } from "../modules/jobs/types/professionalTaxonomy";
+import { assertLocalSeedAllowed } from "./seedPolicy";
 import { createClient } from "redis";
 
 export type SeedTechnologyExperience = {
@@ -391,6 +392,7 @@ async function connectSeedCache(url: string) {
 }
 
 export async function seedCatalogJobs(): Promise<void> {
+  assertLocalSeedAllowed();
   const url = process.env.VALKEY_URL;
 
   if (!url) {

backend/src/scripts/seedPolicy.ts

@@ -0,0 +1,13 @@
+// Demo users, saved jobs and catalog fixtures are only permitted in an
+// explicitly opted-in local environment. NODE_ENV=development also occurs in DEV.
+export function assertLocalSeedAllowed(env = process.env): void {
+  if (
+    env.APP_ENV !== "local" ||
+    env.ALLOW_DEMO_SEED !== "true" ||
+    env.NODE_ENV === "production"
+  ) {
+    throw new Error(
+      "Seed fictício bloqueado: exige APP_ENV=local, ALLOW_DEMO_SEED=true e NODE_ENV diferente de production.",
+    );
+  }
+}

backend/tests/unit/modules/admin/providerDispatch.test.ts

@@ -0,0 +1,38 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { scraperClient, ScraperInvalidSelectionError, ScraperRateLimitedError } from "../../../../src/modules/admin/scrapers/scraperClient";
+import { ScrapersService } from "../../../../src/modules/admin/scrapers/scrapers.service";
+import { ScrapersController } from "../../../../src/modules/admin/scrapers/scrapers.controller";
+import type { Request, Response } from "express";
+
+afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); });
+
+it("preserves Retry-After when a provider is temporarily blocked", async () => {
+  vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 429, headers: { get: () => "120" }, json: async () => ({ message: "rate limited" }) }));
+  await expect(new ScrapersService().triggerScraper("greenhouse")).rejects.toBeInstanceOf(ScraperRateLimitedError);
+  const res = { status: vi.fn().mockReturnThis(), json: vi.fn(), setHeader: vi.fn() } as unknown as Response;
+  await new ScrapersController(new ScrapersService(), { fromRequest: vi.fn() } as never).triggerOne({ params: { id: "greenhouse" } } as unknown as Request, res);
+  expect(res.status).toHaveBeenCalledWith(429);
+  expect(res.setHeader).toHaveBeenCalledWith("Retry-After", "120");
+});
+
+it("forwards an individual ProviderID to Go without a global fallback", async () => {
+  const fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ ok: true, message: "started" }) });
+  vi.stubGlobal("fetch", fetch);
+  await new ScrapersService().triggerScraper("greenhouse");
+  expect(JSON.parse(fetch.mock.calls[0][1].body)).toEqual({ sources: ["greenhouse"] });
+});
+
+it("preserves a Go rejection when the provider is disabled", async () => {
+  vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 400, json: async () => ({ message: "provider is not configured" }) }));
+  await expect(new ScrapersService().triggerScraper("greenhouse")).rejects.toBeInstanceOf(ScraperInvalidSelectionError);
+});
+
+it("rejects invalid selection before sending any request to Go", async () => {
+  const trigger = vi.spyOn(scraperClient, "triggerScrape");
+  const service = new ScrapersService();
+  const res = { status: vi.fn().mockReturnThis(), json: vi.fn() } as unknown as Response;
+  const controller = new ScrapersController(service, { fromRequest: vi.fn() } as never);
+  await controller.trigger({ body: { sources: ["unknown"] } } as Request, res);
+  expect(res.status).toHaveBeenCalledWith(400);
+  expect(trigger).not.toHaveBeenCalled();
+});

backend/tests/unit/modules/admin/providerSchedule.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, it } from "vitest";
+import { ScraperStatusSchema } from "../../../../src/modules/admin/scrapers/scrapers.types";
+
+describe("provider window status contract", () => {
+  it("preserves legacy UTC windows and manual status responses", () => {
+    expect(ScraperStatusSchema.parse({ running: false })).toEqual({ running: false });
+    const window = { provider: "gupy", startMinuteUTC: 60, endMinuteUTC: 120 };
+    expect(ScraperStatusSchema.parse({ running: false, providerWindows: [window] }).providerWindows).toEqual([window]);
+  });
+
+  it("preserves local window identity and timezone", () => {
+    const window = { provider: "greenhouse", id: "greenhouse-1", timezone: "America/Sao_Paulo", startMinuteLocal: 0, endMinuteLocal: 60 };
+    expect(ScraperStatusSchema.parse({ running: true, providerWindows: [window] }).providerWindows).toEqual([window]);
+  });
+
+  it("rejects incomplete local windows and invalid minute bounds", () => {
+    for (const window of [
+      { provider: "gupy", startMinuteLocal: 0, endMinuteLocal: 60 },
+      { provider: "gupy", id: "gupy-1", timezone: "America/Sao_Paulo", startMinuteLocal: 1440, endMinuteLocal: 1500 },
+    ]) {
+      expect(ScraperStatusSchema.safeParse({ running: false, providerWindows: [window] }).success).toBe(false);
+    }
+  });
+});

backend/tests/unit/modules/admin/scraperResult.test.ts

@@ -0,0 +1,14 @@
+import { expect, it } from "vitest";
+import { ScraperStatusSchema } from "../../../../src/modules/admin/scrapers/scrapers.types";
+
+it("retains partial execution counts and accepts an empty initial history", () => {
+  expect(ScraperStatusSchema.parse({ running: false, lastRunAt: null, lastRun: null }).lastRun).toBeNull();
+  const result = ScraperStatusSchema.parse({
+    running: false,
+    lastRunAt: "2026-11-01T00:00:00Z",
+    jobsCollected: 210,
+    lastRun: { runId: "test", status: "partial", finishedAt: "2026-11-01T00:00:00Z", jobsSaved: 210, jobsIndexed: 210, errorType: "timeout" },
+  });
+  expect(result.lastRun?.status).toBe("partial");
+  expect(result.lastRun?.jobsIndexed).toBe(210);
+});

backend/tests/unit/modules/admin/scrapers.test.ts

@@ -211,7 +211,9 @@ describe("ScrapersService", () => {
       ok: true,
       message: "started",
     });
-    await expect(service.triggerScraper("lever")).rejects.toThrow(
+    await expect(service.triggerScraper("lever")).resolves.toEqual({ ok: true, message: "started" });
+    expect(scraperClient.triggerScrape).toHaveBeenLastCalledWith(["lever"]);
+    await expect(service.triggerScraper("invalid")).rejects.toThrow(
       "scraper desconhecido",
     );
   });