From 50e25b16017fb779864ff32a2e318dc566095505 Mon Sep 17 00:00:00 2001 From: AgentLinkOps Date: Tue, 6 Oct 2026 19:48:50 -0400 Subject: [PATCH] Release 0.6.11: consistent check output, ledger validation and import deduplication --- .claude-plugin/plugin.json | 2 +- .codex-plugin/plugin.json | 2 +- CHANGELOG.md | 7 ++ README.md | 8 +- cli/client.js | 2 +- cli/import.js | 5 +- cli/main.js | 15 +-- cli/mcp-client.js | 4 +- package-lock.json | 4 +- package.json | 2 +- .../references/agentlinkops.md | 2 +- skills/agentlinkops-crm/SKILL.md | 2 +- test/cli-check.test.js | 104 +++++++++++++++++- test/cli-client-resilience.test.js | 15 +++ test/cli-import.test.js | 42 ++++++- 15 files changed, 191 insertions(+), 25 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index f4085ef..7ade711 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentlinkops", - "version": "0.6.10", + "version": "0.6.11", "description": "Connect an agent to AgentLinkOps and run backlink work with it: the connect skill (views, the three discovery verbs, evidence rules), local ledger and campaign skills, a portable CRM, the source-cited content pipeline, user-managed disavow lists, HTML reports and monitoring-event sync.", "author": { "name": "AgentLinkOps" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 659f627..fe26d1e 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentlinkops", - "version": "0.6.10", + "version": "0.6.11", "description": "Connect an agent to AgentLinkOps and run backlink work with it: the connect skill (views, the three discovery verbs, evidence rules), local ledger and campaign skills, a portable CRM, the source-cited content pipeline, user-managed disavow lists, HTML reports and monitoring-event sync.", "author": { "name": "AgentLinkOps" diff --git a/CHANGELOG.md b/CHANGELOG.md index 592cf5c..d936f96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ All notable changes to the `agentlinkops` package are recorded here. The version follows the plugin manifests in `.claude-plugin/` and `.codex-plugin/`. +## 0.6.11 (2026-10-06) + +- Keep `check --json` stdout limited to observation rows. Send empty-ledger, not-due and filter notices to stderr. +- Reject malformed ledger rows with line numbers and exit with status 2 before fetching or updating customer records. +- Apply `--fail-on-unknown` to JSON and human output. Unknown observations remain nonfailing by default. +- Detect duplicate placements across import batches and retain their original file line numbers. + ## 0.6.10 (2026-10-06) Inspect the installed version's help before using selectors; older published versions diff --git a/README.md b/README.md index d21a18c..63cc211 100644 --- a/README.md +++ b/README.md @@ -98,10 +98,10 @@ agentlinkops status the source page and appends an observation to the mirror. `status` names every entry whose observations disagree with its intent and proposes the edit; applying it is a commit you make. -For ledger checks, exit codes are `0` when every expected link is present, `1` only -when an expected link is observed absent with complete evidence, and `2` for a usage, -configuration or ledger error. An unknown never fails the run. -Pass `--fail-on-unknown` when your CI wants it to. +Ledger checks return `0` when no expected placement is conclusively absent and `1` when +an expected placement is observed absent with complete evidence. Unknown observations do +not fail a run by default. Pass `--fail-on-unknown` to return `1` for an unknown result +in text or JSON output. Usage, configuration and malformed-ledger errors return `2`. `agentlinkops doctor` verifies the ledger, the verifier self-test, cloud reachability and the token, and prints a fix line for anything that fails. diff --git a/cli/client.js b/cli/client.js index 436019e..a6cc2a9 100644 --- a/cli/client.js +++ b/cli/client.js @@ -5,7 +5,7 @@ export {CloudError} from './cloud-error.js'; // A small HTTP client for the cloud. Scoped keys only; no human session ever reaches here. // DP-0036-T13: the CLI names itself so the usage counters can tell it from other REST callers. // The version follows the plugin manifests; bump it with them. -export const CLI_VERSION = '0.6.10'; +export const CLI_VERSION = '0.6.11'; export const CLI_USER_AGENT = `agentlinkops-cli/${CLI_VERSION}`; const MAX_ATTEMPTS = 3; const MAX_RETRY_WAIT_MS = 5_000; diff --git a/cli/import.js b/cli/import.js index 757a57d..f5fdf71 100644 --- a/cli/import.js +++ b/cli/import.js @@ -74,11 +74,14 @@ export async function readImport(path, { // chunks, so a placement repeated in two different parts of one file is still one candidate. const prepared = { accepted: [], rejected: [], duplicates: [] }; const seen = new Map(); + // Candidate identity includes the run. These bounded slices describe one preview, so keep + // its first-slice identity throughout the file or a repeated placement gets a new ID. + const previewRunId = `${runId}_0`; const chunks = []; for (let index = 0; index < rows.length; index += DISCOVERY_LIMITS.rows) chunks.push(index); for (const offset of chunks) { const part = await prepareImport({ - runId: `${runId}_${offset / DISCOVERY_LIMITS.rows}`, workspaceId, projectId, supplier, + runId: previewRunId, workspaceId, projectId, supplier, rows: rows.slice(offset, offset + DISCOVERY_LIMITS.rows), target: targetUrl, scope, taskId: 'import_preview', retrievedAt: now, generatedAt: supplierGeneratedAt, seen, rowOffset: offset, diff --git a/cli/main.js b/cli/main.js index 7f03213..857cca1 100644 --- a/cli/main.js +++ b/cli/main.js @@ -115,8 +115,9 @@ const USAGE = `agentlinkops — a backlink ledger that lives in your repository Suppliers: ${SUPPLIER_NAMES.join(', ')} A preset is a convenience over --map, never a requirement. -Exit codes: 0 every expected link present · 1 an expected link observed absent with complete -evidence · 2 usage, configuration or ledger error. An unknown never fails. +Exit codes: 0 no expected link conclusively absent · 1 an expected link observed absent with complete +evidence · 2 usage, configuration or ledger error. Unknown is nonfailing by default; +--fail-on-unknown returns 1 when any observation is unknown. Names: \`linktrail\` still runs this CLI, LINKTRAIL_* variables are still read, and an existing .linktrail/ directory is still used, each with a one-line notice, until the DP-0029 cutover.`; @@ -586,10 +587,11 @@ export async function main(argv = process.argv.slice(2), { cwd = process.cwd(), } if (command === 'check') { + if (ledger.problems.length) { reportProblems(ledger.problems, 'ledger line(s)', err); return 2; } const state = await readState(config.paths.state); const selected = selectEntries(ledger.entries, { filter: args.filter === true ? null : args.filter }); const entries = args.all ? selected : dueEntries(selected, lastCheckedMap(state)); - if (!entries.length) { out(`nothing due (${plural(selected.length, 'entry', 'entries')} in scope; pass --all to recheck)`); return 0; } + if (!entries.length) { (args.json ? err : out)(`nothing due (${plural(selected.length, 'entry', 'entries')} in scope; pass --all to recheck)`); return 0; } if (!args.json) err(`checking ${plural(entries.length, 'entry', 'entries')}…`); const rows = await runCheck(entries, { concurrency: Number(args.concurrency ?? config.defaults.concurrency ?? DEFAULTS.concurrency), @@ -603,8 +605,8 @@ export async function main(argv = process.argv.slice(2), { cwd = process.cwd(), await appendObservations(config.paths.observations, changed); await writeState(config.paths.state, applyRun(state, rows)); const summary = summarize(entries, rows); - if (args.json) { out(rows.map(row => JSON.stringify(row)).join('\n')); return summary.exitCode; } - reportProblems(ledger.problems, 'ledger line(s)', out); + const exitCode = args['fail-on-unknown'] && summary.counts.unknown ? 1 : summary.exitCode; + if (args.json) { out(rows.map(row => JSON.stringify(row)).join('\n')); return exitCode; } for (const { entry, row } of summary.appeared) out(`APPEARED ${entry.id} ${entry.source} (${row.occurrences} occurrence(s))`); for (const { entry, row } of summary.failures) out(`ABSENT ${entry.id} ${entry.source} [${row.reason}]`); out(`\n${JSON.stringify(summary.counts)}`); @@ -612,8 +614,7 @@ export async function main(argv = process.argv.slice(2), { cwd = process.cwd(), if (summary.counts.unknown) { out(`${plural(summary.counts.unknown, 'page')} could not be concluded. An unknown is not a lost link; run \`agentlinkops status\` for the reasons.`); } - if (args['fail-on-unknown'] && summary.counts.unknown) return 1; - return summary.exitCode; + return exitCode; } err(`unknown command: ${command}\n\n${USAGE}`); diff --git a/cli/mcp-client.js b/cli/mcp-client.js index ca72d93..17ad9a3 100644 --- a/cli/mcp-client.js +++ b/cli/mcp-client.js @@ -62,7 +62,7 @@ function retryDelay(error, attempt, now) { async function sdkClient({url, token, workspaceId, fetchImpl}) { const {Client, StreamableHTTPClientTransport} = await import('@modelcontextprotocol/client'); - const client = new Client({name: 'agentlinkops-cli', version: '0.6.10'}); + const client = new Client({name: 'agentlinkops-cli', version: '0.6.11'}); const destination = new URL(url); const transport = new StreamableHTTPClientTransport(destination, { authProvider: {token: async () => token}, @@ -71,7 +71,7 @@ async function sdkClient({url, token, workspaceId, fetchImpl}) { if (target.href !== destination.href) fail('OAUTH_TOKEN_DESTINATION_MISMATCH'); const headers = new Headers(init.headers ?? (input instanceof Request ? input.headers : undefined)); if (workspaceId) headers.set('X-Workspace-ID', workspaceId); - headers.set('User-Agent', 'agentlinkops-cli/0.6.10'); + headers.set('User-Agent', 'agentlinkops-cli/0.6.11'); const response = await fetchImpl(input, {...init, headers, redirect: 'error', signal: AbortSignal.timeout(30000)}); if (response.redirected || response.status >= 300 && response.status < 400 || response.url && response.url !== destination.href) fail('OAUTH_TOKEN_DESTINATION_MISMATCH'); diff --git a/package-lock.json b/package-lock.json index a61721f..76149e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agentlinkops", - "version": "0.6.10", + "version": "0.6.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agentlinkops", - "version": "0.6.10", + "version": "0.6.11", "license": "Apache-2.0", "dependencies": { "@modelcontextprotocol/client": "2.2.0", diff --git a/package.json b/package.json index 98119e3..0a210f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agentlinkops", - "version": "0.6.10", + "version": "0.6.11", "description": "Backlink checks with dated evidence and portable results, optional repository ledgers, supplier-export imports and sync to AgentLinkOps.", "author": "AgentLinkOps (https://agentlinkops.com)", "license": "Apache-2.0", diff --git a/skills/agentlinkops-connect/references/agentlinkops.md b/skills/agentlinkops-connect/references/agentlinkops.md index 1b53b4d..1685e76 100644 --- a/skills/agentlinkops-connect/references/agentlinkops.md +++ b/skills/agentlinkops-connect/references/agentlinkops.md @@ -9,7 +9,7 @@ records. This page is the complete agent reference: the same text prints from full once per session, then use `describe` (or `exec describe` on MCP) for any command's exact schema. -Pack version 0.6.10. 139 commands in 15 toolsets. +Pack version 0.6.11. 139 commands in 15 toolsets. ## Install and connect (once per machine) diff --git a/skills/agentlinkops-crm/SKILL.md b/skills/agentlinkops-crm/SKILL.md index bdbd3cf..e5c2cdd 100644 --- a/skills/agentlinkops-crm/SKILL.md +++ b/skills/agentlinkops-crm/SKILL.md @@ -6,7 +6,7 @@ allowed-tools: Read Bash(python3 ../../scripts/agentlinkops.py *) Bash(python3 . **For the optional bundled CLI, the local surface is the ledger file.** `.agentlinkops/links.jsonl` in the user's repository (or a `.linktrail/` directory from an earlier install, read until `agentlinkops migrate` renames it) declares the links they expect, want and have earned, and `agentlinkops check` runs the exact cloud verifier against it without an account. Run the CLI as `node cli/agentlinkops.mjs` from the repository or plugin-package root (`cli/linktrail.mjs` still works and prints a one-line notice); the packaged plugin ships the CLI, its pinned verifier modules and their pinned runtime dependencies inside the package, so the installed package needs no npm install; checking a publisher still requires a read-only network fetch. It needs Node 22 or newer (`adopt` reads SQLite databases through `node:sqlite`); the Python helper below needs Python 3.10+. This CRM is one optional adapter beside CSV and JSONL: it is available, tested and unchanged, and nothing requires it. When a user already keeps links here, `agentlinkops adopt ` reads it into ledger entries — read-only, adding no schema — and previews unless `--write` is given. Do not propose the CRM as the place a user should start keeping links; propose it when they want contacts and outreach activity beside them, or when they already have one. -To record a link in the ledger: `agentlinkops add --source URL --target URL --intent expected [--anchor TEXT] [--ref ID] [--note TEXT]` (`--intent wanted` for a link not yet earned; `agentlinkops init` first in a repository without `.agentlinkops/`). `agentlinkops check` then verifies every expected link and exits 0 when all are present, 1 when one is observed absent with complete evidence, 2 on a usage or ledger error; an unknown never fails. +To record a link in the ledger: `agentlinkops add --source URL --target URL --intent expected [--anchor TEXT] [--ref ID] [--note TEXT]` (`--intent wanted` for a link not yet earned; `agentlinkops init` first in a repository without `.agentlinkops/`). `agentlinkops check` then verifies every expected link and returns 0 when no expected placement is conclusively absent, 1 when one is observed absent with complete evidence, 2 on a usage or ledger error. Unknown is nonfailing by default; `--fail-on-unknown` returns 1 for an unknown result in text or JSON output. Use the deterministic Python helper at `../../scripts/agentlinkops.py`, resolved relative to this skill's directory. It requires Python 3.10+ and only its standard library. Run `--help` for the command surface. Keep the database outside the plugin installation directory; updates can replace that directory. diff --git a/test/cli-check.test.js b/test/cli-check.test.js index 96caa48..388bab6 100644 --- a/test/cli-check.test.js +++ b/test/cli-check.test.js @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtemp, rm, readFile, writeFile, mkdir } from 'node:fs/promises'; +import { mkdtemp, rm, readFile, writeFile, mkdir, readdir } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { main } from '../cli/main.js'; @@ -28,6 +28,27 @@ const run = (dir, argv) => { .then(code => ({ code, out: out.join('\n'), err: err.join('\n') })); }; +async function savedFiles(dir) { + const files = {}; + async function visit(path = '') { + for (const entry of (await readdir(join(dir, path), { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) { + const name = join(path, entry.name); + if (entry.isDirectory()) await visit(name); + else files[name] = (await readFile(join(dir, name))).toString('base64'); + } + } + await visit(); + return files; +} + +function assertSavedFilesUnchanged(before, after, message) { + // The established advisory gate may be created; every other saved file stays identical. + const gate = join('.agentlinkops', '.sync-gate'); + if (Object.hasOwn(before, gate)) assert.equal(before[gate], '', `${message}: existing advisory gate is empty`); + assert.equal(after[gate], '', `${message}: advisory gate remains empty`); + assert.deepEqual(after, { ...before, [gate]: '' }, message); +} + test('the CLI runs the EXACT cloud verifier, asserted on the shared cases', async () => { // Every case here is also asserted against `verifyLink` directly in test/verifier.test.js. // A fork would show two green suites while the two answers drifted, which is the failure @@ -284,3 +305,84 @@ test('a command outside a ledger says so instead of inventing one', async t => { assert.equal(result.code, 2); assert.match(result.err, /agentlinkops init/u); }); + +test('check --json keeps stdout empty when there are no observations due and preserves customer records', async t => { + const space = await workspace(t); + const fetchMock = t.mock.method(globalThis, 'fetch', () => assert.fail('an empty check must not fetch')); + const entry = normalizeEntry({ id: 'lk_jsonempty', intent: 'expected', source: SOURCE, target: TARGET }); + const row = observationRow(entry.id, { state: 'present', reason: 'link_found', checkedAt: new Date().toISOString(), occurrences: [{}], evidence: { complete: true } }); + const scenarios = [ + { name: 'empty ledger', entries: [], args: [], count: 0 }, + { name: 'not due', entries: [entry], args: [], count: 1 }, + { name: 'filter selects nothing', entries: [entry], args: ['--all', '--filter', 'not-in-this-ledger'], count: 0 }, + ]; + for (const scenario of scenarios) { + await writeFile(space.ledger, serializeLedger(scenario.entries)); + await writeFile(space.observations, `${JSON.stringify(row)}\n`); + await writeFile(join(space.dir, '.agentlinkops/state.json'), `${JSON.stringify(applyRun({ v: 1, entries: {} }, [row]))}\n`); + const before = await savedFiles(space.dir); + const result = await run(space.dir, ['check', '--json', ...scenario.args]); + assert.equal(result.code, 0, scenario.name); + assert.equal(result.out, '', scenario.name); + assert.match(result.err, new RegExp(`nothing due \\(${scenario.count} entr(?:y|ies) in scope; pass --all to recheck\\)`), scenario.name); + assertSavedFilesUnchanged(before, await savedFiles(space.dir), scenario.name); + } + assert.equal(fetchMock.mock.callCount(), 0); +}); + +test('malformed ledgers stop human and JSON checks before fetching or changing customer records', async t => { + const space = await workspace(t); + const fetchMock = t.mock.method(globalThis, 'fetch', () => assert.fail('a malformed ledger must not fetch')); + const entry = normalizeEntry({ id: 'lk_parsevalid', intent: 'expected', source: SOURCE, target: TARGET }); + await writeFile(space.observations, ''); + await writeFile(join(space.dir, '.agentlinkops/state.json'), '{"v":1,"entries":{}}\n'); + const scenarios = [ + { name: 'partial JSON failure', ledger: serializeLedger([entry]) + '{not JSON\n', message: /line 2:/u }, + { name: 'invalid intent', ledger: serializeLedger([entry]) + JSON.stringify({ ...entry, id: 'lk_parsebad', intent: 'unsupported' }) + '\n', message: /line 2 \(lk_parsebad\): intent must be one of wanted, expected, retired/u }, + { name: 'duplicate identity', ledger: serializeLedger([entry, { ...entry, source: 'https://other.example.com/' }]), message: /line 2 \(lk_parsevalid\): duplicate id, first seen on line 1/u }, + { name: 'no usable entries', ledger: '{not JSON\n', message: /line 1:/u }, + ]; + for (const scenario of scenarios) for (const json of [false, true]) { + await writeFile(space.ledger, scenario.ledger); + const before = await savedFiles(space.dir); + const result = await run(space.dir, ['check', '--all', ...(json ? ['--json'] : [])]); + assert.equal(result.code, 2, `${scenario.name}; json=${json}`); + assert.equal(result.out, '', scenario.name); + assert.match(result.err, /1 ledger line\(s\) could not be read/u, scenario.name); + assert.match(result.err, scenario.message, scenario.name); + assert.doesNotMatch(result.err, /nothing due/u, scenario.name); + assertSavedFilesUnchanged(before, await savedFiles(space.dir), scenario.name); + } + assert.equal(fetchMock.mock.callCount(), 0); +}); + +test('human and JSON checks honor only the explicit unknown failure flag without claiming absence', async t => { + const space = await workspace(t); + const entry = normalizeEntry({ id: 'lk_unknownflag', intent: 'expected', source: SOURCE, target: TARGET }); + await writeFile(space.ledger, serializeLedger([entry])); + const fetchMock = t.mock.method(globalThis, 'fetch', async url => new URL(url).pathname === '/robots.txt' + ? new Response('', { status: 404 }) + : new Response('', { status: 403, headers: { 'content-type': 'text/html' } })); + for (const json of [false, true]) for (const failOnUnknown of [false, true]) { + const beforeRequests = fetchMock.mock.callCount(); + const result = await run(space.dir, ['check', '--all', '--host-delay', '0', '--concurrency', '1', + ...(json ? ['--json'] : []), ...(failOnUnknown ? ['--fail-on-unknown'] : [])]); + assert.equal(result.code, failOnUnknown ? 1 : 0, `json=${json}; failOnUnknown=${failOnUnknown}`); + assert.ok(fetchMock.mock.callCount() > beforeRequests, 'a selected due check ran the verifier'); + if (json) { + const rows = result.out.split('\n').filter(Boolean).map(line => JSON.parse(line)); + assert.equal(rows.length, 1); + assert.equal(rows[0].id, entry.id); + assert.equal(rows[0].state, 'unknown'); + assert.equal(rows[0].complete, false); + assert.equal(rows[0].occurrences, 0); + assert.equal(rows[0].result.state, 'unknown'); + assert.equal(rows[0].result.httpStatus, 403); + assert.equal(rows[0].result.sourceUrl, SOURCE); + assert.equal(rows[0].result.targetUrl, TARGET); + } else assert.match(result.out, /could not be concluded\. An unknown is not a lost link/u); + assert.doesNotMatch(result.out, /\b(?:ABSENT|LOST)\b/u); + const state = JSON.parse(await readFile(join(space.dir, '.agentlinkops/state.json'), 'utf8')); + assert.equal(state.entries[entry.id].last_state, 'unknown'); + } +}); diff --git a/test/cli-client-resilience.test.js b/test/cli-client-resilience.test.js index cb342e2..8f58527 100644 --- a/test/cli-client-resilience.test.js +++ b/test/cli-client-resilience.test.js @@ -8,6 +8,21 @@ const unavailable=(status,headers={})=>Response.json({error:{code:status===429?' {status,headers:{'X-Request-ID':`req-${status}`,...headers}}); const client=(fetchImpl,options={})=>createClient({origin:ORIGIN,token:TOKEN,fetchImpl,...options}); +test('the REST client sends the 0.6.11 release User-Agent', async () => { + const requests = []; + const result = await client(async (url, init) => { + const request = new Request(url, init); + requests.push(request); + assert.equal(request.url, ORIGIN + '/v1/watches'); + assert.equal(request.method, 'GET'); + assert.equal(request.headers.get('user-agent'), 'agentlinkops-cli/0.6.11'); + assert.equal(request.headers.get('authorization'), 'Bearer ' + TOKEN); + return page(); + }).listWatches(); + assert.equal(requests.length, 1); + assert.deepEqual(result.items, []); +}); + test('a dropped GET response retries within three attempts and returns the read',async()=>{ let calls=0;const waits=[]; const result=await client(async()=>{if(++calls===1)throw new TypeError('fetch failed');return page();}, diff --git a/test/cli-import.test.js b/test/cli-import.test.js index dc6acb3..6bb46e8 100644 --- a/test/cli-import.test.js +++ b/test/cli-import.test.js @@ -1,8 +1,8 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import { parseCsv, readTable, normalizeHeader, CsvError } from '../cli/adapters/csv.js'; import { resolveMapping, toImportRow, readFlag, SUPPLIER_NAMES } from '../cli/adapters/suppliers.js'; import { readImport, parseMap } from '../cli/import.js'; @@ -175,3 +175,41 @@ test('import previews and never writes', async t => { assert.ok(result.accepted.length); assert.equal(result.written, undefined); }); + +test('a large export reports duplicates across normalization slices with original file lines', async t => { + const header = 'Source url,Target url\n'; + const placement = 'https://p.example.com/0,https://example.com/g'; + const rows = Array.from({ length: 2002 }, (_, index) => + `https://p.example.com/${index},https://example.com/g`); + rows[1] = placement; // Same slice as the first occurrence. + rows[1000] = 'https://P.EXAMPLE.COM/0,https://example.com/g'; // Normalized cross-slice duplicate. + rows[1001] = 'https://p.example.com/0,https://example.com/other'; // A different target is distinct. + rows[2000] = rows[1001]; // Repeat that distinct placement in the third slice. + rows[2001] = 'not-a-url,https://example.com/g'; + // A malformed record is not sent to normalization, but still shifts every reported file line. + const contents = `${header}https://short.example.com/\n${rows.join('\n')}\n`; + const path = await file(t, 'large.csv', contents); + const singlePath = await file(t, 'single.csv', `${header}${placement}\n`); + const options = { supplier: 'semrush', target: 'example.com' }; + const single = await readImport(singlePath, options); + const result = await readImport(path, options); + + assert.equal(result.error, undefined); + assert.equal(result.chunks, 3); + assert.deepEqual(result.counts, { read: 2003, accepted: 1998, rejected: 2, duplicates: 3 }); + assert.deepEqual(result.duplicates, [ + { row: 4, first_seen_on_row: 3 }, + { row: 1003, first_seen_on_row: 3 }, + { row: 2003, first_seen_on_row: 1004 }, + ]); + assert.deepEqual(result.rejected.map(entry => entry.row), [2, 2004]); + assert.equal(new Set(result.accepted.map(candidate => candidate.id)).size, result.counts.accepted); + assert.equal(result.accepted[0].id, single.accepted[0].id, 'the existing single-slice identity is preserved'); + assert.ok(result.accepted.every(candidate => candidate.discovery_run_id === single.accepted[0].discovery_run_id)); + assert.ok(result.accepted.some(candidate => candidate.source_url === 'https://p.example.com/0' + && candidate.target_url === 'https://example.com/other')); + assert.ok(result.accepted.every(candidate => candidate.verification_status === 'not_checked' + && candidate.verified_at === null && candidate.observation_id === null && candidate.evidence_id === null)); + assert.equal(await readFile(path, 'utf8'), contents); + assert.deepEqual(await readdir(dirname(path)), ['large.csv'], 'preview creates no ledger or state'); +});