diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index f907881..3d3883f 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -6,7 +6,7 @@ import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL, - annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, inferDistributionFrequency, installSystemCa, + annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa, isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange, parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions, parseProductPageSummary, parseRange, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls, @@ -258,6 +258,21 @@ describe('parsing', () => { expect(parseEdgarAtomFilings(atom)[0].url).toContain('0000000000-26-000001.txt'); }); + test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => { + const headers = ['Name', 'Ticker', 'Weight']; + const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }]; + expect(holdingsGuard(headers, rows, 2)).toBeNull(); + expect(holdingsGuard(headers, [['A', 'A', '60'], ['B', 'B', '40']], null)).toBeNull(); + expect(holdingsGuard([], [], 0)).toBeNull(); // nothing published before: an empty sheet is honest + expect(holdingsGuard(headers, [], 505)).toContain('no holdings rows'); + expect(holdingsGuard(['Name', 'Ticker', 'Wt'], rows, 2)).toContain('no weight column'); + expect(holdingsGuard(headers, rows.map((row) => ({ ...row, Weight: '6\u200B0' })), 2)).toContain('numeric weight'); + const allocation = [{ Name: 'Cash&Collateral', Ticker: '-', Weight: '100' }, { Name: 'Equities', Ticker: '-', Weight: '476' }]; + expect(holdingsGuard(headers, allocation, 2)).toContain('outside 50..150'); + expect(holdingsGuard(headers, allocation, 2, ['sum'])).toBeNull(); + expect(holdingsGuard(headers, rows, 5)).toContain('less than half'); + }); + test('Yahoo chart: close, adjusted close and dividends', () => { const parsed = parseChart({ chart: { result: [{ timestamp: [Date.UTC(2025, 0, 2) / 1000, Date.UTC(2025, 0, 3) / 1000], indicators: { quote: [{ close: [10, 11], volume: [100, 200] }], adjclose: [{ adjclose: [9, 10] }] }, events: { dividends: { '1735776000': { amount: 0.25 } } }, meta: { regularMarketPrice: 11, exchangeName: 'NYSEArca' } }] } }); expect(parsed.days[1]).toMatchObject({ date: '2025-01-03', close: 11, adjClose: 10, volume: 200 }); @@ -409,7 +424,7 @@ function secResponse(url: string, nportAsOf: string): Response | null { if (url.includes('company_tickers_mf')) return new Response(JSON.stringify({ fields: ['cik', 'seriesId', 'classId', 'symbol'], data: [['1350487', 'S000043966', 'C000136444', 'AAA']] })); if (url.includes('company_tickers')) return new Response('{}'); if (url.includes('browse-edgar')) return new Response('NPORT-P0000940400-26-0000012026-07-30'); - if (url.includes('/Archives/edgar/')) return new Response(`0001350487${nportAsOf}Fund AAAS0000439661000000NPORT CO ${nportAsOf}9999999995501EC`); + if (url.includes('/Archives/edgar/')) return new Response(`0001350487${nportAsOf}Fund AAAS0000439661000000NPORT CO ${nportAsOf}999999999100501EC`); return new Response('nope', { status: 404 }); } @@ -696,6 +711,27 @@ describe('pipeline', () => { expect(s.index().funds.map((f: any) => f.ticker)).toContain('CCC'); }); + test('fresh holdings that fail the guard keep the fund exactly as published and report it failed', async () => { + // BBB has no N-PORT series in the SEC world, so its holdings come from the product-page top holdings table + const s = new Scenario(['BBB']); + const base = s.handler; + const serve = (...rows: string[]) => { + const page = `${productPage('BBB')}\n\n### Holdings\n\n#### As of 9/15/2026\n\n| Name | Weight (%) |\n| --- | --- |\n${rows.join('\n')}\n\nView all holdings`; + s.handler = (url, init) => (url.startsWith('https://r.jina.ai/') && !url.endsWith('/us/products') ? new Response(page) : base(url, init)); + }; + serve('| Alpha Corp | 40.00 |', '| Beta Corp | 30.00 |', '| Gamma Corp | 20.00 |', '| Delta Corp | 10.00 |'); + await s.run({ EDGAR_FALLBACK: 'true' }); + expect(s.meta('BBB').holdings.totalRows).toBe(4); + s.backdate(); + const before = s.files().map((path) => readFileSync(path, 'utf8')); + serve('| Alpha Corp | 90.00 |'); // one row of four left: a truncated page, not a new portfolio + process.exitCode = 0; + await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' }); + expect(s.touched()).toEqual([]); + expect(s.files().map((path) => readFileSync(path, 'utf8'))).toEqual(before); + expect(process.exitCode).toBe(1); + }); + test('N-PORT freshness: a filing older than the published holdings never replaces them, a newer one does', async () => { const s = new Scenario(['AAA']); await s.run({ EDGAR_FALLBACK: 'true' }); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 7433379..db2d01d 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -232,6 +232,10 @@ export function setApiRoot(root: URL): void { } const HOLDINGS_HEADERS = ['Name', 'Ticker', 'Identifier', 'Weight', 'Market Value', 'Shares Held', 'Asset Category']; +/** Funds whose holdings legitimately fail a holdingsGuard check (geared, futures, look-through, top-N lists): ticker, or '*' for every fund -> relaxed checks. */ +const HOLDINGS_GUARD_RELAX: Record = { + WSPC: ['sum'], // no N-PORT filing yet: the product page lists only its top 10 holdings (about 47% of the fund) +}; const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; const TRUTHY = new Set(['1', 'true', 'yes', 'y', 'on']); const AUM_BOUNDS = { nano: [0, 10_000_000], micro: [10_000_000, 300_000_000], small: [300_000_000, 2_000_000_000], mid: [2_000_000_000, 10_000_000_000], large: [10_000_000_000, undefined] } as const; @@ -1815,6 +1819,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J let holdingsAsOf: string | null = null; let holdingsSource = 'not available from current public SEC filing'; let nport: ParsedNport | null = null; + let holdingsFresh = false; // false while holdingsRows is the previous run's sheet if (config.edgarFallback) { try { const filing = await resolveNportFiling(fund, config); @@ -1827,6 +1832,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J if (seriesMatches && fresher && parsed.holdings.length) { const names = await loadCompanyTickerTable(config); holdingsRows = fillNportTickers(parsed.holdings, names); + holdingsFresh = true; holdingsHeaders = holdingsRows.some((row) => 'Coupon' in row || 'Maturity' in row) ? [...HOLDINGS_HEADERS, 'Coupon', 'Maturity'] : HOLDINGS_HEADERS; holdingsAsOf = parsed.repPdDate || null; nport = parsed; @@ -1843,6 +1849,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J const top = parseTopHoldingsMarkdown(productPageMarkdown, fund.ticker, companyTickerMap || new Map()); if (top.rows.length && (!previousMeta?.holdings?.totalRows || /^WisdomTree product page/.test(String(previousMeta.holdings.source || '')))) { holdingsRows = top.rows; + holdingsFresh = true; holdingsHeaders = top.headers; holdingsAsOf = top.asOfDate; holdingsSource = 'WisdomTree product page top holdings (issuer fallback; full N-PORT filing unavailable)'; @@ -1909,6 +1916,11 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J return { __skipped: true, ticker: fund.ticker, __skipReasons: returnFilterReasons }; } + // Fresh holdings that look broken (no weight column, non-numeric weights, a bad weight sum, half the rows gone) never replace + // the published ones: the fund fails before anything of it is written, so it stays exactly as published. + const holdingsProblem = holdingsFresh ? holdingsGuard(holdingsHeaders, holdingsRows, numberOrNull(previousMeta?.holdings?.totalRows), HOLDINGS_GUARD_RELAX[fund.ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? []) : null; + if (holdingsProblem) throw new Error(`holdings guard: ${holdingsProblem}; ${previousMeta ? 'kept the previous complete data' : 'no data published for this fund yet'}`); + const history = historyRows(days, previousRows); const historyAsOf = derived.asOfDate || previousMeta?.history?.asOf || null; const holdingPages = await writePages(fundDir, fund.ticker, 'holdings', holdingsHeaders, holdingsRows, config.holdingsPageSize, holdingsAsOf, holdingsSource); @@ -2495,3 +2507,49 @@ if ((import.meta as { main?: boolean }).main) { if (process.argv.some((arg) => ['-h', '--help', 'help'].includes(arg))) console.log(USAGE.trim()); else await main().catch((error) => { console.error(error instanceof Error ? error.stack : String(error)); process.exitCode = 1; }); } + +// shared block of .claude/tools/etfs/etf-std/holdings-guard.block.ts, applied by apply-holdings-guard.ts: never edit it in one repo +/** + * Sanity check of a fund's new holdings before they replace the published ones. Returns why they must be rejected + * (the caller then keeps the fund's previous state and reports it as failed), or null when they may be written. + * An empty sheet passes only when nothing was published before. Otherwise a Name and a weight column (Weight, + * Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to + * 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund + * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'. + * Rows are arrays in header order or objects keyed by the header text. + */ +export function holdingsGuard(headers: ReadonlyArray, rows: ReadonlyArray, previousRows: number | null | undefined, relax: ReadonlyArray = []): string | null { + const previous = Number(previousRows) > 0 ? Number(previousRows) : 0; + if (!rows.length) return previous ? `no holdings rows (${previous} published)` : null; + const names = headers.map((header) => String(header).trim().toLowerCase()); + const nameAt = names.indexOf('name'); + let weightAt = names.indexOf('weight'); + if (weightAt < 0) weightAt = names.indexOf('weight (%)'); + if (weightAt < 0) weightAt = names.indexOf('market weight'); + if (nameAt < 0 || weightAt < 0) return `no ${nameAt < 0 ? 'Name' : 'weight'} column (headers: ${headers.join(', ') || 'none'})`; + let numeric = 0; + let sum = 0; + for (let i = 0; i < rows.length; i++) { + const row: any = rows[i]; + const weight = holdingsGuardNumber(Array.isArray(row) ? row[weightAt] : row && typeof row === 'object' ? row[headers[weightAt]] : undefined); + if (weight !== null) { + numeric++; + sum += weight; + } + } + if (relax.indexOf('numeric') < 0 && numeric < rows.length * 0.8) return `only ${numeric} of ${rows.length} rows have a numeric weight`; + if (relax.indexOf('sum') < 0 && (sum < 50 || sum > 150)) return `weights add up to ${sum.toFixed(2)}, outside 50..150`; + if (relax.indexOf('count') < 0 && previous && rows.length < previous * 0.5) return `${rows.length} rows, less than half of the ${previous} published`; + return null; +} + +/** A weight cell as a number: %, $, commas and spaces are ignored, (x) is negative; null when it is no plain number (a zero-width character inside counts as broken). */ +function holdingsGuardNumber(value: unknown): number | null { + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + let text = String(value === null || value === undefined ? '' : value).replace(/[ \t ,%$]/g, '').replace(/−/g, '-'); + const negative = /^\(.*\)$/.test(text); + if (negative) text = text.slice(1, -1); + if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(text)) return null; + return negative ? -Number(text) : Number(text); +} +//