diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index 3d3883f..1fcadfc 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -9,7 +9,7 @@ import { annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa, isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange, parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions, - parseProductPageSummary, parseRange, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls, + parseProductPageSummary, parseRange, parseTopHoldingsMarkdown, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls, rowFromMeta, runUpdate, runtimeControls, setApiRoot, toIsoDate, writeIndex, yahooChartQuery, } from './update-data'; @@ -56,6 +56,9 @@ const catalogFixture = [ ].join('\n'); const usfrPage = `# USFR WisdomTree Floating Rate Treasury Fund\n\n### 3.68%\n\n30-day SEC yield\n\nAs of 9/15/2026\n\n| Product Overview | As of 9/16/2026 |\n| --- | --- |\n| Expense Ratio | 0.15% |\n| CUSIP | 97717Y527 |\n| Total Assets (000) | $19,560,180.26 |\n| SEC 30-day Yield | 3.68% |\n\n### Net Asset Value\n\n| Net Asset Value | As of 9/16/2026 |\n| --- | --- |\n| NAV | $50.476 |\n| Premium/Discount to NAV | 0.01% |\n\n### Closing Market Price\n\n| Closing Market Value | As of 9/15/2026 |\n| --- | --- |\n| Closing Market Price | $50.470 |\n\n### Total Returns\n\nMonth End Performance (8/31/2026)\n\n| Cumulative | 1 Month | 3 Month | YTD | Since Inception* | |\n| --- | --- | --- | --- | --- | --- |\n| Underlying Index Returns | 0.33% | 1.04% | 2.70% | 30.94% | |\n| NAV Returns | 0.32% | 1.00% | 2.57% | 28.21% | |\n| Market Price Returns | 0.32% | 1.02% | 2.59% | 28.05% | |\n| Average Annual | 1 Year | 3 Year | 5 Year | 10 Year | Since Inception* |\n| Underlying Index Returns | 4.18% | 4.84% | 4.04% | 2.67% | 2.17% |\n| NAV Returns | 3.98% | 4.66% | 3.86% | 2.49% | 2.00% |\n| Market Price Returns | 4.00% | 4.65% | 3.86% | 2.52% | 1.99% |\n\nQuarter End Performance (6/30/2026)\n\n| Cumulative | Since Inception* | | | | |\n| --- | --- | --- | --- | --- | --- |\n| Underlying Index Returns | 30.04% | | | | |\n| NAV Returns | 27.37% | | | | |\n| Market Price Returns | 27.19% | | | | |\n| Average Annual | 1 Year | 3 Year | 5 Year | 10 Year | Since Inception* |\n| Underlying Index Returns | 4.30% | 4.90% | 4.10% | 2.70% | 2.20% |\n| NAV Returns | 4.10% | 4.70% | 3.90% | 2.50% | 2.02% |\n| Market Price Returns | 4.12% | 4.69% | 3.90% | 2.53% | 2.01% |`; const DIST_HEADER = '| Ex-Dividend Date | Record Date | Payable Date | Ordinary Income | Short Term Capital Gains | Long Term Capital Gains | Return of Capital | Total Distribution |\n| --- | --- | --- | --- | --- | --- | --- | --- |'; +/** The product-page holdings section: the top-N list of most funds, or the allocation summary of the long/short funds (WSLS, WTAL). */ +const holdingsSection = (header: string, ...rows: string[]): string => `### Holdings${header.startsWith('| Group') ? ' Summary\n\nAs of 10/22/2026' : '\n\n#### As of 10/22/2026'}\n\n${header}\n| --- | --- |\n${rows.join('\n')}\n\nView all holdings`; +const ALLOCATION_ROWS = ['| Cash&Collateral | 100.00% | 0.00% | 100.00% |', '| Treasuries | 90.68% | 0.00% | 90.68% |', '| Equities | 100.72% | -88.06% | 90.00% |', '| Equity Futures | 90.00% | 0.00% | 90.00% |']; const distPage = (...rows: string[]) => `### Recent Distributions\n\n${DIST_HEADER}\n${rows.join('\n')}\n\n### Thought Leadership\n\n| Name | Ticker |\n| --- | --- |\n| unrelated | table |`; // =========================================================================== @@ -258,6 +261,18 @@ describe('parsing', () => { expect(parseEdgarAtomFilings(atom)[0].url).toContain('0000000000-26-000001.txt'); }); + test('top holdings: only a Name and weight table up to the next heading; an allocation summary or a list adding up to more than 105 gives no rows', () => { + const top = parseTopHoldingsMarkdown(`${holdingsSection('| Name | Weight (%) |', '| Space Exploration Techn-Cl A | 17.06 |', '| Rocket Lab Corp | 4.54 |', '| Remaining Portfolio | 78.40 |')}\n\n### Sector Breakdown\n\n| Name | Weight (%) |\n| --- | --- |\n| Industrials | 63.71 |`, 'WSPC'); + expect([top.rows.map((row) => [row.Name, row.Weight]), top.asOfDate, top.rejected]).toEqual([[['Space Exploration Techn-Cl A', '17.06'], ['Rocket Lab Corp', '4.54']], '2026-10-22', null]); + const allocation = parseTopHoldingsMarkdown(holdingsSection('| Group | Long | Short | Group Weight |', ...ALLOCATION_ROWS), 'WSLS'); + expect([allocation.rows, allocation.rejected]).toEqual([[], 'no Name and weight table (headers: Group, Long, Short, Group Weight)']); + const over = parseTopHoldingsMarkdown(holdingsSection('| Name | Weight (%) |', '| Treasuries | 90.68 |', '| Equities | 100.72 |'), 'WTAL'); + expect([over.rows, over.rejected]).toEqual([[], 'top holdings add up to 191.4, more than 105']); + // a list right before the next heading ends there; a page without a holdings section is no rejection + expect(parseTopHoldingsMarkdown('### Holdings\n\n| Name | Weight |\n| --- | --- |\n| A Corp | 5 |\n### Recent Distributions\n\n| Name | Weight |\n| --- | --- |\n| B Corp | 6 |', 'AAA').rows.map((row) => row.Name)).toEqual(['A Corp']); + expect(parseTopHoldingsMarkdown(usfrPage, 'USFR')).toMatchObject({ rows: [], rejected: null }); + }); + test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => { const headers = ['Name', 'Ticker', 'Weight']; const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }]; @@ -732,6 +747,20 @@ describe('pipeline', () => { expect(process.exitCode).toBe(1); }); + test('a product page that turns its holdings list into an allocation summary keeps the published list and still refreshes the fund', async () => { + const s = new Scenario(['BBB']); + const base = s.handler; + const serve = (ytd: string, section: string) => { s.handler = (url, init) => (url.startsWith('https://r.jina.ai/') && !url.endsWith('/us/products') ? new Response(`${productPage('BBB', ytd)}\n\n${section}`) : base(url, init)); }; + serve('2.59%', holdingsSection('| Name | Weight (%) |', '| Alpha Corp | 60.00 |', '| Beta Corp | 40.00 |')); + await s.run({ EDGAR_FALLBACK: 'true' }); + const published = s.meta('BBB').holdings; + expect(published.totalRows).toBe(2); + serve('9.99%', holdingsSection('| Group | Long | Short | Group Weight |', ...ALLOCATION_ROWS)); + process.exitCode = 0; + await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' }); + expect([s.meta('BBB').holdings, s.meta('BBB').returns.monthEnd.ytd, process.exitCode]).toEqual([published, 9.99, 0]); + }); + test('N-PORT freshness: a filing older than the published holdings never replaces them, a newer one does', async () => { const s = new Scenario(['AAA']); await s.run({ EDGAR_FALLBACK: 'true' }); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index db2d01d..6fbe301 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -1146,30 +1146,53 @@ function distributionRowToCells(row: DistributionRow): string[] { ]; } -export function parseTopHoldingsMarkdown(markdown: string, ticker: string, companyNames: Map = new Map()): { headers: string[]; rows: JsonRecord[]; asOfDate: string | null } { +/** Funds verified as leveraged whose product-page top holdings may add up to more than TOP_HOLDINGS_MAX_SUM. None yet: the + * long/short funds WSLS and WTAL publish only an allocation summary (Group, Long, Short, Group Weight), no holdings list. */ +const TOP_HOLDINGS_LEVERAGED = new Set(); +/** A top-N holdings list covers at most the whole fund; a little slack for rounding. */ +const TOP_HOLDINGS_MAX_SUM = 105; + +/** + * The top holdings list of the product page (`### Holdings` section, up to the next heading of level 1-3). Only a table + * whose header is Name plus a weight column counts as a holdings list; an allocation table, or a list whose weights add + * up to more than TOP_HOLDINGS_MAX_SUM, yields no rows and a `rejected` reason, so the next source or the published sheet applies. + */ +export function parseTopHoldingsMarkdown(markdown: string, ticker: string, companyNames: Map = new Map()): { headers: string[]; rows: JsonRecord[]; asOfDate: string | null; rejected: string | null } { const source = String(markdown ?? '').replace(/\r/g, ''); const heading = /###\s+Holdings\s*\n+####\s+As of\s*([^\n]*)/i.exec(source); const asOfDate = heading ? toIsoDate(heading[1]) : null; + const none = (rejected: string | null) => ({ headers: HOLDINGS_HEADERS, rows: [] as JsonRecord[], asOfDate, rejected }); const start = source.search(/###\s+Holdings\b/i); - if (start < 0) return { headers: HOLDINGS_HEADERS, rows: [], asOfDate }; - const lines = source.slice(start).split('\n'); + if (start < 0) return none(null); + const section: string[] = []; + for (const line of source.slice(start).split('\n').slice(1)) { + if (/^#{1,3}\s/.test(line.trim())) break; + section.push(line.trim()); + } + // Tables are runs of consecutive pipe lines; the first one with a Name and a weight header is the holdings list. + const tables: string[][][] = []; + let previousWasTable = false; + for (const line of section) { + const isTable = line.startsWith('|'); + if (isTable && !previousWasTable) tables.push([]); + if (isTable) tables[tables.length - 1].push(line.replace(/^\|/, '').replace(/\|$/, '').split('|').map(cleanText)); + previousWasTable = isTable; + } + const table = tables.find((rows) => rows[0][0].toLowerCase() === 'name' && rows[0].some((cell, i) => i > 0 && /^weight\b/i.test(cell))); + if (!table) return none(tables.length ? `no Name and weight table (headers: ${tables.map((rows) => rows[0].join(', ')).join('; ')})` : null); + const weightAt = table[0].findIndex((cell, i) => i > 0 && /^weight\b/i.test(cell)); const rows: JsonRecord[] = []; - let tableStarted = false; - for (const line of lines) { - if (!line.trim().startsWith('|')) { - if (tableStarted && rows.length) break; - continue; - } - const cells = line.trim().replace(/^\|/, '').replace(/\|$/, '').split('|').map(cleanText); - if (cells.length < 2 || cells[0].toLowerCase() === 'name' || /^-+$/.test(cells[0])) continue; - tableStarted = true; + let sum = 0; + for (const cells of table.slice(1)) { const name = cells[0]; - if (name.toLowerCase() === 'remaining portfolio') continue; - const normalized = normalizeHoldingName(name); - const resolvedTicker = companyNames.get(normalized) || companyNames.get(normalizeHoldingNameCore(name)) || '-'; - rows.push({ Name: name, Ticker: resolvedTicker || '-', Identifier: '-', Weight: String(numberOrNull(cells[1]) ?? 0), 'Market Value': '-', 'Shares Held': '-', 'Asset Category': '-' }); + if (!name || /^:?-+:?$/.test(name) || name.toLowerCase() === 'remaining portfolio') continue; + const weight = numberOrNull(cells[weightAt]); + sum += weight ?? 0; + const resolvedTicker = companyNames.get(normalizeHoldingName(name)) || companyNames.get(normalizeHoldingNameCore(name)) || '-'; + rows.push({ Name: name, Ticker: resolvedTicker || '-', Identifier: '-', Weight: String(weight ?? 0), 'Market Value': '-', 'Shares Held': '-', 'Asset Category': '-' }); } - return { headers: HOLDINGS_HEADERS, rows, asOfDate }; + if (sum > TOP_HOLDINGS_MAX_SUM && !TOP_HOLDINGS_LEVERAGED.has(ticker)) return none(`top holdings add up to ${round(sum, 2)}, more than ${TOP_HOLDINGS_MAX_SUM}`); + return { headers: HOLDINGS_HEADERS, rows, asOfDate, rejected: null }; } function secHeaders(): Record { @@ -1847,6 +1870,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J if (!holdingsRows.length && config.edgarFallback && productPageMarkdown) { const top = parseTopHoldingsMarkdown(productPageMarkdown, fund.ticker, companyTickerMap || new Map()); + if (top.rejected) outputNote(`[ ${'holdings'.padEnd(9)}] ${fund.ticker}: product page top holdings rejected: ${top.rejected}`); if (top.rows.length && (!previousMeta?.holdings?.totalRows || /^WisdomTree product page/.test(String(previousMeta.holdings.source || '')))) { holdingsRows = top.rows; holdingsFresh = true;