Skip to content

fix(holdings): accept only a name and weight table as product-page top holdings #82

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daggerok merged 1 commit into from
Oct 25, 2026
Merged

fix(holdings): accept only a name and weight table as product-page top holdings #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion scripts/update-data.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa,
isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange,
parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions,
parseProductPageSummary, parseRange, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls,
parseProductPageSummary, parseRange, parseTopHoldingsMarkdown, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls,
rowFromMeta, runUpdate, runtimeControls, setApiRoot, toIsoDate, writeIndex, yahooChartQuery,
} from './update-data';

Expand Down Expand Up @@ -56,6 +56,9 @@ const catalogFixture = [
].join('\n');
const usfrPage = `# USFR WisdomTree Floating Rate Treasury Fund\n\n### 3.68%\n\n30-day SEC yield\n\nAs of 9/15/2026\n\n| Product Overview | As of 9/16/2026 |\n| --- | --- |\n| Expense Ratio | 0.15% |\n| CUSIP | 97717Y527 |\n| Total Assets (000) | $19,560,180.26 |\n| SEC 30-day Yield | 3.68% |\n\n### Net Asset Value\n\n| Net Asset Value | As of 9/16/2026 |\n| --- | --- |\n| NAV | $50.476 |\n| Premium/Discount to NAV | 0.01% |\n\n### Closing Market Price\n\n| Closing Market Value | As of 9/15/2026 |\n| --- | --- |\n| Closing Market Price | $50.470 |\n\n### Total Returns\n\nMonth End Performance (8/31/2026)\n\n| Cumulative | 1 Month | 3 Month | YTD | Since Inception* | |\n| --- | --- | --- | --- | --- | --- |\n| Underlying Index Returns | 0.33% | 1.04% | 2.70% | 30.94% | |\n| NAV Returns | 0.32% | 1.00% | 2.57% | 28.21% | |\n| Market Price Returns | 0.32% | 1.02% | 2.59% | 28.05% | |\n| Average Annual | 1 Year | 3 Year | 5 Year | 10 Year | Since Inception* |\n| Underlying Index Returns | 4.18% | 4.84% | 4.04% | 2.67% | 2.17% |\n| NAV Returns | 3.98% | 4.66% | 3.86% | 2.49% | 2.00% |\n| Market Price Returns | 4.00% | 4.65% | 3.86% | 2.52% | 1.99% |\n\nQuarter End Performance (6/30/2026)\n\n| Cumulative | Since Inception* | | | | |\n| --- | --- | --- | --- | --- | --- |\n| Underlying Index Returns | 30.04% | | | | |\n| NAV Returns | 27.37% | | | | |\n| Market Price Returns | 27.19% | | | | |\n| Average Annual | 1 Year | 3 Year | 5 Year | 10 Year | Since Inception* |\n| Underlying Index Returns | 4.30% | 4.90% | 4.10% | 2.70% | 2.20% |\n| NAV Returns | 4.10% | 4.70% | 3.90% | 2.50% | 2.02% |\n| Market Price Returns | 4.12% | 4.69% | 3.90% | 2.53% | 2.01% |`;
const DIST_HEADER = '| Ex-Dividend Date | Record Date | Payable Date | Ordinary Income | Short Term Capital Gains | Long Term Capital Gains | Return of Capital | Total Distribution |\n| --- | --- | --- | --- | --- | --- | --- | --- |';
/** The product-page holdings section: the top-N list of most funds, or the allocation summary of the long/short funds (WSLS, WTAL). */
const holdingsSection = (header: string, ...rows: string[]): string => `### Holdings${header.startsWith('| Group') ? ' Summary\n\nAs of 10/22/2026' : '\n\n#### As of 10/22/2026'}\n\n${header}\n| --- | --- |\n${rows.join('\n')}\n\nView all holdings`;
const ALLOCATION_ROWS = ['| Cash&Collateral | 100.00% | 0.00% | 100.00% |', '| Treasuries | 90.68% | 0.00% | 90.68% |', '| Equities | 100.72% | -88.06% | 90.00% |', '| Equity Futures | 90.00% | 0.00% | 90.00% |'];
const distPage = (...rows: string[]) => `### Recent Distributions\n\n${DIST_HEADER}\n${rows.join('\n')}\n\n### Thought Leadership\n\n| Name | Ticker |\n| --- | --- |\n| unrelated | table |`;

// ===========================================================================
Expand Down Expand Up @@ -258,6 +261,18 @@ describe('parsing', () => {
expect(parseEdgarAtomFilings(atom)[0].url).toContain('0000000000-26-000001.txt');
});

test('top holdings: only a Name and weight table up to the next heading; an allocation summary or a list adding up to more than 105 gives no rows', () => {
const top = parseTopHoldingsMarkdown(`${holdingsSection('| Name | Weight (%) |', '| Space Exploration Techn-Cl A | 17.06 |', '| Rocket Lab Corp | 4.54 |', '| Remaining Portfolio | 78.40 |')}\n\n### Sector Breakdown\n\n| Name | Weight (%) |\n| --- | --- |\n| Industrials | 63.71 |`, 'WSPC');
expect([top.rows.map((row) => [row.Name, row.Weight]), top.asOfDate, top.rejected]).toEqual([[['Space Exploration Techn-Cl A', '17.06'], ['Rocket Lab Corp', '4.54']], '2026-10-22', null]);
const allocation = parseTopHoldingsMarkdown(holdingsSection('| Group | Long | Short | Group Weight |', ...ALLOCATION_ROWS), 'WSLS');
expect([allocation.rows, allocation.rejected]).toEqual([[], 'no Name and weight table (headers: Group, Long, Short, Group Weight)']);
const over = parseTopHoldingsMarkdown(holdingsSection('| Name | Weight (%) |', '| Treasuries | 90.68 |', '| Equities | 100.72 |'), 'WTAL');
expect([over.rows, over.rejected]).toEqual([[], 'top holdings add up to 191.4, more than 105']);
// a list right before the next heading ends there; a page without a holdings section is no rejection
expect(parseTopHoldingsMarkdown('### Holdings\n\n| Name | Weight |\n| --- | --- |\n| A Corp | 5 |\n### Recent Distributions\n\n| Name | Weight |\n| --- | --- |\n| B Corp | 6 |', 'AAA').rows.map((row) => row.Name)).toEqual(['A Corp']);
expect(parseTopHoldingsMarkdown(usfrPage, 'USFR')).toMatchObject({ rows: [], rejected: null });
});

test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => {
const headers = ['Name', 'Ticker', 'Weight'];
const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }];
Expand Down Expand Up @@ -732,6 +747,20 @@ describe('pipeline', () => {
expect(process.exitCode).toBe(1);
});

test('a product page that turns its holdings list into an allocation summary keeps the published list and still refreshes the fund', async () => {
const s = new Scenario(['BBB']);
const base = s.handler;
const serve = (ytd: string, section: string) => { s.handler = (url, init) => (url.startsWith('https://r.jina.ai/') && !url.endsWith('/us/products') ? new Response(`${productPage('BBB', ytd)}\n\n${section}`) : base(url, init)); };
serve('2.59%', holdingsSection('| Name | Weight (%) |', '| Alpha Corp | 60.00 |', '| Beta Corp | 40.00 |'));
await s.run({ EDGAR_FALLBACK: 'true' });
const published = s.meta('BBB').holdings;
expect(published.totalRows).toBe(2);
serve('9.99%', holdingsSection('| Group | Long | Short | Group Weight |', ...ALLOCATION_ROWS));
process.exitCode = 0;
await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' });
expect([s.meta('BBB').holdings, s.meta('BBB').returns.monthEnd.ytd, process.exitCode]).toEqual([published, 9.99, 0]);
});

test('N-PORT freshness: a filing older than the published holdings never replaces them, a newer one does', async () => {
const s = new Scenario(['AAA']);
await s.run({ EDGAR_FALLBACK: 'true' });
Expand Down
58 changes: 41 additions & 17 deletions scripts/update-data.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1146,30 +1146,53 @@ function distributionRowToCells(row: DistributionRow): string[] {
];
}

export function parseTopHoldingsMarkdown(markdown: string, ticker: string, companyNames: Map<string, string> = new Map()): { headers: string[]; rows: JsonRecord[]; asOfDate: string | null } {
/** Funds verified as leveraged whose product-page top holdings may add up to more than TOP_HOLDINGS_MAX_SUM. None yet: the
* long/short funds WSLS and WTAL publish only an allocation summary (Group, Long, Short, Group Weight), no holdings list. */
const TOP_HOLDINGS_LEVERAGED = new Set<string>();
/** A top-N holdings list covers at most the whole fund; a little slack for rounding. */
const TOP_HOLDINGS_MAX_SUM = 105;

/**
* The top holdings list of the product page (`### Holdings` section, up to the next heading of level 1-3). Only a table
* whose header is Name plus a weight column counts as a holdings list; an allocation table, or a list whose weights add
* up to more than TOP_HOLDINGS_MAX_SUM, yields no rows and a `rejected` reason, so the next source or the published sheet applies.
*/
export function parseTopHoldingsMarkdown(markdown: string, ticker: string, companyNames: Map<string, string> = new Map()): { headers: string[]; rows: JsonRecord[]; asOfDate: string | null; rejected: string | null } {
const source = String(markdown ?? '').replace(/\r/g, '');
const heading = /###\s+Holdings\s*\n+####\s+As of\s*([^\n]*)/i.exec(source);
const asOfDate = heading ? toIsoDate(heading[1]) : null;
const none = (rejected: string | null) => ({ headers: HOLDINGS_HEADERS, rows: [] as JsonRecord[], asOfDate, rejected });
const start = source.search(/###\s+Holdings\b/i);
if (start < 0) return { headers: HOLDINGS_HEADERS, rows: [], asOfDate };
const lines = source.slice(start).split('\n');
if (start < 0) return none(null);
const section: string[] = [];
for (const line of source.slice(start).split('\n').slice(1)) {
if (/^#{1,3}\s/.test(line.trim())) break;
section.push(line.trim());
}
// Tables are runs of consecutive pipe lines; the first one with a Name and a weight header is the holdings list.
const tables: string[][][] = [];
let previousWasTable = false;
for (const line of section) {
const isTable = line.startsWith('|');
if (isTable && !previousWasTable) tables.push([]);
if (isTable) tables[tables.length - 1].push(line.replace(/^\|/, '').replace(/\|$/, '').split('|').map(cleanText));
previousWasTable = isTable;
}
const table = tables.find((rows) => rows[0][0].toLowerCase() === 'name' && rows[0].some((cell, i) => i > 0 && /^weight\b/i.test(cell)));
if (!table) return none(tables.length ? `no Name and weight table (headers: ${tables.map((rows) => rows[0].join(', ')).join('; ')})` : null);
const weightAt = table[0].findIndex((cell, i) => i > 0 && /^weight\b/i.test(cell));
const rows: JsonRecord[] = [];
let tableStarted = false;
for (const line of lines) {
if (!line.trim().startsWith('|')) {
if (tableStarted && rows.length) break;
continue;
}
const cells = line.trim().replace(/^\|/, '').replace(/\|$/, '').split('|').map(cleanText);
if (cells.length < 2 || cells[0].toLowerCase() === 'name' || /^-+$/.test(cells[0])) continue;
tableStarted = true;
let sum = 0;
for (const cells of table.slice(1)) {
const name = cells[0];
if (name.toLowerCase() === 'remaining portfolio') continue;
const normalized = normalizeHoldingName(name);
const resolvedTicker = companyNames.get(normalized) || companyNames.get(normalizeHoldingNameCore(name)) || '-';
rows.push({ Name: name, Ticker: resolvedTicker || '-', Identifier: '-', Weight: String(numberOrNull(cells[1]) ?? 0), 'Market Value': '-', 'Shares Held': '-', 'Asset Category': '-' });
if (!name || /^:?-+:?$/.test(name) || name.toLowerCase() === 'remaining portfolio') continue;
const weight = numberOrNull(cells[weightAt]);
sum += weight ?? 0;
const resolvedTicker = companyNames.get(normalizeHoldingName(name)) || companyNames.get(normalizeHoldingNameCore(name)) || '-';
rows.push({ Name: name, Ticker: resolvedTicker || '-', Identifier: '-', Weight: String(weight ?? 0), 'Market Value': '-', 'Shares Held': '-', 'Asset Category': '-' });
}
return { headers: HOLDINGS_HEADERS, rows, asOfDate };
if (sum > TOP_HOLDINGS_MAX_SUM && !TOP_HOLDINGS_LEVERAGED.has(ticker)) return none(`top holdings add up to ${round(sum, 2)}, more than ${TOP_HOLDINGS_MAX_SUM}`);
return { headers: HOLDINGS_HEADERS, rows, asOfDate, rejected: null };
}

function secHeaders(): Record<string, string> {
Expand Down Expand Up @@ -1847,6 +1870,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J

if (!holdingsRows.length && config.edgarFallback && productPageMarkdown) {
const top = parseTopHoldingsMarkdown(productPageMarkdown, fund.ticker, companyTickerMap || new Map());
if (top.rejected) outputNote(`[ ${'holdings'.padEnd(9)}] ${fund.ticker}: product page top holdings rejected: ${top.rejected}`);
if (top.rows.length && (!previousMeta?.holdings?.totalRows || /^WisdomTree product page/.test(String(previousMeta.holdings.source || '')))) {
holdingsRows = top.rows;
holdingsFresh = true;
Expand Down
Loading