diff --git a/components/LegislatorProfile/TabComponents/FinanceTab.tsx b/components/LegislatorProfile/TabComponents/FinanceTab.tsx index 18a5ce105..f46aa5867 100644 --- a/components/LegislatorProfile/TabComponents/FinanceTab.tsx +++ b/components/LegislatorProfile/TabComponents/FinanceTab.tsx @@ -91,7 +91,7 @@ export function FinanceTab({ finance }: { finance?: MembersFinance }) { label: t("finance.stats.totalRaised"), value: formatCurrency(finance.totalRaised), subtitle: t("finance.stats.totalRaisedSubtitle", { - count: finance.contributorCount ?? 0 + count: finance.contributionsCount ?? 0 }), footnote: t("finance.stats.totalRaisedFootnote", { date: bankDataAsOf }) }, diff --git a/components/db/membersFinance.ts b/components/db/membersFinance.ts index b2196d37e..e36dfd633 100644 --- a/components/db/membersFinance.ts +++ b/components/db/membersFinance.ts @@ -50,7 +50,7 @@ export interface MembersFinance { totalRaised: number totalSpent: number cashOnHand: number - contributorCount: number + contributionsCount: number lastUpdated: Timestamp // End_Date of the most recent Bank Report (type 70) — the basis for totalRaised/cashOnHand. bankDataAsOf?: Timestamp diff --git a/docs/ocpf-data-inclusion-summary.md b/docs/ocpf-data-inclusion-summary.md new file mode 100644 index 000000000..9f083ca65 --- /dev/null +++ b/docs/ocpf-data-inclusion-summary.md @@ -0,0 +1,98 @@ +## MAPLE Finance Tab — Data Inclusion Summary + +This summarizes what campaign finance data MAPLE uses from OCPF's public bulk data files, what we include vs. exclude, and why. + +Source data: OCPF's `reports.txt` (one row per filed report) and `report-items.txt` (one row per individual transaction). Our research and decisions were based on files for the current election cycle only (2025–2026). + +### Report types: what we include/exclude from `reports.txt` (uses blocklist strategy, inclusive by default) + +| Report Type | Included in Total Raised/Spent? | Why | +| ------------------------------------------------------------------------------ | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **60 – Deposit Report** | No (totals excluded) — but its items ARE used for the breakdown | Carries gross amounts, before processing fees. The same money reappears net-of-fees on the Bank Report; counting both would double it. We still pull its individual transactions (see below) and its unitemized total. | +| **70 – Bank Report** | **Yes — primary source** | This is the actual bank reconciliation: net of fees, matches real cash. Total Raised, Total Spent, and Cash on Hand all come from here. Total Raised then has non-contribution receipts (type 204, see below) netted back out, to match OCPF's own public "Receipts" definition. | +| **80 – Credit Card Report** | No | Supplemental — itemizes spending the Bank Report already counts. Verified empirically: summing Bank Reports alone matches Year-End totals exactly without these. | +| **90 – Reimbursement Report** | No | Same reasoning as Credit Card reports. | +| **11 / 24 / 32 / 36 / 45 / 52 / 113 – Year-End Reports** (all committee types) | No | Annual rollups that duplicate the periodic (Bank Report) totals already summed. Used only as a sanity check — we compare our summed totals against the Year-End figure and log a warning if they don't match to the penny. | +| **12 – Dissolution Report, 14/15 – Transition-Out/In Report** | No | **Same double-counting risk as Year-End, confirmed with a real example**: one committee's Dissolution Report exactly matched the sum of its prior two Bank Reports ($1,583.64 to the penny). Not tied to a calendar year, so tracked separately from the Year-End reconciliation. | +| **62 – Segregated Account Report** | N/A | Never filed by any state legislator — used by a different class of committee (18 filers statewide, all non-legislators). Never reaches our logic since we only process CPF IDs already mapped to a sitting legislator. | +| Anything else not listed above | **Included by default** | ⚠️ **Flag for reviewer**: our logic is a blocklist (exclude known rollup/supplemental types), not an allowlist. An unfamiliar report type we haven't seen yet would be summed into totals automatically. Hasn't caused a known problem, but worth knowing. | + +### Where the data from `reports.txt` is used in the UI: + +| UI field | Source in `reports.txt` | +| ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Total Raised** | Sum of `Receipts_Total`, from every report type _except_ 60, Year-End rollups, Dissolution/Transition rollups, and 80/90, then reduced by subtracting the dollar amount of any item coded 204 (Non-contribution receipts, such as refunds, etc.) found in `report-items.txt` | +| **Total Spent** | Sum of `Expenditures_Total`, same exclusion set as Total Raised (60, Year-End, Dissolution/Transition, 80/90). No item-level adjustment | +| **Cash on Hand** | `End_Balance` of the type-70 Bank Report with the latest `End_Date` | +| **Contributions Breakdown → "Micro-donations (under $50)"** | `Receipts_Unitemized_Total`, from every report type _except_ the Year-End/Dissolution/Transition/80/90 rollups. Unlike Total Raised/Spent, type 60 is included here, since this field is blank in the bank reports | +| **"As of" date on Total Raised/Spent/Cash on Hand** | taken from the type-70 Bank Report's `End_Date`. (`bankDataAsOf` in code) | +| **"As of" date on Contributions Breakdown** | taken from the type-60 Deposit Report's `End_Date`. (`depositDataAsOf` in code) | +| **"Small Donors" %** | Partly — combines unitemized amount above with item code 201 (<$200) | + +- `startBalance` is also captured from `reports.txt` (type-70 `Start_Balance`, earliest `Start_Date`) and written to Firestore, but per caveat #1 below, it is **not currently displayed** on the tab. + +- `years[y].finalized`, whether a type-11/24/32/36/45/52/113 Year-End report has been seen for that year, is also captured and written to Firestore, but it is not currently displayed on the tab, and nothing in the frontend reads it yet. + +- Everything else on the tab (Individual/Committee/Union donors, Candidate's own funds, In-kind, contributions count) comes from `report-items.txt` instead. + +### Item/transaction codes: what we include/exclude from `report-items.txt` (uses allowlist strategy, exclusive by default) + +**Included:** + +| Code | Description | Where it's used | +| --------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 201 | Individual Contribution | "Individual donors"; also counted toward Small Donors if under $200; counts toward contributions count | +| 202 | Committee Contribution | "Committees/PACs" | +| 203 | Union/Association Contribution | "Unions & associations" | +| 204 | Non-contribution receipt (refunds, misc.) | Subtracted from Total Raised, and also excluded from the Contributions Breakdown too. It's not a contribution, and OCPF's own public "Receipts" figure excludes it as well. Not stored separately — netted out at the point it's found. | +| 206, 331 | Candidate Loan / out-of-pocket expense (as loan) | "Candidate's own funds" (loans) | +| 332 | Out-of-pocket expense (as contribution) | "Candidate's own funds" (contributions) | +| 401 / 402 / 403 | Individual / Committee / Union In-kind | "In-kind contributions" | +| 420 | Aggregated un-itemized in-kind | "In-kind contributions" | +| 319 | Payment-processor fee | Tracked and disclosed via footnote. Not part of the Contributions Breakdown total | + +**Not included, with reasoning:** + +| Code | Description | Why excluded | +| ------------------------------ | ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 205 | Bank Interest | Not itemized here — if present, it's already folded into a report's overall receipts total. | +| 220 | Aggregated Un-itemized Receipts | Our "Micro-donations (under $50)" figure comes from the report-level unitemized field instead, not from summing this code. This code is essentially never used; exactly 1 occurrence statewide across 2 years of data ($201.01, for one legislator), and in that case it wasn't even reflected in that report's own unitemized field. | +| 211 | Business/Corporation Contribution | MA law prohibits direct corporate contributions to candidate committees. **Confirmed empirically**: this code exists in the statewide data ($3.8M across 82 transactions in 2025) but _only_ for Independent Expenditure PACs and ballot-question committees, never for a candidate. Correctly never appears for any legislator we track. | +| 230 | Donation (Segregated Account Reports) | Only ever attached to Report Type 62 (see above) — never reaches a legislator's data. | +| 310 | Candidate Loan Repayment | Only appears on committee lifecycle reports (Dissolution/Transition/Year-End), which we already exclude entirely (see above). The dollar amount is still captured in Total Spent via that report's own totals when it's a normal periodic report; it's just not broken into its own visible spending category. | +| 311, 351, 354, 801, and others | Bank-report placeholder subtotals, credit card/reimbursement sub-items | These duplicate report-level totals we already use directly, or belong to report types (80/90), which we already exclude. | + +### A note about inclusion/exclusion models: + +The scraper uses different default-inclusion rules for `reports.txt` and `report-items.txt`. + +For `reports.txt` (report type codes), we include entries by default and use a blocklist to exclude specific reports. Every report's totals are counted in Total Raised/Spent unless it is in one of 3 exclude sets, Year-End rollups, Dissolution/Transition rollups, or Credit Card/Reimbursement supplements. (Detailed in above table). A report type OCPF introduces that we haven't seen before would be summed in automatically. + +For `report-items.txt` (transaction/record codes), we only allow explicitly named codes. A transaction only counts toward a category if its code is explicitly included (201, 202, 203, 204, 206/331, 332, 401–403, 420, 319) (Detailed in above table). Any other code, including any OCPF introduces later, is silently skipped. + +Reasoning for this choice: + +Each row in `reports.txt` includes OCPF-computed totals representing real committee activity for that report. The risk of excluding a legitimate report type is undercounting a candidate's real fundraising. A small, enumerable set of known duplicates (rollups, supplements) is blocklisted instead. + +`report-items.txt`, by contrast, is used for the Contributions Breakdown list, adding only to specific categories (Individual donors, Committees/PACs, etc). Since there's no generic "misc" bucket a transaction can safely fall into, a code is only counted once we know which category it belongs to. + +Missing an item code leaves a potential gap in the breakdown chart. Missing a report type would corrupt the headline number. That risk analysis led to the different inclusion defaults. + +### What we show on a legislator's Finance tab + +- **Total Raised** — net receipts, "as of" the most recent Bank Report on file, minus non-contribution receipts (refunds/misc). Matches OCPF's own public "Receipts" figure; verified to the penny against ocpf.us for CPF 16883 (Rausch). +- **Total Spent** — net expenditures for the cycle +- **Cash on Hand** — most recent Bank Report's ending balance +- **Small Donors** — % of contribution money from donations under $200 +- **Contributions Breakdown** — a table/chart of: Individual donors, Committees/PACs, Unions & associations, Micro-donations (under $50), Candidate's own funds, In-kind contributions — each shown with dollar amount and % share, "as of" the most recent Deposit Report on file +- Footnote disclosing processing fees (why the breakdown total is larger than Total Raised) + +### Known caveats, already disclosed on the tab + +1. **Scope is 2025–2026 only.** We don't track or display money carried over from a prior cycle ("Start Balance"). This means **Total Spent can legitimately exceed Total Raised** for a legislator who started the cycle with leftover cash. (Confirmed with a real example: one legislator's numbers only balance once you back into a ~$45K starting balance from the prior cycle.) +2. **Total Raised (net) will typically differ from the sum of the Contributions Breakdown (gross)** by a validated, exact identity: + `Total Raised = Breakdown Sum − Processing Fees − Money Itemized But Not Yet Bank-Reported` + The last term is a normal, ongoing lag (Deposit Reports are filed per-transaction; Bank Reports are filed on a fixed periodic schedule). Both figures carry their own "as of" date on the tab for transparency. +3. **Cash on Hand will not exactly equal Start Balance + Total Raised − Total Spent**, off by the amount of non-contribution receipts for the period. Cash on Hand and Total Spent both come straight from the Bank Report (which includes non-contribution cash that actually hit the account), while Total Raised has that same amount netted out (see above) to match OCPF's own "Receipts" figure. This is expected — OCPF's own displayed numbers have the identical property (confirmed against ocpf.us for CPF 16883). +4. **"Small Donors" (<$200) is defined as**: itemized individual contributions under $200 (record type 201, amount < $200) plus the report-level unitemized aggregate (contributions ≤ $50, which by law don't need to be itemized at all). This lines up with MA's real $200/year cumulative threshold for when a committee must start disclosing a donor's occupation/employer. +5. **Contributions count is a transaction count, not a unique-donor count** — a person giving 5 separate times counts as 5, not 1. diff --git a/functions/src/ocpf/scrapeOcpfFinance.ts b/functions/src/ocpf/scrapeOcpfFinance.ts index bb39e1edc..a25109937 100644 --- a/functions/src/ocpf/scrapeOcpfFinance.ts +++ b/functions/src/ocpf/scrapeOcpfFinance.ts @@ -64,7 +64,7 @@ interface MemberAccumulator { startBalance: number startBalanceStartDateMs: number // Start_Date (as ms) of the earliest Bank Report (type 70) seen depositEndDateMs: number // End_Date (as ms) of the most recent Deposit Report (type 60) seen - contributorCount: number + contributionsCount: number breakdown: { individual: MutableBreakdownEntry committee: MutableBreakdownEntry @@ -130,7 +130,7 @@ function newAccumulator(cpfId: number): MemberAccumulator { startBalance: 0, startBalanceStartDateMs: Infinity, depositEndDateMs: 0, - contributorCount: 0, + contributionsCount: 0, breakdown: { individual: emptyEntry(), committee: emptyEntry(), @@ -292,7 +292,7 @@ export const scrapeOcpfFinance = functions totalSpent: acc.totalSpent, cashOnHand: acc.cashOnHand, startBalance: acc.startBalance, - contributorCount: acc.contributorCount, + contributionsCount: acc.contributionsCount, lastUpdated: now, bankDataAsOf: Timestamp.fromMillis(acc.cashOnHandEndDateMs), depositDataAsOf: Timestamp.fromMillis(acc.depositEndDateMs), @@ -583,7 +583,7 @@ function accumulateItem( switch (recordTypeId) { case 201: // Individual Contribution addTo(acc.breakdown.individual, yb?.individual) - acc.contributorCount++ + acc.contributionsCount++ if (amount < 200) { addTo(acc.breakdown.smallDonors.itemized, yb?.smallDonors?.itemized) } diff --git a/functions/src/ocpf/types.ts b/functions/src/ocpf/types.ts index 89b5b1b1f..0b81e5545 100644 --- a/functions/src/ocpf/types.ts +++ b/functions/src/ocpf/types.ts @@ -82,7 +82,7 @@ export interface MembersFinance { // i.e. cash on hand at the start of the current election cycle. // TODO: Surface this on the Finance tab. startBalance: number - contributorCount: number // count of type-201 rows (row = one itemized contribution) + contributionsCount: number // count of type-201 rows (row = one itemized contribution) lastUpdated: FirebaseFirestore.Timestamp // End_Date of the most recent Bank Report (type 70) — the basis for totalRaised/cashOnHand. bankDataAsOf?: FirebaseFirestore.Timestamp diff --git a/public/locales/en/legislators.json b/public/locales/en/legislators.json index 79d2da6fc..7e7f142f0 100644 --- a/public/locales/en/legislators.json +++ b/public/locales/en/legislators.json @@ -30,7 +30,7 @@ "smallDonorsSubtitle": "Under $200 contributions", "totalRaised": "Total Raised*", "totalRaisedFootnote": "*after processing fees, through {{date}}", - "totalRaisedSubtitle": "From {{count}} contributors", + "totalRaisedSubtitle": "From {{count}} contributions", "totalSpent": "Total Spent", "totalSpentSubtitle": "As of {{date}}" },