From d7fa8adf44dfb66d6ceb3bdee97d31b3b30ac480 Mon Sep 17 00:00:00 2001 From: Aaron Lisman Date: Fri, 23 Oct 2026 02:40:25 +0000 Subject: [PATCH] "nan" and "inf" in stored values are missing numbers, not NaN: one numeric read for every query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some studies record a missing number as the text "nan", "NaN" or "inf". On the public warehouse: prostate_pcbm_swiss_2019 has PSA in 160 patients plus CGA, SYP and Gleason; hcc_meric_2021 has survival months and age; pancan_pdx_uthsa_2023 has telomere measures. toFloat64OrNull reads those as NaN or ±Infinity, not NULL, so they passed every "IS NOT NULL". Explore's histogram binning then failed outright: "Code: 70 … Unexpected inf or nan to integer conversion". This broke Explore for any selection containing such a study, on the internal instance and here alike. Elsewhere the same read was a quiet wrong answer: a NaN minimum or median, a NaN survival time, a NaN plot point. lib/sql-number.ts's finiteSql() is now THE numeric read of a stored value: a finite Float64, or NULL for anything that is not a number, NaN and ±Infinity included. All 49 such reads in lib use it. That covers Explore's statistics and histograms, cohort range clauses, plots, survival months, oncoprint heatmaps, OQL's expression thresholds, composition and timeline lab values. The generic-assay summary already guarded with isFinite and keeps its own form. A unit test fails on any bare toFloat64OrNull in lib outside those two, and the two tests that matched the old SQL text now build their expectation from finiteSql. Verified: Explore on prostate_pcbm_swiss_2019 alone (the failing case), pooled with prad_tcga, and on hcc_meric_2021 and pancan_pdx_uthsa_2023 loads with no error and no non-finite minimum, maximum or median. No gallery example's pinned count moved. scripts/gate.sh green: 172 checks, 1,437 unit tests, 37 cohort-logic checks. A first run had one save-cohort browser step fail, at a menu read unrelated to this change; it passed on the re-run with no code change. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TyVfAhLnjshk2kWewGnLum --- lib/clinical-table.ts | 3 ++- lib/cohort2.ts | 9 +++++---- lib/compiler.ts | 3 ++- lib/composition.ts | 3 ++- lib/explore.ts | 17 +++++++++-------- lib/federation-views.ts | 17 +++++++++-------- lib/federation.ts | 27 ++++++++++++++------------- lib/oncoprint.ts | 5 +++-- lib/oql.ts | 3 ++- lib/plot.ts | 13 +++++++------ lib/sql-number.ts | 10 ++++++++++ test/clinical-table.test.ts | 6 ++++-- test/plot.test.ts | 4 +++- test/sql-number.test.ts | 18 ++++++++++++++++++ 14 files changed, 90 insertions(+), 48 deletions(-) create mode 100644 lib/sql-number.ts create mode 100644 test/sql-number.test.ts diff --git a/lib/clinical-table.ts b/lib/clinical-table.ts index 13284859..925874c7 100644 --- a/lib/clinical-table.ts +++ b/lib/clinical-table.ts @@ -5,6 +5,7 @@ import { sq } from "./compiler.ts"; import type { QueryColumn, RowSource } from "./table-query.ts"; import { clinicalTable, clinicalMetaTable } from "./custom-data.ts"; +import { finiteSql } from "./sql-number.ts"; export interface TableColumn { id: string; name: string; @@ -81,7 +82,7 @@ export function clinicalRowSource(parts: TablePart[], db: string, unit: "sample" if (!parts.length) throw new Error("clinicalRowSource needs at least one part"); const attr = (c: TableColumn): QueryColumn => { const v = "m[" + sq(c.id) + "]"; - return { id: c.id, name: c.name, numeric: c.numeric, text: v, ...(c.numeric ? { num: "toFloat64OrNull(" + v + ")" } : {}) }; + return { id: c.id, name: c.name, numeric: c.numeric, text: v, ...(c.numeric ? { num: finiteSql(v) } : {}) }; }; return { with: allRows(parts, db, unit, columns), diff --git a/lib/cohort2.ts b/lib/cohort2.ts index aa96e000..3d28bdac 100644 --- a/lib/cohort2.ts +++ b/lib/cohort2.ts @@ -26,6 +26,7 @@ import type { ProfiledLeaf, CaseListLeaf, GenomicDataLeaf, GenericAssayLeaf, SampleListLeaf, PatientListLeaf, SplitLeaf, AlterationLeaf, EventLeafFields, ProteinPositionLeaf } from "./types.ts"; +import { finiteSql } from "./sql-number.ts"; /** Every leaf compiler receives a fully-resolved context. */ type Ctx = CompileCtx & { study: string }; @@ -69,7 +70,7 @@ function clinicalLeafSql(leaf: ClinicalLeaf, ctx: Ctx): string { leaf.values.slice(0, 50).map(v => clinicalValueFoldSql(sq(String(v)))).join(",") + ")"); } if (leaf.range && typeof leaf.range === "object") { - const r = leaf.range as Record, num = "toFloat64OrNull(attribute_value)"; + const r = leaf.range as Record, num = finiteSql("attribute_value"); for (const [k, opSym] of [["gt", ">"], ["gte", ">="], ["lt", "<"], ["lte", "<="]]) { if (r[k] != null) { if (!Number.isFinite(Number(r[k]))) throw new Error("clinical range." + k + " must be a number"); @@ -84,7 +85,7 @@ function clinicalLeafSql(leaf: ClinicalLeaf, ctx: Ctx): string { const op = CLIN_OP[leaf.op]; if (!op) throw new Error("invalid clinical op (use = != > >= < <=)"); conds.push(typeof leaf.value === "number" - ? "toFloat64OrNull(attribute_value) " + op + " " + Number(leaf.value) + ? finiteSql("attribute_value") + " " + op + " " + Number(leaf.value) // Equality folds case and whitespace, matching how the facet that offered // this value counted it. Ordering comparisons do not: they are for // numbers, and folding a string range would change what it means. @@ -320,7 +321,7 @@ function genomicDataLeafSql(leaf: GenomicDataLeaf, ctx: Ctx): string { const pt = String(leaf.profileType || ""); if (ctx.profiles && !ctx.profiles.includes(pt)) throw new Error("genomicData: profile '" + pt + "' not in this study (available: " + (ctx.profiles.join(", ") || "none") + ")"); - const conds = numRangeConds(leaf.range, "toFloat64OrNull(ga.alteration_value)"); + const conds = numRangeConds(leaf.range, finiteSql("ga.alteration_value")); if (Array.isArray(leaf.values) && leaf.values.length) conds.push("ga.alteration_value IN (" + leaf.values.slice(0, 20).map(v => sq(String(v))).join(",") + ")"); if (!conds.length) throw new Error("genomicData clause needs range{gt,gte,lt,lte} or values[]"); @@ -339,7 +340,7 @@ function genericAssayLeafSql(leaf: GenericAssayLeaf, ctx: Ctx): string { if (!pt || !entity) throw new Error("genericAssay clause needs profileType and entity"); if (ctx.gaProfiles && !ctx.gaProfiles.includes(pt)) throw new Error("genericAssay: profile '" + pt + "' not in this study (available: " + ctx.gaProfiles.join(", ") + ")"); - const conds = numRangeConds(leaf.range, "toFloat64OrNull(value)"); + const conds = numRangeConds(leaf.range, finiteSql("value")); if (Array.isArray(leaf.values) && leaf.values.length) conds.push("value IN (" + leaf.values.slice(0, 20).map(v => sq(String(v))).join(",") + ")"); if (!conds.length) throw new Error("genericAssay clause needs range or values[]"); diff --git a/lib/compiler.ts b/lib/compiler.ts index 201ec940..5f3b2d9d 100644 --- a/lib/compiler.ts +++ b/lib/compiler.ts @@ -13,6 +13,7 @@ import type { EventSelector, EventRel, Occurrence } from "./types.ts"; import { foldedEqualsSql, canonicalEventName } from "./event-names.ts"; import { linesCtes, compositionKeySql, DEFAULT_LINE_GAP, LINE_TYPE } from "./treatment-lines.ts"; +import { finiteSql } from "./sql-number.ts"; /** A selector after runtime validation: defaults applied, fields narrowed. */ export interface ValidSelector { @@ -88,7 +89,7 @@ function selectorCte(name: string, sel: ValidSelector, db: string, priorGuard: b }); if (sel.numeric) { joins.push("JOIN " + db + ".clinical_event_data nx ON nx.clinical_event_id = ce.clinical_event_id AND " + foldedEqualsSql("nx.key", sel.numeric.key)); - conds.push("toFloat64OrNull(nx.value) " + sel.numeric.op + " " + sel.numeric.value); + conds.push(finiteSql("nx.value") + " " + sel.numeric.op + " " + sel.numeric.value); } if (sel.minDurationDays) { conds.push("ce.stop_date IS NOT NULL AND ce.stop_date - ce.start_date >= " + sel.minDurationDays); diff --git a/lib/composition.ts b/lib/composition.ts index 51dff070..9a4f821c 100644 --- a/lib/composition.ts +++ b/lib/composition.ts @@ -9,6 +9,7 @@ import { sq } from "./compiler.ts"; import { axisSelect } from "./plot.ts"; import { ATTR_NAME_RE } from "./custom-data.ts"; +import { finiteSql } from "./sql-number.ts"; /** the most parts one figure can tell apart: the categorical palette, then its tints */ export const COMPOSITION_MAX_PARTS = 16; @@ -57,7 +58,7 @@ export function partsSql(s: CompositionSource, study: string, db: string, cohort const idCol = s.unit === "patient" ? "patient_unique_id" : "sample_unique_id"; return "SELECT id, c, assumeNotNull(avg(val)) AS v FROM (\n" + " SELECT " + idCol + " AS id, toUInt16(indexOf([" + s.parts.map(sq).join(",") + "], entity_stable_id) - 1) AS c,\n" + - " toFloat64OrNull(value) AS val\n" + + " " + finiteSql("value") + " AS val\n" + " FROM " + db + ".generic_assay_data_derived\n" + " WHERE profile_type = " + sq(s.profileType!) + " AND profile_stable_id = " + sq(study + "_" + s.profileType) + "\n" + " AND entity_stable_id IN (" + s.parts.map(sq).join(",") + ")\n" + diff --git a/lib/explore.ts b/lib/explore.ts index 49f238ce..de636036 100644 --- a/lib/explore.ts +++ b/lib/explore.ts @@ -149,6 +149,7 @@ const GENE_TABLE_LIMIT = 500; /** how many genes may carry their own alteration facet at once */ const GENE_FACET_LIMIT = 6; import { isClinicalLeaf } from "./types.ts"; +import { finiteSql } from "./sql-number.ts"; export interface ExploreArgs { @@ -969,14 +970,14 @@ async function prefetchFacets(opts: { "WHERE cs.cancer_study_identifier IN " + IN, conn).catch(() => ({ data: [] as ChRow[] })), chQuery( "SELECT cancer_study_identifier AS s, type, attribute_name AS a, uniqExact(" + clinicalValueFoldSql("attribute_value") + ") AS nv,\n" + - " countIf(toFloat64OrNull(attribute_value) IS NOT NULL) / count() AS numratio,\n" + + " countIf(" + finiteSql("attribute_value") + " IS NOT NULL) / count() AS numratio,\n" + " uniqExact(if(type = 'sample', sample_unique_id, patient_unique_id)) AS np,\n" + - " min(toFloat64OrNull(attribute_value)) AS mn, max(toFloat64OrNull(attribute_value)) AS mx,\n" + - " quantileExact(0.5)(toFloat64OrNull(attribute_value)) AS md,\n" + - " quantileExact(0.25)(toFloat64OrNull(attribute_value)) AS q1,\n" + - " quantileExact(0.75)(toFloat64OrNull(attribute_value)) AS q3,\n" + - " countIf(toFloat64OrNull(attribute_value) IS NULL) AS na,\n" + - " countIf(toFloat64OrNull(attribute_value) = round(toFloat64OrNull(attribute_value))) = countIf(toFloat64OrNull(attribute_value) IS NOT NULL) AS allints\n" + + " min(" + finiteSql("attribute_value") + ") AS mn, max(" + finiteSql("attribute_value") + ") AS mx,\n" + + " quantileExact(0.5)(" + finiteSql("attribute_value") + ") AS md,\n" + + " quantileExact(0.25)(" + finiteSql("attribute_value") + ") AS q1,\n" + + " quantileExact(0.75)(" + finiteSql("attribute_value") + ") AS q3,\n" + + " countIf(" + finiteSql("attribute_value") + " IS NULL) AS na,\n" + + " countIf(" + finiteSql("attribute_value") + " = round(" + finiteSql("attribute_value") + ")) = countIf(" + finiteSql("attribute_value") + " IS NOT NULL) AS allints\n" + "FROM " + clinicalTable(db, attrAllow.length ? attrAllow : "all") + "\n" + "WHERE cancer_study_identifier IN " + IN + " AND attribute_value != ''" + cond + (attrAllow.length ? "\n AND attribute_name IN (" + attrAllow.map(sq).join(",") + ")" : "") + "\n" + @@ -1086,7 +1087,7 @@ async function pooledHistograms(opts: { const r = await chQuery( "SELECT multiIf(v <= " + edges[0] + ", -1, v > " + lastEdge + ", -2, toInt32(ceil((v - " + edges[0] + ") / " + interval + ")) - 1) AS b,\n" + " uniqExact(id) AS n\n" + - "FROM (SELECT toFloat64OrNull(attribute_value) AS v, " + unit + " AS id\n" + + "FROM (SELECT " + finiteSql("attribute_value") + " AS v, " + unit + " AS id\n" + " FROM " + clinicalTable(db, [f.attribute]) + "\n" + " WHERE cancer_study_identifier IN (" + studies.map(sq).join(",") + ") AND attribute_name = " + sq(f.attribute) + " AND type = " + sq(f.per) + " AND attribute_value != ''" + cond + ")\n" + diff --git a/lib/federation-views.ts b/lib/federation-views.ts index 1e0a025e..60f62e47 100644 --- a/lib/federation-views.ts +++ b/lib/federation-views.ts @@ -9,6 +9,7 @@ import { clinicalValueFoldSql, nameFoldSql, foldedEqualsSql } from "./event-name import { clinicalTable } from "./custom-data.ts"; import { splitBucketSql } from "./cohort2.ts"; import { casesCte, type CompositionSource } from "./composition.ts"; +import { finiteSql } from "./sql-number.ts"; /** bare patient id for a *_unique_id column of this study */ const pidOf = (col: string, study: string): string => "replaceOne(" + col + ", " + sq(study + "_") + ", '')"; @@ -169,7 +170,7 @@ export function geneSetCountsSql(cohortSql: string, sets: { name: string; genes: */ export function profileMomentsAllSql(cohortSql: string, study: string, db: string, profileType: string, log2: boolean): string { const pid = pidOf("sd.patient_unique_id", study); - const raw = "toFloat64OrNull(ga.alteration_value)"; + const raw = finiteSql("ga.alteration_value"); const val = log2 ? "log2(" + raw + " + 1)" : raw; return ( "WITH cohort AS (" + cohortSql + "),\n" + @@ -382,12 +383,12 @@ export function facetsHistogramSql(cohortSql: string, study: string, db: string, return ( facetsSamplesCte(cohortSql, study, db, qc) + "\n" + "SELECT attribute_name AS attr,\n" + - " toInt32(floor(least(if(attribute_name = 'FACETS_PURITY', 0.9999, 7.9999), greatest(0, toFloat64OrNull(attribute_value))) /\n" + + " toInt32(floor(least(if(attribute_name = 'FACETS_PURITY', 0.9999, 7.9999), greatest(0, " + finiteSql("attribute_value") + ")) /\n" + " if(attribute_name = 'FACETS_PURITY', 0.05, 0.25))) AS bin,\n" + " count() AS n\n" + "FROM " + clinicalTable(db, ["FACETS_PURITY", "FACETS_PLOIDY"]) + "\n" + "WHERE cancer_study_identifier = " + sq(study) + " AND type = 'sample' AND attribute_name IN ('FACETS_PURITY', 'FACETS_PLOIDY')\n" + - " AND sample_unique_id IN (SELECT suid FROM samp) AND toFloat64OrNull(attribute_value) IS NOT NULL\n" + + " AND sample_unique_id IN (SELECT suid FROM samp) AND " + finiteSql("attribute_value") + " IS NOT NULL\n" + "GROUP BY attr, bin ORDER BY attr, bin" ); } @@ -452,7 +453,7 @@ export function incidenceCountsSql(cohortSql: string, study: string, db: string, "WITH cohort AS (" + cohortSql + "),\n" + "os AS (\n" + " SELECT " + pid + " AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name = 'OS_MONTHS')) AS months,\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name = 'OS_MONTHS')") + " AS months,\n" + " anyIf(attribute_value, attribute_name = 'OS_STATUS') AS status\n" + " FROM " + clinicalTable(db, ["OS_MONTHS", "OS_STATUS"]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type = 'patient' AND attribute_name IN ('OS_MONTHS', 'OS_STATUS')\n" + @@ -529,14 +530,14 @@ function labCtes(cohortSql: string, study: string, db: string, lab: LabSpec, anc " GROUP BY pid\n" + "),\n" + "lab AS (\n" + - " SELECT p.stable_id AS pid, ce.start_date AS day, toFloat64OrNull(r.value) AS v\n" + + " SELECT p.stable_id AS pid, ce.start_date AS day, " + finiteSql("r.value") + " AS v\n" + " FROM " + db + ".clinical_event ce\n" + " JOIN " + db + ".patient p ON p.internal_id = ce.patient_id\n" + " JOIN " + db + ".cancer_study cs ON cs.cancer_study_id = p.cancer_study_id\n" + " JOIN " + db + ".clinical_event_data t ON t.clinical_event_id = ce.clinical_event_id AND " + foldedEqualsSql("t.key", lab.testKey) + " AND " + foldedEqualsSql("t.value", lab.test) + "\n" + " JOIN " + db + ".clinical_event_data r ON r.clinical_event_id = ce.clinical_event_id AND " + foldedEqualsSql("r.key", lab.resultKey) + "\n" + " WHERE cs.cancer_study_identifier = " + sq(study) + " AND " + foldedEqualsSql("ce.event_type", lab.eventType) + "\n" + - " AND p.stable_id IN (SELECT pid FROM anch) AND toFloat64OrNull(r.value) IS NOT NULL\n" + + " AND p.stable_id IN (SELECT pid FROM anch) AND " + finiteSql("r.value") + " IS NOT NULL\n" + "),\n" + "base AS (\n" + // the baseline is the mean of the readings on the last day with any, so several readings that day @@ -899,7 +900,7 @@ export function compositionEntitiesSql(study: string, db: string, profileType: s */ export function correlatesMomentsSql(cohortSql: string, study: string, db: string, gene: string, profileType: string, otherProfile: string): string { const pid = pidOf("sd.patient_unique_id", study); - const raw = "toFloat64OrNull(ga.alteration_value)"; + const raw = finiteSql("ga.alteration_value"); const vals = (pt: string, geneCond: string) => " SELECT " + pid + " AS pid, ga.hugo_gene_symbol AS gene, avg(" + raw + ") AS v\n" + " FROM " + db + ".genetic_alteration_derived ga\n" + @@ -981,7 +982,7 @@ export function riskmodelRowsSql(cohortSql: string, study: string, db: string, o "WITH cohort AS (" + cohortSql + "),\n" + "os AS (\n" + " SELECT " + pid + " AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name=" + sq(months) + ")) AS months,\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name=" + sq(months) + ")") + " AS months,\n" + " anyIf(attribute_value, attribute_name=" + sq(status) + ") AS status,\n" + " " + splitBucketSql(pid, study) + " AS half\n" + " FROM " + clinicalTable(db, [months, status]) + "\n" + diff --git a/lib/federation.ts b/lib/federation.ts index 2c33ae70..c1ac377b 100644 --- a/lib/federation.ts +++ b/lib/federation.ts @@ -56,6 +56,7 @@ import type { StudyAccess } from "./view-coverage.ts"; import { PATIENT_ID, patientEventsSql, patientSamplesSql, toEvents, toSamples } from "./patient-timeline.ts"; import { clinicalTable, clinicalMetaTable, ATTR_NAME_RE } from "./custom-data.ts"; import { PATIENT_KINDS, type PatientKind } from "./disclosure.ts"; +import { finiteSql } from "./sql-number.ts"; export type InstanceMode = "patient-level" | "aggregate-only"; @@ -425,12 +426,12 @@ export function profileMomentsSql(cohortSql: string, genes: string[], profileTyp "WITH cohort AS (" + cohortSql + "),\n" + "vals AS (\n" + " SELECT " + pid + " AS pid, ga.hugo_gene_symbol AS gene,\n" + - " avg(toFloat64OrNull(ga.alteration_value)) AS v\n" + + " avg(" + finiteSql("ga.alteration_value") + ") AS v\n" + " FROM " + db + ".genetic_alteration_derived ga\n" + " JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + " WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(profileType) + "\n" + " AND ga.hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL\n" + " AND " + pid + " IN (SELECT pid FROM cohort)\n" + " GROUP BY pid, gene\n" + ")\n" + @@ -567,12 +568,12 @@ export function momentPairsSql(cohortSql: string, pairs: [string, string][], pro "WITH cohort AS (" + cohortSql + "),\n" + "vals AS (\n" + " SELECT " + pid + " AS pid, ga.hugo_gene_symbol AS gene,\n" + - " avg(toFloat64OrNull(ga.alteration_value)) AS v\n" + + " avg(" + finiteSql("ga.alteration_value") + ") AS v\n" + " FROM " + db + ".genetic_alteration_derived ga\n" + " JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + " WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(profileType) + "\n" + " AND ga.hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL\n" + " AND " + pid + " IN (SELECT pid FROM cohort)\n" + " GROUP BY pid, gene\n" + "),\n" + @@ -632,7 +633,7 @@ export function clinicalMomentsSql(cohortSql: string, attrs: string[], study: st "WITH cohort AS (" + cohortSql + ")\n" + "SELECT a, per, count() AS n, sum(x) AS sx, sum(x * x) AS sxx, min(x) AS mn, max(x) AS mx\n" + "FROM (\n" + - " SELECT attribute_name AS a, type AS per, toFloat64OrNull(attribute_value) AS x\n" + + " SELECT attribute_name AS a, type AS per, " + finiteSql("attribute_value") + " AS x\n" + " FROM " + clinicalTable(db, attrs) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + "\n" + " AND attribute_name IN (" + attrs.map(sq).join(",") + ")\n" + @@ -663,7 +664,7 @@ export function clinicalHistogramSql(cohortSql: string, attrs: string[], study: return ( "WITH cohort AS (" + cohortSql + "),\n" + " raw AS (\n" + - " SELECT attribute_name AS a, type AS per, toFloat64OrNull(attribute_value) AS x\n" + + " SELECT attribute_name AS a, type AS per, " + finiteSql("attribute_value") + " AS x\n" + " FROM " + clinicalTable(db, attrs) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + "\n" + " AND attribute_name IN (" + attrs.map(sq).join(",") + ")\n" + @@ -691,7 +692,7 @@ export function attributeQuantileSql(cohortSql: string, attribute: string, per: const pid = "replaceOne(patient_unique_id, " + sq(study + "_") + ", '')"; return ( "WITH cohort AS (" + cohortSql + ")\n" + - "SELECT quantileExact(" + p + ")(toFloat64OrNull(attribute_value)) AS q\n" + + "SELECT quantileExact(" + p + ")(" + finiteSql("attribute_value") + ") AS q\n" + "FROM " + clinicalTable(db, [attribute]) + "\n" + "WHERE cancer_study_identifier = " + sq(study) + " AND attribute_name = " + sq(attribute) + "\n" + " AND type = " + sq(per) + " AND attribute_value != ''\n" + @@ -732,8 +733,8 @@ export function survivalCountsSql(cohortSql: string, study: string, db: string = " AND " + pid + " IN (SELECT pid FROM cohort)\n" + " GROUP BY pid\n" + ")\n" + - "SELECT toFloat64OrNull(months) AS m, if(status LIKE '1%', 1, 0) AS ev, count() AS n\n" + - "FROM os WHERE toFloat64OrNull(months) IS NOT NULL\n" + + "SELECT " + finiteSql("months") + " AS m, if(status LIKE '1%', 1, 0) AS ev, count() AS n\n" + + "FROM os WHERE " + finiteSql("months") + " IS NOT NULL\n" + "GROUP BY m, ev ORDER BY m" ); } @@ -769,7 +770,7 @@ export function anchoredSurvivalSql( "WITH cohort AS (" + cohortSql + "),\n" + "os AS (\n" + " SELECT " + pid + " AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name=" + sq(months) + ")) AS months,\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name=" + sq(months) + ")") + " AS months,\n" + " anyIf(attribute_value, attribute_name=" + sq(status) + ") AS status\n" + " FROM " + clinicalTable(db, [months, status]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type='patient'\n" + @@ -839,7 +840,7 @@ export function coxRowsSql( "WITH cohort AS (" + cohortSql + "),\n" + "os AS (\n" + " SELECT " + pid + " AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name=" + sq(months) + ")) AS months,\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name=" + sq(months) + ")") + " AS months,\n" + " anyIf(attribute_value, attribute_name=" + sq(status) + ") AS status\n" + " FROM " + clinicalTable(db, [months, status]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type='patient'\n" + @@ -1026,7 +1027,7 @@ export function eventCourseSql( // Observation window: first recorded event to last contact. "os AS (\n" + " SELECT replaceOne(patient_unique_id, " + sq(study + "_") + ", '') AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name = 'OS_MONTHS')) * " + DAYS_PER_MONTH + " AS os_day\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name = 'OS_MONTHS')") + " * " + DAYS_PER_MONTH + " AS os_day\n" + " FROM " + clinicalTable(db, ["OS_MONTHS"]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type = 'patient'\n" + " AND attribute_name = 'OS_MONTHS'\n" + @@ -1141,7 +1142,7 @@ export function sequenceSql( // records simply stop" — see the fate expression below. "os AS (\n" + " SELECT replaceOne(patient_unique_id, " + sq(study + "_") + ", '') AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name = 'OS_MONTHS')) * " + DAYS_PER_MONTH + " AS os_day\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name = 'OS_MONTHS')") + " * " + DAYS_PER_MONTH + " AS os_day\n" + " FROM " + clinicalTable(db, ["OS_MONTHS"]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type = 'patient'\n" + " AND attribute_name = 'OS_MONTHS'\n" + diff --git a/lib/oncoprint.ts b/lib/oncoprint.ts index f4897156..e1a9f909 100644 --- a/lib/oncoprint.ts +++ b/lib/oncoprint.ts @@ -27,6 +27,7 @@ import { PROFILED_TYPES, type ProfiledType } from "./profile-names.ts"; import { assembleContract, normalizeDisplay, normalizeSort, type AssembleInput, type EventRow } from "./oncoprint-assemble.ts"; import { normalizeGeneTracks, toGeneList } from "./gene-tracks.ts"; import { clinicalTable, assertCustomDataPresent } from "./custom-data.ts"; +import { finiteSql } from "./sql-number.ts"; export { normalizeDisplay, normalizeSort } from "./oncoprint-assemble.ts"; const GENE_RE = /^[A-Z0-9][A-Z0-9.-]{0,19}$/; @@ -200,12 +201,12 @@ export function vafSql(genes: string[], study: string = STUDY, db: string = DB, export function heatmapSql(profileType: string, genes: string[], study: string = STUDY, db: string = DB, unit: Unit = "patient"): string { // values are stored per sample; only patient mode has to average them return "SELECT " + (unit === "sample" ? "replaceOne(ga.sample_unique_id, " + sq(study + "_") + ", '')" : "replaceOne(sd.patient_unique_id, " + sq(study + "_") + ", '')") + " AS pid,\n" + - " ga.hugo_gene_symbol AS gene, avg(toFloat64OrNull(ga.alteration_value)) AS v\n" + + " ga.hugo_gene_symbol AS gene, avg(" + finiteSql("ga.alteration_value") + ") AS v\n" + "FROM " + db + ".genetic_alteration_derived ga\n" + (unit === "sample" ? "" : "JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n") + "WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(profileType) + "\n" + " AND ga.hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL\n" + "GROUP BY pid, gene"; } diff --git a/lib/oql.ts b/lib/oql.ts index 251bb623..a53e9ac5 100644 --- a/lib/oql.ts +++ b/lib/oql.ts @@ -20,6 +20,7 @@ export { parseOql } from "./oql-parse.ts"; export type { OqlSpecType, OqlRange, OqlSpec, OqlEntry } from "./oql-parse.ts"; import type { OqlRange, OqlSpec } from "./oql-parse.ts"; import { parseOql } from "./oql-parse.ts"; +import { finiteSql } from "./sql-number.ts"; /** * A mutation's class as cBioPortal's OQL sees it — cbioportal-frontend's getMutationSubType and @@ -136,7 +137,7 @@ export function oqlCasesSql(q: string, { oncokbPin, profiles, study = STUDY, db " FROM " + db + ".genetic_alteration_derived ga\n" + " JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + " WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.hugo_gene_symbol = " + sq(gene) + "\n" + - " AND ga.profile_type = " + sq(prof) + " AND toFloat64OrNull(ga.alteration_value) " + spec.op + " " + spec.z); + " AND ga.profile_type = " + sq(prof) + " AND " + finiteSql("ga.alteration_value") + " " + spec.op + " " + spec.z); continue; } parts.push( diff --git a/lib/plot.ts b/lib/plot.ts index 34f14efe..f68522df 100644 --- a/lib/plot.ts +++ b/lib/plot.ts @@ -88,6 +88,7 @@ export { POINTS_CAP, POINT_LAYER_CAP, LAYER_COLORS, LAYER_BASE, DEFAULT_POINT_LA import { POINTS_CAP, POINT_RULE_N_MAX } from "./plot-layers.ts"; export { isAbsent, jitterFor } from "./plot-draw.ts"; import { isAbsent } from "./plot-draw.ts"; +import { finiteSql } from "./sql-number.ts"; /** * Over the points cap, a scatter cell holding at most this many cases is drawn as its cases * rather than as a shade: the sparse fringe of the density, which is where the outliers are. @@ -246,8 +247,8 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s // are combined: a number is their mean, a category is the samples' common value, or "Multiple" // when they differ (combineNote says so in the caption). // Only the study knows whether the attribute is a number, so the caller says which. - const val = kind === "num" ? logOf(a, "toFloat64OrNull(attribute_value)") : "attribute_value"; - const valCd = kind === "num" ? logOf(a, "toFloat64OrNull(cd.attribute_value)") : "cd.attribute_value"; + const val = kind === "num" ? logOf(a, finiteSql("attribute_value")) : "attribute_value"; + const valCd = kind === "num" ? logOf(a, finiteSql("cd.attribute_value")) : "cd.attribute_value"; const defined = (e: string) => kind === "num" ? " AND " + e + " IS NOT NULL" : ""; const own = "SELECT " + strip(sample ? "sample_unique_id" : "patient_unique_id", study) + " AS id, " + val + " AS v, " + "1 AS pref\n" + @@ -276,12 +277,12 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s const gene = String(a.gene).toUpperCase(); const level = "multiIf(x = -2, 'Deep deletion', x = -1, 'Shallow deletion', x = 0, 'Diploid', x = 1, 'Gain', x = 2, 'Amplification', toString(x))"; return "SELECT id, if(uniqExact(lv) > 1, " + sq(MULTIPLE) + ", any(lv)) AS v FROM (\n" + - "SELECT " + strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study) + " AS id, " + level.replace(/\bx\b/g, "toFloat64OrNull(ga.alteration_value)") + " AS lv\n" + + "SELECT " + strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study) + " AS id, " + level.replace(/\bx\b/g, finiteSql("ga.alteration_value")) + " AS lv\n" + "FROM " + db + ".genetic_alteration_derived ga\n" + "JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + "WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(a.profileType) + " AND ga.hugo_gene_symbol = " + sq(gene) + "\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL AND " + inCohort("sd.patient_unique_id") + ")\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL AND " + inCohort("sd.patient_unique_id") + ")\n" + "GROUP BY id"; } if (a.source === "molecular") { @@ -289,12 +290,12 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s // patient mode averages a patient's samples, the same rule the oncoprint's heatmap uses; a // value the log leaves undefined drops the case, so coverage counts it as having none const q = "SELECT " + strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study) + " AS id,\n" + - " " + logOf(a, sample ? "argMax(toFloat64OrNull(ga.alteration_value), 1)" : "avg(toFloat64OrNull(ga.alteration_value))") + " AS v\n" + + " " + logOf(a, sample ? "argMax(" + finiteSql("ga.alteration_value") + ", 1)" : "avg(" + finiteSql("ga.alteration_value") + ")") + " AS v\n" + "FROM " + db + ".genetic_alteration_derived ga\n" + "JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + "WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(a.profileType) + " AND ga.hugo_gene_symbol = " + sq(gene) + "\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL AND " + inCohort("sd.patient_unique_id") + "\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL AND " + inCohort("sd.patient_unique_id") + "\n" + "GROUP BY id"; return a.log ? "SELECT id, v FROM (" + q + ") WHERE v IS NOT NULL" : q; } diff --git a/lib/sql-number.ts b/lib/sql-number.ts new file mode 100644 index 00000000..9958669e --- /dev/null +++ b/lib/sql-number.ts @@ -0,0 +1,10 @@ +// THE way a stored value is read as a number in SQL. Clinical values, assay and expression values +// are text in the warehouse, and some studies record a missing number as "nan", "NaN" or "inf": +// toFloat64OrNull reads those as NaN or ±Infinity, not NULL, so they would pass every IS NOT NULL +// test and then either fail a query (a NaN cannot be cast to a histogram bin) or quietly become a +// minimum, a median or a survival time. Here they are missing, like any other value that is not a +// number. Browser-safe: it only builds SQL text. + +/** `expr` as a finite Float64, or NULL when it is not a number (including NaN and ±Infinity) */ +export const finiteSql = (expr: string): string => + "if(isFinite(ifNull(toFloat64OrNull(" + expr + "), nan)), toFloat64OrNull(" + expr + "), NULL)"; diff --git a/test/clinical-table.test.ts b/test/clinical-table.test.ts index b2a199c2..bfe647c2 100644 --- a/test/clinical-table.test.ts +++ b/test/clinical-table.test.ts @@ -4,6 +4,7 @@ import { test, describe } from "node:test"; import assert from "node:assert/strict"; import { clinicalRowSource, type TableColumn } from "../lib/clinical-table.ts"; import { pageSql, countsSql } from "../lib/table-query.ts"; +import { finiteSql } from "../lib/sql-number.ts"; const cols: TableColumn[] = [{ id: "AGE", name: "Age", per: "patient", numeric: true }, { id: "TYPE", name: "Type", per: "sample", numeric: false }]; const part = (cohort: string, study: string) => ({ cohort, study, cohortSql: "SELECT '' AS pid" }); @@ -12,8 +13,9 @@ const defaults = ["patient", "sample", "AGE", "TYPE"]; describe("clinicalRowSource", () => { test("numbers sort with NULLS LAST, blanks last, then patient, sample and document order; the offset is global", () => { const src = clinicalRowSource([part("a", "s")], "db", "sample", cols); - assert.match(pageSql(src, { sort: { column: "AGE", desc: true } }, defaults, 10, 20), - /ORDER BY toFloat64OrNull\(m\['AGE'\]\) DESC NULLS LAST, patient, sample, ord\nLIMIT 10 OFFSET 20$/); + // a number sorts as its finite value; "nan" and "inf" read as missing (lib/sql-number.ts) + assert.ok(pageSql(src, { sort: { column: "AGE", desc: true } }, defaults, 10, 20) + .endsWith("ORDER BY " + finiteSql("m['AGE']") + " DESC NULLS LAST, patient, sample, ord\nLIMIT 10 OFFSET 20")); assert.match(pageSql(src, { sort: { column: "TYPE" } }, defaults, 10, 0), /ORDER BY \(m\['TYPE'\] = ''\) ASC, m\['TYPE'\] ASC, patient, sample, ord/); assert.match(pageSql(src, {}, defaults, 10, 0), /ORDER BY patient, sample, patient, sample, ord/); }); diff --git a/test/plot.test.ts b/test/plot.test.ts index 427b8249..6ea349c3 100644 --- a/test/plot.test.ts +++ b/test/plot.test.ts @@ -6,6 +6,7 @@ import { axisKind, axisLabel, axisSelect, corrPValue, plotBinsSql, plotBoxSql, p plotShape, plotStatsSql, plotTableSql, plotPointsSql, plotViolinSql, categoryColor, isAbsent, tukey, validateAxis, CATEGORY_CAP, DEFAULT_BINS , coverageNote, combineNote, plotCoverageSql, plotScatterPointsSql, type PlotData } from "../lib/plot.ts"; import { STYLE } from "../lib/oncoprint-style.ts"; +import { finiteSql } from "../lib/sql-number.ts"; const ctx = { study: "x", db: "db", unit: "sample" as const, cohortSql: "SELECT pid FROM p" }; const num = { source: "clinical" as const, attribute: "MUTATION_COUNT" }; @@ -203,7 +204,8 @@ test("a plot says how many cases the pair left out, and on which axis", () => { test("log: true reads log10(v + 1), leaves out what has no such log, and says so in the label", () => { const ctx = { study: "s", db: "db", unit: "sample" as const, cohortSql: "SELECT '' AS pid" }; const clin = axisSelect({ source: "clinical", attribute: "MUTATION_COUNT", log: true }, ctx, "num"); - assert.match(clin, /if\(toFloat64OrNull\(attribute_value\) >= 0, log10\(toFloat64OrNull\(attribute_value\) \+ 1\), NULL\) IS NOT NULL/); + const v = finiteSql("attribute_value"); + assert.ok(clin.includes("if(" + v + " >= 0, log10(" + v + " + 1), NULL) IS NOT NULL"), clin); assert.doesNotMatch(axisSelect({ source: "clinical", attribute: "MUTATION_COUNT" }, ctx, "num"), /log10/); const mol = axisSelect({ source: "molecular", profileType: "mrna", gene: "ESR1", log: true }, ctx, "num"); assert.match(mol, /^SELECT id, v FROM \(/); diff --git a/test/sql-number.test.ts b/test/sql-number.test.ts new file mode 100644 index 00000000..03c8102f --- /dev/null +++ b/test/sql-number.test.ts @@ -0,0 +1,18 @@ +// THE numeric read of a stored value (lib/sql-number.ts): NaN and ±Infinity are missing, like any +// other text that is not a number, so they never reach a histogram bin, a minimum or a survival time. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import { finiteSql } from "../lib/sql-number.ts"; + +test("finiteSql keeps a finite number and reads NaN and infinity as NULL", () => { + assert.equal(finiteSql("v"), "if(isFinite(ifNull(toFloat64OrNull(v), nan)), toFloat64OrNull(v), NULL)"); +}); + +test("every numeric read of a value in lib goes through finiteSql", () => { + // one module may name toFloat64OrNull bare: its own; a summary that guards with isFinite itself is the other + const bare = (readdirSync("lib", { recursive: true }) as string[]).filter((f) => f.endsWith(".ts") && f !== "sql-number.ts" && f !== "dsl-reference.ts") + .flatMap((f) => readFileSync("lib/" + f, "utf8").split("\n").map((line, i) => ({ f, i: i + 1, line }))) + .filter(({ line }) => line.includes("toFloat64OrNull(") && !line.includes("isFinite(")); + assert.deepEqual(bare.map(({ f, i }) => f + ":" + i), []); +});