diff --git a/lib/clinical-table.ts b/lib/clinical-table.ts index 1328485..925874c 100644 --- a/lib/clinical-table.ts +++ b/lib/clinical-table.ts @@ -5,6 +5,7 @@ import { sq } from "./compiler.ts"; import type { QueryColumn, RowSource } from "./table-query.ts"; import { clinicalTable, clinicalMetaTable } from "./custom-data.ts"; +import { finiteSql } from "./sql-number.ts"; export interface TableColumn { id: string; name: string; @@ -81,7 +82,7 @@ export function clinicalRowSource(parts: TablePart[], db: string, unit: "sample" if (!parts.length) throw new Error("clinicalRowSource needs at least one part"); const attr = (c: TableColumn): QueryColumn => { const v = "m[" + sq(c.id) + "]"; - return { id: c.id, name: c.name, numeric: c.numeric, text: v, ...(c.numeric ? { num: "toFloat64OrNull(" + v + ")" } : {}) }; + return { id: c.id, name: c.name, numeric: c.numeric, text: v, ...(c.numeric ? { num: finiteSql(v) } : {}) }; }; return { with: allRows(parts, db, unit, columns), diff --git a/lib/cohort2.ts b/lib/cohort2.ts index aa96e00..3d28bda 100644 --- a/lib/cohort2.ts +++ b/lib/cohort2.ts @@ -26,6 +26,7 @@ import type { ProfiledLeaf, CaseListLeaf, GenomicDataLeaf, GenericAssayLeaf, SampleListLeaf, PatientListLeaf, SplitLeaf, AlterationLeaf, EventLeafFields, ProteinPositionLeaf } from "./types.ts"; +import { finiteSql } from "./sql-number.ts"; /** Every leaf compiler receives a fully-resolved context. */ type Ctx = CompileCtx & { study: string }; @@ -69,7 +70,7 @@ function clinicalLeafSql(leaf: ClinicalLeaf, ctx: Ctx): string { leaf.values.slice(0, 50).map(v => clinicalValueFoldSql(sq(String(v)))).join(",") + ")"); } if (leaf.range && typeof leaf.range === "object") { - const r = leaf.range as Record, num = "toFloat64OrNull(attribute_value)"; + const r = leaf.range as Record, num = finiteSql("attribute_value"); for (const [k, opSym] of [["gt", ">"], ["gte", ">="], ["lt", "<"], ["lte", "<="]]) { if (r[k] != null) { if (!Number.isFinite(Number(r[k]))) throw new Error("clinical range." + k + " must be a number"); @@ -84,7 +85,7 @@ function clinicalLeafSql(leaf: ClinicalLeaf, ctx: Ctx): string { const op = CLIN_OP[leaf.op]; if (!op) throw new Error("invalid clinical op (use = != > >= < <=)"); conds.push(typeof leaf.value === "number" - ? "toFloat64OrNull(attribute_value) " + op + " " + Number(leaf.value) + ? finiteSql("attribute_value") + " " + op + " " + Number(leaf.value) // Equality folds case and whitespace, matching how the facet that offered // this value counted it. Ordering comparisons do not: they are for // numbers, and folding a string range would change what it means. @@ -320,7 +321,7 @@ function genomicDataLeafSql(leaf: GenomicDataLeaf, ctx: Ctx): string { const pt = String(leaf.profileType || ""); if (ctx.profiles && !ctx.profiles.includes(pt)) throw new Error("genomicData: profile '" + pt + "' not in this study (available: " + (ctx.profiles.join(", ") || "none") + ")"); - const conds = numRangeConds(leaf.range, "toFloat64OrNull(ga.alteration_value)"); + const conds = numRangeConds(leaf.range, finiteSql("ga.alteration_value")); if (Array.isArray(leaf.values) && leaf.values.length) conds.push("ga.alteration_value IN (" + leaf.values.slice(0, 20).map(v => sq(String(v))).join(",") + ")"); if (!conds.length) throw new Error("genomicData clause needs range{gt,gte,lt,lte} or values[]"); @@ -339,7 +340,7 @@ function genericAssayLeafSql(leaf: GenericAssayLeaf, ctx: Ctx): string { if (!pt || !entity) throw new Error("genericAssay clause needs profileType and entity"); if (ctx.gaProfiles && !ctx.gaProfiles.includes(pt)) throw new Error("genericAssay: profile '" + pt + "' not in this study (available: " + ctx.gaProfiles.join(", ") + ")"); - const conds = numRangeConds(leaf.range, "toFloat64OrNull(value)"); + const conds = numRangeConds(leaf.range, finiteSql("value")); if (Array.isArray(leaf.values) && leaf.values.length) conds.push("value IN (" + leaf.values.slice(0, 20).map(v => sq(String(v))).join(",") + ")"); if (!conds.length) throw new Error("genericAssay clause needs range or values[]"); diff --git a/lib/compiler.ts b/lib/compiler.ts index 201ec94..5f3b2d9 100644 --- a/lib/compiler.ts +++ b/lib/compiler.ts @@ -13,6 +13,7 @@ import type { EventSelector, EventRel, Occurrence } from "./types.ts"; import { foldedEqualsSql, canonicalEventName } from "./event-names.ts"; import { linesCtes, compositionKeySql, DEFAULT_LINE_GAP, LINE_TYPE } from "./treatment-lines.ts"; +import { finiteSql } from "./sql-number.ts"; /** A selector after runtime validation: defaults applied, fields narrowed. */ export interface ValidSelector { @@ -88,7 +89,7 @@ function selectorCte(name: string, sel: ValidSelector, db: string, priorGuard: b }); if (sel.numeric) { joins.push("JOIN " + db + ".clinical_event_data nx ON nx.clinical_event_id = ce.clinical_event_id AND " + foldedEqualsSql("nx.key", sel.numeric.key)); - conds.push("toFloat64OrNull(nx.value) " + sel.numeric.op + " " + sel.numeric.value); + conds.push(finiteSql("nx.value") + " " + sel.numeric.op + " " + sel.numeric.value); } if (sel.minDurationDays) { conds.push("ce.stop_date IS NOT NULL AND ce.stop_date - ce.start_date >= " + sel.minDurationDays); diff --git a/lib/composition.ts b/lib/composition.ts index 51dff07..9a4f821 100644 --- a/lib/composition.ts +++ b/lib/composition.ts @@ -9,6 +9,7 @@ import { sq } from "./compiler.ts"; import { axisSelect } from "./plot.ts"; import { ATTR_NAME_RE } from "./custom-data.ts"; +import { finiteSql } from "./sql-number.ts"; /** the most parts one figure can tell apart: the categorical palette, then its tints */ export const COMPOSITION_MAX_PARTS = 16; @@ -57,7 +58,7 @@ export function partsSql(s: CompositionSource, study: string, db: string, cohort const idCol = s.unit === "patient" ? "patient_unique_id" : "sample_unique_id"; return "SELECT id, c, assumeNotNull(avg(val)) AS v FROM (\n" + " SELECT " + idCol + " AS id, toUInt16(indexOf([" + s.parts.map(sq).join(",") + "], entity_stable_id) - 1) AS c,\n" + - " toFloat64OrNull(value) AS val\n" + + " " + finiteSql("value") + " AS val\n" + " FROM " + db + ".generic_assay_data_derived\n" + " WHERE profile_type = " + sq(s.profileType!) + " AND profile_stable_id = " + sq(study + "_" + s.profileType) + "\n" + " AND entity_stable_id IN (" + s.parts.map(sq).join(",") + ")\n" + diff --git a/lib/explore.ts b/lib/explore.ts index 49f238c..de63603 100644 --- a/lib/explore.ts +++ b/lib/explore.ts @@ -149,6 +149,7 @@ const GENE_TABLE_LIMIT = 500; /** how many genes may carry their own alteration facet at once */ const GENE_FACET_LIMIT = 6; import { isClinicalLeaf } from "./types.ts"; +import { finiteSql } from "./sql-number.ts"; export interface ExploreArgs { @@ -969,14 +970,14 @@ async function prefetchFacets(opts: { "WHERE cs.cancer_study_identifier IN " + IN, conn).catch(() => ({ data: [] as ChRow[] })), chQuery( "SELECT cancer_study_identifier AS s, type, attribute_name AS a, uniqExact(" + clinicalValueFoldSql("attribute_value") + ") AS nv,\n" + - " countIf(toFloat64OrNull(attribute_value) IS NOT NULL) / count() AS numratio,\n" + + " countIf(" + finiteSql("attribute_value") + " IS NOT NULL) / count() AS numratio,\n" + " uniqExact(if(type = 'sample', sample_unique_id, patient_unique_id)) AS np,\n" + - " min(toFloat64OrNull(attribute_value)) AS mn, max(toFloat64OrNull(attribute_value)) AS mx,\n" + - " quantileExact(0.5)(toFloat64OrNull(attribute_value)) AS md,\n" + - " quantileExact(0.25)(toFloat64OrNull(attribute_value)) AS q1,\n" + - " quantileExact(0.75)(toFloat64OrNull(attribute_value)) AS q3,\n" + - " countIf(toFloat64OrNull(attribute_value) IS NULL) AS na,\n" + - " countIf(toFloat64OrNull(attribute_value) = round(toFloat64OrNull(attribute_value))) = countIf(toFloat64OrNull(attribute_value) IS NOT NULL) AS allints\n" + + " min(" + finiteSql("attribute_value") + ") AS mn, max(" + finiteSql("attribute_value") + ") AS mx,\n" + + " quantileExact(0.5)(" + finiteSql("attribute_value") + ") AS md,\n" + + " quantileExact(0.25)(" + finiteSql("attribute_value") + ") AS q1,\n" + + " quantileExact(0.75)(" + finiteSql("attribute_value") + ") AS q3,\n" + + " countIf(" + finiteSql("attribute_value") + " IS NULL) AS na,\n" + + " countIf(" + finiteSql("attribute_value") + " = round(" + finiteSql("attribute_value") + ")) = countIf(" + finiteSql("attribute_value") + " IS NOT NULL) AS allints\n" + "FROM " + clinicalTable(db, attrAllow.length ? attrAllow : "all") + "\n" + "WHERE cancer_study_identifier IN " + IN + " AND attribute_value != ''" + cond + (attrAllow.length ? "\n AND attribute_name IN (" + attrAllow.map(sq).join(",") + ")" : "") + "\n" + @@ -1086,7 +1087,7 @@ async function pooledHistograms(opts: { const r = await chQuery( "SELECT multiIf(v <= " + edges[0] + ", -1, v > " + lastEdge + ", -2, toInt32(ceil((v - " + edges[0] + ") / " + interval + ")) - 1) AS b,\n" + " uniqExact(id) AS n\n" + - "FROM (SELECT toFloat64OrNull(attribute_value) AS v, " + unit + " AS id\n" + + "FROM (SELECT " + finiteSql("attribute_value") + " AS v, " + unit + " AS id\n" + " FROM " + clinicalTable(db, [f.attribute]) + "\n" + " WHERE cancer_study_identifier IN (" + studies.map(sq).join(",") + ") AND attribute_name = " + sq(f.attribute) + " AND type = " + sq(f.per) + " AND attribute_value != ''" + cond + ")\n" + diff --git a/lib/federation-views.ts b/lib/federation-views.ts index 1e0a025..60f62e4 100644 --- a/lib/federation-views.ts +++ b/lib/federation-views.ts @@ -9,6 +9,7 @@ import { clinicalValueFoldSql, nameFoldSql, foldedEqualsSql } from "./event-name import { clinicalTable } from "./custom-data.ts"; import { splitBucketSql } from "./cohort2.ts"; import { casesCte, type CompositionSource } from "./composition.ts"; +import { finiteSql } from "./sql-number.ts"; /** bare patient id for a *_unique_id column of this study */ const pidOf = (col: string, study: string): string => "replaceOne(" + col + ", " + sq(study + "_") + ", '')"; @@ -169,7 +170,7 @@ export function geneSetCountsSql(cohortSql: string, sets: { name: string; genes: */ export function profileMomentsAllSql(cohortSql: string, study: string, db: string, profileType: string, log2: boolean): string { const pid = pidOf("sd.patient_unique_id", study); - const raw = "toFloat64OrNull(ga.alteration_value)"; + const raw = finiteSql("ga.alteration_value"); const val = log2 ? "log2(" + raw + " + 1)" : raw; return ( "WITH cohort AS (" + cohortSql + "),\n" + @@ -382,12 +383,12 @@ export function facetsHistogramSql(cohortSql: string, study: string, db: string, return ( facetsSamplesCte(cohortSql, study, db, qc) + "\n" + "SELECT attribute_name AS attr,\n" + - " toInt32(floor(least(if(attribute_name = 'FACETS_PURITY', 0.9999, 7.9999), greatest(0, toFloat64OrNull(attribute_value))) /\n" + + " toInt32(floor(least(if(attribute_name = 'FACETS_PURITY', 0.9999, 7.9999), greatest(0, " + finiteSql("attribute_value") + ")) /\n" + " if(attribute_name = 'FACETS_PURITY', 0.05, 0.25))) AS bin,\n" + " count() AS n\n" + "FROM " + clinicalTable(db, ["FACETS_PURITY", "FACETS_PLOIDY"]) + "\n" + "WHERE cancer_study_identifier = " + sq(study) + " AND type = 'sample' AND attribute_name IN ('FACETS_PURITY', 'FACETS_PLOIDY')\n" + - " AND sample_unique_id IN (SELECT suid FROM samp) AND toFloat64OrNull(attribute_value) IS NOT NULL\n" + + " AND sample_unique_id IN (SELECT suid FROM samp) AND " + finiteSql("attribute_value") + " IS NOT NULL\n" + "GROUP BY attr, bin ORDER BY attr, bin" ); } @@ -452,7 +453,7 @@ export function incidenceCountsSql(cohortSql: string, study: string, db: string, "WITH cohort AS (" + cohortSql + "),\n" + "os AS (\n" + " SELECT " + pid + " AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name = 'OS_MONTHS')) AS months,\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name = 'OS_MONTHS')") + " AS months,\n" + " anyIf(attribute_value, attribute_name = 'OS_STATUS') AS status\n" + " FROM " + clinicalTable(db, ["OS_MONTHS", "OS_STATUS"]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type = 'patient' AND attribute_name IN ('OS_MONTHS', 'OS_STATUS')\n" + @@ -529,14 +530,14 @@ function labCtes(cohortSql: string, study: string, db: string, lab: LabSpec, anc " GROUP BY pid\n" + "),\n" + "lab AS (\n" + - " SELECT p.stable_id AS pid, ce.start_date AS day, toFloat64OrNull(r.value) AS v\n" + + " SELECT p.stable_id AS pid, ce.start_date AS day, " + finiteSql("r.value") + " AS v\n" + " FROM " + db + ".clinical_event ce\n" + " JOIN " + db + ".patient p ON p.internal_id = ce.patient_id\n" + " JOIN " + db + ".cancer_study cs ON cs.cancer_study_id = p.cancer_study_id\n" + " JOIN " + db + ".clinical_event_data t ON t.clinical_event_id = ce.clinical_event_id AND " + foldedEqualsSql("t.key", lab.testKey) + " AND " + foldedEqualsSql("t.value", lab.test) + "\n" + " JOIN " + db + ".clinical_event_data r ON r.clinical_event_id = ce.clinical_event_id AND " + foldedEqualsSql("r.key", lab.resultKey) + "\n" + " WHERE cs.cancer_study_identifier = " + sq(study) + " AND " + foldedEqualsSql("ce.event_type", lab.eventType) + "\n" + - " AND p.stable_id IN (SELECT pid FROM anch) AND toFloat64OrNull(r.value) IS NOT NULL\n" + + " AND p.stable_id IN (SELECT pid FROM anch) AND " + finiteSql("r.value") + " IS NOT NULL\n" + "),\n" + "base AS (\n" + // the baseline is the mean of the readings on the last day with any, so several readings that day @@ -899,7 +900,7 @@ export function compositionEntitiesSql(study: string, db: string, profileType: s */ export function correlatesMomentsSql(cohortSql: string, study: string, db: string, gene: string, profileType: string, otherProfile: string): string { const pid = pidOf("sd.patient_unique_id", study); - const raw = "toFloat64OrNull(ga.alteration_value)"; + const raw = finiteSql("ga.alteration_value"); const vals = (pt: string, geneCond: string) => " SELECT " + pid + " AS pid, ga.hugo_gene_symbol AS gene, avg(" + raw + ") AS v\n" + " FROM " + db + ".genetic_alteration_derived ga\n" + @@ -981,7 +982,7 @@ export function riskmodelRowsSql(cohortSql: string, study: string, db: string, o "WITH cohort AS (" + cohortSql + "),\n" + "os AS (\n" + " SELECT " + pid + " AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name=" + sq(months) + ")) AS months,\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name=" + sq(months) + ")") + " AS months,\n" + " anyIf(attribute_value, attribute_name=" + sq(status) + ") AS status,\n" + " " + splitBucketSql(pid, study) + " AS half\n" + " FROM " + clinicalTable(db, [months, status]) + "\n" + diff --git a/lib/federation.ts b/lib/federation.ts index 2c33ae7..c1ac377 100644 --- a/lib/federation.ts +++ b/lib/federation.ts @@ -56,6 +56,7 @@ import type { StudyAccess } from "./view-coverage.ts"; import { PATIENT_ID, patientEventsSql, patientSamplesSql, toEvents, toSamples } from "./patient-timeline.ts"; import { clinicalTable, clinicalMetaTable, ATTR_NAME_RE } from "./custom-data.ts"; import { PATIENT_KINDS, type PatientKind } from "./disclosure.ts"; +import { finiteSql } from "./sql-number.ts"; export type InstanceMode = "patient-level" | "aggregate-only"; @@ -425,12 +426,12 @@ export function profileMomentsSql(cohortSql: string, genes: string[], profileTyp "WITH cohort AS (" + cohortSql + "),\n" + "vals AS (\n" + " SELECT " + pid + " AS pid, ga.hugo_gene_symbol AS gene,\n" + - " avg(toFloat64OrNull(ga.alteration_value)) AS v\n" + + " avg(" + finiteSql("ga.alteration_value") + ") AS v\n" + " FROM " + db + ".genetic_alteration_derived ga\n" + " JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + " WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(profileType) + "\n" + " AND ga.hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL\n" + " AND " + pid + " IN (SELECT pid FROM cohort)\n" + " GROUP BY pid, gene\n" + ")\n" + @@ -567,12 +568,12 @@ export function momentPairsSql(cohortSql: string, pairs: [string, string][], pro "WITH cohort AS (" + cohortSql + "),\n" + "vals AS (\n" + " SELECT " + pid + " AS pid, ga.hugo_gene_symbol AS gene,\n" + - " avg(toFloat64OrNull(ga.alteration_value)) AS v\n" + + " avg(" + finiteSql("ga.alteration_value") + ") AS v\n" + " FROM " + db + ".genetic_alteration_derived ga\n" + " JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + " WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(profileType) + "\n" + " AND ga.hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL\n" + " AND " + pid + " IN (SELECT pid FROM cohort)\n" + " GROUP BY pid, gene\n" + "),\n" + @@ -632,7 +633,7 @@ export function clinicalMomentsSql(cohortSql: string, attrs: string[], study: st "WITH cohort AS (" + cohortSql + ")\n" + "SELECT a, per, count() AS n, sum(x) AS sx, sum(x * x) AS sxx, min(x) AS mn, max(x) AS mx\n" + "FROM (\n" + - " SELECT attribute_name AS a, type AS per, toFloat64OrNull(attribute_value) AS x\n" + + " SELECT attribute_name AS a, type AS per, " + finiteSql("attribute_value") + " AS x\n" + " FROM " + clinicalTable(db, attrs) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + "\n" + " AND attribute_name IN (" + attrs.map(sq).join(",") + ")\n" + @@ -663,7 +664,7 @@ export function clinicalHistogramSql(cohortSql: string, attrs: string[], study: return ( "WITH cohort AS (" + cohortSql + "),\n" + " raw AS (\n" + - " SELECT attribute_name AS a, type AS per, toFloat64OrNull(attribute_value) AS x\n" + + " SELECT attribute_name AS a, type AS per, " + finiteSql("attribute_value") + " AS x\n" + " FROM " + clinicalTable(db, attrs) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + "\n" + " AND attribute_name IN (" + attrs.map(sq).join(",") + ")\n" + @@ -691,7 +692,7 @@ export function attributeQuantileSql(cohortSql: string, attribute: string, per: const pid = "replaceOne(patient_unique_id, " + sq(study + "_") + ", '')"; return ( "WITH cohort AS (" + cohortSql + ")\n" + - "SELECT quantileExact(" + p + ")(toFloat64OrNull(attribute_value)) AS q\n" + + "SELECT quantileExact(" + p + ")(" + finiteSql("attribute_value") + ") AS q\n" + "FROM " + clinicalTable(db, [attribute]) + "\n" + "WHERE cancer_study_identifier = " + sq(study) + " AND attribute_name = " + sq(attribute) + "\n" + " AND type = " + sq(per) + " AND attribute_value != ''\n" + @@ -732,8 +733,8 @@ export function survivalCountsSql(cohortSql: string, study: string, db: string = " AND " + pid + " IN (SELECT pid FROM cohort)\n" + " GROUP BY pid\n" + ")\n" + - "SELECT toFloat64OrNull(months) AS m, if(status LIKE '1%', 1, 0) AS ev, count() AS n\n" + - "FROM os WHERE toFloat64OrNull(months) IS NOT NULL\n" + + "SELECT " + finiteSql("months") + " AS m, if(status LIKE '1%', 1, 0) AS ev, count() AS n\n" + + "FROM os WHERE " + finiteSql("months") + " IS NOT NULL\n" + "GROUP BY m, ev ORDER BY m" ); } @@ -769,7 +770,7 @@ export function anchoredSurvivalSql( "WITH cohort AS (" + cohortSql + "),\n" + "os AS (\n" + " SELECT " + pid + " AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name=" + sq(months) + ")) AS months,\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name=" + sq(months) + ")") + " AS months,\n" + " anyIf(attribute_value, attribute_name=" + sq(status) + ") AS status\n" + " FROM " + clinicalTable(db, [months, status]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type='patient'\n" + @@ -839,7 +840,7 @@ export function coxRowsSql( "WITH cohort AS (" + cohortSql + "),\n" + "os AS (\n" + " SELECT " + pid + " AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name=" + sq(months) + ")) AS months,\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name=" + sq(months) + ")") + " AS months,\n" + " anyIf(attribute_value, attribute_name=" + sq(status) + ") AS status\n" + " FROM " + clinicalTable(db, [months, status]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type='patient'\n" + @@ -1026,7 +1027,7 @@ export function eventCourseSql( // Observation window: first recorded event to last contact. "os AS (\n" + " SELECT replaceOne(patient_unique_id, " + sq(study + "_") + ", '') AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name = 'OS_MONTHS')) * " + DAYS_PER_MONTH + " AS os_day\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name = 'OS_MONTHS')") + " * " + DAYS_PER_MONTH + " AS os_day\n" + " FROM " + clinicalTable(db, ["OS_MONTHS"]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type = 'patient'\n" + " AND attribute_name = 'OS_MONTHS'\n" + @@ -1141,7 +1142,7 @@ export function sequenceSql( // records simply stop" — see the fate expression below. "os AS (\n" + " SELECT replaceOne(patient_unique_id, " + sq(study + "_") + ", '') AS pid,\n" + - " toFloat64OrNull(anyIf(attribute_value, attribute_name = 'OS_MONTHS')) * " + DAYS_PER_MONTH + " AS os_day\n" + + " " + finiteSql("anyIf(attribute_value, attribute_name = 'OS_MONTHS')") + " * " + DAYS_PER_MONTH + " AS os_day\n" + " FROM " + clinicalTable(db, ["OS_MONTHS"]) + "\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND type = 'patient'\n" + " AND attribute_name = 'OS_MONTHS'\n" + diff --git a/lib/oncoprint.ts b/lib/oncoprint.ts index f489715..e1a9f90 100644 --- a/lib/oncoprint.ts +++ b/lib/oncoprint.ts @@ -27,6 +27,7 @@ import { PROFILED_TYPES, type ProfiledType } from "./profile-names.ts"; import { assembleContract, normalizeDisplay, normalizeSort, type AssembleInput, type EventRow } from "./oncoprint-assemble.ts"; import { normalizeGeneTracks, toGeneList } from "./gene-tracks.ts"; import { clinicalTable, assertCustomDataPresent } from "./custom-data.ts"; +import { finiteSql } from "./sql-number.ts"; export { normalizeDisplay, normalizeSort } from "./oncoprint-assemble.ts"; const GENE_RE = /^[A-Z0-9][A-Z0-9.-]{0,19}$/; @@ -200,12 +201,12 @@ export function vafSql(genes: string[], study: string = STUDY, db: string = DB, export function heatmapSql(profileType: string, genes: string[], study: string = STUDY, db: string = DB, unit: Unit = "patient"): string { // values are stored per sample; only patient mode has to average them return "SELECT " + (unit === "sample" ? "replaceOne(ga.sample_unique_id, " + sq(study + "_") + ", '')" : "replaceOne(sd.patient_unique_id, " + sq(study + "_") + ", '')") + " AS pid,\n" + - " ga.hugo_gene_symbol AS gene, avg(toFloat64OrNull(ga.alteration_value)) AS v\n" + + " ga.hugo_gene_symbol AS gene, avg(" + finiteSql("ga.alteration_value") + ") AS v\n" + "FROM " + db + ".genetic_alteration_derived ga\n" + (unit === "sample" ? "" : "JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n") + "WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(profileType) + "\n" + " AND ga.hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL\n" + "GROUP BY pid, gene"; } diff --git a/lib/oql.ts b/lib/oql.ts index 251bb62..a53e9ac 100644 --- a/lib/oql.ts +++ b/lib/oql.ts @@ -20,6 +20,7 @@ export { parseOql } from "./oql-parse.ts"; export type { OqlSpecType, OqlRange, OqlSpec, OqlEntry } from "./oql-parse.ts"; import type { OqlRange, OqlSpec } from "./oql-parse.ts"; import { parseOql } from "./oql-parse.ts"; +import { finiteSql } from "./sql-number.ts"; /** * A mutation's class as cBioPortal's OQL sees it — cbioportal-frontend's getMutationSubType and @@ -136,7 +137,7 @@ export function oqlCasesSql(q: string, { oncokbPin, profiles, study = STUDY, db " FROM " + db + ".genetic_alteration_derived ga\n" + " JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + " WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.hugo_gene_symbol = " + sq(gene) + "\n" + - " AND ga.profile_type = " + sq(prof) + " AND toFloat64OrNull(ga.alteration_value) " + spec.op + " " + spec.z); + " AND ga.profile_type = " + sq(prof) + " AND " + finiteSql("ga.alteration_value") + " " + spec.op + " " + spec.z); continue; } parts.push( diff --git a/lib/plot.ts b/lib/plot.ts index 34f14ef..f68522d 100644 --- a/lib/plot.ts +++ b/lib/plot.ts @@ -88,6 +88,7 @@ export { POINTS_CAP, POINT_LAYER_CAP, LAYER_COLORS, LAYER_BASE, DEFAULT_POINT_LA import { POINTS_CAP, POINT_RULE_N_MAX } from "./plot-layers.ts"; export { isAbsent, jitterFor } from "./plot-draw.ts"; import { isAbsent } from "./plot-draw.ts"; +import { finiteSql } from "./sql-number.ts"; /** * Over the points cap, a scatter cell holding at most this many cases is drawn as its cases * rather than as a shade: the sparse fringe of the density, which is where the outliers are. @@ -246,8 +247,8 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s // are combined: a number is their mean, a category is the samples' common value, or "Multiple" // when they differ (combineNote says so in the caption). // Only the study knows whether the attribute is a number, so the caller says which. - const val = kind === "num" ? logOf(a, "toFloat64OrNull(attribute_value)") : "attribute_value"; - const valCd = kind === "num" ? logOf(a, "toFloat64OrNull(cd.attribute_value)") : "cd.attribute_value"; + const val = kind === "num" ? logOf(a, finiteSql("attribute_value")) : "attribute_value"; + const valCd = kind === "num" ? logOf(a, finiteSql("cd.attribute_value")) : "cd.attribute_value"; const defined = (e: string) => kind === "num" ? " AND " + e + " IS NOT NULL" : ""; const own = "SELECT " + strip(sample ? "sample_unique_id" : "patient_unique_id", study) + " AS id, " + val + " AS v, " + "1 AS pref\n" + @@ -276,12 +277,12 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s const gene = String(a.gene).toUpperCase(); const level = "multiIf(x = -2, 'Deep deletion', x = -1, 'Shallow deletion', x = 0, 'Diploid', x = 1, 'Gain', x = 2, 'Amplification', toString(x))"; return "SELECT id, if(uniqExact(lv) > 1, " + sq(MULTIPLE) + ", any(lv)) AS v FROM (\n" + - "SELECT " + strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study) + " AS id, " + level.replace(/\bx\b/g, "toFloat64OrNull(ga.alteration_value)") + " AS lv\n" + + "SELECT " + strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study) + " AS id, " + level.replace(/\bx\b/g, finiteSql("ga.alteration_value")) + " AS lv\n" + "FROM " + db + ".genetic_alteration_derived ga\n" + "JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + "WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(a.profileType) + " AND ga.hugo_gene_symbol = " + sq(gene) + "\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL AND " + inCohort("sd.patient_unique_id") + ")\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL AND " + inCohort("sd.patient_unique_id") + ")\n" + "GROUP BY id"; } if (a.source === "molecular") { @@ -289,12 +290,12 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s // patient mode averages a patient's samples, the same rule the oncoprint's heatmap uses; a // value the log leaves undefined drops the case, so coverage counts it as having none const q = "SELECT " + strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study) + " AS id,\n" + - " " + logOf(a, sample ? "argMax(toFloat64OrNull(ga.alteration_value), 1)" : "avg(toFloat64OrNull(ga.alteration_value))") + " AS v\n" + + " " + logOf(a, sample ? "argMax(" + finiteSql("ga.alteration_value") + ", 1)" : "avg(" + finiteSql("ga.alteration_value") + ")") + " AS v\n" + "FROM " + db + ".genetic_alteration_derived ga\n" + "JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + "WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(a.profileType) + " AND ga.hugo_gene_symbol = " + sq(gene) + "\n" + - " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL AND " + inCohort("sd.patient_unique_id") + "\n" + + " AND " + finiteSql("ga.alteration_value") + " IS NOT NULL AND " + inCohort("sd.patient_unique_id") + "\n" + "GROUP BY id"; return a.log ? "SELECT id, v FROM (" + q + ") WHERE v IS NOT NULL" : q; } diff --git a/lib/sql-number.ts b/lib/sql-number.ts new file mode 100644 index 0000000..9958669 --- /dev/null +++ b/lib/sql-number.ts @@ -0,0 +1,10 @@ +// THE way a stored value is read as a number in SQL. Clinical values, assay and expression values +// are text in the warehouse, and some studies record a missing number as "nan", "NaN" or "inf": +// toFloat64OrNull reads those as NaN or ±Infinity, not NULL, so they would pass every IS NOT NULL +// test and then either fail a query (a NaN cannot be cast to a histogram bin) or quietly become a +// minimum, a median or a survival time. Here they are missing, like any other value that is not a +// number. Browser-safe: it only builds SQL text. + +/** `expr` as a finite Float64, or NULL when it is not a number (including NaN and ±Infinity) */ +export const finiteSql = (expr: string): string => + "if(isFinite(ifNull(toFloat64OrNull(" + expr + "), nan)), toFloat64OrNull(" + expr + "), NULL)"; diff --git a/test/clinical-table.test.ts b/test/clinical-table.test.ts index b2a199c..bfe647c 100644 --- a/test/clinical-table.test.ts +++ b/test/clinical-table.test.ts @@ -4,6 +4,7 @@ import { test, describe } from "node:test"; import assert from "node:assert/strict"; import { clinicalRowSource, type TableColumn } from "../lib/clinical-table.ts"; import { pageSql, countsSql } from "../lib/table-query.ts"; +import { finiteSql } from "../lib/sql-number.ts"; const cols: TableColumn[] = [{ id: "AGE", name: "Age", per: "patient", numeric: true }, { id: "TYPE", name: "Type", per: "sample", numeric: false }]; const part = (cohort: string, study: string) => ({ cohort, study, cohortSql: "SELECT '' AS pid" }); @@ -12,8 +13,9 @@ const defaults = ["patient", "sample", "AGE", "TYPE"]; describe("clinicalRowSource", () => { test("numbers sort with NULLS LAST, blanks last, then patient, sample and document order; the offset is global", () => { const src = clinicalRowSource([part("a", "s")], "db", "sample", cols); - assert.match(pageSql(src, { sort: { column: "AGE", desc: true } }, defaults, 10, 20), - /ORDER BY toFloat64OrNull\(m\['AGE'\]\) DESC NULLS LAST, patient, sample, ord\nLIMIT 10 OFFSET 20$/); + // a number sorts as its finite value; "nan" and "inf" read as missing (lib/sql-number.ts) + assert.ok(pageSql(src, { sort: { column: "AGE", desc: true } }, defaults, 10, 20) + .endsWith("ORDER BY " + finiteSql("m['AGE']") + " DESC NULLS LAST, patient, sample, ord\nLIMIT 10 OFFSET 20")); assert.match(pageSql(src, { sort: { column: "TYPE" } }, defaults, 10, 0), /ORDER BY \(m\['TYPE'\] = ''\) ASC, m\['TYPE'\] ASC, patient, sample, ord/); assert.match(pageSql(src, {}, defaults, 10, 0), /ORDER BY patient, sample, patient, sample, ord/); }); diff --git a/test/plot.test.ts b/test/plot.test.ts index 427b824..6ea349c 100644 --- a/test/plot.test.ts +++ b/test/plot.test.ts @@ -6,6 +6,7 @@ import { axisKind, axisLabel, axisSelect, corrPValue, plotBinsSql, plotBoxSql, p plotShape, plotStatsSql, plotTableSql, plotPointsSql, plotViolinSql, categoryColor, isAbsent, tukey, validateAxis, CATEGORY_CAP, DEFAULT_BINS , coverageNote, combineNote, plotCoverageSql, plotScatterPointsSql, type PlotData } from "../lib/plot.ts"; import { STYLE } from "../lib/oncoprint-style.ts"; +import { finiteSql } from "../lib/sql-number.ts"; const ctx = { study: "x", db: "db", unit: "sample" as const, cohortSql: "SELECT pid FROM p" }; const num = { source: "clinical" as const, attribute: "MUTATION_COUNT" }; @@ -203,7 +204,8 @@ test("a plot says how many cases the pair left out, and on which axis", () => { test("log: true reads log10(v + 1), leaves out what has no such log, and says so in the label", () => { const ctx = { study: "s", db: "db", unit: "sample" as const, cohortSql: "SELECT '' AS pid" }; const clin = axisSelect({ source: "clinical", attribute: "MUTATION_COUNT", log: true }, ctx, "num"); - assert.match(clin, /if\(toFloat64OrNull\(attribute_value\) >= 0, log10\(toFloat64OrNull\(attribute_value\) \+ 1\), NULL\) IS NOT NULL/); + const v = finiteSql("attribute_value"); + assert.ok(clin.includes("if(" + v + " >= 0, log10(" + v + " + 1), NULL) IS NOT NULL"), clin); assert.doesNotMatch(axisSelect({ source: "clinical", attribute: "MUTATION_COUNT" }, ctx, "num"), /log10/); const mol = axisSelect({ source: "molecular", profileType: "mrna", gene: "ESR1", log: true }, ctx, "num"); assert.match(mol, /^SELECT id, v FROM \(/); diff --git a/test/sql-number.test.ts b/test/sql-number.test.ts new file mode 100644 index 0000000..03c8102 --- /dev/null +++ b/test/sql-number.test.ts @@ -0,0 +1,18 @@ +// THE numeric read of a stored value (lib/sql-number.ts): NaN and ±Infinity are missing, like any +// other text that is not a number, so they never reach a histogram bin, a minimum or a survival time. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import { finiteSql } from "../lib/sql-number.ts"; + +test("finiteSql keeps a finite number and reads NaN and infinity as NULL", () => { + assert.equal(finiteSql("v"), "if(isFinite(ifNull(toFloat64OrNull(v), nan)), toFloat64OrNull(v), NULL)"); +}); + +test("every numeric read of a value in lib goes through finiteSql", () => { + // one module may name toFloat64OrNull bare: its own; a summary that guards with isFinite itself is the other + const bare = (readdirSync("lib", { recursive: true }) as string[]).filter((f) => f.endsWith(".ts") && f !== "sql-number.ts" && f !== "dsl-reference.ts") + .flatMap((f) => readFileSync("lib/" + f, "utf8").split("\n").map((line, i) => ({ f, i: i + 1, line }))) + .filter(({ line }) => line.includes("toFloat64OrNull(") && !line.includes("isFinite(")); + assert.deepEqual(bare.map(({ f, i }) => f + ":" + i), []); +});