Skip to content

Stored "nan"/"inf" values are missing numbers: fix Explore's Code 70 error, one numeric read for every query #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alisman merged 1 commit into from
Oct 23, 2026
Merged

Stored "nan"/"inf" values are missing numbers: fix Explore's Code 70 error, one numeric read for every query #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion lib/clinical-table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import { sq } from "./compiler.ts";
import type { QueryColumn, RowSource } from "./table-query.ts";
import { clinicalTable, clinicalMetaTable } from "./custom-data.ts";
import { finiteSql } from "./sql-number.ts";

export interface TableColumn {
id: string; name: string;
Expand Down Expand Up @@ -81,7 +82,7 @@ export function clinicalRowSource(parts: TablePart[], db: string, unit: "sample"
if (!parts.length) throw new Error("clinicalRowSource needs at least one part");
const attr = (c: TableColumn): QueryColumn => {
const v = "m[" + sq(c.id) + "]";
return { id: c.id, name: c.name, numeric: c.numeric, text: v, ...(c.numeric ? { num: "toFloat64OrNull(" + v + ")" } : {}) };
return { id: c.id, name: c.name, numeric: c.numeric, text: v, ...(c.numeric ? { num: finiteSql(v) } : {}) };
};
return {
with: allRows(parts, db, unit, columns),
Expand Down
9 changes: 5 additions & 4 deletions lib/cohort2.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import type {
ProfiledLeaf, CaseListLeaf, GenomicDataLeaf, GenericAssayLeaf,
SampleListLeaf, PatientListLeaf, SplitLeaf, AlterationLeaf, EventLeafFields, ProteinPositionLeaf
} from "./types.ts";
import { finiteSql } from "./sql-number.ts";

/** Every leaf compiler receives a fully-resolved context. */
type Ctx = CompileCtx & { study: string };
Expand Down Expand Up @@ -69,7 +70,7 @@ function clinicalLeafSql(leaf: ClinicalLeaf, ctx: Ctx): string {
leaf.values.slice(0, 50).map(v => clinicalValueFoldSql(sq(String(v)))).join(",") + ")");
}
if (leaf.range && typeof leaf.range === "object") {
const r = leaf.range as Record<string, number | undefined>, num = "toFloat64OrNull(attribute_value)";
const r = leaf.range as Record<string, number | undefined>, num = finiteSql("attribute_value");
for (const [k, opSym] of [["gt", ">"], ["gte", ">="], ["lt", "<"], ["lte", "<="]]) {
if (r[k] != null) {
if (!Number.isFinite(Number(r[k]))) throw new Error("clinical range." + k + " must be a number");
Expand All @@ -84,7 +85,7 @@ function clinicalLeafSql(leaf: ClinicalLeaf, ctx: Ctx): string {
const op = CLIN_OP[leaf.op];
if (!op) throw new Error("invalid clinical op (use = != > >= < <=)");
conds.push(typeof leaf.value === "number"
? "toFloat64OrNull(attribute_value) " + op + " " + Number(leaf.value)
? finiteSql("attribute_value") + " " + op + " " + Number(leaf.value)
// Equality folds case and whitespace, matching how the facet that offered
// this value counted it. Ordering comparisons do not: they are for
// numbers, and folding a string range would change what it means.
Expand Down Expand Up @@ -320,7 +321,7 @@ function genomicDataLeafSql(leaf: GenomicDataLeaf, ctx: Ctx): string {
const pt = String(leaf.profileType || "");
if (ctx.profiles && !ctx.profiles.includes(pt))
throw new Error("genomicData: profile '" + pt + "' not in this study (available: " + (ctx.profiles.join(", ") || "none") + ")");
const conds = numRangeConds(leaf.range, "toFloat64OrNull(ga.alteration_value)");
const conds = numRangeConds(leaf.range, finiteSql("ga.alteration_value"));
if (Array.isArray(leaf.values) && leaf.values.length)
conds.push("ga.alteration_value IN (" + leaf.values.slice(0, 20).map(v => sq(String(v))).join(",") + ")");
if (!conds.length) throw new Error("genomicData clause needs range{gt,gte,lt,lte} or values[]");
Expand All @@ -339,7 +340,7 @@ function genericAssayLeafSql(leaf: GenericAssayLeaf, ctx: Ctx): string {
if (!pt || !entity) throw new Error("genericAssay clause needs profileType and entity");
if (ctx.gaProfiles && !ctx.gaProfiles.includes(pt))
throw new Error("genericAssay: profile '" + pt + "' not in this study (available: " + ctx.gaProfiles.join(", ") + ")");
const conds = numRangeConds(leaf.range, "toFloat64OrNull(value)");
const conds = numRangeConds(leaf.range, finiteSql("value"));
if (Array.isArray(leaf.values) && leaf.values.length)
conds.push("value IN (" + leaf.values.slice(0, 20).map(v => sq(String(v))).join(",") + ")");
if (!conds.length) throw new Error("genericAssay clause needs range or values[]");
Expand Down
3 changes: 2 additions & 1 deletion lib/compiler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import type { EventSelector, EventRel, Occurrence } from "./types.ts";
import { foldedEqualsSql, canonicalEventName } from "./event-names.ts";
import { linesCtes, compositionKeySql, DEFAULT_LINE_GAP, LINE_TYPE } from "./treatment-lines.ts";
import { finiteSql } from "./sql-number.ts";

/** A selector after runtime validation: defaults applied, fields narrowed. */
export interface ValidSelector {
Expand Down Expand Up @@ -88,7 +89,7 @@ function selectorCte(name: string, sel: ValidSelector, db: string, priorGuard: b
});
if (sel.numeric) {
joins.push("JOIN " + db + ".clinical_event_data nx ON nx.clinical_event_id = ce.clinical_event_id AND " + foldedEqualsSql("nx.key", sel.numeric.key));
conds.push("toFloat64OrNull(nx.value) " + sel.numeric.op + " " + sel.numeric.value);
conds.push(finiteSql("nx.value") + " " + sel.numeric.op + " " + sel.numeric.value);
}
if (sel.minDurationDays) {
conds.push("ce.stop_date IS NOT NULL AND ce.stop_date - ce.start_date >= " + sel.minDurationDays);
Expand Down
3 changes: 2 additions & 1 deletion lib/composition.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import { sq } from "./compiler.ts";
import { axisSelect } from "./plot.ts";
import { ATTR_NAME_RE } from "./custom-data.ts";
import { finiteSql } from "./sql-number.ts";

/** the most parts one figure can tell apart: the categorical palette, then its tints */
export const COMPOSITION_MAX_PARTS = 16;
Expand Down Expand Up @@ -57,7 +58,7 @@ export function partsSql(s: CompositionSource, study: string, db: string, cohort
const idCol = s.unit === "patient" ? "patient_unique_id" : "sample_unique_id";
return "SELECT id, c, assumeNotNull(avg(val)) AS v FROM (\n" +
" SELECT " + idCol + " AS id, toUInt16(indexOf([" + s.parts.map(sq).join(",") + "], entity_stable_id) - 1) AS c,\n" +
" toFloat64OrNull(value) AS val\n" +
" " + finiteSql("value") + " AS val\n" +
" FROM " + db + ".generic_assay_data_derived\n" +
" WHERE profile_type = " + sq(s.profileType!) + " AND profile_stable_id = " + sq(study + "_" + s.profileType) + "\n" +
" AND entity_stable_id IN (" + s.parts.map(sq).join(",") + ")\n" +
Expand Down
17 changes: 9 additions & 8 deletions lib/explore.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ const GENE_TABLE_LIMIT = 500;
/** how many genes may carry their own alteration facet at once */
const GENE_FACET_LIMIT = 6;
import { isClinicalLeaf } from "./types.ts";
import { finiteSql } from "./sql-number.ts";


export interface ExploreArgs {
Expand Down Expand Up @@ -969,14 +970,14 @@ async function prefetchFacets(opts: {
"WHERE cs.cancer_study_identifier IN " + IN, conn).catch(() => ({ data: [] as ChRow[] })),
chQuery<ChRow>(
"SELECT cancer_study_identifier AS s, type, attribute_name AS a, uniqExact(" + clinicalValueFoldSql("attribute_value") + ") AS nv,\n" +
" countIf(toFloat64OrNull(attribute_value) IS NOT NULL) / count() AS numratio,\n" +
" countIf(" + finiteSql("attribute_value") + " IS NOT NULL) / count() AS numratio,\n" +
" uniqExact(if(type = 'sample', sample_unique_id, patient_unique_id)) AS np,\n" +
" min(toFloat64OrNull(attribute_value)) AS mn, max(toFloat64OrNull(attribute_value)) AS mx,\n" +
" quantileExact(0.5)(toFloat64OrNull(attribute_value)) AS md,\n" +
" quantileExact(0.25)(toFloat64OrNull(attribute_value)) AS q1,\n" +
" quantileExact(0.75)(toFloat64OrNull(attribute_value)) AS q3,\n" +
" countIf(toFloat64OrNull(attribute_value) IS NULL) AS na,\n" +
" countIf(toFloat64OrNull(attribute_value) = round(toFloat64OrNull(attribute_value))) = countIf(toFloat64OrNull(attribute_value) IS NOT NULL) AS allints\n" +
" min(" + finiteSql("attribute_value") + ") AS mn, max(" + finiteSql("attribute_value") + ") AS mx,\n" +
" quantileExact(0.5)(" + finiteSql("attribute_value") + ") AS md,\n" +
" quantileExact(0.25)(" + finiteSql("attribute_value") + ") AS q1,\n" +
" quantileExact(0.75)(" + finiteSql("attribute_value") + ") AS q3,\n" +
" countIf(" + finiteSql("attribute_value") + " IS NULL) AS na,\n" +
" countIf(" + finiteSql("attribute_value") + " = round(" + finiteSql("attribute_value") + ")) = countIf(" + finiteSql("attribute_value") + " IS NOT NULL) AS allints\n" +
"FROM " + clinicalTable(db, attrAllow.length ? attrAllow : "all") + "\n" +
"WHERE cancer_study_identifier IN " + IN + " AND attribute_value != ''" + cond +
(attrAllow.length ? "\n AND attribute_name IN (" + attrAllow.map(sq).join(",") + ")" : "") + "\n" +
Expand Down Expand Up @@ -1086,7 +1087,7 @@ async function pooledHistograms(opts: {
const r = await chQuery<ChRow>(
"SELECT multiIf(v <= " + edges[0] + ", -1, v > " + lastEdge + ", -2, toInt32(ceil((v - " + edges[0] + ") / " + interval + ")) - 1) AS b,\n" +
" uniqExact(id) AS n\n" +
"FROM (SELECT toFloat64OrNull(attribute_value) AS v, " + unit + " AS id\n" +
"FROM (SELECT " + finiteSql("attribute_value") + " AS v, " + unit + " AS id\n" +
" FROM " + clinicalTable(db, [f.attribute]) + "\n" +
" WHERE cancer_study_identifier IN (" + studies.map(sq).join(",") + ") AND attribute_name = " + sq(f.attribute) +
" AND type = " + sq(f.per) + " AND attribute_value != ''" + cond + ")\n" +
Expand Down
17 changes: 9 additions & 8 deletions lib/federation-views.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { clinicalValueFoldSql, nameFoldSql, foldedEqualsSql } from "./event-name
import { clinicalTable } from "./custom-data.ts";
import { splitBucketSql } from "./cohort2.ts";
import { casesCte, type CompositionSource } from "./composition.ts";
import { finiteSql } from "./sql-number.ts";

/** bare patient id for a *_unique_id column of this study */
const pidOf = (col: string, study: string): string => "replaceOne(" + col + ", " + sq(study + "_") + ", '')";
Expand Down Expand Up @@ -169,7 +170,7 @@ export function geneSetCountsSql(cohortSql: string, sets: { name: string; genes:
*/
export function profileMomentsAllSql(cohortSql: string, study: string, db: string, profileType: string, log2: boolean): string {
const pid = pidOf("sd.patient_unique_id", study);
const raw = "toFloat64OrNull(ga.alteration_value)";
const raw = finiteSql("ga.alteration_value");
const val = log2 ? "log2(" + raw + " + 1)" : raw;
return (
"WITH cohort AS (" + cohortSql + "),\n" +
Expand Down Expand Up @@ -382,12 +383,12 @@ export function facetsHistogramSql(cohortSql: string, study: string, db: string,
return (
facetsSamplesCte(cohortSql, study, db, qc) + "\n" +
"SELECT attribute_name AS attr,\n" +
" toInt32(floor(least(if(attribute_name = 'FACETS_PURITY', 0.9999, 7.9999), greatest(0, toFloat64OrNull(attribute_value))) /\n" +
" toInt32(floor(least(if(attribute_name = 'FACETS_PURITY', 0.9999, 7.9999), greatest(0, " + finiteSql("attribute_value") + ")) /\n" +
" if(attribute_name = 'FACETS_PURITY', 0.05, 0.25))) AS bin,\n" +
" count() AS n\n" +
"FROM " + clinicalTable(db, ["FACETS_PURITY", "FACETS_PLOIDY"]) + "\n" +
"WHERE cancer_study_identifier = " + sq(study) + " AND type = 'sample' AND attribute_name IN ('FACETS_PURITY', 'FACETS_PLOIDY')\n" +
" AND sample_unique_id IN (SELECT suid FROM samp) AND toFloat64OrNull(attribute_value) IS NOT NULL\n" +
" AND sample_unique_id IN (SELECT suid FROM samp) AND " + finiteSql("attribute_value") + " IS NOT NULL\n" +
"GROUP BY attr, bin ORDER BY attr, bin"
);
}
Expand Down Expand Up @@ -452,7 +453,7 @@ export function incidenceCountsSql(cohortSql: string, study: string, db: string,
"WITH cohort AS (" + cohortSql + "),\n" +
"os AS (\n" +
" SELECT " + pid + " AS pid,\n" +
" toFloat64OrNull(anyIf(attribute_value, attribute_name = 'OS_MONTHS')) AS months,\n" +
" " + finiteSql("anyIf(attribute_value, attribute_name = 'OS_MONTHS')") + " AS months,\n" +
" anyIf(attribute_value, attribute_name = 'OS_STATUS') AS status\n" +
" FROM " + clinicalTable(db, ["OS_MONTHS", "OS_STATUS"]) + "\n" +
" WHERE cancer_study_identifier = " + sq(study) + " AND type = 'patient' AND attribute_name IN ('OS_MONTHS', 'OS_STATUS')\n" +
Expand Down Expand Up @@ -529,14 +530,14 @@ function labCtes(cohortSql: string, study: string, db: string, lab: LabSpec, anc
" GROUP BY pid\n" +
"),\n" +
"lab AS (\n" +
" SELECT p.stable_id AS pid, ce.start_date AS day, toFloat64OrNull(r.value) AS v\n" +
" SELECT p.stable_id AS pid, ce.start_date AS day, " + finiteSql("r.value") + " AS v\n" +
" FROM " + db + ".clinical_event ce\n" +
" JOIN " + db + ".patient p ON p.internal_id = ce.patient_id\n" +
" JOIN " + db + ".cancer_study cs ON cs.cancer_study_id = p.cancer_study_id\n" +
" JOIN " + db + ".clinical_event_data t ON t.clinical_event_id = ce.clinical_event_id AND " + foldedEqualsSql("t.key", lab.testKey) + " AND " + foldedEqualsSql("t.value", lab.test) + "\n" +
" JOIN " + db + ".clinical_event_data r ON r.clinical_event_id = ce.clinical_event_id AND " + foldedEqualsSql("r.key", lab.resultKey) + "\n" +
" WHERE cs.cancer_study_identifier = " + sq(study) + " AND " + foldedEqualsSql("ce.event_type", lab.eventType) + "\n" +
" AND p.stable_id IN (SELECT pid FROM anch) AND toFloat64OrNull(r.value) IS NOT NULL\n" +
" AND p.stable_id IN (SELECT pid FROM anch) AND " + finiteSql("r.value") + " IS NOT NULL\n" +
"),\n" +
"base AS (\n" +
// the baseline is the mean of the readings on the last day with any, so several readings that day
Expand Down Expand Up @@ -899,7 +900,7 @@ export function compositionEntitiesSql(study: string, db: string, profileType: s
*/
export function correlatesMomentsSql(cohortSql: string, study: string, db: string, gene: string, profileType: string, otherProfile: string): string {
const pid = pidOf("sd.patient_unique_id", study);
const raw = "toFloat64OrNull(ga.alteration_value)";
const raw = finiteSql("ga.alteration_value");
const vals = (pt: string, geneCond: string) =>
" SELECT " + pid + " AS pid, ga.hugo_gene_symbol AS gene, avg(" + raw + ") AS v\n" +
" FROM " + db + ".genetic_alteration_derived ga\n" +
Expand Down Expand Up @@ -981,7 +982,7 @@ export function riskmodelRowsSql(cohortSql: string, study: string, db: string, o
"WITH cohort AS (" + cohortSql + "),\n" +
"os AS (\n" +
" SELECT " + pid + " AS pid,\n" +
" toFloat64OrNull(anyIf(attribute_value, attribute_name=" + sq(months) + ")) AS months,\n" +
" " + finiteSql("anyIf(attribute_value, attribute_name=" + sq(months) + ")") + " AS months,\n" +
" anyIf(attribute_value, attribute_name=" + sq(status) + ") AS status,\n" +
" " + splitBucketSql(pid, study) + " AS half\n" +
" FROM " + clinicalTable(db, [months, status]) + "\n" +
Expand Down
Loading
Loading