diff --git a/app/lab/custom-data-client.ts b/app/lab/custom-data-client.ts index 3a19114..8e243a8 100644 --- a/app/lab/custom-data-client.ts +++ b/app/lab/custom-data-client.ts @@ -16,7 +16,7 @@ // (`uploads`, see bindUploads) resolves to that version; a body that // carries a spec rather than the document is given the pins with it. import type { Sharing } from "../../lib/access.ts"; -import { CUSTOM_PREFIX, LIMITS, type CustomColumn, type CustomDataset, type CustomLevel } from "../../lib/custom-data-names.ts"; +import { CUSTOM_PREFIX, CUSTOM_REF_RE, LIMITS, type CustomColumn, type CustomDataset, type CustomLevel } from "../../lib/custom-data-names.ts"; import type { DatasetMeta } from "../../lib/custom-store.ts"; import { EXAMPLES } from "../../lib/dsl-reference.ts"; import type { MembersBody } from "../../lib/api-schemas.ts"; @@ -42,7 +42,7 @@ export function datasetStudies(d: CustomDataset): string[] { return [...new Set( export function referencedDatasets(x: unknown): string[] { const text = typeof x === "string" ? x : JSON.stringify(x ?? null); if (!text.includes(CUSTOM_PREFIX)) return []; - return [...new Set([...text.matchAll(/"custom:([a-z0-9][a-z0-9-]{0,39})\//g)].map((m) => m[1]))]; + return [...new Set([...text.matchAll(CUSTOM_REF_RE)].map((m) => m[1]))]; } // --------------------------------------------------------------------------- diff --git a/app/lab/custom-data-panel.tsx b/app/lab/custom-data-panel.tsx index 846bce4..1ecf94c 100644 --- a/app/lab/custom-data-panel.tsx +++ b/app/lab/custom-data-panel.tsx @@ -13,7 +13,7 @@ import { } from "./custom-data-client.ts"; import type { DatasetMeta } from "../../lib/custom-store.ts"; import { LIMITS } from "../../lib/custom-data-names.ts"; -import { Download, LoaderCircle, Share2, Trash2 } from "lucide-react"; +import { Download, LoaderCircle, Share2, Trash2, Users } from "lucide-react"; import { Button } from "../_ui/button"; import { Dialog, DialogClose, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "../_ui/dialog"; import { Input } from "../_ui/input"; @@ -22,6 +22,8 @@ import { Textarea } from "../_ui/textarea"; import { cn } from "../../lib/cn"; import { OptionSelect } from "../_ui/choice.tsx"; import { TB } from "../_ui/toolbar.ts"; +import { encodeParam } from "../portal/page-doc.ts"; +import { DOC_VERSION } from "../../lib/doc-version.ts"; /** a tab of the custom-data manager */ const TAB = "h-auto flex-none rounded-none px-3 pt-1.5 pb-1.5 text-md after:bottom-[-1px] data-[state=active]:text-accent-foreground data-[state=active]:after:bg-primary"; @@ -245,6 +247,14 @@ function AddCustomData({ studies, onAdded }: { studies: string[]; onAdded: (name ); } +/** A new analysis of the patients with a row in this upload, pinned to its latest version */ +function analyzeUploadHref(d: DatasetMeta, studies: string[]): string { + const where = studies.length > 1 ? { studies } : { study: studies[0] }; + const doc = { dsl: DOC_VERSION, study: studies[0], uploads: [{ as: d.slug, dataset: d.id, version: d.latestVersion }], + cohorts: { [d.name]: { ...where, all: [{ kind: "upload", dataset: "custom:" + d.slug }] } }, views: [] }; + return "/lab?" + new URLSearchParams({ tab: "analysis", doc: encodeParam(doc) }).toString(); +} + function SavedCustomData({ onChanged, onAdd }: { onChanged: () => void; onAdd: () => void }) { const server = useCustomDataState(); const [serverBusy, setServerBusy] = useState(null); @@ -291,6 +301,9 @@ function SavedCustomData({ onChanged, onAdd }: { onChanged: () => void; onAdd: ( {v?.rowCount.toLocaleString()} {d.level}s {v?.studies.join(", ")} + {v && v.studies.length > 0 && <>{" "}} {" "} {d.rights.share && <>{" "} } diff --git a/app/lab/pills.tsx b/app/lab/pills.tsx index 7b807fb..f9ea844 100644 --- a/app/lab/pills.tsx +++ b/app/lab/pills.tsx @@ -21,7 +21,8 @@ const KIND_META: Record = { genomicData: { label: "data", color: "#CC79A7" }, genericAssay: { label: "assay", color: "#CC79A7" }, derived: { label: "derived", color: "#D55E00" }, - split: { label: "held-out", color: "#E69F00" } + split: { label: "held-out", color: "#E69F00" }, + upload: { label: "custom data", color: "#0072B2" } }; function leafKind(leaf: any) { @@ -89,6 +90,7 @@ function leafText(leaf: any) { // mutations at all", where the clause a gene-status facet builds means "no // panel of theirs covers THIS gene". Two different cohorts, and the narrower // one is the whole point of that facet. + if (k === "upload") return "a row in " + String(leaf.dataset).slice("custom:".length); if (k === "profiled") return "profiled: " + leaf.type + (leaf.gene ? " · " + leaf.gene : ""); if (k === "caseList") return leaf.id; if (k === "split") return leaf.part + " half"; diff --git a/docs/dsl.md b/docs/dsl.md index 83f9839..8fc4d09 100644 --- a/docs/dsl.md +++ b/docs/dsl.md @@ -119,6 +119,7 @@ rather than silently ignoring what it does not understand. | `genericAssay` | `{"kind":"genericAssay","profileType":P,"entity":E,"range"\|"values"}` | generic assay values (mutational signatures, …) | | `proteinPosition` | `{"kind":"proteinPosition","gene":G,"range":{gt,gte,lt,lte}}` | patient carries a mutation in `gene` whose protein position (the stored proteinPosStart, same column the lollipop draws from) falls in `range` — for an N-/C-terminal split when a study doesn't deposit one as a clinical attribute. Position-only: AND with an `oql` leaf (e.g. `{"kind":"oql","q":"APC: TRUNC"}`) to also restrict by mutation type — the two can then be met by different mutations; an OQL position range (`APC: TRUNC_(1400-)`) requires both of one mutation | | `split` | `{"kind":"split","part":"discovery"|"validation"}` | one fixed half of the study's patients, decided inside the database by a salted hash of the patient id — the same half for every caller, reproducible, and never passed through the client as an id list. It exists for **held-out replication**: test on `discovery`, confirm the same document on `validation`. A hypothesis campaign (`/campaign`, MCP `campaign_*`) adds it itself and refuses documents that already carry one, so a generator cannot choose its own half | +| `upload` | `{"kind":"upload","dataset":"custom:"}` | the patients with a row in an upload: a value in any of its columns, patient- or sample-level (a blank cell is not a value). `dataset` names the upload as its attributes do, and the document's `uploads` pins it like any of them; a document naming an upload the request cannot read is refused | | `alteration` | *(deprecated)* | translated to OQL internally; do not write new ones | **`sampleList`/`patientList` ids.** Each id is `":"` — a colon, not the diff --git a/e2e-saved.cjs b/e2e-saved.cjs index 7742eb6..40ada15 100644 --- a/e2e-saved.cjs +++ b/e2e-saved.cjs @@ -165,7 +165,20 @@ async function savedLists(browser, ok, BASE, LOGIN) { return /rows matched/.test(t) && t; }, null, { timeout: 60000 }).then((h) => h.jsonValue()).catch(() => ""); ok("the upload example matches the explored study in full", /(\d+) of \1 rows matched a patient of msk_chord_2024/.test(ex), ex.slice(0, 120)); - await page.keyboard.press("Escape"); + // saved, its "Analyze" opens a new analysis whose one cohort is every patient with a row in it + const rows = Number((ex.match(/(\d[\d,]*) of/) || [])[1]?.replace(/,/g, "")); + let cohort = ""; + try { + await page.locator("[role=dialog].customdatamanager button", { hasText: /^Add data to/ }).click(); + await page.locator("[role=dialog].customdatamanager .customlist a", { hasText: "Analyze" }).first().click({ timeout: 60000 }); + cohort = await page.waitForFunction(() => { + const b = [...document.querySelectorAll(".cohortbox")].find((x) => /a row in /.test(x.textContent || "")); + return b && /\d+ patients/.test(b.textContent) && b.textContent.replace(/\s+/g, " "); + }, null, { timeout: 120000 }).then((h) => h.jsonValue()); + } catch (e) { cohort = cohort || String(e.message).split("\n")[0]; } + const n = Number((cohort.match(/([\d,]+) patients/) || [])[1]?.replace(/,/g, "")); + ok("a saved upload's Analyze opens an analysis of the patients with a row in it", rows > 0 && n === rows, + rows + " rows · " + cohort.slice(0, 140)); } ok("Saved tab: no page errors", errors.length === 0, errors.join(" | ").slice(0, 200)); diff --git a/lib/cohort-partitions.ts b/lib/cohort-partitions.ts index 87c8922..9cc75d1 100644 --- a/lib/cohort-partitions.ts +++ b/lib/cohort-partitions.ts @@ -67,7 +67,7 @@ function leafEvaluability(leaf: Leaf): Evaluability { // Membership facts (a list, a case list, a profiling status, the held-out half) are known for // every patient. Measurements and timelines are not, and this engine has no "was measured" // condition for them yet, so they are named rather than silently treated as known. - if (kind === "patientList" || kind === "sampleList" || kind === "caseList" || kind === "profiled" || kind === "split") + if (kind === "patientList" || kind === "sampleList" || kind === "caseList" || kind === "profiled" || kind === "split" || kind === "upload") return { leaves: [], unchecked: [] }; return { leaves: [], unchecked: [describeLeafBrief(leaf)] }; } diff --git a/lib/cohort2.ts b/lib/cohort2.ts index 3d28bda..f9df0b6 100644 --- a/lib/cohort2.ts +++ b/lib/cohort2.ts @@ -18,13 +18,14 @@ import { compileCohort, sq, LINE_TYPE } from "./compiler.ts"; import { oqlLeafSql } from "./oql.ts"; import { clinicalValueFoldSql, foldedEqualsSql } from "./event-names.ts"; import { linesCtes, DEFAULT_LINE_GAP } from "./treatment-lines.ts"; -import { clinicalTable } from "./custom-data.ts"; +import { activeCustomData, clinicalTable } from "./custom-data.ts"; +import { UPLOAD_DATASET_RE, customAttr } from "./custom-data-names.ts"; import { idSetTable, idSetIfLoaded, ID_SET_UUID } from "./id-sets.ts"; import { lockClause } from "./cohort-lock.ts"; import type { CohortDef, Clause, Leaf, CompileCtx, ClinicalLeaf, ClinicalRange, ProfiledLeaf, CaseListLeaf, GenomicDataLeaf, GenericAssayLeaf, - SampleListLeaf, PatientListLeaf, SplitLeaf, AlterationLeaf, EventLeafFields, ProteinPositionLeaf + SampleListLeaf, PatientListLeaf, SplitLeaf, UploadLeaf, AlterationLeaf, EventLeafFields, ProteinPositionLeaf } from "./types.ts"; import { finiteSql } from "./sql-number.ts"; @@ -100,6 +101,18 @@ function clinicalLeafSql(leaf: ClinicalLeaf, ctx: Ctx): string { " AND attribute_name = " + sq(attr) + " AND " + conds.join(" AND "); } +/** The patients with any value in an upload: a row of any of its columns, patient or sample level. */ +function uploadLeafSql(leaf: UploadLeaf, ctx: Ctx): string { + const name = UPLOAD_DATASET_RE.exec(String(leaf.dataset || ""))?.[1]; + if (!name) throw new Error('upload clause needs dataset:"custom:"'); + const d = activeCustomData()?.datasets.find((x) => x.id === name); + if (!d) throw new Error("this analysis uses custom data that was not sent with it: '" + name + "'"); + const attrs = d.columns.map((c) => customAttr(d.id, c.name)); + return "SELECT DISTINCT " + PID("patient_unique_id", ctx.study) + " AS pid\n" + + " FROM " + clinicalTable(dbOf(ctx), attrs) + "\n" + + " WHERE cancer_study_identifier = " + sq(ctx.study) + " AND attribute_name IN (" + attrs.map(sq).join(",") + ")"; +} + const PROFILED_TYPE: Record = { MUTATION: "MUTATION_EXTENDED", CNA: "COPY_NUMBER_ALTERATION", SV: "STRUCTURAL_VARIANT" }; /** @@ -517,6 +530,7 @@ function leafSql(leaf: Leaf, ctx: Ctx): string { case "sampleList": return sampleListLeafSql(leaf, ctx); case "patientList": return patientListLeafSql(leaf, ctx); case "split": return splitLeafSql(leaf, ctx); + case "upload": return uploadLeafSql(leaf, ctx); default: return unknownLeaf(leaf); } } @@ -606,6 +620,7 @@ function leafText(leaf: any): string { } if (leaf.kind === "caseList") return "in case list " + leaf.id; if (leaf.kind === "split") return "in the " + leaf.part + " half (fixed held-out split)"; + if (leaf.kind === "upload") return "with a row in upload " + String(leaf.dataset).slice("custom:".length); if (leaf.kind === "genomicData") return leaf.gene + " " + leaf.profileType + " " + JSON.stringify(leaf.range || leaf.values); if (leaf.kind === "genericAssay") return leaf.entity + " " + JSON.stringify(leaf.range || leaf.values); if (leaf.kind === "proteinPosition") return leaf.gene + " protein position " + JSON.stringify(leaf.range || {}); diff --git a/lib/custom-data-names.ts b/lib/custom-data-names.ts index 15ef89b..12bdc57 100644 --- a/lib/custom-data-names.ts +++ b/lib/custom-data-names.ts @@ -22,6 +22,12 @@ export const ATTR_NAME_RE = /^(?:[A-Za-z0-9_]{1,64}|custom:[a-z0-9][a-z0-9-]{0,3 /** An attribute name a view setting may hold: ATTR_NAME_RE's two forms, a built-in name also taking "." and "-" (HLA-A1). */ export const VIEW_ATTR_RE = /^(?:[A-Za-z0-9_.-]{1,64}|custom:[a-z0-9][a-z0-9-]{0,39}\/[A-Za-z0-9_]{1,64})$/; +/** A document's mention of an upload: an attribute of it ("custom:/") or the upload + * itself ("custom:", an upload clause's dataset). Group 1 is the upload's name. */ +export const CUSTOM_REF_RE = /"custom:([a-z0-9][a-z0-9-]{0,39})[/"]/g; +/** An upload clause's dataset: the upload's name as its attributes carry it */ +export const UPLOAD_DATASET_RE = /^custom:([a-z0-9][a-z0-9-]{0,39})$/; + export const LIMITS = { datasets: 10, columns: 64, diff --git a/lib/custom-data.ts b/lib/custom-data.ts index 198d67d..2a5d079 100644 --- a/lib/custom-data.ts +++ b/lib/custom-data.ts @@ -26,7 +26,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { - CUSTOM_PREFIX, LIMITS, customAttr, isCustomAttr, + CUSTOM_PREFIX, CUSTOM_REF_RE, LIMITS, customAttr, isCustomAttr, type CustomColumn, type CustomRow, type CustomDataset, type CustomDatatype, type CustomLevel } from "./custom-data-names.ts"; export * from "./custom-data-names.ts"; @@ -272,10 +272,10 @@ export function assertCustomDataPresent(x: unknown): void { const have = new Map>(); for (const d of activeCustomData()?.datasets ?? []) have.set(d.id, new Set(d.columns.map((c) => c.name))); const missing = new Set(), unknownCols: string[] = []; + for (const m of text.matchAll(CUSTOM_REF_RE)) if (!have.has(m[1])) missing.add(m[1]); for (const m of text.matchAll(/"custom:([a-z0-9][a-z0-9-]{0,39})\/([A-Za-z0-9_]{1,64})"/g)) { const cols = have.get(m[1]); - if (!cols) missing.add(m[1]); - else if (!cols.has(m[2])) unknownCols.push("custom:" + m[1] + "/" + m[2]); + if (cols && !cols.has(m[2])) unknownCols.push("custom:" + m[1] + "/" + m[2]); } if (missing.size) throw new Error("this analysis uses custom data that was not sent with it: " + [...missing].map((m) => "'" + m + "'").join(", ") + diff --git a/lib/dsl-reference.ts b/lib/dsl-reference.ts index 39c6502..e301ff7 100644 --- a/lib/dsl-reference.ts +++ b/lib/dsl-reference.ts @@ -1667,6 +1667,7 @@ rather than silently ignoring what it does not understand. | \`genericAssay\` | \`{"kind":"genericAssay","profileType":P,"entity":E,"range"\\|"values"}\` | generic assay values (mutational signatures, …) | | \`proteinPosition\` | \`{"kind":"proteinPosition","gene":G,"range":{gt,gte,lt,lte}}\` | patient carries a mutation in \`gene\` whose protein position (the stored proteinPosStart, same column the lollipop draws from) falls in \`range\` — for an N-/C-terminal split when a study doesn't deposit one as a clinical attribute. Position-only: AND with an \`oql\` leaf (e.g. \`{"kind":"oql","q":"APC: TRUNC"}\`) to also restrict by mutation type — the two can then be met by different mutations; an OQL position range (\`APC: TRUNC_(1400-)\`) requires both of one mutation | | \`split\` | \`{"kind":"split","part":"discovery"\|"validation"}\` | one fixed half of the study's patients, decided inside the database by a salted hash of the patient id — the same half for every caller, reproducible, and never passed through the client as an id list. It exists for **held-out replication**: test on \`discovery\`, confirm the same document on \`validation\`. A hypothesis campaign (\`/campaign\`, MCP \`campaign_*\`) adds it itself and refuses documents that already carry one, so a generator cannot choose its own half | +| \`upload\` | \`{"kind":"upload","dataset":"custom:"}\` | the patients with a row in an upload: a value in any of its columns, patient- or sample-level (a blank cell is not a value). \`dataset\` names the upload as its attributes do, and the document's \`uploads\` pins it like any of them; a document naming an upload the request cannot read is refused | | \`alteration\` | *(deprecated)* | translated to OQL internally; do not write new ones | **\`sampleList\`/\`patientList\` ids.** Each id is \`":"\` — a colon, not the diff --git a/lib/types.ts b/lib/types.ts index 1e86909..f2418cf 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -72,6 +72,12 @@ export interface PatientListLeaf { kind: "patientList"; ids?: string[]; set?: st * and never passes through the client as an id list. */ export interface SplitLeaf { kind: "split"; part: "discovery" | "validation" } +/** + * The patients with a row in an upload: a value in any of its columns (blank cells are not + * stored). `dataset` names the upload as its attributes do, "custom:", pinned by the + * document's `uploads` like any of its attributes. + */ +export interface UploadLeaf { kind: "upload"; dataset: string } export interface GeneStateFacet { gene: string; type: "MUTATION" | "CNA"; @@ -195,7 +201,7 @@ export interface DerivedLeaf { export type Leaf = | OqlLeaf | ClinicalLeaf | EventLeaf | SampleListLeaf | PatientListLeaf | ProfiledLeaf | CaseListLeaf | GenomicDataLeaf | GenericAssayLeaf | AlterationLeaf | DerivedLeaf - | ProteinPositionLeaf | SplitLeaf + | ProteinPositionLeaf | SplitLeaf | UploadLeaf /** back-compat: the flat v1 event shape, accepted without a `kind` */ | EventLeafFields; diff --git a/test/custom-data.test.ts b/test/custom-data.test.ts index 1127bb3..8dd13ec 100644 --- a/test/custom-data.test.ts +++ b/test/custom-data.test.ts @@ -139,6 +139,28 @@ describe("custom data: a document that names an upload it was not sent with", () }); }); +describe("custom data: the upload clause", () => { + const upload = { kind: "upload" as const, dataset: "custom:brain-mets" }; + test("selects the patients with a row of any of the upload's columns", () => withCustomData(ctx(), async () => { + const sql = compileCohortDef({ all: [upload] }, { study: "s1", db: "db1" }); + assert.match(sql, /_custom_clinical_[0-9a-f]{8}/); + assert.match(sql, /attribute_name IN \('custom:brain-mets\/MET_SITE','custom:brain-mets\/KPS'\)/); + // patient and sample rows alike + assert.doesNotMatch(sql, /type = /); + })); + test("names the upload as the document's other mentions of it do", async () => { + const { assertCustomDataPresent } = await import("../lib/custom-data.ts"); + const { referencedDatasets } = await import("../app/lab/custom-data-client.ts"); + const doc = { study: "s1", cohorts: { a: { all: [upload] } }, views: [] }; + assert.deepEqual(referencedDatasets(doc), ["brain-mets"]); + assert.throws(() => assertCustomDataPresent(doc), /custom data that was not sent with it: 'brain-mets'/); + await withCustomData(ctx(), async () => assert.doesNotThrow(() => assertCustomDataPresent(doc))); + }); + test("refuses a dataset that is not an upload's name", () => withCustomData(ctx(), async () => { + assert.throws(() => compileCohortDef({ all: [{ kind: "upload", dataset: "brain-mets" }] }, { study: "s1", db: "db1" }), /dataset:"custom:"/); + })); +}); + describe("custom data: every reader of the clinical tables goes through the merge", () => { test("no engine module names clinical_data_derived or clinical_attribute_meta in SQL directly", async () => { const { readdirSync, readFileSync } = await import("node:fs");