Custom data: "Analyze" opens an analysis of the patients with a row in an upload

app/lab/custom-data-client.ts

@@ -16,7 +16,7 @@
 //                           (`uploads`, see bindUploads) resolves to that version; a body that
 //                           carries a spec rather than the document is given the pins with it.
 import type { Sharing } from "../../lib/access.ts";
-import { CUSTOM_PREFIX, LIMITS, type CustomColumn, type CustomDataset, type CustomLevel } from "../../lib/custom-data-names.ts";
+import { CUSTOM_PREFIX, CUSTOM_REF_RE, LIMITS, type CustomColumn, type CustomDataset, type CustomLevel } from "../../lib/custom-data-names.ts";
 import type { DatasetMeta } from "../../lib/custom-store.ts";
 import { EXAMPLES } from "../../lib/dsl-reference.ts";
 import type { MembersBody } from "../../lib/api-schemas.ts";
@@ -42,7 +42,7 @@ export function datasetStudies(d: CustomDataset): string[] { return [...new Set(
 export function referencedDatasets(x: unknown): string[] {
   const text = typeof x === "string" ? x : JSON.stringify(x ?? null);
   if (!text.includes(CUSTOM_PREFIX)) return [];
-  return [...new Set([...text.matchAll(/"custom:([a-z0-9][a-z0-9-]{0,39})\//g)].map((m) => m[1]))];
+  return [...new Set([...text.matchAll(CUSTOM_REF_RE)].map((m) => m[1]))];
 }
 
 // ---------------------------------------------------------------------------

app/lab/custom-data-panel.tsx

@@ -13,7 +13,7 @@ import {
 } from "./custom-data-client.ts";
 import type { DatasetMeta } from "../../lib/custom-store.ts";
 import { LIMITS } from "../../lib/custom-data-names.ts";
-import { Download, LoaderCircle, Share2, Trash2 } from "lucide-react";
+import { Download, LoaderCircle, Share2, Trash2, Users } from "lucide-react";
 import { Button } from "../_ui/button";
 import { Dialog, DialogClose, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "../_ui/dialog";
 import { Input } from "../_ui/input";
@@ -22,6 +22,8 @@ import { Textarea } from "../_ui/textarea";
 import { cn } from "../../lib/cn";
 import { OptionSelect } from "../_ui/choice.tsx";
 import { TB } from "../_ui/toolbar.ts";
+import { encodeParam } from "../portal/page-doc.ts";
+import { DOC_VERSION } from "../../lib/doc-version.ts";
 
 /** a tab of the custom-data manager */
 const TAB = "h-auto flex-none rounded-none px-3 pt-1.5 pb-1.5 text-md after:bottom-[-1px] data-[state=active]:text-accent-foreground data-[state=active]:after:bg-primary";
@@ -245,6 +247,14 @@ function AddCustomData({ studies, onAdded }: { studies: string[]; onAdded: (name
   );
 }
 
+/** A new analysis of the patients with a row in this upload, pinned to its latest version */
+function analyzeUploadHref(d: DatasetMeta, studies: string[]): string {
+  const where = studies.length > 1 ? { studies } : { study: studies[0] };
+  const doc = { dsl: DOC_VERSION, study: studies[0], uploads: [{ as: d.slug, dataset: d.id, version: d.latestVersion }],
+    cohorts: { [d.name]: { ...where, all: [{ kind: "upload", dataset: "custom:" + d.slug }] } }, views: [] };
+  return "/lab?" + new URLSearchParams({ tab: "analysis", doc: encodeParam(doc) }).toString();
+}
+
 function SavedCustomData({ onChanged, onAdd }: { onChanged: () => void; onAdd: () => void }) {
   const server = useCustomDataState();
   const [serverBusy, setServerBusy] = useState<string | null>(null);
@@ -291,6 +301,9 @@ function SavedCustomData({ onChanged, onAdd }: { onChanged: () => void; onAdd: (
                     <td className="meta">{v?.rowCount.toLocaleString()} {d.level}s</td>
                     <td className="meta">{v?.studies.join(", ")}</td>
                     <td style={{ whiteSpace: "nowrap" }}>
+                      {v && v.studies.length > 0 && <><Button variant="outline" size="sm" className={TB} asChild>
+                        <a href={analyzeUploadHref(d, v.studies)} title={"start an analysis with one cohort: the " + v.rowCount.toLocaleString() + " " + d.level + "s' patients with a row in this upload"}>
+                          <Users />Analyze</a></Button>{" "}</>}
                       <Button variant="outline" size="sm" className={TB} disabled={!!serverBusy} onClick={() => exportServer(d)}><Download />Export</Button>{" "}
                       {d.rights.share && <><Button variant="outline" size="sm" className={TB} disabled={!!serverBusy} onClick={() => setSharing(d)}><Share2 />Share…</Button>{" "}
                         <Button variant="outline" size="sm" className={cn(TB, "text-destructive hover:text-destructive")} disabled={!!serverBusy} onClick={() => removeServer(d)}><Trash2 />Delete</Button></>}

app/lab/pills.tsx

@@ -21,7 +21,8 @@ const KIND_META: Record<string, { label: string; color: string }> = {
   genomicData: { label: "data", color: "#CC79A7" },
   genericAssay: { label: "assay", color: "#CC79A7" },
   derived: { label: "derived", color: "#D55E00" },
-  split: { label: "held-out", color: "#E69F00" }
+  split: { label: "held-out", color: "#E69F00" },
+  upload: { label: "custom data", color: "#0072B2" }
 };
 
 function leafKind(leaf: any) {
@@ -89,6 +90,7 @@ function leafText(leaf: any) {
   // mutations at all", where the clause a gene-status facet builds means "no
   // panel of theirs covers THIS gene". Two different cohorts, and the narrower
   // one is the whole point of that facet.
+  if (k === "upload") return "a row in " + String(leaf.dataset).slice("custom:".length);
   if (k === "profiled") return "profiled: " + leaf.type + (leaf.gene ? " · " + leaf.gene : "");
   if (k === "caseList") return leaf.id;
   if (k === "split") return leaf.part + " half";

docs/dsl.md

@@ -119,6 +119,7 @@ rather than silently ignoring what it does not understand.
 | `genericAssay` | `{"kind":"genericAssay","profileType":P,"entity":E,"range"\|"values"}` | generic assay values (mutational signatures, …) |
 | `proteinPosition` | `{"kind":"proteinPosition","gene":G,"range":{gt,gte,lt,lte}}` | patient carries a mutation in `gene` whose protein position (the stored proteinPosStart, same column the lollipop draws from) falls in `range` — for an N-/C-terminal split when a study doesn't deposit one as a clinical attribute. Position-only: AND with an `oql` leaf (e.g. `{"kind":"oql","q":"APC: TRUNC"}`) to also restrict by mutation type — the two can then be met by different mutations; an OQL position range (`APC: TRUNC_(1400-)`) requires both of one mutation |
 | `split` | `{"kind":"split","part":"discovery"|"validation"}` | one fixed half of the study's patients, decided inside the database by a salted hash of the patient id — the same half for every caller, reproducible, and never passed through the client as an id list. It exists for **held-out replication**: test on `discovery`, confirm the same document on `validation`. A hypothesis campaign (`/campaign`, MCP `campaign_*`) adds it itself and refuses documents that already carry one, so a generator cannot choose its own half |
+| `upload` | `{"kind":"upload","dataset":"custom:<name>"}` | the patients with a row in an upload: a value in any of its columns, patient- or sample-level (a blank cell is not a value). `dataset` names the upload as its attributes do, and the document's `uploads` pins it like any of them; a document naming an upload the request cannot read is refused |
 | `alteration` | *(deprecated)* | translated to OQL internally; do not write new ones |
 
 **`sampleList`/`patientList` ids.** Each id is `"<study>:<bare-id>"` — a colon, not the

e2e-saved.cjs

@@ -165,7 +165,20 @@ async function savedLists(browser, ok, BASE, LOGIN) {
         return /rows matched/.test(t) && t;
       }, null, { timeout: 60000 }).then((h) => h.jsonValue()).catch(() => "");
       ok("the upload example matches the explored study in full", /(\d+) of \1 rows matched a patient of msk_chord_2024/.test(ex), ex.slice(0, 120));
-      await page.keyboard.press("Escape");
+      // saved, its "Analyze" opens a new analysis whose one cohort is every patient with a row in it
+      const rows = Number((ex.match(/(\d[\d,]*) of/) || [])[1]?.replace(/,/g, ""));
+      let cohort = "";
+      try {
+        await page.locator("[role=dialog].customdatamanager button", { hasText: /^Add data to/ }).click();
+        await page.locator("[role=dialog].customdatamanager .customlist a", { hasText: "Analyze" }).first().click({ timeout: 60000 });
+        cohort = await page.waitForFunction(() => {
+          const b = [...document.querySelectorAll(".cohortbox")].find((x) => /a row in /.test(x.textContent || ""));
+          return b && /\d+ patients/.test(b.textContent) && b.textContent.replace(/\s+/g, " ");
+        }, null, { timeout: 120000 }).then((h) => h.jsonValue());
+      } catch (e) { cohort = cohort || String(e.message).split("\n")[0]; }
+      const n = Number((cohort.match(/([\d,]+) patients/) || [])[1]?.replace(/,/g, ""));
+      ok("a saved upload's Analyze opens an analysis of the patients with a row in it", rows > 0 && n === rows,
+        rows + " rows · " + cohort.slice(0, 140));
     }
 
     ok("Saved tab: no page errors", errors.length === 0, errors.join(" | ").slice(0, 200));

lib/cohort-partitions.ts

@@ -67,7 +67,7 @@ function leafEvaluability(leaf: Leaf): Evaluability {
   // Membership facts (a list, a case list, a profiling status, the held-out half) are known for
   // every patient. Measurements and timelines are not, and this engine has no "was measured"
   // condition for them yet, so they are named rather than silently treated as known.
-  if (kind === "patientList" || kind === "sampleList" || kind === "caseList" || kind === "profiled" || kind === "split")
+  if (kind === "patientList" || kind === "sampleList" || kind === "caseList" || kind === "profiled" || kind === "split" || kind === "upload")
     return { leaves: [], unchecked: [] };
   return { leaves: [], unchecked: [describeLeafBrief(leaf)] };
 }

lib/cohort2.ts

@@ -18,13 +18,14 @@ import { compileCohort, sq, LINE_TYPE } from "./compiler.ts";
 import { oqlLeafSql } from "./oql.ts";
 import { clinicalValueFoldSql, foldedEqualsSql } from "./event-names.ts";
 import { linesCtes, DEFAULT_LINE_GAP } from "./treatment-lines.ts";
-import { clinicalTable } from "./custom-data.ts";
+import { activeCustomData, clinicalTable } from "./custom-data.ts";
+import { UPLOAD_DATASET_RE, customAttr } from "./custom-data-names.ts";
 import { idSetTable, idSetIfLoaded, ID_SET_UUID } from "./id-sets.ts";
 import { lockClause } from "./cohort-lock.ts";
 import type {
   CohortDef, Clause, Leaf, CompileCtx, ClinicalLeaf, ClinicalRange,
   ProfiledLeaf, CaseListLeaf, GenomicDataLeaf, GenericAssayLeaf,
-  SampleListLeaf, PatientListLeaf, SplitLeaf, AlterationLeaf, EventLeafFields, ProteinPositionLeaf
+  SampleListLeaf, PatientListLeaf, SplitLeaf, UploadLeaf, AlterationLeaf, EventLeafFields, ProteinPositionLeaf
 } from "./types.ts";
 import { finiteSql } from "./sql-number.ts";
 
@@ -100,6 +101,18 @@ function clinicalLeafSql(leaf: ClinicalLeaf, ctx: Ctx): string {
     "    AND attribute_name = " + sq(attr) + " AND " + conds.join(" AND ");
 }
 
+/** The patients with any value in an upload: a row of any of its columns, patient or sample level. */
+function uploadLeafSql(leaf: UploadLeaf, ctx: Ctx): string {
+  const name = UPLOAD_DATASET_RE.exec(String(leaf.dataset || ""))?.[1];
+  if (!name) throw new Error('upload clause needs dataset:"custom:<name>"');
+  const d = activeCustomData()?.datasets.find((x) => x.id === name);
+  if (!d) throw new Error("this analysis uses custom data that was not sent with it: '" + name + "'");
+  const attrs = d.columns.map((c) => customAttr(d.id, c.name));
+  return "SELECT DISTINCT " + PID("patient_unique_id", ctx.study) + " AS pid\n" +
+    "  FROM " + clinicalTable(dbOf(ctx), attrs) + "\n" +
+    "  WHERE cancer_study_identifier = " + sq(ctx.study) + " AND attribute_name IN (" + attrs.map(sq).join(",") + ")";
+}
+
 const PROFILED_TYPE: Record<string, string> = { MUTATION: "MUTATION_EXTENDED", CNA: "COPY_NUMBER_ALTERATION", SV: "STRUCTURAL_VARIANT" };
 
 /**
@@ -517,6 +530,7 @@ function leafSql(leaf: Leaf, ctx: Ctx): string {
     case "sampleList": return sampleListLeafSql(leaf, ctx);
     case "patientList": return patientListLeafSql(leaf, ctx);
     case "split": return splitLeafSql(leaf, ctx);
+    case "upload": return uploadLeafSql(leaf, ctx);
     default: return unknownLeaf(leaf);
   }
 }
@@ -606,6 +620,7 @@ function leafText(leaf: any): string {
   }
   if (leaf.kind === "caseList") return "in case list " + leaf.id;
   if (leaf.kind === "split") return "in the " + leaf.part + " half (fixed held-out split)";
+  if (leaf.kind === "upload") return "with a row in upload " + String(leaf.dataset).slice("custom:".length);
   if (leaf.kind === "genomicData") return leaf.gene + " " + leaf.profileType + " " + JSON.stringify(leaf.range || leaf.values);
   if (leaf.kind === "genericAssay") return leaf.entity + " " + JSON.stringify(leaf.range || leaf.values);
   if (leaf.kind === "proteinPosition") return leaf.gene + " protein position " + JSON.stringify(leaf.range || {});

lib/custom-data-names.ts

@@ -22,6 +22,12 @@ export const ATTR_NAME_RE = /^(?:[A-Za-z0-9_]{1,64}|custom:[a-z0-9][a-z0-9-]{0,3
 /** An attribute name a view setting may hold: ATTR_NAME_RE's two forms, a built-in name also taking "." and "-" (HLA-A1). */
 export const VIEW_ATTR_RE = /^(?:[A-Za-z0-9_.-]{1,64}|custom:[a-z0-9][a-z0-9-]{0,39}\/[A-Za-z0-9_]{1,64})$/;
 
+/** A document's mention of an upload: an attribute of it ("custom:<name>/<COLUMN>") or the upload
+ *  itself ("custom:<name>", an upload clause's dataset). Group 1 is the upload's name. */
+export const CUSTOM_REF_RE = /"custom:([a-z0-9][a-z0-9-]{0,39})[/"]/g;
+/** An upload clause's dataset: the upload's name as its attributes carry it */
+export const UPLOAD_DATASET_RE = /^custom:([a-z0-9][a-z0-9-]{0,39})$/;
+
 export const LIMITS = {
   datasets: 10,
   columns: 64,

lib/custom-data.ts

@@ -26,7 +26,7 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 
 import {
-  CUSTOM_PREFIX, LIMITS, customAttr, isCustomAttr,
+  CUSTOM_PREFIX, CUSTOM_REF_RE, LIMITS, customAttr, isCustomAttr,
   type CustomColumn, type CustomRow, type CustomDataset, type CustomDatatype, type CustomLevel
 } from "./custom-data-names.ts";
 export * from "./custom-data-names.ts";
@@ -272,10 +272,10 @@ export function assertCustomDataPresent(x: unknown): void {
   const have = new Map<string, Set<string>>();
   for (const d of activeCustomData()?.datasets ?? []) have.set(d.id, new Set(d.columns.map((c) => c.name)));
   const missing = new Set<string>(), unknownCols: string[] = [];
+  for (const m of text.matchAll(CUSTOM_REF_RE)) if (!have.has(m[1])) missing.add(m[1]);
   for (const m of text.matchAll(/"custom:([a-z0-9][a-z0-9-]{0,39})\/([A-Za-z0-9_]{1,64})"/g)) {
     const cols = have.get(m[1]);
-    if (!cols) missing.add(m[1]);
-    else if (!cols.has(m[2])) unknownCols.push("custom:" + m[1] + "/" + m[2]);
+    if (cols && !cols.has(m[2])) unknownCols.push("custom:" + m[1] + "/" + m[2]);
   }
   if (missing.size)
     throw new Error("this analysis uses custom data that was not sent with it: " + [...missing].map((m) => "'" + m + "'").join(", ") +

lib/dsl-reference.ts

@@ -1667,6 +1667,7 @@ rather than silently ignoring what it does not understand.
 | \`genericAssay\` | \`{"kind":"genericAssay","profileType":P,"entity":E,"range"\\|"values"}\` | generic assay values (mutational signatures, …) |
 | \`proteinPosition\` | \`{"kind":"proteinPosition","gene":G,"range":{gt,gte,lt,lte}}\` | patient carries a mutation in \`gene\` whose protein position (the stored proteinPosStart, same column the lollipop draws from) falls in \`range\` — for an N-/C-terminal split when a study doesn't deposit one as a clinical attribute. Position-only: AND with an \`oql\` leaf (e.g. \`{"kind":"oql","q":"APC: TRUNC"}\`) to also restrict by mutation type — the two can then be met by different mutations; an OQL position range (\`APC: TRUNC_(1400-)\`) requires both of one mutation |
 | \`split\` | \`{"kind":"split","part":"discovery"\|"validation"}\` | one fixed half of the study's patients, decided inside the database by a salted hash of the patient id — the same half for every caller, reproducible, and never passed through the client as an id list. It exists for **held-out replication**: test on \`discovery\`, confirm the same document on \`validation\`. A hypothesis campaign (\`/campaign\`, MCP \`campaign_*\`) adds it itself and refuses documents that already carry one, so a generator cannot choose its own half |
+| \`upload\` | \`{"kind":"upload","dataset":"custom:<name>"}\` | the patients with a row in an upload: a value in any of its columns, patient- or sample-level (a blank cell is not a value). \`dataset\` names the upload as its attributes do, and the document's \`uploads\` pins it like any of them; a document naming an upload the request cannot read is refused |
 | \`alteration\` | *(deprecated)* | translated to OQL internally; do not write new ones |
 
 **\`sampleList\`/\`patientList\` ids.** Each id is \`"<study>:<bare-id>"\` — a colon, not the

lib/types.ts

@@ -72,6 +72,12 @@ export interface PatientListLeaf { kind: "patientList"; ids?: string[]; set?: st
  * and never passes through the client as an id list.
  */
 export interface SplitLeaf { kind: "split"; part: "discovery" | "validation" }
+/**
+ * The patients with a row in an upload: a value in any of its columns (blank cells are not
+ * stored). `dataset` names the upload as its attributes do, "custom:<name>", pinned by the
+ * document's `uploads` like any of its attributes.
+ */
+export interface UploadLeaf { kind: "upload"; dataset: string }
 export interface GeneStateFacet {
   gene: string;
   type: "MUTATION" | "CNA";
@@ -195,7 +201,7 @@ export interface DerivedLeaf {
 export type Leaf =
   | OqlLeaf | ClinicalLeaf | EventLeaf | SampleListLeaf | PatientListLeaf | ProfiledLeaf
   | CaseListLeaf | GenomicDataLeaf | GenericAssayLeaf | AlterationLeaf | DerivedLeaf
-  | ProteinPositionLeaf | SplitLeaf
+  | ProteinPositionLeaf | SplitLeaf | UploadLeaf
   /** back-compat: the flat v1 event shape, accepted without a `kind` */
   | EventLeafFields;
 

test/custom-data.test.ts

@@ -139,6 +139,28 @@ describe("custom data: a document that names an upload it was not sent with", ()
   });
 });
 
+describe("custom data: the upload clause", () => {
+  const upload = { kind: "upload" as const, dataset: "custom:brain-mets" };
+  test("selects the patients with a row of any of the upload's columns", () => withCustomData(ctx(), async () => {
+    const sql = compileCohortDef({ all: [upload] }, { study: "s1", db: "db1" });
+    assert.match(sql, /_custom_clinical_[0-9a-f]{8}/);
+    assert.match(sql, /attribute_name IN \('custom:brain-mets\/MET_SITE','custom:brain-mets\/KPS'\)/);
+    // patient and sample rows alike
+    assert.doesNotMatch(sql, /type = /);
+  }));
+  test("names the upload as the document's other mentions of it do", async () => {
+    const { assertCustomDataPresent } = await import("../lib/custom-data.ts");
+    const { referencedDatasets } = await import("../app/lab/custom-data-client.ts");
+    const doc = { study: "s1", cohorts: { a: { all: [upload] } }, views: [] };
+    assert.deepEqual(referencedDatasets(doc), ["brain-mets"]);
+    assert.throws(() => assertCustomDataPresent(doc), /custom data that was not sent with it: 'brain-mets'/);
+    await withCustomData(ctx(), async () => assert.doesNotThrow(() => assertCustomDataPresent(doc)));
+  });
+  test("refuses a dataset that is not an upload's name", () => withCustomData(ctx(), async () => {
+    assert.throws(() => compileCohortDef({ all: [{ kind: "upload", dataset: "brain-mets" }] }, { study: "s1", db: "db1" }), /dataset:"custom:<name>"/);
+  }));
+});
+
 describe("custom data: every reader of the clinical tables goes through the merge", () => {
   test("no engine module names clinical_data_derived or clinical_attribute_meta in SQL directly", async () => {
     const { readdirSync, readFileSync } = await import("node:fs");