From aa4dfb51a95a36d6e4a512a6654674169ae58a2a Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:02 -0700 Subject: [PATCH 1/2] Fix Docker image builds and keep scratch off the bind mount --- coderbuild/docker/Dockerfile.beataml | 21 +++++++ coderbuild/docker/Dockerfile.bladder | 21 +++++++ coderbuild/docker/Dockerfile.broad_sanger_exp | 62 ++++++++++++++++++- .../docker/Dockerfile.broad_sanger_omics | 48 +++++++++++++- coderbuild/docker/Dockerfile.colorectal | 21 +++++++ coderbuild/docker/Dockerfile.cptac | 21 +++++++ coderbuild/docker/Dockerfile.genes | 21 +++++++ coderbuild/docker/Dockerfile.hcmi | 21 +++++++ coderbuild/docker/Dockerfile.liver | 21 +++++++ coderbuild/docker/Dockerfile.mpnst | 32 +++++++++- coderbuild/docker/Dockerfile.mpnstpdx | 21 +++++++ coderbuild/docker/Dockerfile.novartis | 21 +++++++ coderbuild/docker/Dockerfile.pancreatic | 21 +++++++ coderbuild/docker/Dockerfile.sarcoma | 21 +++++++ coderbuild/docker/Dockerfile.upload | 21 +++++++ coderbuild/docker/docker-compose.yml | 20 +++++- 16 files changed, 406 insertions(+), 8 deletions(-) diff --git a/coderbuild/docker/Dockerfile.beataml b/coderbuild/docker/Dockerfile.beataml index d5b16b9f..63c3b9c6 100644 --- a/coderbuild/docker/Dockerfile.beataml +++ b/coderbuild/docker/Dockerfile.beataml @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app # Set MPLCONFIGDIR to a writable directory diff --git a/coderbuild/docker/Dockerfile.bladder b/coderbuild/docker/Dockerfile.bladder index 2cebce01..4312be7d 100644 --- a/coderbuild/docker/Dockerfile.bladder +++ b/coderbuild/docker/Dockerfile.bladder @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.broad_sanger_exp b/coderbuild/docker/Dockerfile.broad_sanger_exp index d25a90e3..21aed7e5 100755 --- a/coderbuild/docker/Dockerfile.broad_sanger_exp +++ b/coderbuild/docker/Dockerfile.broad_sanger_exp @@ -2,9 +2,57 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive -# Update and upgrade packages, then install required packages. + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + +# Install required packages. +# +# Do NOT add `apt-get upgrade -y` here. r-base:4.4.1 pins R at 4.4.1, but its +# apt sources point at Debian sid, where the r-cran-* packages have since been +# rebuilt against R 4.5.x. Upgrading therefore replaces the base image's +# consistent R library set with binaries built for a different R, and loading +# any of them fails with: +# +# /usr/lib/R/library/Matrix/libs/Matrix.so: undefined symbol: ANY_ATTRIB +# +# which cascades: Matrix breaks survival, which breaks reticulate, tidyverse +# and PharmacoGx, ~50 minutes into the image build. Verified: on a clean +# r-base:4.4.1 Matrix loads; after `apt-get upgrade -y` it does not. +# +# The upgrade is also unnecessary -- this apt list installs cleanly without it, +# and 4.4.1 already ships libssl3t64 3.6.3-1 so there is no openssl conflict. +# +# The lib*-dev entries below are build dependencies for CRAN packages that +# compile from source here. They are not optional: +# libuv1-dev fs. Without it fs fails to configure, taking down +# sass -> bslib -> rmarkdown/shiny -> DT/htmlwidgets -> +# piano -> CoreGx -> PharmacoGx. +# libfontconfig1-dev systemfonts (needs fontconfig + freetype2) +# libharfbuzz-dev textshaping +# libfribidi-dev textshaping +# libfreetype6-dev ragg -> tidyverse +# libpng/tiff/jpeg ragg image backends +# Each failure surfaces only as "dependency X is not available", an hour in, +# so the root cause is easy to misread as a CRAN outage. RUN apt-get update --fix-missing && \ - apt-get upgrade -y && \ apt-get install -y --fix-missing --allow-unauthenticated \ build-essential \ python3-pip \ @@ -15,7 +63,15 @@ RUN apt-get update --fix-missing && \ libglpk-dev \ libxml2-dev \ libpq-dev \ - ca-certificates && \ + ca-certificates \ + libuv1-dev \ + libfontconfig1-dev \ + libharfbuzz-dev \ + libfribidi-dev \ + libfreetype6-dev \ + libpng-dev \ + libtiff5-dev \ + libjpeg-dev && \ apt-get clean && rm -rf /var/lib/apt/lists/* # Create and upgrade the Python virtual environment. diff --git a/coderbuild/docker/Dockerfile.broad_sanger_omics b/coderbuild/docker/Dockerfile.broad_sanger_omics index 6b6cfc2e..23f73ca8 100755 --- a/coderbuild/docker/Dockerfile.broad_sanger_omics +++ b/coderbuild/docker/Dockerfile.broad_sanger_omics @@ -2,9 +2,43 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive -# Update and upgrade packages, then install required packages. + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + +# Install required packages. +# +# Do NOT add `apt-get upgrade -y` here. r-base:4.4.1 pins R at 4.4.1, but its +# apt sources point at Debian sid, where the r-cran-* packages have since been +# rebuilt against R 4.5.x. Upgrading therefore replaces the base image's +# consistent R library set with binaries built for a different R, and loading +# any of them fails with: +# +# /usr/lib/R/library/Matrix/libs/Matrix.so: undefined symbol: ANY_ATTRIB +# +# which cascades through every package that depends on Matrix. Verified: on a +# clean r-base:4.4.1 Matrix loads; after `apt-get upgrade -y` it does not. +# +# The upgrade is also unnecessary -- this apt list installs cleanly without it, +# and 4.4.1 already ships libssl3t64 3.6.3-1 so there is no openssl conflict. RUN apt-get update --fix-missing && \ - apt-get upgrade -y && \ apt-get install -y --fix-missing --allow-unauthenticated \ build-essential \ python3-pip \ @@ -15,7 +49,15 @@ RUN apt-get update --fix-missing && \ libglpk-dev \ libxml2-dev \ libpq-dev \ - ca-certificates && \ + ca-certificates \ + libuv1-dev \ + libfontconfig1-dev \ + libharfbuzz-dev \ + libfribidi-dev \ + libfreetype6-dev \ + libpng-dev \ + libtiff5-dev \ + libjpeg-dev && \ apt-get clean && rm -rf /var/lib/apt/lists/* # Create and upgrade the Python virtual environment. diff --git a/coderbuild/docker/Dockerfile.colorectal b/coderbuild/docker/Dockerfile.colorectal index c4b7b872..506d1f35 100644 --- a/coderbuild/docker/Dockerfile.colorectal +++ b/coderbuild/docker/Dockerfile.colorectal @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.cptac b/coderbuild/docker/Dockerfile.cptac index 95d11391..652b7e56 100755 --- a/coderbuild/docker/Dockerfile.cptac +++ b/coderbuild/docker/Dockerfile.cptac @@ -1,4 +1,25 @@ FROM python:3.11-slim + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + #ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --allow-insecure-repositories RUN apt-get install -y --allow-unauthenticated --fix-missing gcc-12 build-essential libpq-dev python3-venv diff --git a/coderbuild/docker/Dockerfile.genes b/coderbuild/docker/Dockerfile.genes index aa6c63af..ed87e121 100755 --- a/coderbuild/docker/Dockerfile.genes +++ b/coderbuild/docker/Dockerfile.genes @@ -1,5 +1,26 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + RUN apt-get update --fix-missing RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libglpk-dev libxml2-dev libpq-dev diff --git a/coderbuild/docker/Dockerfile.hcmi b/coderbuild/docker/Dockerfile.hcmi index 176f3c86..c465aeb7 100644 --- a/coderbuild/docker/Dockerfile.hcmi +++ b/coderbuild/docker/Dockerfile.hcmi @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app # Set MPLCONFIGDIR to a writable directory diff --git a/coderbuild/docker/Dockerfile.liver b/coderbuild/docker/Dockerfile.liver index ce0133b6..1b2e864a 100644 --- a/coderbuild/docker/Dockerfile.liver +++ b/coderbuild/docker/Dockerfile.liver @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.mpnst b/coderbuild/docker/Dockerfile.mpnst index 37c75f59..2f3eff70 100755 --- a/coderbuild/docker/Dockerfile.mpnst +++ b/coderbuild/docker/Dockerfile.mpnst @@ -1,8 +1,38 @@ -FROM r-base:4.3.3 +# r-base:4.4.1 to match every other R image in this build (bladder, colorectal, +# genes, liver, novartis, broad_sanger_exp, broad_sanger_omics). +# +# 4.3.3 could no longer build: its resident libssl3t64 (3.2.1-3) owns +# /usr/lib/x86_64-linux-gnu/ossl-modules/legacy.so, while Debian sid has moved +# that file into openssl-provider-legacy (3.6.3-1). Installing libssl-dev then +# failed the layer with "trying to overwrite ... which is also in package +# libssl3t64". 4.4.1 already ships libssl3t64 3.6.3-1, so libssl-dev installs +# cleanly with no dpkg workaround. +FROM r-base:4.4.1 # Set environment to noninteractive ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.mpnstpdx b/coderbuild/docker/Dockerfile.mpnstpdx index ddc6b1e3..08354c56 100755 --- a/coderbuild/docker/Dockerfile.mpnstpdx +++ b/coderbuild/docker/Dockerfile.mpnstpdx @@ -3,6 +3,27 @@ FROM r-base:4.3.2 # Set environment to noninteractive ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.novartis b/coderbuild/docker/Dockerfile.novartis index 9687b410..bf9edf79 100644 --- a/coderbuild/docker/Dockerfile.novartis +++ b/coderbuild/docker/Dockerfile.novartis @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.pancreatic b/coderbuild/docker/Dockerfile.pancreatic index bf797c9c..95f088e8 100644 --- a/coderbuild/docker/Dockerfile.pancreatic +++ b/coderbuild/docker/Dockerfile.pancreatic @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app diff --git a/coderbuild/docker/Dockerfile.sarcoma b/coderbuild/docker/Dockerfile.sarcoma index 3b741c39..ac25f26a 100644 --- a/coderbuild/docker/Dockerfile.sarcoma +++ b/coderbuild/docker/Dockerfile.sarcoma @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app # Set MPLCONFIGDIR to a writable directory diff --git a/coderbuild/docker/Dockerfile.upload b/coderbuild/docker/Dockerfile.upload index e08bb0c3..7862cef4 100644 --- a/coderbuild/docker/Dockerfile.upload +++ b/coderbuild/docker/Dockerfile.upload @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app RUN python -m pip install --upgrade pip pyyaml requests linkml diff --git a/coderbuild/docker/docker-compose.yml b/coderbuild/docker/docker-compose.yml index 51eb56ee..0a87e0e5 100644 --- a/coderbuild/docker/docker-compose.yml +++ b/coderbuild/docker/docker-compose.yml @@ -96,6 +96,15 @@ services: platform: linux/amd64 image: genes:latest + phosphosites: + build: + context: ../../ + dockerfile: coderbuild/docker/Dockerfile.phosphosites + args: + HTTPS_PROXY: ${HTTPS_PROXY} + platform: linux/amd64 + image: phosphosites:latest + upload: build: context: ../../ @@ -130,4 +139,13 @@ services: args: HTTPS_PROXY: ${HTTPS_PROXY} platform: linux/amd64 - image: novartis:latest \ No newline at end of file + image: novartis:latest + + cnf: + build: + context: ../../ + dockerfile: coderbuild/docker/Dockerfile.cnf + args: + HTTPS_PROXY: ${HTTPS_PROXY} + platform: linux/amd64 + image: cnf:latest \ No newline at end of file From 5be9072052b5fd58a79dd02cd68cb81fdb2e536d Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:02 -0700 Subject: [PATCH 2/2] Add build resume support and DepMap readiness gate Also fixes three orchestration bugs found in the v19 full build, and redacts credentials from the build log. Every process_* function awaited only the PREVIOUS future before submitting the next, so the LAST dataset's exception was never retrieved -- and an unretrieved exception in a ThreadPoolExecutor is discarded, not raised. In --high_mem mode nothing was awaited at all. In v19 that let `hcmi omics` fail all three attempts while the build logged "All omics files completed", then published and validated a release with hcmi transcriptomics, copy_number and mutations missing entirely. Every future is now collected and retrieved through _await_all(), so no dataset failure can be dropped. The resulting `validate failed` printed only "None": stderr is passed through to sys.stderr, so subprocess never captures it and res.stderr is always None. Report the exit code and point at the container output. Finally, docker command lines were logged verbatim, and credentials are passed as `-e NAME=value`, so build logs contained the live SYNAPSE_AUTH_TOKEN in cleartext. Added _redact() and gitignored the build logs and PID files. --- .gitignore | 8 + coderbuild/build_all.py | 755 +++++++++++++++++++++++++++++++----- coderbuild/build_dataset.py | 112 +++++- 3 files changed, 773 insertions(+), 102 deletions(-) diff --git a/.gitignore b/.gitignore index fe663199..bfd44a7a 100755 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,11 @@ build/lib build/local coderdata/_version.py local/ + +# Build logs and PID files. +# Build logs capture full `docker run` command lines. Those command lines pass +# credentials as `-e NAME=value`, so older logs contain a live +# SYNAPSE_AUTH_TOKEN in cleartext (build_all_v19.log had 47 copies). +# coderbuild/build_all.py now redacts them via _redact(), but never commit these. +*.log +build_v*.pid diff --git a/coderbuild/build_all.py b/coderbuild/build_all.py index 067dd5dc..70e50bc4 100644 --- a/coderbuild/build_all.py +++ b/coderbuild/build_all.py @@ -12,7 +12,34 @@ from glob import glob import sys import requests - +import threading +import atexit +import signal +import tempfile +import yaml +from datetime import datetime + +def _log(*args, **kwargs): + ts = datetime.now().strftime('[%Y-%m-%d %H:%M:%S]') + print(ts, *args, **kwargs) + +def _redact(cmd): + """Render a command for logging with secret values masked. + + Tokens are passed to containers as `-e NAME=value`, so logging the command + verbatim wrote them in cleartext. build_all_v19.log contained the live + SYNAPSE_AUTH_TOKEN 47 times, and the build logs sit untracked in the repo + root where a `git add -A` would commit them. + """ + out = [] + for part in cmd: + if '=' in part: + key, _, val = part.partition('=') + if val and any(m in key.upper() for m in ('TOKEN', 'SECRET', 'PASSWORD', 'KEY')): + part = f'{key}=***REDACTED***' + out.append(part) + return ' '.join(out) + def main(): parser=argparse.ArgumentParser( description="This script initializes all docker containers, builds datasets, validates them, and uploads to Figshare.", @@ -35,19 +62,246 @@ def main(): parser.add_argument('--samples',dest='samples',default=False,action='store_true', help="Build all sample files.") parser.add_argument('--omics',dest='omics',default=False,action='store_true', help="Build all omics files.") parser.add_argument('--drugs',dest='drugs',default=False,action='store_true', help="Build all drug files") + parser.add_argument('--prev_drugs', type=str, default='', help='Comma-separated list of previously built drug files (e.g. local_old/broad_sanger_drugs.tsv,local_old/beataml_drugs.tsv). Each matching dataset is still rebuilt, but its previous file seeds pubchem_retrieval so only genuinely new drugs are queried. Existing drugs are preserved; removed drugs fall off naturally.') parser.add_argument('--misc', action='store_true', help="Run the final misc post-build step (e.g., split broad_sanger datasets).") parser.add_argument('--exp',dest='exp',default=False,action='store_true', help="Build all experiment file.") parser.add_argument('--validate', action='store_true', help="Run schema checker on all local files. Note this will be run, whether specified or not, if figshare arguments are included.") parser.add_argument('--figshare', action='store_true', help="Upload all local data to Figshare. FIGSHARE_TOKEN must be set in local environment.") parser.add_argument('--all',dest='all',default=False,action='store_true', help="Run all data build commands. This includes docker, samples, omics, drugs, exp arguments. This does not run the validate or figshare commands") parser.add_argument('--high_mem',dest='high_mem',default=False,action='store_true',help = "If you have 32 or more CPUs, this option is recommended. It will run many code portions in parallel. If you don't have enough memory, this will cause a run failure.") - parser.add_argument('--dataset',dest='datasets',default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,hcmi',help='Datasets to process. Defaults to all available.') + parser.add_argument('--dataset',dest='datasets',default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,cptac,hcmi',help='Datasets to process. Defaults to all available.') parser.add_argument('--version', type=str, required=False, help='Version number for the Figshare upload title (e.g., "0.1.29"). This is required for Figshare upload. This must be a higher version than previously published versions.') parser.add_argument('--github-username', type=str, required=False, help='GitHub username for the repository.') parser.add_argument('--github-email', type=str, required=False, help='GitHub email for the repository.') - + parser.add_argument('--depmap-ready', dest='depmap_ready', default=False, action='store_true', + help='Confirm that the static DepMap files on Synapse are up to date for ' + 'the current DepMap release. Required for any build that includes the ' + 'broad_sanger dataset, because DepMap files can no longer be downloaded ' + 'programmatically and must be refreshed by hand.') + parser.add_argument('--continue', dest='resume', default=False, action='store_true', + help='Resume a partial build: skip steps whose sentinel output already ' + 'exists in local/, rebuild Docker images only for datasets that ' + 'failed in the previous run, and log all activity to ' + 'local/build_progress.log.') + args = parser.parse_args() - + + # ------------------------------------------------------------------------- + # Resume / progress tracking helpers + # ------------------------------------------------------------------------- + PROGRESS_LOG = 'local/build_progress.log' + _progress_lock = threading.Lock() + + def _step_sentinel(step_name): + """Return the file/dir whose existence proves this step completed.""" + if step_name == 'genes file': + return 'local/genes.csv' + if step_name == 'phosphosites file': + return 'local/phosphosites.csv' + # All other steps are "{dataset} {type}" e.g. "colorectal omics" + da, _, stype = step_name.rpartition(' ') + if not stype: + return None + if stype == 'samples': + return f'local/{da}_samples.csv' + if stype == 'drugs': + return f'local/{da}_drugs.tsv' + if stype == 'omics': + # transcriptomics is the last large file written — proves full completion + for ext in ('.csv.gz', '.csv'): + candidate = f'local/{da}_transcriptomics{ext}' + if os.path.exists(candidate): + return candidate + # default expectation for first-time check + gzip_ds = {'broad_sanger', 'beataml', 'pancreatic', 'hcmi'} + ext = '.csv.gz' if da in gzip_ds else '.csv' + return f'local/{da}_transcriptomics{ext}' + if stype == 'experiments': + return f'local/{da}_experiments.tsv' + if stype == 'misc': + return 'local/all_files_dir' + return None + + def _sentinel_done(step_name): + """Return True if the sentinel for this step exists and is non-trivial.""" + s = _step_sentinel(step_name) + if s is None: + return False + if os.path.isdir(s): + return bool(os.listdir(s)) + return os.path.isfile(s) and os.path.getsize(s) > 100 + + def _step_cleanup_globs(step_name): + """Glob patterns (relative to cwd) to remove when a step fails.""" + if step_name == 'genes file': + return ['local/genes.csv'] + if step_name == 'phosphosites file': + return ['local/phosphosites.csv'] + da, _, stype = step_name.rpartition(' ') + if not stype: + return [] + if stype == 'samples': + return [f'local/{da}_samples.csv'] + if stype == 'drugs': + return [f'local/{da}_drugs.tsv', f'local/{da}_drug_descriptors.tsv.gz'] + if stype == 'omics': + if da == 'broad_sanger': + # broad_sanger omics produces both broad_* and sanger_* files + return [ + 'local/broad_mutations*', 'local/broad_transcriptomics*', + 'local/broad_copy_number*', 'local/broad_proteomics*', + 'local/sanger_mutations*', 'local/sanger_transcriptomics*', + 'local/sanger_copy_number*', 'local/sanger_proteomics*', + ] + return [ + f'local/{da}_mutations*', f'local/{da}_transcriptomics*', + f'local/{da}_copy_number*', f'local/{da}_proteomics*', + f'local/{da}_phosphoproteomics*', + ] + if stype == 'experiments': + return [f'local/{da}_experiments.tsv'] + if stype == 'misc': + return ['local/all_files_dir'] + return [] + + def _cleanup_step(step_name): + """Remove partial outputs from a failed step; log what was removed.""" + removed = [] + for pattern in _step_cleanup_globs(step_name): + for f in glob(pattern): + try: + if os.path.isdir(f): + shutil.rmtree(f) + else: + os.remove(f) + removed.append(f) + except Exception as exc: + _log(f'[cleanup] could not remove {f}: {exc}') + if removed: + _log(f'[cleanup] {step_name}: removed {len(removed)} file(s): {", ".join(removed)}') + _write_progress(step_name, 'CLEANUP', f'removed: {", ".join(removed)}') + else: + _log(f'[cleanup] {step_name}: nothing to remove') + + def _clean_local_intermediates(reason='', protect_prefixes=()): + """Move leftover intermediate files in local/ into a quarantine + directory (local/unexpected_intermediates/) to keep the working set + small (raw downloads like rnaseq_all_data_*.csv, cell.xml, + proteomics_all_*.csv, *.xlsx, etc. are never needed after the step that + consumed them and otherwise accumulate until a later step hits ENOSPC). + + Files are MOVED, not deleted, so a mis-declared build output is never + silently lost -- if something you expected to be uploaded ends up in + local/unexpected_intermediates/, it means it was not declared in + schema/expected_files.yaml. NOTE: because the quarantine directory is on + the same filesystem, moving does NOT reclaim disk space during the + build; clear local/unexpected_intermediates/ (or point it at another + volume) if you hit disk pressure. + + Safety: a file is KEPT in place if it is a schema-declared output (per + schema/expected_files.yaml -- these may still be sitting in local/ before + the move to all_files_dir), a reference file, an incremental-build cache, + build infrastructure, or matches one of protect_prefixes (e.g. + broad*/sanger*, which build_misc.sh still needs to split broad_sanger). + Only regular files matching local/*.* are considered (mirrors the move + step's own glob), so directories like all_files_dir are never touched. + """ + try: + with open('schema/expected_files.yaml') as _f: + _schema = yaml.safe_load(_f) + except Exception as exc: + _log(f'[cleanup-intermediates] {reason}: skipped, cannot read schema: {exc}') + return + keep = set() + for _entries in _schema.get('datasets', {}).values(): + for _entry in _entries: + _bn = os.path.basename(_entry['file']) + keep.add(_bn) + keep.add(_bn + '.gz') + if _bn.endswith('.gz'): + keep.add(_bn[:-3]) + protected_exact = {'genes.csv', 'phosphosites.csv', 'prev_drug_files.txt'} + quarantine = os.path.join('local', 'unexpected_intermediates') + moved, moved_bytes = 0, 0 + for f in glob(os.path.join('local', '*.*')): + if os.path.isdir(f): + continue + bn = os.path.basename(f) + if bn in keep or bn in protected_exact: + continue + if '_prev.' in bn or bn.endswith(('.cid', '.log', '.jsonl')): + continue + if any(bn.startswith(p) for p in protect_prefixes): + continue + try: + os.makedirs(quarantine, exist_ok=True) + sz = os.path.getsize(f) + dest = os.path.join(quarantine, bn) + if os.path.exists(dest): + os.remove(dest) # replace a stale copy from an earlier pass + shutil.move(f, dest) + moved += 1 + moved_bytes += sz + except Exception as exc: + _log(f'[cleanup-intermediates] could not move {f}: {exc}') + if moved: + _log(f'[cleanup-intermediates] {reason}: moved {moved} file(s) ' + f'({moved_bytes / 1e9:.2f} GB) to {quarantine}/') + else: + _log(f'[cleanup-intermediates] {reason}: nothing to move') + + def _write_progress(step, status, note=''): + """Append one line to local/build_progress.log (thread-safe).""" + ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + line = f'{ts} | {status:<8} | {step}' + if note: + line += f' ({note})' + line += '\n' + with _progress_lock: + try: + with open(PROGRESS_LOG, 'a') as f: + f.write(line) + except Exception: + pass # never let logging crash the build + + def _step_to_docker_dataset(step_name): + """Map a step name to the dataset name used by process_docker.""" + if step_name in ('genes file', 'gene file'): + return 'genes' + if step_name == 'phosphosites file': + return 'phosphosites' + da, _, stype = step_name.rpartition(' ') + if not stype: + return None + return da # e.g. "colorectal", "broad_sanger", "liver" + + def _last_run_failures(): + """ + Parse build_progress.log and return the set of step names that + FAILED in the most recent run (since the last START line) and were + not subsequently fixed (no SUCCESS line after the FAILURE). + """ + if not os.path.exists(PROGRESS_LOG): + return set() + current_failures = set() + in_run = False + with open(PROGRESS_LOG) as f: + for line in f: + parts = [p.strip() for p in line.split('|', 2)] + if len(parts) < 3: + continue + _ts, status, rest = parts + step = rest.split('(')[0].strip() # strip any trailing note + if status == 'START': + in_run = True + current_failures = set() + elif in_run and status == 'FAILURE': + current_failures.add(step) + elif in_run and status == 'SUCCESS': + current_failures.discard(step) + return current_failures + + # ------------------------------------------------------------------------- # Simulation command for testing order of everything: # def run_docker_cmd(cmd_arr, filename): # # Simulate running the command by printing what would be run @@ -56,37 +310,91 @@ def main(): # time.sleep(2) # print(f'Completed: {filename}') - def run_docker_cmd(cmd_arr,filename): + # --- Container cleanup on exit/interrupt --- + _active_cid_files = [] + _cid_lock = threading.Lock() + + def _kill_all_containers(): + with _cid_lock: + cids = list(_active_cid_files) + for cid_file in cids: + try: + with open(cid_file) as f: + cid = f.read().strip() + if cid: + _log(f"Cleaning up container {cid[:12]}...") + subprocess.run(['docker', 'kill', cid], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except Exception: + pass + + def _signal_handler(_signum, _frame): + _log("\nBuild interrupted — killing all running containers...") + _kill_all_containers() + sys.exit(1) + + atexit.register(_kill_all_containers) + signal.signal(signal.SIGINT, _signal_handler) + signal.signal(signal.SIGTERM, _signal_handler) + + def run_docker_cmd(cmd_arr, filename): ''' Essentially a wrapper for 'docker run'. Also provides output. + Tracks each container via --cidfile so they can be killed on exit/interrupt. + With --continue: skips steps whose sentinel file already exists, and + cleans up partial outputs after all retries are exhausted. ''' - retries=3 + # --- Resume: skip if already done --- + if args.resume and _sentinel_done(filename): + sentinel = _step_sentinel(filename) + _log(f'[{filename}] skipping — already complete ({sentinel})') + _write_progress(filename, 'SKIPPED', f'sentinel: {sentinel}') + return + + retries = 3 delays = [3 * 60, 10 * 60] # 3 minutes, 10 minutes - print('running...'+filename) + _log('running...' + filename) env = os.environ.copy() if 'SYNAPSE_AUTH_TOKEN' not in env.keys(): - print('You need to set the SYNAPSE_AUTH_TOKEN to acess the MPNST, beatAML, bladder, pancreatic, liver, or sarcoma datasets') - docker_run = ['docker','run','--rm','-v',env['PWD']+'/local/:/tmp/','--platform=linux/amd64'] + _log('You need to set the SYNAPSE_AUTH_TOKEN to acess the MPNST, beatAML, bladder, pancreatic, liver, sarcoma, cnf datasets') + docker_run = ['docker', 'run', '--rm', '-v', env['PWD']+'/local/:/tmp/', '--platform=linux/amd64'] else: - docker_run = ['docker','run','--rm','-v',env['PWD']+'/local/:/tmp/','-e','SYNAPSE_AUTH_TOKEN='+env['SYNAPSE_AUTH_TOKEN'],'--platform=linux/amd64'] - cmd = docker_run+cmd_arr - print(cmd) - + docker_run = ['docker', 'run', '--rm', '-v', env['PWD']+'/local/:/tmp/', '-e', 'SYNAPSE_AUTH_TOKEN='+env['SYNAPSE_AUTH_TOKEN'], '--platform=linux/amd64'] + attempt = 1 while attempt <= retries: - print(f"[{filename}] Attempt {attempt}/{retries}: {' '.join(cmd)}") - res = subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr) - if res.returncode == 0: - print(f"[{filename}] succeeded on attempt {attempt}.") - return - else: - print(f"[{filename}] failed (exit {res.returncode}).") - if attempt < retries: - delay = delays[attempt - 1] - print(f"[{filename}] waiting {delay//60} minutes before retrying...") - print(cmd) - time.sleep(delay) + with tempfile.NamedTemporaryFile(suffix='.cid', delete=False) as _tf: + cid_file = _tf.name + os.remove(cid_file) # Docker requires the file to not exist before writing + cmd = docker_run + ['--cidfile', cid_file] + cmd_arr + with _cid_lock: + _active_cid_files.append(cid_file) + try: + _log(f"[{filename}] Attempt {attempt}/{retries}: {_redact(cmd)}") + res = subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr) + if res.returncode == 0: + _log(f"[{filename}] succeeded on attempt {attempt}.") + _write_progress(filename, 'SUCCESS') + return + else: + _log(f"[{filename}] failed (exit {res.returncode}).") + if attempt < retries: + delay = delays[attempt - 1] + _log(f"[{filename}] waiting {delay//60} minutes before retrying...") + time.sleep(delay) + finally: + with _cid_lock: + if cid_file in _active_cid_files: + _active_cid_files.remove(cid_file) + try: + os.remove(cid_file) + except OSError: + pass attempt += 1 + + # All retries exhausted — log failure and remove partial outputs + _write_progress(filename, 'FAILURE', f'all {retries} attempts failed') + _cleanup_step(filename) raise RuntimeError(f"{filename} failed after {retries} attempts") @@ -113,13 +421,17 @@ def process_docker(datasets): 'colorectal': ['colorectal'], 'cptac': ['cptac'], 'genes': ['genes'], + 'phosphosites': ['phosphosites'], 'upload': ['upload'], 'liver': ['liver'], - 'novartis': ['novartis'] + 'novartis': ['novartis'], + 'cnf': ['cnf'] } # Collect container names to build based on the datasets provided. Always build genes and upload. datasets_to_build = ['genes', 'upload'] + if 'cnf' in datasets: + datasets_to_build.append('phosphosites') for dataset in datasets: datasets_to_build.extend(dataset_map.get(dataset, [])) @@ -129,50 +441,153 @@ def process_docker(datasets): log_file_path = 'local/docker.log' env = os.environ.copy() - print(f"Docker-compose is building images for {', '.join(datasets_to_build)}. View output in {log_file_path}.") + _log(f"Docker-compose is building images for {', '.join(datasets_to_build)}. View output in {log_file_path}.") with open(log_file_path, 'w') as log_file: try: # Execute the docker-compose command res = subprocess.run(compose_command, env=env, stdout=log_file, stderr=log_file, text=True, check=True) log_file.write("Docker images built successfully.\n") - print(f"Docker images for {', '.join(datasets_to_build)} built successfully. Details logged in {log_file_path}.") + _log(f"Docker images for {', '.join(datasets_to_build)} built successfully. Details logged in {log_file_path}.") except subprocess.CalledProcessError as e: log_file.write(f"Docker compose build failed with error: {e}\n") - print(f"Docker compose build failed. See {log_file_path} for details.") + _log(f"Docker compose build failed. See {log_file_path} for details.") raise - def process_drugs(executor, datasets): + + # target_class values in schema/expected_files.yaml, grouped by the build + # step that produces them. + _STEP_CLASSES = { + 'samples': {'Sample'}, + 'omics': {'Transcriptomics', 'Proteomics', 'Mutations', 'Copy Number', + 'Phosphoproteomics'}, + 'drugs': {'Drug', 'Drug Descriptor'}, + 'experiments': {'Experiments', 'Combinations'}, + } + + def _verify_step_outputs(dataset, step): + """Fail a step that exited 0 without writing the files it declares. + + build_all only checks a container's exit status, so a script that + returns 0 after doing nothing counts as success. beataml omics did + exactly that on 2026-09-13: it ran for 88 seconds (against 4m24s in a + known-good build), printed "Starting Transcriptomics Data", never + reached proteomics, exited 0, and wrote none of its three files. The + build carried on through experiments, misc and three upload stages and + only fell over ~11 hours later in validate. + + schema/expected_files.yaml already declares what each dataset must + produce, so check it here and fail at the step that went wrong, while + the log still shows why. + """ + want = _STEP_CLASSES.get(step) + if not want: + return + try: + with open('schema/expected_files.yaml') as fh: + declared = yaml.safe_load(fh)['datasets'].get(dataset, []) + except Exception as exc: + _log(f'[verify] could not read expected_files.yaml ({exc}); skipping output check.') + return + + missing = [] + for entry in declared: + if entry.get('target_class') not in want: + continue + base = os.path.basename(entry['file']) + stem = base[:-3] if base.endswith('.gz') else base + # the build writes either the plain or the gzipped form + if not any(os.path.exists(os.path.join('local', c)) + for c in (stem, stem + '.gz')): + missing.append(stem) + + if missing: + raise RuntimeError( + f"{dataset} {step} reported success but produced none of: " + f"{', '.join(missing)}. The step exited 0 without writing the " + f"files schema/expected_files.yaml declares for it, so the build " + f"is stopping here rather than failing later in validate.") + + def _await_all(futures, step_label): + """Retrieve every future so no step failure can be silently dropped. + + This is load-bearing. Each process_* function used to await only the + PREVIOUS future before submitting the next, so the LAST dataset's + exception was never retrieved -- and an unretrieved exception in a + ThreadPoolExecutor is discarded, not raised. In --high_mem mode nothing + was awaited at all. + + On 2026-09-04 that let `hcmi omics` fail all three attempts while the + build logged "All omics files completed" and carried on to publish and + validate a release with hcmi transcriptomics missing entirely. + + Raises the first failure after collecting the rest, so one broken step + cannot hide behind a later success. + """ + errors = [] + for fut in futures: + try: + fut.result() + except Exception as exc: + errors.append(exc) + if errors: + if len(errors) > 1: + _log(f'{step_label}: {len(errors)} steps failed; raising the first.') + raise errors[0] + + def process_drugs(executor, datasets, prev_drugs=None): ''' - Build all drug files sequentially + Build all drug files sequentially. + + prev_drugs: list of local paths (e.g. ['local_old/broad_sanger_drugs.tsv', ...]) + Files are copied into local/ and listed in local/prev_drug_files.txt. + pubchem_retrieval.py auto-loads this file on import and uses it as a + local cache: known drugs skip HTTP requests entirely, but ID assignment + proceeds exactly as in a fresh run (dflist chain is unchanged). ''' last_drug_future = None - dflist = [] - - # WE NEED A METHOD TO CONFIRM THAT DRUG FILES ARE NOT INCOMPLETE - ##THIS IS BUILT IN- always rerun drug code to check - # Check for existing files and update dflist with processed files - for da in datasets: - if da not in ['cptac', 'hcmi']: - file_path = f'local/{da}_drugs.tsv' - desc_path = f'local/{da}_drug_descriptor.tsv.gz' - #if os.path.exists(file_path): ##always rerun drug process - # dflist.append(f'/tmp/{da}_drugs.tsv') # Add to dflist if already processed + _futures = [] + dflist = [] + + if prev_drugs: + docker_paths = [] + for p in prev_drugs: + base = os.path.basename(p) + if base.endswith('.tsv.gz'): + stem = base[:-len('.tsv.gz')] + prev_name = f'{stem}_prev.tsv.gz' + elif base.endswith('.tsv'): + stem = base[:-len('.tsv')] + prev_name = f'{stem}_prev.tsv' + else: + stem = os.path.splitext(base)[0] + prev_name = f'{stem}_prev.tsv' + shutil.copy(p, f'local/{prev_name}') + docker_paths.append(f'/tmp/{prev_name}') + _log(f'Copied prev drug cache: {p} → local/{prev_name}') + with open('local/prev_drug_files.txt', 'w') as f: + f.write('\n'.join(docker_paths) + '\n') + _log(f'Wrote local/prev_drug_files.txt with {len(docker_paths)} cache file(s).') for da in datasets: if da not in ['cptac', 'hcmi']: di = 'broad_sanger_exp' if da == 'broad_sanger' else da - #if not os.path.exists(f'local/{da}_drugs.tsv'): if last_drug_future: last_drug_future.result() # Ensure the last drug process is completed before starting the next - last_drug_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_drugs.sh', ','.join(dflist)], f'{da} drugs') + def _drugs_step(image=di, arg=','.join(dflist), ds=da): + run_docker_cmd([image, 'bash', 'build_drugs.sh', arg], f'{ds} drugs') + _verify_step_outputs(ds, 'drugs') + last_drug_future = executor.submit(_drugs_step) + _futures.append(last_drug_future) dflist.append(f'/tmp/{da}_drugs.tsv') - + _await_all(_futures, 'drugs') + def process_samples(executor, datasets): ''' Build all samples files sequentially ''' last_sample_future = None + _futures = [] sf = '' for da in datasets: file_path = f'local/{da}_samples.csv' @@ -185,49 +600,77 @@ def process_samples(executor, datasets): if not os.path.exists(f'local/{da}_samples.csv'): if last_sample_future: last_sample_future.result() - last_sample_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_samples.sh', sf], f'{da} samples') + def _samples_step(image=di, arg=sf, ds=da): + run_docker_cmd([image, 'bash', 'build_samples.sh', arg], f'{ds} samples') + _verify_step_outputs(ds, 'samples') + last_sample_future = executor.submit(_samples_step) + _futures.append(last_sample_future) sf = f'/tmp/{da}_samples.csv' - - def process_omics(executor, datasets,high_mem): + _await_all(_futures, 'samples') + + def process_phosphosites(executor): + ''' + Build the phosphosites reference file if it does not exist. + Caller must ensure genes.csv exists first. Returns a Future (or None). + ''' + if not os.path.exists('local/phosphosites.csv'): + return executor.submit( + run_docker_cmd, + ['phosphosites', 'bash', 'build_phosphosites.sh', '/tmp/genes.csv'], + 'phosphosites file', + ) + return None + + def process_omics(executor, datasets, high_mem): ''' Build all omics files concurrently ''' last_omics_future = None + _futures = [] for da in datasets: di = 'broad_sanger_omics' if da == 'broad_sanger' else da - #Run all at once: + omics_cmd = [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{da}_samples.csv'] + if da == 'cnf': + omics_cmd.append('/tmp/phosphosites.csv') + def _omics_step(cmd=omics_cmd, ds=da): + run_docker_cmd(cmd, f'{ds} omics') + _verify_step_outputs(ds, 'omics') if high_mem: - executor.submit(run_docker_cmd, [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{da}_samples.csv'], f'{da} omics') - #Run one at a time. + _futures.append(executor.submit(_omics_step)) else: if last_omics_future: - last_omics_future.result() - last_omics_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{da}_samples.csv'], f'{da} omics') - + last_omics_future.result() + last_omics_future = executor.submit(_omics_step) + _futures.append(last_omics_future) + _await_all(_futures, 'omics') + def process_experiments(executor, datasets, high_mem): ''' Build all experiments files concurrently ''' last_experiments_future = None + _futures = [] for da in datasets: if da not in ['cptac', 'hcmi']: di = 'broad_sanger_exp' if da == 'broad_sanger' else da if not os.path.exists(f'local/{da}_experiments.tsv'): #Run all at once if high_mem: - executor.submit(run_docker_cmd, [di, 'bash', 'build_exp.sh', f'/tmp/{da}_samples.csv', f'/tmp/{da}_drugs.tsv'], f'{da} experiments') + _futures.append(executor.submit(run_docker_cmd, [di, 'bash', 'build_exp.sh', f'/tmp/{da}_samples.csv', f'/tmp/{da}_drugs.tsv'], f'{da} experiments')) #Run one at a time else: if last_experiments_future: last_experiments_future.result() last_experiments_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_exp.sh', f'/tmp/{da}_samples.csv', f'/tmp/{da}_drugs.tsv'], f'{da} experiments') - + _futures.append(last_experiments_future) + _await_all(_futures, 'experiments') def process_misc(executor, datasets, high_mem): ''' Run all misc scripts concurrently or one at a time. ''' last_misc_future = None + _futures = [] #Currently this only applies to broad_sanger. Add others here if they need a final step. if "broad_sanger" in datasets: datasets = ["broad_sanger"] @@ -238,17 +681,19 @@ def process_misc(executor, datasets, high_mem): di = 'broad_sanger_omics' if da == 'broad_sanger' else da #Run all at once: if high_mem: - executor.submit(run_docker_cmd, [di, 'bash', 'build_misc.sh'], f'{da} misc') + _futures.append(executor.submit(run_docker_cmd, [di, 'bash', 'build_misc.sh'], f'{da} misc')) #Run one at a time. else: if last_misc_future: last_misc_future.result() last_misc_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_misc.sh'], f'{da} misc') - + _futures.append(last_misc_future) + _await_all(_futures, 'misc') def process_genes(executor): - if not os.path.exists('/tmp/genes.csv'): - executor.submit(run_docker_cmd,['genes','bash','build_genes.sh'],'gene file') + if not os.path.exists('local/genes.csv'): + return executor.submit(run_docker_cmd,['genes','bash','build_genes.sh'],'gene file') + return None def run_docker_upload_cmd(cmd_arr, all_files_dir, name, version): @@ -270,14 +715,20 @@ def run_docker_upload_cmd(cmd_arr, all_files_dir, name, version): # Full command to run including version update docker_run.extend(cmd_arr) - print('Executing:', ' '.join(docker_run)) + _log('Executing:', _redact(docker_run)) # res = subprocess.run(docker_run, stdout=subprocess.PIPE, stderr=subprocess.PIPE) res = subprocess.run(docker_run, stdout=sys.stdout, stderr=sys.stderr) if res.returncode != 0: - print(res.stderr) + # Do NOT log res.stderr here. stderr is passed through to sys.stderr + # above, so subprocess does not capture it and res.stderr is always + # None -- which is exactly what this line used to print, hiding the + # real cause of a `validate failed` behind the word "None". + # The container's own output is already interleaved above. + _log(f'{name} failed: exited with code {res.returncode}.' + f' See the container output immediately above for the cause.') exit(f'{name} failed') else: - print(f'{name} successful') + _log(f'{name} successful') def decompress_file(file_path): @@ -320,10 +771,78 @@ def get_latest_commit_hash(owner, repo, branch='main'): # Error handling for required tokens if args.figshare and not figshare_token: raise ValueError("FIGSHARE_TOKEN environment variable is not set.") - if any(dataset in args.datasets for dataset in ['beataml', 'mpnst', 'bladder', 'pancreatic','sarcoma','liver','novartis','colorectal']) and not synapse_auth_token: + # NOTE: broad_sanger now needs a Synapse token too -- its DepMap inputs are + # read from Synapse (syn75028495) because the DepMap portal added a + # Cloudflare challenge. See the DepMap gate below. + if any(dataset in args.datasets for dataset in ['beataml', 'mpnst', 'bladder', 'pancreatic','sarcoma','liver','novartis','colorectal','cnf','broad_sanger']) and not synapse_auth_token: if args.docker or args.samples or args.omics or args.drugs or args.exp or args.all: # Token only required if building data, not upload or validate. - raise ValueError("SYNAPSE_AUTH_TOKEN is required for accessing MPNST, beatAML, bladder, pancreatic, liver, novartis, colorectal or sarcoma datasets.") - + raise ValueError("SYNAPSE_AUTH_TOKEN is required for accessing MPNST, beatAML, bladder, pancreatic, liver, novartis, colorectal, sarcoma, cnf, broad_sanger (DepMap files) datasets.") + + # ------------------------------------------------------------------------- + # DepMap freshness gate. + # + # DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in + # front of https://depmap.org/portal/api/download/files, so the broad_sanger + # samples and omics steps can no longer fetch DepMap data from the portal. + # coderdata instead reads a static copy from the "DepMap Raw" Synapse folder + # (syn75028495); coderbuild/utils/fetch_depmap.py downloads it inside the + # container at the start of build_samples.sh and build_omics.sh. + # + # That copy must be refreshed BY HAND for each new DepMap release. Because a + # stale copy fails silently -- the build succeeds and quietly publishes the + # previous release's cell lines -- we require an explicit confirmation flag + # rather than trusting that someone remembered. + # + # Only the steps that actually read the DepMap files are gated (samples and + # omics); --docker on its own still works for image debugging. + # ------------------------------------------------------------------------- + DEPMAP_SYNAPSE_FOLDER = 'syn75028495' + DEPMAP_REQUIRED_FILES = [ + 'Model.csv', + 'OmicsSomaticMutations.csv', + 'OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv', + 'PortalOmicsCNGeneLog2.csv', + ] + if 'broad_sanger' in args.datasets and (args.samples or args.omics or args.all): + if not args.depmap_ready: + raise SystemExit( + "\n" + "=====================================================================\n" + " STOP: DepMap files must be confirmed up to date before building\n" + "=====================================================================\n" + " This build includes broad_sanger, which requires DepMap data.\n" + "\n" + " DepMap files can NO LONGER be downloaded from the DepMap portal --\n" + " it now serves a Cloudflare 'verify you are a person' challenge.\n" + " coderdata reads a STATIC COPY from Synapse instead, which must be\n" + " updated MANUALLY for each new DepMap release.\n" + "\n" + f" Synapse folder: https://www.synapse.org/Synapse:{DEPMAP_SYNAPSE_FOLDER}\n" + "\n" + " To refresh the files for a new DepMap release:\n" + " 1. Download them from the DepMap download page:\n" + " https://depmap.org/portal/data_page/?tab=allData\n" + " (accept the terms, then use the download button on each file)\n" + f" 2. Upload them to {DEPMAP_SYNAPSE_FOLDER} as NEW VERSIONS of the\n" + " existing entities (this keeps the syn IDs stable)\n" + " 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py\n" + "\n" + " Files tracked (4 total, ~1.29 GB):\n" + + ''.join(f" - {f}\n" for f in DEPMAP_REQUIRED_FILES) + + "\n" + " Repository: https://github.com/PNNL-CompBio/coderdata\n" + "\n" + " Once you have confirmed the DepMap files on Synapse are current,\n" + " re-run this command with:\n" + "\n" + " --depmap-ready\n" + "\n" + "=====================================================================\n" + ) + _log(f"DepMap files confirmed current on Synapse ({DEPMAP_SYNAPSE_FOLDER}) " + f"via --depmap-ready; fetch_depmap.py will download them in-container.") + + ###### ### Begin Pipeline ##### @@ -332,6 +851,9 @@ def get_latest_commit_hash(owner, repo, branch='main'): if not os.path.exists('local'): os.mkdir('local') + # Record this run in the progress log + _write_progress('BUILD START', 'START', f'args={sys.argv[1:]}') + # Get dataset names - default is all. datasets = args.datasets.split(',') @@ -339,28 +861,56 @@ def get_latest_commit_hash(owner, repo, branch='main'): # Ouput is logged at local/docker.log if args.docker or args.all: process_docker(datasets) - print("Docker image generation completed") + _log("Docker image generation completed") + + # --continue: rebuild Docker images only for datasets that failed last run + if args.resume: + failed_steps = _last_run_failures() + if failed_steps: + failed_datasets = { + _step_to_docker_dataset(s) for s in failed_steps + if _step_to_docker_dataset(s) + } + # Only rebuild images for datasets we're actually processing + failed_datasets &= set(datasets) | {'genes', 'phosphosites'} + if failed_datasets: + _log(f'--continue: rebuilding Docker images for failed datasets: {sorted(failed_datasets)}') + process_docker(list(failed_datasets)) + else: + _log('--continue: no failed datasets from last run match current dataset list — skipping Docker rebuild') + else: + _log('--continue: no failures found in build_progress.log — skipping Docker rebuild') - ### Build Drugs files, Samples files, and Genes file. These two steps are run in Parallel. + ### Build Drugs files, Samples files, and Genes file. These two steps are run in Parallel. ### Within each step, sequential running is required. with ThreadPoolExecutor() as executor: if args.samples or args.all: sample_thread = executor.submit(process_samples,executor, datasets) if args.drugs or args.all: - drug_thread = executor.submit(process_drugs,executor, datasets) + prev_drugs = [p.strip() for p in args.prev_drugs.split(',') if p.strip()] + drug_thread = executor.submit(process_drugs, executor, datasets, prev_drugs or None) + + # Genes must finish before phosphosites can start (phosphosites reads genes.csv). + # Run genes now and wait; then submit phosphosites (which can overlap with samples/drugs). if args.samples or args.omics or args.exp or args.all: - gene_thread = executor.submit(process_genes,executor) - - # Wait for both processes to complete before proceeding to omics and experiments + genes_future = process_genes(executor) + if genes_future is not None: + genes_future.result() + + phosphosite_future = None + if (args.omics or args.all) and 'cnf' in datasets: + phosphosite_future = process_phosphosites(executor) + + # Wait for all remaining tasks to complete before proceeding to omics and experiments if args.drugs or args.all: drug_thread.result() - if args.samples or args.all:##need to wait for samples for all of these + if args.samples or args.all: sample_thread.result() - if args.samples or args.omics or args.exp or args.all: - gene_thread.result() - - print("All samples, drugs files, and genes file completed or skipped") + if phosphosite_future is not None: + phosphosite_future.result() + + _log("All samples, drugs files, genes, and phosphosites files completed or skipped") ### At this point in the pipeline, all samples and drugs files have been created. There are no blockers to proceed. @@ -375,10 +925,16 @@ def get_latest_commit_hash(owner, repo, branch='main'): if args.omics or args.all: omics_thread.result() - print("All omics files completed") + _log("All omics files completed") if args.exp or args.all: exp_thread.result() - print("All experiments files completed") + _log("All experiments files completed") + + # Omics + experiments are done. Drop their large raw intermediates now + # (e.g. rnaseq_all_data_*.csv ~5GB, cell.xml, proteomics_all_*.csv), but keep + # broad*/sanger* — build_misc.sh still needs them to split broad_sanger. + _clean_local_intermediates(reason='after omics/experiments', + protect_prefixes=('broad', 'sanger')) ### Final Step, some datasets may need an additional post build step. Add this here @@ -389,7 +945,13 @@ def get_latest_commit_hash(owner, repo, branch='main'): misc_thread = executor.submit(process_misc, executor, datasets, args.high_mem) if args.misc or args.all: misc_thread.result() - print("Final build step complete.") + _log("Final build step complete.") + + # misc has split broad_sanger into its per-dataset outputs, so the broad*/ + # sanger* intermediates are no longer needed. Clear all remaining + # intermediates from local/ before the disk-heavy map/align/upload steps + # (this is where the previous build hit "No space left on device"). + _clean_local_intermediates(reason='after misc (pre-upload)') ###### @@ -399,10 +961,8 @@ def get_latest_commit_hash(owner, repo, branch='main'): # if args.figshare or args.validate: # FigShare File Prefixes: - prefixes = ['beataml', 'hcmi', 'cptac', 'pancreatic', 'bladder','sarcoma', 'genes', 'drugs', 'liver','novartis','colorectal','mpnst'] broad_sanger_datasets = ["ccle","ctrpv2","fimm","gdscv1","gdscv2","gcsi","prism","nci60"] if "broad_sanger" in datasets: - prefixes.extend(broad_sanger_datasets) datasets.extend(broad_sanger_datasets) datasets.remove("broad_sanger") @@ -420,9 +980,21 @@ def get_latest_commit_hash(owner, repo, branch='main'): if args.figshare and not args.version: raise ValueError("Version must be specified when pushing to figshare") - # Move relevant files to a designated directory + # Build exact allowlist from schema/expected_files.yaml so only + # schema-declared output files (not _prev caches, docker.log, etc.) + # end up in all_files_dir. + with open('schema/expected_files.yaml') as _f: + _schema = yaml.safe_load(_f) + expected_basenames = set() + for _entries in _schema.get('datasets', {}).values(): + for _entry in _entries: + _bn = os.path.basename(_entry['file']) + expected_basenames.add(_bn) + expected_basenames.add(_bn + '.gz') + + # Move only expected files to a designated directory for file in glob(os.path.join("local", '*.*')): - if any(file.startswith(os.path.join("local", prefix)) for prefix in prefixes): + if os.path.basename(file) in expected_basenames: shutil.move(file, os.path.join(all_files_dir, os.path.basename(file))) # Decompress all compressed files in the directory for schema checking @@ -430,20 +1002,21 @@ def get_latest_commit_hash(owner, repo, branch='main'): decompress_file(file) ### These should be done before schema checking. - sample_mapping_command = ['python3', 'scripts/map_improve_sample_ids.py', '--local_dir', "/tmp", '--version', args.version] + version_args = ['--version', args.version] if args.version is not None else [] + sample_mapping_command = ['python3', 'scripts/map_improve_sample_ids.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(sample_mapping_command, 'all_files_dir', 'Map_Samples', args.version) - - drug_mapping_command = ['python3', 'scripts/map_improve_drug_ids.py', '--local_dir', "/tmp", '--version', args.version] + + drug_mapping_command = ['python3', 'scripts/map_improve_drug_ids.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(drug_mapping_command, 'all_files_dir', 'Map_Drugs', args.version) - - drug_mapping_command_2 = ['python3', 'scripts/align_drug_descriptors.py', '--local_dir', "/tmp", '--version', args.version] + + drug_mapping_command_2 = ['python3', 'scripts/align_drug_descriptors.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(drug_mapping_command_2, 'all_files_dir', 'Align_Drug_Descriptors', args.version) # Run schema checker - This will always run if uploading data. schema_check_command = ['python3', 'scripts/check_schema.py', '--datasets'] + datasets run_docker_upload_cmd(schema_check_command, 'all_files_dir', 'validate', args.version) - print("Validation complete. Proceeding with file compression/decompression adjustments") + _log("Validation complete. Proceeding with file compression/decompression adjustments") # Compress or decompress files based on specific conditions after checking for file in glob(os.path.join(all_files_dir, '*')): @@ -453,7 +1026,7 @@ def get_latest_commit_hash(owner, repo, branch='main'): elif not ('samples' in file or 'figshare' in file) and not is_compressed: compress_file(file) - print("File compression and decompression adjustments are complete.") + _log("File compression and decompression adjustments are complete.") ### Upload to Figshare using Docker if args.figshare and args.version and figshare_token: diff --git a/coderbuild/build_dataset.py b/coderbuild/build_dataset.py index 7c5ed10c..c1a0bcb1 100644 --- a/coderbuild/build_dataset.py +++ b/coderbuild/build_dataset.py @@ -45,11 +45,13 @@ def process_docker(dataset,validate): 'cptac': ['cptac'], 'sarcoma': ['sarcoma'], 'genes': ['genes'], + 'phosphosites': ['phosphosites'], 'upload': ['upload'], - 'colorectal': ['colorectal'], + 'colorectal': ['colorectal'], 'bladder': ['bladder'], 'liver': ['liver'], - 'novartis': ['novartis'] + 'novartis': ['novartis'], + 'cnf': ['cnf'] } # Collect container names to build based on the dataset provided. Always build 'genes'. @@ -57,7 +59,10 @@ def process_docker(dataset,validate): # Append upload if validation step is included if validate is True: datasets_to_build.append('upload') - + # phosphosites container is required when building cnf + if dataset == 'cnf': + datasets_to_build.append('phosphosites') + datasets_to_build.extend(dataset_map.get(dataset, [])) compose_command = ['docker', 'compose', '-f', compose_file, 'build'] + datasets_to_build @@ -79,10 +84,26 @@ def process_docker(dataset,validate): def process_genes(executor): ''' - Build the genes file if it does not exist. + Build the genes file if it does not exist. Returns a Future (or None if already built). ''' if not os.path.exists('local/genes.csv'): - executor.submit(run_docker_cmd, ['genes', 'bash', 'build_genes.sh'], 'genes file') + return executor.submit(run_docker_cmd, ['genes', 'bash', 'build_genes.sh'], 'genes file') + return None + + +def process_phosphosites(executor): + ''' + Build the phosphosites reference file if it does not exist. + Only needed when the cnf dataset is being built. Returns a Future (or None). + Caller must ensure genes.csv exists before calling this. + ''' + if not os.path.exists('local/phosphosites.csv'): + return executor.submit( + run_docker_cmd, + ['phosphosites', 'bash', 'build_phosphosites.sh', '/tmp/genes.csv'], + 'phosphosites file', + ) + return None def process_samples(executor, dataset, use_prev_dataset, should_continue): ''' @@ -133,7 +154,8 @@ def process_omics(executor, dataset, should_continue): 'bladder': ['copy_number', 'mutations', 'transcriptomics'], 'colorectal':['copy_number', 'mutations', 'transcriptomics'], 'novartis':['copy_number', 'mutations', 'transcriptomics'], - 'liver':['copy_number', 'mutations', 'transcriptomics','proteomics'] + 'liver':['copy_number', 'mutations', 'transcriptomics','proteomics'], + 'cnf': ['transcriptomics', 'proteomics', 'phosphoproteomics'], } expected_omics = dataset_omics_files.get(dataset, []) @@ -167,7 +189,10 @@ def process_omics(executor, dataset, should_continue): di = 'broad_sanger_omics' if dataset == 'broad_sanger' else dataset filename = f'{dataset} omics' - executor.submit(run_docker_cmd, [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{dataset}_samples.csv'], filename) + omics_cmd = [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{dataset}_samples.csv'] + if dataset == 'cnf': + omics_cmd.append('/tmp/phosphosites.csv') + executor.submit(run_docker_cmd, omics_cmd, filename) def process_experiments(executor, dataset, should_continue): @@ -245,6 +270,8 @@ def run_schema_checker(dataset): ''' # Prepare the directory with the built files prefixes = ['genes', dataset] + if dataset == 'cnf': + prefixes.append('phosphosites') datasets = [dataset] broad_sanger_datasets = ["ccle","ctrpv2","fimm","gdscv1","gdscv2","gcsi","prism","nci60"] all_files_dir = 'all_files_dir' @@ -280,9 +307,63 @@ def main(): parser.add_argument('--build', action='store_true', help='Run data build.') parser.add_argument('--validate', action='store_true', help='Run schema checker on the built files') parser.add_argument('--continue', dest='should_continue', action='store_true', help='Continue from where the build left off by skipping existing files') + parser.add_argument('--depmap-ready', dest='depmap_ready', default=False, action='store_true', + help='Confirm that the static DepMap files on Synapse are up to date for the ' + 'current DepMap release. Required when building broad_sanger.') args = parser.parse_args() + # DepMap freshness gate -- see the matching block in build_all.py for the + # full rationale. DepMap files can no longer be fetched from the DepMap + # portal (Cloudflare challenge), so coderdata reads a static copy from + # Synapse (fetch_depmap.py downloads it in-container). That copy is + # hand-maintained and must be confirmed current before a broad_sanger build. + DEPMAP_SYNAPSE_FOLDER = 'syn75028495' + DEPMAP_REQUIRED_FILES = [ + 'Model.csv', + 'OmicsSomaticMutations.csv', + 'OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv', + 'PortalOmicsCNGeneLog2.csv', + ] + if args.dataset == 'broad_sanger' and args.build: + if not args.depmap_ready: + raise SystemExit( + "\n" + "=====================================================================\n" + " STOP: DepMap files must be confirmed up to date before building\n" + "=====================================================================\n" + " broad_sanger requires DepMap data.\n" + "\n" + " DepMap files can NO LONGER be downloaded from the DepMap portal --\n" + " it now serves a Cloudflare 'verify you are a person' challenge.\n" + " coderdata reads a STATIC COPY from Synapse instead, which must be\n" + " updated MANUALLY for each new DepMap release.\n" + "\n" + f" Synapse folder: https://www.synapse.org/Synapse:{DEPMAP_SYNAPSE_FOLDER}\n" + "\n" + " To refresh the files for a new DepMap release:\n" + " 1. Download them from the DepMap download page:\n" + " https://depmap.org/portal/data_page/?tab=allData\n" + " (accept the terms, then use the download button on each file)\n" + f" 2. Upload them to {DEPMAP_SYNAPSE_FOLDER} as NEW VERSIONS of the\n" + " existing entities (this keeps the syn IDs stable)\n" + " 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py\n" + "\n" + " Files tracked (4 total, ~1.29 GB):\n" + + ''.join(f" - {f}\n" for f in DEPMAP_REQUIRED_FILES) + + "\n" + " Repository: https://github.com/PNNL-CompBio/coderdata\n" + "\n" + " Once you have confirmed the DepMap files on Synapse are current,\n" + " re-run this command with:\n" + "\n" + " --depmap-ready\n" + "\n" + "=====================================================================\n" + ) + print(f"DepMap files confirmed current on Synapse ({DEPMAP_SYNAPSE_FOLDER}) " + f"via --depmap-ready; fetch_depmap.py will download them in-container.") + if not os.path.exists('local'): os.mkdir('local') @@ -292,16 +373,25 @@ def main(): if args.build: # Use ThreadPoolExecutor for parallel execution with ThreadPoolExecutor() as executor: - # Always build genes file - process_genes(executor) + # Genes must finish before phosphosites can start (phosphosites reads genes.csv) + genes_future = process_genes(executor) + if genes_future is not None: + genes_future.result() + + # Now safe to start phosphosites (genes.csv is present) + phosphosites_future = None + if args.dataset == 'cnf': + phosphosites_future = process_phosphosites(executor) - # Build samples and drugs + # Build samples and drugs in parallel while phosphosites runs samples_future = executor.submit(process_samples, executor, args.dataset, args.use_prev_dataset, args.should_continue) drugs_future = executor.submit(process_drugs, executor, args.dataset, args.use_prev_dataset, args.should_continue) samples_future.result() drugs_future.result() - + if phosphosites_future is not None: + phosphosites_future.result() + print("Samples and Drugs Files Completed.") with ThreadPoolExecutor() as executor: