Modelling

genomics/.env.example

@@ -0,0 +1,34 @@
+# ProGenome genomics: configuration template.
+#   cp .env.example .env      # then fill in what you use; .env is git-ignored and read by config.py and every script
+# Alternative location: ~/.progenome.env (same format, kept outside the repository).
+# Precedence: variables exported in your shell > genomics/.env > ~/.progenome.env > defaults in config.py.
+# Keep comments on their own lines. Check what is in effect with:  make config
+
+# --- LLM decoder (make decode, run_v2.sh) --------------------------------------------------------
+# Key from https://build.nvidia.com (sign in, any model page, "Get API Key"); starts with nvapi-
+NVIDIA_API_KEY=nvapi-REPLACE_ME
+# Model id on the endpoint (default: Nemotron 3 Super)
+NIM_MODEL=nvidia/nemotron-3-super-120b-a12b
+# Any OpenAI-compatible chat-completions endpoint (a local NIM container, vLLM, ...)
+NIM_URL=https://integrate.api.nvidia.com/v1/chat/completions
+
+# --- Neo4j browser (make neo4j-load) ----------------------------------------------------------------
+# Defaults match docker-compose.yml; change both if you change one
+NEO4J_URI=bolt://localhost:7687
+NEO4J_USER=neo4j
+NEO4J_PASSWORD=progenome
+
+# --- Data and defaults ------------------------------------------------------------------------------
+# Where fetch_data.sh downloads the HaploGraph from
+HAPLOBLOCKS_BASE=https://data.haploblocks.org
+# Chromosome for every stage (also: CHROM=chr21 make run)
+CHROM=chr22
+# Optional: move the 15 MB of data and the ~400 MB of outputs elsewhere
+# PROGENOME_DATA_DIR=/data/progenome/data
+# PROGENOME_OUTPUTS_DIR=/data/progenome/outputs
+
+# --- NVIDIA Brev (make brev) -------------------------------------------------------------------------
+# The CLI stores its own login (brev login --api-key ...); these only choose the instance
+BREV_INSTANCE=progenome-gpu
+BREV_TYPE=g2-standard-4:nvidia-l4:1
+# A100 example:  BREV_INSTANCE=progenome-a100  BREV_TYPE=a100-80gb.1x

genomics/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+.pytest_cache/
+.venv/
+*.pyc
+# downloaded inputs (re-fetch with fetch_data.sh) and generated outputs
+data/
+outputs_brev/
+outputs/
+.env

genomics/Dockerfile

@@ -0,0 +1,34 @@
+# ProGenome genomics pipeline: knowledge graph -> co-occurrence analysis -> GNN (PyTorch Geometric).
+# CUDA image for NVIDIA Brev / any GPU box; the same image runs on CPU when no GPU is present.
+#
+#   Build from the REPOSITORY ROOT so the two proteomics inputs are baked in (make docker does this):
+#     docker build -f genomics/Dockerfile -t progenome-genomics .
+#   Run (data/ and outputs/ bind-mounted; .env passed for the decoder):
+#     docker run --rm --gpus all --env-file genomics/.env -v $PWD/genomics/data:/app/genomics/data -v $PWD/genomics/outputs:/app/genomics/outputs progenome-genomics run_all.sh
+#     docker run --rm --gpus all -v ... progenome-genomics run_v2.sh
+#     docker run --rm --gpus all -v ... progenome-genomics -c "python train_gnn.py --target ancestry --init node2vec"
+ARG BASE=pytorch/pytorch:2.14.0-cuda12.6-cudnn9-runtime
+FROM ${BASE}
+ARG PYG_WHEELS=https://data.pyg.org/whl/torch-2.14.0+cu126.html
+
+ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 PIP_BREAK_SYSTEM_PACKAGES=1 NX_CUGRAPH_AUTOCONFIG=True
+RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app/genomics
+COPY genomics/requirements.txt .
+RUN pip install -r requirements.txt \
+ && (pip install pyg_lib -f ${PYG_WHEELS} \
+     || echo "no pyg_lib wheel for this torch build -> Node2Vec falls back to SVD") \
+ && (pip install nx-cugraph-cu12 --extra-index-url https://pypi.nvidia.com \
+     || echo "nx-cugraph not installed -> NetworkX runs on CPU") \
+ && (pip install torch-tensorrt --extra-index-url https://download.pytorch.org/whl/cu126 \
+     || echo "torch-tensorrt not installed -> infer.py --compile tensorrt falls back to inductor")
+
+COPY proteomics/uniprot_chr22.bed /app/proteomics/uniprot_chr22.bed
+COPY proteomics/synthetic_proteomics_chr22/gene_symbol_cache.csv /app/proteomics/synthetic_proteomics_chr22/gene_symbol_cache.csv
+COPY genomics/ /app/genomics/
+RUN python -m pytest tests -q
+
+# data/ and outputs/ are bind-mounted at run time (see docker-compose.yml)
+ENTRYPOINT ["bash"]
+CMD ["run_all.sh"]

genomics/Dockerfile.dockerignore

@@ -0,0 +1,15 @@
+# Build context is the repository root (docker build -f genomics/Dockerfile ..); keep the context small.
+# BuildKit reads <Dockerfile>.dockerignore next to the Dockerfile.
+*
+!genomics/**
+!proteomics/uniprot_chr22.bed
+!proteomics/synthetic_proteomics_chr22/gene_symbol_cache.csv
+genomics/data
+genomics/outputs
+genomics/outputs_brev
+genomics/.venv
+genomics/.env
+genomics/docs/report/*.pdf
+genomics/docs/report/*.docx
+**/__pycache__
+**/.pytest_cache

genomics/Makefile

@@ -0,0 +1,93 @@
+# Clone-and-go entry points for the genomics pipeline.  `make help` lists them.
+CHROM ?= chr22
+INIT  ?= svd
+PY    := .venv/bin/python
+
+.PHONY: help setup data kg analysis baseline graph gnn embeddings run run-v2 eda decode federated test docker docker-run docker-run-v2 docker-federated docker-shell config neo4j neo4j-load neo4j-down brev report clean
+
+help:            ## show this help
+	@grep -E '^[a-z-]+:.*##' $(MAKEFILE_LIST) | awk -F':.*## ' '{printf "  make %-12s %s\n", $$1, $$2}'
+
+setup:           ## create .venv with torch (CPU or CUDA) + pinned deps, run unit tests
+	bash setup.sh
+
+data:            ## download the HaploGraph, phenotypes and block stats for $(CHROM)
+	bash fetch_data.sh $(CHROM)
+
+kg: data         ## build the knowledge graph (tables, sparse carriers, PyG HeteroData)
+	$(PY) build_kg.py --chrom $(CHROM)
+
+analysis: kg     ## cluster/edge/block vs phenotype co-occurrence tests + plots
+	$(PY) cooccurrence_analysis.py --chrom $(CHROM)
+
+baseline: kg     ## logistic-regression baseline, writes the shared train/val/test split
+	$(PY) baseline.py --chrom $(CHROM)
+
+graph: analysis  ## NetworkX stats, GraphML export and plots of the graph
+	$(PY) graph_explore.py --chrom $(CHROM)
+
+gnn: baseline    ## train the hetero-GNN on ancestry, population and sex (control)
+	for t in ancestry population sex; do $(PY) train_gnn.py --chrom $(CHROM) --target $$t --init $(INIT); done
+
+embeddings: gnn  ## evaluate/plot SVD and GNN embeddings
+	$(PY) embeddings.py --chrom $(CHROM)
+
+run:             ## the whole pipeline (same as run_all.sh)
+	PYTHON=$(PY) CHROM=$(CHROM) INIT=$(INIT) bash run_all.sh
+
+run-v2: run      ## schema v2: proteomics on 1000G IDs, genes/proteins, genome+proteome GNN, EDA, decoder dry-run
+	PYTHON=$(PY) CHROM=$(CHROM) bash run_v2.sh
+
+eda:             ## exploratory data analysis report -> outputs/eda/$(CHROM)/EDA.md
+	$(PY) eda.py --chrom $(CHROM)
+
+decode:          ## GraphRAG insight for one person via NVIDIA NIM (needs NVIDIA_API_KEY; WHO=HG00103)
+	$(PY) graphrag_decoder.py --chrom $(CHROM) --individual $(or $(WHO),HG00103) --run phenotype_both_raw
+
+federated:       ## NVFlare FedAvg over 3 sites (simulator), score the global model centrally, compare with each site alone
+	$(PY) federated/job.py --chrom $(CHROM) --rounds $(or $(ROUNDS),10) --local-epochs $(or $(LOCAL_EPOCHS),5)
+	$(PY) federated/evaluate_global.py --chrom $(CHROM)
+	$(PY) federated/local_only.py --chrom $(CHROM) --steps $$(( $(or $(ROUNDS),10) * $(or $(LOCAL_EPOCHS),5) ))
+
+test:            ## unit tests
+	$(PY) -m pytest tests -q
+
+DOCKER_RUN = docker run --rm $$(command -v nvidia-smi >/dev/null 2>&1 && echo --gpus all) $$( [ -f .env ] && echo --env-file .env ) \
+	  -e CHROM=$(CHROM) -e INIT=$(INIT) -v $(PWD)/data:/app/genomics/data -v $(PWD)/outputs:/app/genomics/outputs progenome-genomics
+
+docker:          ## build the CUDA image from the repository root (runs on CPU too); proteomics inputs are baked in
+	docker build -f Dockerfile -t progenome-genomics ..
+
+docker-run:      ## v1 pipeline inside the image (GPU if available; .env passed if present)
+	$(DOCKER_RUN) run_all.sh
+
+docker-run-v2:   ## v2 chain inside the image (after docker-run)
+	$(DOCKER_RUN) run_v2.sh
+
+docker-federated: ## NVFlare FedAvg inside the image (after docker-run-v2)
+	$(DOCKER_RUN) -c "python federated/job.py --chrom $(CHROM) --rounds $(or $(ROUNDS),30) --local-epochs $(or $(LOCAL_EPOCHS),5) && python federated/evaluate_global.py --chrom $(CHROM) && python federated/local_only.py --chrom $(CHROM) --steps $$(( $(or $(ROUNDS),30) * $(or $(LOCAL_EPOCHS),5) ))"
+
+docker-shell:    ## interactive shell inside the image with data/ and outputs/ mounted
+	$(DOCKER_RUN:--rm=--rm -it)
+
+config:          ## show the effective configuration (.env / ~/.progenome.env / defaults), secrets masked
+	$(PY) config.py
+
+neo4j:           ## start Neo4j community at http://localhost:7474 (neo4j / progenome)
+	docker compose up -d neo4j
+
+neo4j-load: neo4j ## load the knowledge graph into Neo4j
+	$(PY) neo4j_load.py --chrom $(CHROM)
+
+neo4j-down:      ## stop Neo4j (keeps its data volume)
+	docker compose down
+
+brev:            ## create/use a Brev GPU instance, build there, run with Node2Vec + TensorRT benchmark, copy outputs back
+	bash brev_deploy.sh
+
+report:          ## rebuild docs/report (figures -> .tex + .docx -> .pdf); needs node with the docx package and tectonic
+	$(PY) docs/report/make_report_figures.py
+	cd docs/report && node build_report.js && (command -v tectonic >/dev/null && tectonic ProGenome_KT.tex || echo "tectonic not installed: compile ProGenome_KT.tex with pdflatex/xelatex")
+
+clean:           ## remove generated outputs (keeps downloaded data)
+	rm -rf outputs