From 038dbd158509587439e8032b04bd042d04d49cc1 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:03 -0700 Subject: [PATCH] Update package dataset metadata and loader --- coderdata/dataset.yml | 15 ++++++- coderdata/dataset/dataset.py | 71 +++++++++++++++++++++++++++++++- coderdata/download/downloader.py | 4 +- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/coderdata/dataset.yml b/coderdata/dataset.yml index 15e91270..2be843f1 100644 --- a/coderdata/dataset.yml +++ b/coderdata/dataset.yml @@ -1,5 +1,5 @@ -figshare: https://api.figshare.com/v2/articles/29923646/files?page=1&page_size=500 -version: 2.2.1 +figshare: https://api.figshare.com/v2/articles/32906495/files?page=1&page_size=500 +version: 2.3 datasets: beataml: description: Beat acute myeloid leukemia (BeatAML) focuses on acute myeloid leukemia tumor data. Data includes drug response, proteomics, and transcriptomics datasets. @@ -46,6 +46,17 @@ datasets: - drug descriptor - experiments + cnf: + description: Cutaneous Neurofibroma (cNF) patient-derived organoid drug screen. Cutaneous neurofibromas are benign nerve-sheath tumors that develop in patients with neurofibromatosis type 1 (NF1). Data includes transcriptomics, proteomics, phosphoproteomics, and drug response data derived from Synapse. + modalities: + - sample + - transcriptomics + - proteomics + - phosphoproteomics + - drug + - drug descriptor + - experiments + cptac: description: The Clinical Proteomic Tumor Analysis Consortium (CPTAC) project is a collaborative network funded by the National Cancer Institute (NCI) focused on improving our understanding of cancer biology through the integration of transcriptomic, proteomic, and genomic data. references: diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 695db22c..41ac73fd 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -44,6 +44,8 @@ def __init__( name: Optional[str]=None, transcriptomics: Optional[pd.DataFrame]=None, proteomics: Optional[pd.DataFrame]=None, + phosphoproteomics: Optional[pd.DataFrame]=None, + phosphosites: Optional[pd.DataFrame]=None, mutations: Optional[pd.DataFrame]=None, copy_number: Optional[pd.DataFrame]=None, samples: Optional[pd.DataFrame]=None, @@ -95,6 +97,8 @@ def __init__( self.name = name self.transcriptomics = transcriptomics self.proteomics = proteomics + self.phosphoproteomics = phosphoproteomics + self.phosphosites = phosphosites self.mutations = mutations self.copy_number = copy_number self.samples = samples @@ -151,6 +155,32 @@ def proteomics(self): del self._proteomics + @property + def phosphoproteomics(self): + return self._phosphoproteomics + + @phosphoproteomics.setter + def phosphoproteomics(self, value): + self._phosphoproteomics = value + + @phosphoproteomics.deleter + def phosphoproteomics(self): + del self._phosphoproteomics + + + @property + def phosphosites(self): + return self._phosphosites + + @phosphosites.setter + def phosphosites(self, value): + self._phosphosites = value + + @phosphosites.deleter + def phosphosites(self): + del self._phosphosites + + @property def mutations(self): return self._mutations @@ -513,6 +543,8 @@ def types(self) -> list: data_types = [ 'transcriptomics', 'proteomics', + 'phosphoproteomics', + 'phosphosites', 'mutations', 'copy_number', 'samples', @@ -521,6 +553,7 @@ def types(self) -> list: 'experiments', 'methylation', 'metabolomics', + 'combinations', 'genes', ] data_types_present = [] @@ -609,6 +642,7 @@ def load( data_types_to_load = ( 'transcriptomics', 'proteomics', + 'phosphoproteomics', 'mutations', 'copy_number', 'samples', @@ -618,7 +652,11 @@ def load( 'experiments', 'methylation', 'metabolomics', + 'combinations', + # reference tables are loaded last so they can be subset to the + # identifiers present in the data types loaded above 'genes', + 'phosphosites', ) if type(local_path) is not Path: @@ -649,6 +687,9 @@ def load( for p in local_path.glob(f'genes*'): if p.name.endswith(accepted_file_endings) and p.is_file(): files['genes'] = p + for p in local_path.glob(f'phosphosites*'): + if p.name.endswith(accepted_file_endings) and p.is_file(): + files['phosphosites'] = p for dataset_type in data_types_to_load: if dataset_type not in files: @@ -659,7 +700,7 @@ def load( ) continue file = files[dataset_type] - if dataset_type != 'genes': + if dataset_type not in ('genes', 'phosphosites'): print( f"Importing '{dataset_type}' from {file} ...", end=' ', @@ -668,6 +709,34 @@ def load( if hasattr(dataset, dataset_type): setattr(dataset, dataset_type, _load_file(file)) print("DONE", file=sys.stderr) + elif dataset_type == 'phosphosites': + ''' + The phosphosites table is a universal reference (analogous + to 'genes') mapping phosphosite_id -> gene / residue info. + It is only relevant when the dataset has phosphoproteomics, + and is subset to the phosphosite_ids present in that data. + ''' + if dataset.phosphoproteomics is None: + print( + f"'phosphosites' skipped for {name} " + f"(no phosphoproteomics present)", + file=sys.stderr + ) + continue + print( + f"Importing 'phosphosites' from {file} ...", + end=' ', + file=sys.stderr + ) + dataset.phosphosites = _load_file(file) + if 'phosphosite_id' in dataset.phosphoproteomics.columns: + site_ids = set( + dataset.phosphoproteomics['phosphosite_id'].unique() + ) + dataset.phosphosites = dataset.phosphosites[ + dataset.phosphosites['phosphosite_id'].isin(site_ids) + ] + print("DONE", file=sys.stderr) else: ''' The genes dataset available in the online repository is diff --git a/coderdata/download/downloader.py b/coderdata/download/downloader.py index 552cd9ba..b6b367f9 100644 --- a/coderdata/download/downloader.py +++ b/coderdata/download/downloader.py @@ -110,7 +110,9 @@ def download( if name != "all": filtered_files = [ f for f in all_files - if (f.get('name', '').casefold().startswith(name)) or ('genes' in f.get('name', '').casefold()) + if (f.get('name', '').casefold().startswith(name)) + or ('genes' in f.get('name', '').casefold()) + or ('phosphosites' in f.get('name', '').casefold()) ] else: filtered_files = all_files