Merge pull request #46 from GilbertLabUCSF/abe-dev

debug, new modules, and code improvements

.github/workflows/python-package.yml

@@ -16,7 +16,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- python-version: ["3.9"] # ["3.8", "3.9", "3.10"]
+ python-version: ["3.11"]
 
  steps:
  - uses: actions/checkout@v3

.github/workflows/python-publish.yml

@@ -15,7 +15,7 @@ jobs:
  fail-fast: false
  matrix:
  os-version: ["ubuntu-latest"]
- python-version: ["3.9"] # ["3.8", "3.9", "3.10"]
+ python-version: ["3.11"]
 
  steps:
  - uses: actions/checkout@v3

CanDI/__version__.py

@@ -1 +1 @@
-version = "0.1.1"
+version = "0.2.0"

CanDI/candi/__init__.py

@@ -1,4 +1,6 @@
+from . import load
 from . import data
+
 data = data.Data() #Global object data instantiated on import required for access by GeneQuery Objects
-from . import (Gene, CellLine, Organelle, Cancer, CellLineCluster, GeneCluster)
 
+from .candi import (Gene, CellLine, Organelle, Cancer, CellLineCluster, GeneCluster)

CanDI/candi/candi.py

@@ -1,11 +1,11 @@
 # Classes for handling data aggregations
 import operator
-from collections import OrderedDict, MutableSequence
+from collections.abc import MutableSequence
 import itertools as it
 import pandas as pd
 import numpy as np
 from . import data, grabber
-from . import entity
+from ..structures import entity
 
 class SubsetHandler(object):
 

CanDI/candi/data.py

@@ -14,21 +14,27 @@ class Data(object):
  can be tuned to load specific datasets upon import by editing config.ini
  can call Data.load() to load any specific dataset
  """
- def __init__(self):
+ def __init__(self, config_path='auto', verbose=False):
 
- self._file_path = Path(os.path.dirname(os.path.realpath(__file__))).parent.absolute() / 'setup'
- config_path = self._file_path / 'data/config.ini'
+ if config_path == 'auto':
+ self._file_path = Path(os.path.dirname(os.path.realpath(__file__))).parent.absolute() / 'setup'
+ config_path = self._file_path / 'data/config.ini'
+ elif os.path.exists(config_path) == False:
+ raise FileNotFoundError("Config file not found at {}".format(config_path))
+ elif os.path.exists(config_path) == True:
+ if verbose: print("Using config file at {}".format(config_path))
 
  parser = configparser.ConfigParser() #parses config for data sources
  parser.read(config_path)
 
  self._parser = parser
- #self._verify_install()
+ self._verify_install()
  self._init_sources()
  self._init_depmap_paths()
- # self._init_index_tables()
+ self._init_index_tables()
 
  def _verify_install(self): #ensures data being loaded is present
+ #TODO: add more checks for different data sources
  try:
  assert "depmap_urls" in self._parser.sections()
  except AssertionError:
@@ -91,6 +97,7 @@ def _handle_autoload(method, path):
  df = pd.read_csv(path,
  memory_map=True,
  low_memory=False,
+ sep='\t',
  index_col="DepMap_ID")
 
  elif method == "locations":

CanDI/pipelines/diffexp.py

@@ -0,0 +1,52 @@
+import numpy as np
+import pandas as pd
+import anndata as ad
+
+from pydeseq2.dds import DeseqDataSet
+from pydeseq2.default_inference import DefaultInference
+from pydeseq2.ds import DeseqStats
+from adpbulk import ADPBulk
+
+
+def pseudobulk_by_group(adt, groups, method="mean"):
+ # initialize the object
+ adpb = ADPBulk(adt, groupby=groups, method=method)
+
+ # perform the pseudobulking
+ pseudobulk_matrix = adpb.fit_transform()
+
+ # retrieve the sample metadata (useful for easy incorporation with edgeR)
+ sample_meta = adpb.get_meta()
+
+ out = ad.AnnData(
+ X=pseudobulk_matrix,
+ obs=sample_meta.set_index('SampleName')
+ )
+
+ return out
+
+
+def run_deseq(adata, design, tested_level, ref_level, n_cpus=8):
+
+ inference = DefaultInference(n_cpus=n_cpus)
+
+ dds = DeseqDataSet(
+ counts=adata.to_df().astype(int),
+ metadata=adata.obs,
+ design_factors=design, # compare samples based on the "condition"
+ refit_cooks=True,
+ inference=inference,
+ )
+
+ dds.deseq2()
+
+ stat_res = DeseqStats(
+ dds, 
+ contrast=[design, tested_level, ref_level], 
+ inference=inference
+ )
+ stat_res.summary()
+
+ df = stat_res.results_df
+
+ return df

CanDI/setup/dataverse.py

@@ -9,6 +9,18 @@
 
 CANDI_DATAVERSE_DOI = 'doi:10.7910/DVN/JIAT0H'
 
+
+### Datasets Metadata ###
+
+coessentiality_dataset_names = [
+ 'genes',
+ # 10273535
+ 'GLS_p',
+ # 10273534
+ 'GLS_sign',
+ # 10273533
+]
+
 depmap_dataset_names = [
  'CCLE_expression',
  'CCLE_fusions',
@@ -22,6 +34,11 @@
 ]
 
 name2type = {
+ # Coessentiality datasets
+ 'genes': 'txt',
+ 'GLS_p': 'npy',
+ 'GLS_sign': 'npy',
+ # DepMap datasets
  'CCLE_expression': 'csv',
  'CCLE_fusions': 'csv',
  'CCLE_gene_cn': 'csv',
@@ -34,6 +51,11 @@
 }
 
 name2id = {
+ # Coessentiality datasets
+ 'genes': 10273535,
+ 'GLS_p': 10273534,
+ 'GLS_sign': 10273533,
+ # DepMap datasets
  'CCLE_expression': 8076862,
  'CCLE_fusions': 10085763,
  'CCLE_gene_cn': 8076861,
@@ -46,6 +68,7 @@
 }
 
 
+### Utility functions ###
 def print_sys(s):
  """system print
 
@@ -55,80 +78,102 @@ def print_sys(s):
  print(s, flush = True, file = sys.stderr)
 
 
-def dataverse_download(url, path, name, types):
- """dataverse download helper with progress bar
-
- Args:
- url (str): the url of the dataset
- path (str): the path to save the dataset
- name (str): the dataset name
- types (dict): a dictionary mapping from the dataset name to the file format
- """
- save_path = os.path.join(path, f"{name}.{types[name]}")
- response = requests.get(url, stream=True)
- total_size_in_bytes = int(response.headers.get("content-length", 0))
- block_size = 1024
- progress_bar = tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True)
- with open(save_path, "wb") as file:
- for data in response.iter_content(block_size):
- progress_bar.update(len(data))
- file.write(data)
- progress_bar.close()
-
-
-def download_wrapper(name, path, return_type=None):
- """wrapper for downloading a dataset given the name and path, for csv,pkl,tsv or similar files
-
- Args:
- name (str): the rough dataset query name
- path (str): the path to save the dataset
- return_type (str, optional): the return type. Defaults to None. Can be "url", "name", or ["url", "name"]
+### Downloading scripts ###
+
+class Downloader:
+ def __init__(self):
+ pass
+
+ def _dataverse_download(self, url, path, name, types):
+ """dataverse download helper with progress bar
+
+ Args:
+ url (str): the url of the dataset
+ path (str): the path to save the dataset
+ name (str): the dataset name
+ types (dict): a dictionary mapping from the dataset name to the file format
+ """
+ save_path = os.path.join(path, f"{name}.{types[name]}")
+ response = requests.get(url, stream=True)
+ total_size_in_bytes = int(response.headers.get("content-length", 0))
+ block_size = 1024
+ progress_bar = tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True)
+ with open(save_path, "wb") as file:
+ for data in response.iter_content(block_size):
+ progress_bar.update(len(data))
+ file.write(data)
+ progress_bar.close()
+
+
+ def _download_wrapper(self, name, path, return_type=None):
+ """wrapper for downloading a dataset given the name and path, for csv,pkl,tsv or similar files
+
+ Args:
+ name (str): the rough dataset query name
+ path (str): the path to save the dataset
+ return_type (str, optional): the return type. Defaults to None. Can be "url", "name", or ["url", "name"]
+
+ Returns:
+ str: the exact dataset query name
+ """
+ server_path = "https://dataverse.harvard.edu/api/access/datafile/"
+
+ url = server_path + str(name2id[name])
+
+ if not os.path.exists(path):
+ os.mkdir(path)
+
+ file_name = f"{name}.{name2type[name]}"
+
+ if os.path.exists(os.path.join(path, file_name)):
+ print_sys("Found local copy...")
+ os.path.join(path, file_name)
+ else:
+ print_sys("Downloading...")
+ self._dataverse_download(url, path, name, name2type)
+
+ if return_type == "url":
+ return url
+ elif return_type == "name":
+ return file_name
+ elif return_type == ["url", "name"]:
+ return url, file_name
 
- Returns:
- str: the exact dataset query name
- """
- server_path = "https://dataverse.harvard.edu/api/access/datafile/"
-
- url = server_path + str(name2id[name])
-
- if not os.path.exists(path):
- os.mkdir(path)
-
- file_name = f"{name}.{name2type[name]}"
-
- if os.path.exists(os.path.join(path, file_name)):
- print_sys("Found local copy...")
- os.path.join(path, file_name)
- else:
- print_sys("Downloading...")
- dataverse_download(url, path, name, name2type)
 
- if return_type == "url":
- return url
- elif return_type == "name":
- return file_name
- elif return_type == ["url", "name"]:
- return url, file_name
-
-
-def depmap_dataverse_download(path, return_type=None):
- """download all datasets to the path
+ def run(self, path, datasets, return_type=None):
+ """download all datasets to the path
+
+ Args:
+ path (str): the path to save the datasets
+ return_type (str, optional): the return type. Defaults to None. Can be "url", "name", or ["url", "name"]
+ """
+ url_list = []
+ file_names = []
+
+ for name in datasets:
+ url, file_name = self._download_wrapper(name, path, return_type=["url", "name"])
+ url_list.append(url)
+ file_names.append(file_name)
+
+ if return_type == "url":
+ return url_list
+ elif return_type == "name":
+ return file_names
+ elif return_type == ["url", "name"]:
+ return url_list, file_names
+
+
+class DepMapDownloader(Downloader):
+ def __init__(self):
+ super().__init__()
+
+ def download(self, path, return_type=None):
+ return self.run(path, depmap_dataset_names, return_type)
 
- Args:
- path (str): the path to save the datasets
- return_type (str, optional): the return type. Defaults to None. Can be "url", "name", or ["url", "name"]
- """
- url_list = []
- file_names = []
 
- for name in depmap_dataset_names:
- url, file_name = download_wrapper(name, path, return_type=["url", "name"])
- url_list.append(url)
- file_names.append(file_name)
+class CoessentialityDownloader(Downloader):
+ def __init__(self):
+ super().__init__()
 
- if return_type == "url":
- return url_list
- elif return_type == "name":
- return file_names
- elif return_type == ["url", "name"]:
- return url_list, file_names
+ def download(self, path, return_type=None):
+ return self.run(path, coessentiality_dataset_names, return_type)