Reformatting generation scripts

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+- repo: https:/python/black
+ rev: 19.10b0
+ hooks:
+ - id: black
+ pass_filenames: true
+- repo: https://gitlab.com/pycqa/flake8
+ rev: 3.7.9
+ hooks:
+ - id: flake8
+ pass_filenames: true
+ # this seems to need to be here in addition to setup.cfg
+ exclude: __init__.py

brainatlas_api/atlas_gen/__init__.py

@@ -0,0 +1,4 @@
+from brainatlas_api.atlas_gen.wrapup import wrapup_atlas_from_dir
+from brainatlas_api.atlas_gen.stacks import save_anatomy, save_annotation
+
+from brainatlas_api.atlas_gen import descriptors

brainatlas_api/atlas_gen/atlas_scripts/allenbrain_atlas.py

@@ -0,0 +1,91 @@
+from allensdk.api.queries.ontologies_api import OntologiesApi
+from allensdk.api.queries.reference_space_api import ReferenceSpaceApi
+from allensdk.core.reference_space_cache import ReferenceSpaceCache
+
+from requests import exceptions
+from pathlib import Path
+import tempfile
+import json
+
+import tifffile
+import pandas as pd
+
+RES_UM = 25
+ATLAS_NAME = f"allenbrain{RES_UM}um"
+
+# Generated atlas path:
+bg_root_dir = Path.home() / "brainglobe"
+bg_root_dir.mkdir(exist_ok=True)
+
+# Temporary folder for nrrd files download:
+temp_path = Path(tempfile.mkdtemp())
+downloading_path = temp_path / "downloading_path"
+downloading_path.mkdir()
+
+# Temporary folder for files before compressing:
+uncompr_atlas_path = temp_path / ATLAS_NAME
+uncompr_atlas_path.mkdir()
+
+# Download annotated and template volume:
+#########################################
+spacecache = ReferenceSpaceCache(
+ manifest=downloading_path / "manifest.json",
+ # downloaded files are stored relative to here
+ resolution=RES_UM,
+ reference_space_key="annotation/ccf_2017"
+ # use the latest version of the CCF
+)
+
+# Download
+annotated_volume, _ = spacecache.get_annotation_volume()
+template_volume, _ = spacecache.get_template_volume()
+print("Download completed...")
+# Save tiff stacks:
+tifffile.imsave(str(uncompr_atlas_path / "reference.tiff"), template_volume)
+tifffile.imsave(str(uncompr_atlas_path / "annotated.tiff"), annotated_volume)
+
+# Download structures tree and meshes:
+######################################
+oapi = OntologiesApi() # ontologies
+struct_tree = spacecache.get_structure_tree() # structures tree
+
+# Find id of set of regions with mesh:
+select_set = "Structures whose surfaces are represented by a precomputed mesh"
+
+all_sets = pd.DataFrame(oapi.get_structure_sets())
+mesh_set_id = all_sets[all_sets.description == select_set].id.values[0]
+
+structs_with_mesh = struct_tree.get_structures_by_set_id([mesh_set_id])
+
+meshes_dir = uncompr_atlas_path / "meshes" # directory to save meshes into
+space = ReferenceSpaceApi()
+for s in structs_with_mesh:
+ name = s["id"]
+ try:
+ space.download_structure_mesh(
+ structure_id=s["id"],
+ ccf_version="annotation/ccf_2017",
+ file_name=meshes_dir / f"{name}.obj",
+ )
+ except (exceptions.HTTPError, ConnectionError):
+ print(s)
+
+# Loop over structures, remove entries not used in brainglobe:
+for struct in structs_with_mesh:
+ [struct.pop(k) for k in ["graph_id", "structure_set_ids", "graph_order"]]
+
+with open(uncompr_atlas_path / "structures.json", "w") as f:
+ json.dump(structs_with_mesh, f)
+
+metadata_dict = {
+ "name": ATLAS_NAME,
+ "citation": "Wang et al 2020, https://doi.org/10.1016/j.cell.2020.04.007",
+ "atlas_link": "www.brain-map.org.com",
+ "species": "Mus musculus",
+ "symmetric": True,
+ "resolution": (RES_UM, RES_UM, RES_UM),
+ "shape": template_volume.shape,
+}
+
+with open(uncompr_atlas_path / "atlas_metadata.json", "w") as f:
+ json.dump(metadata_dict, f)

brainatlas_api/atlas_gen/atlas_scripts/fishatlas_atlas.py

@@ -0,0 +1,117 @@
+from atlas_scripts.fishatlas_utils import add_path_inplace, collect_all_inplace
+
+from pathlib import Path
+import tempfile
+from brainatlas_api.utils import retrieve_over_http
+import json
+import tarfile
+
+import nrrd
+import numpy as np
+import tifffile
+import requests
+
+ATLAS_NAME = "fishatlas"
+
+base_url = r"https://fishatlas.neuro.mpg.de"
+
+
+# Generated atlas path:
+bg_root_dir = Path.home() / "brainglobe"
+bg_root_dir.mkdir(exist_ok=True)
+
+# Temporary folder for nrrd files download:
+temp_path = Path(tempfile.mkdtemp())
+download_dir_path = temp_path / "downloading_path"
+download_dir_path.mkdir()
+
+# Temporary folder for files before compressing:
+uncompr_atlas_path = temp_path / ATLAS_NAME
+uncompr_atlas_path.mkdir()
+
+# Download reference:
+#####################
+reference_url = f"{base_url}/media/brain_browser/Brain/MovieViewBrain/standard_brain_fixed_SYP_T_GAD1b.nrrd"
+out_file_path = download_dir_path / "reference.nrrd"
+
+retrieve_over_http(reference_url, out_file_path)
+
+# Cleanup to have in brainglobe order:
+refstack_axes = (1, 2, 0)
+refstack_flips = [False, True, False]
+
+refstack, h = nrrd.read(str(out_file_path))
+
+refstack = refstack.transpose(refstack_axes)
+for i, flip in enumerate(refstack_flips):
+ if flip:
+ refstack = np.flip(refstack, i)
+
+
+tifffile.imsave(str(uncompr_atlas_path / "reference.tiff"), refstack)
+
+# Download structures tree and meshes:
+######################################
+regions_url = f"{base_url}/neurons/get_brain_regions"
+
+meshes_dir_path = uncompr_atlas_path / "meshes"
+meshes_dir_path.mkdir(exist_ok=True)
+
+# Download structures hierarchy:
+regions = requests.get(regions_url).json()["brain_regions"]
+
+# Initiate dictionary with root info:
+regions_dict = {
+ "name": "root",
+ "id": 0,
+ "sub_regions": regions.copy(),
+ "structure_id_path": [],
+ "acronym": "root",
+ "files": {
+ "file_3D": "/media/Neurons_database/Brain_and_regions/Brains/Outline/Outline_new.txt"
+ },
+ "color": "#ffffff",
+}
+
+# Go through the regions hierarchy and create the structure path entry:
+add_path_inplace(regions_dict)
+
+# Create empty list and collect all regions traversing the regions hierarchy:
+regions_list = []
+collect_all_inplace(
+ regions_dict,
+ regions_list,
+ meshes_dir_path,
+ refstack_axes,
+ refstack_flips,
+ refstack.shape,
+)
+
+# save regions list json:
+with open(uncompr_atlas_path / "structures.json", "w") as f:
+ json.dump(regions_list, f)
+
+# Write metadata:
+#################
+metadata_dict = {
+ "name": ATLAS_NAME,
+ "citation": "Kunst et al 2019, https://doi.org/10.1016/j.neuron.2019.04.034",
+ "atlas_link": "https://fishatlas.neuro.mpg.de",
+ "species": "Danio rerio",
+ "symmetric": False,
+ "resolution": (0.994, 1, 0.994),
+ "shape": refstack.shape,
+}
+
+with open(uncompr_atlas_path / "atlas_metadata.json", "w") as f:
+ json.dump(metadata_dict, f)
+
+# Compress folder:
+output_filename = bg_root_dir / f"{uncompr_atlas_path.name}.tar.gz"
+with tarfile.open(output_filename, "w:gz") as tar:
+ tar.add(uncompr_atlas_path, arcname=uncompr_atlas_path.name)
+
+# Clean temporary directory and remove it:
+for f in download_dir_path.glob("*"):
+ f.unlink()
+download_dir_path.rmdir()

brainatlas_api/atlas_gen/atlas_scripts/fishatlas_utils.py

@@ -0,0 +1,132 @@
+from brainatlas_api.utils import retrieve_over_http
+import trimesh
+from brainatlas_api.structures import StructureTree
+import requests
+import warnings
+
+BASE_URL = r"https://fishatlas.neuro.mpg.de"
+
+
+def download_convert_mesh(
+ url,
+ interm_file_path,
+ obj_file_path,
+ refstack_axes,
+ refstack_flips,
+ ref_shape,
+ cleanup=True,
+):
+ """
+ Parameters
+ ----------
+ url : str
+ mesh url for download
+ interm_file_path : Path obj
+ path of the intermediate .stl mesh
+ obj_file_path : Path obj
+ path of the final .obj object
+ cleanup : bool (default True)
+ if True, intermediate file is unlinked
+
+ Returns
+ -------
+
+ """
+ retrieve_over_http(url, interm_file_path)
+
+ mesh = trimesh.load(interm_file_path)
+ mesh.vertices = mesh.vertices[:, refstack_axes]
+ for i, (f, size) in enumerate(zip(refstack_flips, ref_shape)):
+ if f:
+ mesh.vertices[:, i] = size - mesh.vertices[:, i]
+
+ mesh.export(obj_file_path)
+
+ if cleanup:
+ interm_file_path.unlink()
+
+
+def add_path_inplace(parent):
+ """ Recursively traverse hierarchy of regions and append for each region
+ the full path of substructures in brainglobe standard list.
+ Parameters
+ ----------
+ parent : dict
+ node parsed from fishatlas website containing a "sub_regions" key;
+
+ Returns
+ -------
+
+ """
+ for ch in parent["sub_regions"]:
+ new_root = parent["structure_id_path"] + [
+ parent["id"],
+ ]
+
+ ch["structure_id_path"] = new_root
+
+ add_path_inplace(ch)
+
+
+def collect_all_inplace(
+ node,
+ traversing_list,
+ download_path,
+ refstack_axes,
+ refstack_flips,
+ ref_shape,
+):
+ """ Recursively traverse a region hierarchy, download meshes, and append
+ regions to a list inplace.
+
+ Parameters
+ ----------
+ node
+ traversing_list
+ download_path
+
+ Returns
+ -------
+
+ """
+
+ # Append clean dictionary with brainglobe standard info:
+ traversing_list.append(
+ {
+ "name": node["name"],
+ "acronym": node["name"],
+ "id": node["id"],
+ "rgb_triplet": StructureTree.hex_to_rgb(node["color"]),
+ "structure_id_path": node["structure_id_path"],
+ }
+ )
+
+ # Url for the mesh:
+ mesh_url = (
+ BASE_URL + node["files"]["file_3D"][:-4].replace("\\", "/") + ".stl"
+ )
+
+ # Try download, if mesh does not exist region is removed:
+ try:
+ download_convert_mesh(
+ mesh_url,
+ download_path / "mesh.stl",
+ download_path / "{}.obj".format(node["id"]),
+ refstack_axes,
+ refstack_flips,
+ ref_shape,
+ )
+ except requests.exceptions.ConnectionError:
+ # Pop region from list:
+ message = "No mesh found for {}".format(traversing_list.pop()["name"])
+ warnings.warn(message)
+
+ for region in node["sub_regions"]:
+ collect_all_inplace(
+ region,
+ traversing_list,
+ download_path,
+ refstack_axes,
+ refstack_flips,
+ ref_shape,
+ )