diff --git a/.github/workflows/conda-windows.yml b/.github/workflows/conda-windows.yml new file mode 100644 index 0000000..62ef9bf --- /dev/null +++ b/.github/workflows/conda-windows.yml @@ -0,0 +1,41 @@ +name: Windows Conda + +on: + pull_request: + branches: + - master + - dev + push: + branches: [ master ] + +jobs: + example-6: + name: Ex6 Mamba + runs-on: "windows-latest" + steps: + - uses: actions/checkout@v2 + - uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.9 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true + activate-environment: mescore + environment-file: environment.yml + - shell: bash -l {0} + run: | + conda info + conda list + conda config --show-sources + conda config --show + printenv | sort + + - shell: bash -l {0} + run: | + conda activate mescore + mamba install pytest + caimanmanager.py install + pip install https://pypi.anaconda.org/scipy-wheels-nightly/simple/pandas/1.5.0.dev0%2B1172.ga7c5773d19/pandas-1.5.0.dev0%2B1172.ga7c5773d19-cp39-cp39-win_amd64.whl + pip install . + DOWNLOAD_GROUND_TRUTHS=1 pytest -s . + diff --git a/environment.yml b/environment.yml index 0e4037c..5c9d500 100644 --- a/environment.yml +++ b/environment.yml @@ -5,5 +5,4 @@ dependencies: - pandas - requests - click - - jupyterlab - - pydata-sphinx-theme + diff --git a/mesmerize_core/algorithms/cnmf.py b/mesmerize_core/algorithms/cnmf.py index ca92fbf..66ef469 100644 --- a/mesmerize_core/algorithms/cnmf.py +++ b/mesmerize_core/algorithms/cnmf.py @@ -13,6 +13,7 @@ # prevent circular import if __name__ == "__main__": from mesmerize_core import set_parent_raw_data_path, load_batch + from mesmerize_core.utils import IS_WINDOWS @click.command() @@ -97,7 +98,8 @@ def main(batch_path, uuid, data_path: str = None): d = dict() cnmf_memmap_path = output_dir.joinpath(Path(fname_new).name) - + if IS_WINDOWS: + Yr._mmap.close() # accessing private attr but windows is annoying otherwise move_file(fname_new, cnmf_memmap_path) cnmf_hdf5_path = output_path.relative_to(output_dir.parent) diff --git a/mesmerize_core/algorithms/cnmfe.py b/mesmerize_core/algorithms/cnmfe.py index 8030cbf..b3d3415 100644 --- a/mesmerize_core/algorithms/cnmfe.py +++ b/mesmerize_core/algorithms/cnmfe.py @@ -11,6 +11,7 @@ if __name__ == "__main__": from mesmerize_core import set_parent_raw_data_path, load_batch + from mesmerize_core.utils import IS_WINDOWS @click.command() @@ -104,7 +105,8 @@ def main(batch_path, uuid, data_path: str = None): ) cnmf_memmap_path = output_dir.joinpath(Path(fname_new).name) - + if IS_WINDOWS: + Yr._mmap.close() # accessing private attr but windows is annoying otherwise move_file(fname_new, cnmf_memmap_path) cnmfe_memmap_path = cnmf_memmap_path.relative_to(output_dir.parent) diff --git a/mesmerize_core/caiman_extensions/common.py b/mesmerize_core/caiman_extensions/common.py index 91c3b25..69df933 100644 --- a/mesmerize_core/caiman_extensions/common.py +++ b/mesmerize_core/caiman_extensions/common.py @@ -150,8 +150,10 @@ def _run_subprocess( # Get the dir that contains the input movie parent_path = self._series.paths.resolve(self._series.input_movie_path).parent - - self.process = Popen(runfile_path, cwd=parent_path) + if not IS_WINDOWS: + self.process = Popen(runfile_path, cwd=parent_path) + else: + self.process = Popen(f"powershell {runfile_path}", cwd=parent_path) return self.process def _run_slurm( @@ -199,8 +201,12 @@ def run( batch_path = self._series.paths.get_batch_path() # Create the runfile in the batch dir using this Series' UUID as the filename + if IS_WINDOWS: + runfile_ext = ".ps1" + else: + runfile_ext = ".runfile" runfile_path = str( - batch_path.parent.joinpath(self._series["uuid"] + ".runfile") + batch_path.parent.joinpath(self._series["uuid"] + runfile_ext) ) args_str = f"--batch-path {batch_path} --uuid {self._series.uuid}" @@ -208,7 +214,7 @@ def run( args_str += f" --data-path {get_parent_raw_data_path()}" # make the runfile - runfile = make_runfile( + runfile_path = make_runfile( module_path=os.path.abspath( ALGO_MODULES[self._series["algo"]].__file__ ), # caiman algorithm @@ -217,7 +223,7 @@ def run( ) try: self.process = getattr(self, f"_run_{backend}")( - runfile, **kwargs + runfile_path, **kwargs ) except: with open(runfile_path, "r") as f: diff --git a/mesmerize_core/utils.py b/mesmerize_core/utils.py index fd42618..3dc3fc9 100644 --- a/mesmerize_core/utils.py +++ b/mesmerize_core/utils.py @@ -14,7 +14,9 @@ import re as regex from pathlib import Path from warnings import warn - +import sys +from tempfile import NamedTemporaryFile +from subprocess import check_call if os.name == "nt": IS_WINDOWS = True @@ -236,9 +238,18 @@ def make_runfile( else: with open(sh_file, "w") as f: for k, v in os.environ.items(): # copy the current environment - f.write(f'$env:{k}="{v}";\n') - - f.write(f"python {module_path} {args_str}") + if regex.match("^.*[\(\)]", str(k)) or regex.match("^.*[\(\)]", str(v)): + continue + with NamedTemporaryFile(suffix=".ps1", delete=False) as tmp: + try: # windows powershell is stupid so make sure all the env var names work + tmp.write(f'$env:{k}="{v}";\n') + tmp.close() + check_call(f"powershell {tmp.name}") + os.unlink(tmp.name) + except: + continue + f.write(f'$env:{k}="{v}";\n') # write only env vars that powershell likes + f.write(f"{sys.executable} {module_path} {args_str}") st = os.stat(sh_file) os.chmod(sh_file, st.st_mode | S_IEXEC) diff --git a/tests/test_core.py b/tests/test_core.py index e5abf56..255395f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -13,6 +13,7 @@ set_parent_raw_data_path, ) from mesmerize_core.batch_utils import DATAFRAME_COLUMNS, COMPUTE_BACKEND_SUBPROCESS, get_full_raw_data_path +from mesmerize_core.utils import IS_WINDOWS from uuid import uuid4 from typing import * import pytest @@ -119,6 +120,8 @@ def download_data(fname: str): def teardown_module(): + if IS_WINDOWS: # because windows is stupid with permissions + return clear_tmp()