Merge pull request #95 from nel-lab/annoying-windows

things for windows compatibility

.github/workflows/conda-windows.yml

@@ -0,0 +1,41 @@
+name: Windows Conda
+
+on:
+ pull_request:
+ branches:
+ - master
+ - dev
+ push:
+ branches: [ master ]
+
+jobs:
+ example-6:
+ name: Ex6 Mamba
+ runs-on: "windows-latest"
+ steps:
+ - uses: actions/checkout@v2
+ - uses: conda-incubator/setup-miniconda@v2
+ with:
+ python-version: 3.9
+ mamba-version: "*"
+ channels: conda-forge,defaults
+ channel-priority: true
+ activate-environment: mescore
+ environment-file: environment.yml
+ - shell: bash -l {0}
+ run: |
+ conda info
+ conda list
+ conda config --show-sources
+ conda config --show
+ printenv | sort
+ 
+ - shell: bash -l {0}
+ run: |
+ conda activate mescore
+ mamba install pytest
+ caimanmanager.py install
+ pip install https://pypi.anaconda.org/scipy-wheels-nightly/simple/pandas/1.5.0.dev0%2B1172.ga7c5773d19/pandas-1.5.0.dev0%2B1172.ga7c5773d19-cp39-cp39-win_amd64.whl
+ pip install .
+ DOWNLOAD_GROUND_TRUTHS=1 pytest -s .
+

environment.yml

@@ -5,5 +5,4 @@ dependencies:
  - pandas
  - requests
  - click
- - jupyterlab
- - pydata-sphinx-theme
+

mesmerize_core/algorithms/cnmf.py

@@ -13,6 +13,7 @@
 # prevent circular import
 if __name__ == "__main__":
  from mesmerize_core import set_parent_raw_data_path, load_batch
+ from mesmerize_core.utils import IS_WINDOWS
 
 
 @click.command()
@@ -97,7 +98,8 @@ def main(batch_path, uuid, data_path: str = None):
  d = dict()
 
  cnmf_memmap_path = output_dir.joinpath(Path(fname_new).name)
-
+ if IS_WINDOWS:
+ Yr._mmap.close() # accessing private attr but windows is annoying otherwise
  move_file(fname_new, cnmf_memmap_path)
 
  cnmf_hdf5_path = output_path.relative_to(output_dir.parent)

mesmerize_core/algorithms/cnmfe.py

@@ -11,6 +11,7 @@
 
 if __name__ == "__main__":
  from mesmerize_core import set_parent_raw_data_path, load_batch
+ from mesmerize_core.utils import IS_WINDOWS
 
 
 @click.command()
@@ -104,7 +105,8 @@ def main(batch_path, uuid, data_path: str = None):
  )
 
  cnmf_memmap_path = output_dir.joinpath(Path(fname_new).name)
-
+ if IS_WINDOWS:
+ Yr._mmap.close() # accessing private attr but windows is annoying otherwise
  move_file(fname_new, cnmf_memmap_path)
 
  cnmfe_memmap_path = cnmf_memmap_path.relative_to(output_dir.parent)

mesmerize_core/caiman_extensions/common.py

@@ -150,8 +150,10 @@ def _run_subprocess(
 
  # Get the dir that contains the input movie
  parent_path = self._series.paths.resolve(self._series.input_movie_path).parent
-
- self.process = Popen(runfile_path, cwd=parent_path)
+ if not IS_WINDOWS:
+ self.process = Popen(runfile_path, cwd=parent_path)
+ else:
+ self.process = Popen(f"powershell {runfile_path}", cwd=parent_path)
  return self.process
 
  def _run_slurm(
@@ -199,16 +201,20 @@ def run(
  batch_path = self._series.paths.get_batch_path()
 
  # Create the runfile in the batch dir using this Series' UUID as the filename
+ if IS_WINDOWS:
+ runfile_ext = ".ps1"
+ else:
+ runfile_ext = ".runfile"
  runfile_path = str(
- batch_path.parent.joinpath(self._series["uuid"] + ".runfile")
+ batch_path.parent.joinpath(self._series["uuid"] + runfile_ext)
  )
 
  args_str = f"--batch-path {batch_path} --uuid {self._series.uuid}"
  if get_parent_raw_data_path() is not None:
  args_str += f" --data-path {get_parent_raw_data_path()}"
 
  # make the runfile
- runfile = make_runfile(
+ runfile_path = make_runfile(
  module_path=os.path.abspath(
  ALGO_MODULES[self._series["algo"]].__file__
  ), # caiman algorithm
@@ -217,7 +223,7 @@ def run(
  )
  try:
  self.process = getattr(self, f"_run_{backend}")(
- runfile, **kwargs
+ runfile_path, **kwargs
  )
  except:
  with open(runfile_path, "r") as f:

mesmerize_core/utils.py

@@ -14,7 +14,9 @@
 import re as regex
 from pathlib import Path
 from warnings import warn
-
+import sys
+from tempfile import NamedTemporaryFile
+from subprocess import check_call
 
 if os.name == "nt":
  IS_WINDOWS = True
@@ -236,9 +238,18 @@ def make_runfile(
  else:
  with open(sh_file, "w") as f:
  for k, v in os.environ.items(): # copy the current environment
- f.write(f'$env:{k}="{v}";\n')
-
- f.write(f"python {module_path} {args_str}")
+ if regex.match("^.*[\(\)]", str(k)) or regex.match("^.*[\(\)]", str(v)):
+ continue
+ with NamedTemporaryFile(suffix=".ps1", delete=False) as tmp:
+ try: # windows powershell is stupid so make sure all the env var names work
+ tmp.write(f'$env:{k}="{v}";\n')
+ tmp.close()
+ check_call(f"powershell {tmp.name}")
+ os.unlink(tmp.name)
+ except:
+ continue
+ f.write(f'$env:{k}="{v}";\n') # write only env vars that powershell likes
+ f.write(f"{sys.executable} {module_path} {args_str}")
 
  st = os.stat(sh_file)
  os.chmod(sh_file, st.st_mode | S_IEXEC)

tests/test_core.py

@@ -13,6 +13,7 @@
  set_parent_raw_data_path,
 )
 from mesmerize_core.batch_utils import DATAFRAME_COLUMNS, COMPUTE_BACKEND_SUBPROCESS, get_full_raw_data_path
+from mesmerize_core.utils import IS_WINDOWS
 from uuid import uuid4
 from typing import *
 import pytest
@@ -119,6 +120,8 @@ def download_data(fname: str):
 
 
 def teardown_module():
+ if IS_WINDOWS: # because windows is stupid with permissions
+ return
  clear_tmp()