ShervanGharari · ShervanGharari · Apr 21, 2023 · Jun 8, 2023 · Jun 18, 2023 · Jun 19, 2023
diff --git a/easymore/easymore.py b/easymore/easymore.py
diff --git a/examples/00_test_example.ipynb b/examples/00_test_example.ipynb
diff --git a/examples/01_ERA5_Regular_Lat_Lon.ipynb b/examples/01_ERA5_Regular_Lat_Lon.ipynb
diff --git a/examples/02_ERA5_Regular_Lat_Lon_NaN.ipynb b/examples/02_ERA5_Regular_Lat_Lon_NaN.ipynb
diff --git a/examples/03_WRF_Rotated_Lat_Lon.ipynb b/examples/03_WRF_Rotated_Lat_Lon.ipynb
diff --git a/examples/05_irregular_shapes.ipynb b/examples/05_irregular_shapes.ipynb
diff --git a/examples/10_Calendar_365.ipynb b/examples/10_Calendar_365.ipynb
diff --git a/examples/data/Alberta_population/census_2016_DA_Canada.nc b/examples/data/Alberta_population/census_2016_DA_Canada.nc
diff --git a/examples/data/Source_nc_CanRCM4/CanRCM4_prcp.nc b/examples/data/Source_nc_CanRCM4/CanRCM4_prcp.nc
diff --git a/examples/data/Source_nc_ERA5/ERA5_NA_19790101.ncNaN b/examples/data/Source_nc_ERA5/ERA5_NA_19790101.ncNaN
diff --git a/examples/data/Source_nc_ERA5/ERA5_NA_19790102.ncNaN b/examples/data/Source_nc_ERA5/ERA5_NA_19790102.ncNaN
diff --git a/examples/data/Source_nc_ERA5/ERA5_NA_19790103.ncNaN b/examples/data/Source_nc_ERA5/ERA5_NA_19790103.ncNaN
diff --git a/examples/parallel/ERA5_Regular_Lat_Lon_parallel.py b/examples/parallel/ERA5_Regular_Lat_Lon_parallel.py
@@ -0,0 +1,48 @@
+import multiprocessing
+import os
+import glob
+from easymore.easymore import easymore
+
+def easymore_config():
+ esmr = easymore() # initial the easymore
+ #esmr.read_config_dict('config.txt') # read the configuration file
+ esmr.case_name = "ERA5_capitals_north_america"
+ esmr.temp_dir = "../temporary/"
+ esmr.target_shp = "../data/target_shapefiles/Capitals_point.shp"
+ esmr.source_nc = "../data/Source_nc_ERA5/ERA5_NA_*.nc"
+ esmr.var_names = ["airtemp"]
+ esmr.var_lon = "longitude"
+ esmr.var_lat = "latitude"
+ esmr.var_time = "time"
+ esmr.output_dir = "../output/"
+ return esmr
+
+def process_nc(nc_name):
+ print("Processing", nc_name)
+ print(str(os.getpid()))
+
+ esmr = easymore_config()
+ esmr.only_create_remap_csv = False # back to false/default in case if it reamins true (de)
+ esmr.source_nc = nc_name # update the file name
+ esmr.remap_csv = esmr.temp_dir+esmr.case_name +'_remapping.csv' # pass the remapping file name to skip remapping craetion
+ esmr.nc_remapper() # create remap file
+
+if __name__ == '__main__':
+
+ # initialise easymore and read configuration from the file
+ esmr = easymore_config()
+ esmr.only_create_remap_csv = True # only crate the remapping file
+
+ # get the nc file names
+ nc_names = sorted(glob.glob(esmr.source_nc))
+ # Create a multiprocessing.Pool with the desired number of processes
+ num_processes = multiprocessing.cpu_count() # Use the number of available CPU cores
+ num_processes = min(len(nc_names), num_processes) # limit the worker if number of files is smaller
+ pool = multiprocessing.Pool(processes=num_processes) # assign the number of workers
+
+ # Use pool.map() to parallelize the for loop
+ pool.map(process_nc, nc_names)
+
+ # Close the pool to free up resources
+ pool.close()
+ pool.join()
diff --git a/examples/parallel/SLURM_job.submit b/examples/parallel/SLURM_job.submit
@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH --account=rpp-kshook
+#SBATCH --ntasks=4
+#SBATCH --mem-per-cpu=2G
+#SBATCH --time=00:30:00 # time (DD-HH:MM)
+#SBATCH --job-name=easymore_era5_remapping
+#SBATCH --error=easymore_era5_remapping
+
+# load needed modules
+module load StdEnv/2020 gcc/9.3.0 openmpi/4.0.3
+module load gdal/3.5.1 libspatialindex/1.8.5
+module load python/3.8.10 scipy-stack/2022a mpi4py/3.0.3
+
+# create virtual env inside the job
+virtualenv --no-download $SLURM_TMPDIR/env
+source $SLURM_TMPDIR/env/bin/activate
+pip install --no-index --upgrade pip
+pip install --no-index easymore
+
+# OR use your locally created virtual env in home directory (created as explained on log in node; above)
+# source ~/easymore-env/bin/activate # when this is uncommneted, then commnet above (virtualenv ...)
+
+# run python script that include easymore remapper
+python ERA5_Regular_Lat_Lon_parallel.py
diff --git a/examples/parallel/config.txt b/examples/parallel/config.txt
@@ -0,0 +1,11 @@
+{
+"case_name" : "ERA5_capitals_north_america",
+"temp_dir" : "./../temporary/",
+"target_shp" : "./../data/target_shapefiles/Capitals_point.shp",
+"source_nc" : "./../data/Source_nc_ERA5/ERA5_NA_*.nc",
+"var_names" : ["airtemp"],
+"var_lon" : "longitude",
+"var_lat" : "latitude",
+"var_time" : "time",
+"output_dir" : "./../output/"
+}
diff --git a/setup.py b/setup.py
@@ -45,7 +45,7 @@
 
 setup(
  name='easymore',
- version='1.0.0',
+ version='1.1.0',
  license='GPLv3',
  author=('Shervan Gharari'),
  author_email = '[email protected]',