From dace7964387f119b143a40ddb6076a8f0757966c Mon Sep 17 00:00:00 2001 From: Peter Battaglia Date: Tue, 29 Mar 2022 13:24:42 +0100 Subject: [PATCH] Fixes issue (127) where append_date_dirs wasn't working properly. * Fixed bug where partition_keys was being modified in-place, and causing problems laters. * Adjusted how target_path is constructed so that trailing forward slashes in target_path are all removed. * Added typing.cast to comply with pytype. --- weather_dl/download_pipeline/parsers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/weather_dl/download_pipeline/parsers.py b/weather_dl/download_pipeline/parsers.py index 617608ee..90c169a1 100644 --- a/weather_dl/download_pipeline/parsers.py +++ b/weather_dl/download_pipeline/parsers.py @@ -328,8 +328,9 @@ def require(condition: bool, message: str) -> None: num_partition_keys = len(partition_keys) if use_date_as_directory(config): num_partition_keys -= 1 - if str(params.get('target_path')).endswith('/'): - params['target_path'] = params.get('target_path')[:-1] + target_path = t.cast(str, params.get('target_path', '')) + if target_path != '': + params['target_path'] = target_path.rstrip('/') require(num_template_replacements == num_partition_keys, """ @@ -352,10 +353,12 @@ def prepare_target_name(config: Config) -> str: target_path = t.cast(str, parameters.get('target_path', '')) target_filename = t.cast(str, parameters.get('target_filename', '')) - partition_keys = t.cast(t.List[str], parameters.get('partition_keys', list())) + partition_keys = t.cast(t.List[str], + cp.copy(parameters.get('partition_keys', list()))) if use_date_as_directory(config): - date_vals = config['selection']['date'][0].split('-') + date = t.cast(str, config['selection']['date'][0]) + date_vals = date.split('-') target_path = os.path.join(target_path, *date_vals) partition_keys.remove('date')