diff --git a/metaflow/plugins/pypi/conda_environment.py b/metaflow/plugins/pypi/conda_environment.py index 9e0de9f0be..71a1706cf2 100644 --- a/metaflow/plugins/pypi/conda_environment.py +++ b/metaflow/plugins/pypi/conda_environment.py @@ -20,7 +20,7 @@ from metaflow.metaflow_profile import profile from . import MAGIC_FILE, _datastore_packageroot -from .utils import conda_platform +from .utils import conda_platform, generate_cache_path, parse_filename_from_url class CondaEnvironmentException(MetaflowException): @@ -107,7 +107,7 @@ def cache(storage, results, type_): local_packages = { url: { # Path to package in datastore. - "path": urlparse(url).netloc + urlparse(url).path, + "path": generate_cache_path(url, local_path), # Path to package on local disk. "local_path": local_path, } @@ -122,9 +122,8 @@ def cache(storage, results, type_): # Cache only those packages that manifest is unaware of local_packages.pop(package["url"], None) else: - package["path"] = ( - urlparse(package["url"]).netloc - + urlparse(package["url"]).path + package["path"] = generate_cache_path( + package["url"], parse_filename_from_url(package["url"]) ) dirty.add(id_) diff --git a/metaflow/plugins/pypi/pip.py b/metaflow/plugins/pypi/pip.py index e715e6bd25..938e96e6ca 100644 --- a/metaflow/plugins/pypi/pip.py +++ b/metaflow/plugins/pypi/pip.py @@ -9,7 +9,7 @@ from metaflow.util import which from .micromamba import Micromamba -from .utils import pip_tags +from .utils import parse_filename_from_url, pip_tags class PipException(MetaflowException): @@ -118,7 +118,7 @@ def download(self, id_, packages, python, platform): for package in packages: cmd.append("{url}".format(**package)) metadata["{url}".format(**package)] = "{prefix}/.pip/wheels/{wheel}".format( - prefix=prefix, wheel=package["url"].split("/")[-1] + prefix=prefix, wheel=parse_filename_from_url(package["url"]) ) self._call(prefix, cmd) # write the url to wheel mappings in a magic location diff --git a/metaflow/plugins/pypi/utils.py b/metaflow/plugins/pypi/utils.py index 96e6c1268e..e9c8829f22 100644 --- a/metaflow/plugins/pypi/utils.py +++ b/metaflow/plugins/pypi/utils.py @@ -1,3 +1,4 @@ +import os import platform import sys @@ -13,6 +14,7 @@ class Tags: from metaflow._vendor.packaging import tags from metaflow.exception import MetaflowException +from urllib.parse import unquote, urlparse def conda_platform(): @@ -73,3 +75,16 @@ def pip_tags(python_version, mamba_platform): supported.extend(tags.cpython_tags(py_version, abis, platforms)) supported.extend(tags.compatible_tags(py_version, interpreter, platforms)) return supported + + +def parse_filename_from_url(url): + # Separate method as it might require additional checks for the parsing. + filename = url.split("/")[-1] + return unquote(filename) + + +def generate_cache_path(url, local_path): + base, _ = os.path.split(urlparse(url).path) + _, localfile = os.path.split(local_path) + unquoted_base = unquote(base) + return urlparse(url).netloc + os.path.join(unquoted_base, localfile)