Support GIT packages through conda-lock

This will now build packages coming from GIT just like in the pure PYPI path.
Netflix · Aug 29, 2024 · ef5229d · ef5229d
1 parent 1c534c5
commit ef5229d
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 29 deletions.
diff --git a/metaflow_extensions/netflix_ext/plugins/conda/resolvers/conda_lock_resolver.py b/metaflow_extensions/netflix_ext/plugins/conda/resolvers/conda_lock_resolver.py
@@ -1,16 +1,12 @@
 # pyright: strict, reportTypeCommentUsage=false, reportMissingTypeStubs=false
-import json
 import os
-import subprocess
-import sys
 import tempfile
-import uuid
 
 from itertools import chain
 from typing import Dict, List, Optional, Set, Tuple, cast
 
+from urllib.parse import urlparse
 from metaflow.debug import debug
-from metaflow.metaflow_config import CONDA_LOCAL_PATH
 
 from metaflow._vendor.packaging.requirements import Requirement
 
@@ -27,14 +23,14 @@
 
 from ..utils import (
     CondaException,
-    WithDir,
     arch_id,
     channel_or_url,
     parse_explicit_url_conda,
     parse_explicit_url_pypi,
     pypi_tags_from_arch,
 )
 from . import Resolver
+from .pip_resolver import _FAKE_WHEEL
 
 
 class CondaLockResolver(Resolver):
@@ -197,8 +193,13 @@ def resolve(
             for pypi_name, info in pypi_dep_lines.items():
                 if "url" in info:
                     toml_lines.append(
-                        '"%s" = {url = "%s", %s source="pypi"}\n'
-                        % (pypi_name, info["url"], info["url_extras"])
+                        '"%s" = {%s = "%s", %s source="pypi"}\n'
+                        % (
+                            pypi_name,
+                            "git" if info["url"].startswith("git+") else "url",
+                            info["url"],
+                            info["url_extras"],
+                        )
                     )
                 else:
                     toml_lines.append(
@@ -293,6 +294,34 @@ def resolve(
                                 raise CondaException(
                                     "Unexpected package specification line: %s" % l
                                 )
+                            debug.conda_exec("Got PYPI package %s" % l)
+                            if components[4].startswith("git+"):
+                                # This is a GIT URL so we have to build the package
+                                # See pip_resolver.py for vcs_info on an explanation
+                                # on how we deal with git packages
+                                # Looks like gi+https://<url>@<commit>
+                                base_build_url = components[4]
+                                parse = urlparse(base_build_url)
+                                clean_path, clean_commit = parse.path.split("@")
+                                clean_url = parse.scheme[4:] + parse.netloc + clean_path
+                                base_pkg_url = "%s/%s" % (clean_url, clean_commit)
+                                # TODO: Do we need to handle subdirectories
+                                cache_base_url = (
+                                    PypiCachePackage.make_partial_cache_url(
+                                        base_pkg_url, is_real_url=False
+                                    )
+                                )
+                                packages_to_build[cache_base_url] = PackageToBuild(
+                                    base_build_url,
+                                    PypiPackageSpecification(
+                                        _FAKE_WHEEL,
+                                        base_pkg_url,
+                                        is_real_url=False,
+                                        url_format=".whl",
+                                    ),
+                                )
+                                continue
+                            # Non-GIT URL
                             parse_result = parse_explicit_url_pypi(components[4])
                             if parse_result.url_format != ".whl":
                                 cache_base_url = (
@@ -370,27 +399,24 @@ def resolve(
                     supported_tags = pypi_tags_from_arch(
                         python_version, architecture, glibc_version
                     )
-                    if self._conda.storage:
-                        built_pypi_packages, builder_envs = build_pypi_packages(
-                            self._conda,
-                            self._conda.storage,
-                            python_version,
-                            packages_to_build,
-                            builder_envs,
-                            build_dir,
-                            architecture,
-                            supported_tags,
-                            sources.get("pypi", []),
-                        )
-                        packages.extend(built_pypi_packages)
-                    else:
-                        # Here it was just URLs so we are good
-                        packages.extend(
-                            [
-                                cast(PackageSpecification, v.spec)
-                                for v in packages_to_build.values()
-                            ]
+                    if not self._conda.storage:
+                        raise CondaException(
+                            "Cannot create a relocatable environment as it depends on "
+                            "local files or non wheels and no storage backend is defined: %s"
+                            % ", ".join([v.url for v in packages_to_build.values()])
                         )
+                    built_pypi_packages, builder_envs = build_pypi_packages(
+                        self._conda,
+                        self._conda.storage,
+                        python_version,
+                        packages_to_build,
+                        builder_envs,
+                        build_dir,
+                        architecture,
+                        supported_tags,
+                        sources.get("pypi", []),
+                    )
+                    packages.extend(built_pypi_packages)
             return (
                 ResolvedEnvironment(
                     deps,

diff --git a/metaflow_extensions/netflix_ext/plugins/conda/utils.py b/metaflow_extensions/netflix_ext/plugins/conda/utils.py
@@ -362,7 +362,12 @@ def parse_explicit_url_pypi(url: str) -> ParseExplicitResult:
     filename = None
     url_format = None
 
-    url_clean, url_hash = url.rsplit("#", 1)
+    splits = url.rsplit("#", 1)
+    if len(splits) > 1:
+        url_clean, url_hash = splits
+    else:
+        url_clean = url
+        url_hash = None
     if url_hash:
         if not url_hash.startswith("sha256="):
             raise CondaException("URL '%s' has a SHA type which is not supported" % url)