Merge pull request #790 from conda-forge/dot-conda

ENH support .conda artifacts

bin/conda-forge-scan-artifacts

@@ -117,6 +117,8 @@ def _munge_validate_yamls():
 def _process_artifact(pkg, repodata, libcfgraph_path, subdir, validate_yamls, verbose):
     if pkg.endswith(".tar.bz2"):
         pkg_json = pkg[:-len(".tar.bz2")] + ".json"
+    elif pkg.endswith(".conda"):
+        pkg_json = pkg[:-len(".conda")] + ".json"
     else:
         pkg_json = pkg
 

conda_forge_artifact_validation/generate_validate_yamls.py

@@ -40,10 +40,10 @@
 ]
 
 
-def _get_subdir_pkg_from_libcfgraph_artifact(artifact_pth):
+def _get_subdir_pkg_from_libcfgraph_artifact(artifact_pth, tail):
     subdir_pkg = "/".join(artifact_pth.split('/')[-2:])
     if subdir_pkg.endswith(".json"):
-        subdir_pkg = subdir_pkg[:-len(".json")] + ".tar.bz2"
+        subdir_pkg = subdir_pkg[:-len(".json")] + tail
 
     subdir, pkg = os.path.split(subdir_pkg)
     return subdir, pkg
@@ -63,9 +63,9 @@ def _get_all_json_blobs_for_artifact(artifact_name, verbose=0):
     sentinel = os.path.join("artifacts", artifact_name) + "/"
     artifact_pths = [pth for pth in LIBCFGRAPH_INDEX if pth.startswith(sentinel)]
 
-    def _download_jsob_blob(artifact_pth):
+    def _download_jsob_blob(artifact_pth, tail):
         # ignore things not on the main channel
-        subdir, pkg = _get_subdir_pkg_from_libcfgraph_artifact(artifact_pth)
+        subdir, pkg = _get_subdir_pkg_from_libcfgraph_artifact(artifact_pth, tail)
         if pkg not in REPODATA_CACHE[subdir]["packages"]:
             return None
 
@@ -79,10 +79,11 @@ def _download_jsob_blob(artifact_pth):
         except Exception:
             return None
 
-    jobs = [
-        joblib.delayed(_download_jsob_blob)(artifact_pth)
-        for artifact_pth in artifact_pths
-    ]
+    jobs = []
+    for artifact_pth in artifact_pths:
+        for tail in [".tar.bz2", ".conda"]:
+            jobs.append(joblib.delayed(_download_jsob_blob)(artifact_pth, tail))
+
     artifacts = joblib.Parallel(n_jobs=5, backend="threading", verbose=verbose)(jobs)
 
     return [a for a in artifacts if a is not None]

conda_forge_artifact_validation/tests/test_generate_validate_yamls.py

@@ -7,11 +7,19 @@
 
 def test_get_subdir_pkg_from_libcfgraph_artifact():
     subdir, pkg = _get_subdir_pkg_from_libcfgraph_artifact(
-        "artifacts/foo/conda-forge/osx-64/foo.json"
+        "artifacts/foo/conda-forge/osx-64/foo.json",
+        ".tar.bz2",
     )
     assert subdir == "osx-64"
     assert pkg == "foo.tar.bz2"
 
+    subdir, pkg = _get_subdir_pkg_from_libcfgraph_artifact(
+        "artifacts/foo/conda-forge/osx-64/foo.json",
+        ".conda",
+    )
+    assert subdir == "osx-64"
+    assert pkg == "foo.conda"
+
 
 def test_generate_validate_yaml_from_libcfgraph():
     validate_yaml = generate_validate_yaml_from_libcfgraph(

conda_forge_artifact_validation/tests/test_split_pkg.py

@@ -1,7 +1,7 @@
 from ..utils import split_pkg
 
 
-def test_split_pkg():
+def test_split_pkg_tar_bz2():
     pkg = "linux-64/python-3.7.6-py37djfa_0.tar.bz2"
     res = split_pkg(pkg)
 
@@ -11,3 +11,15 @@ def test_split_pkg():
         "3.7.6",
         "py37djfa_0",
     )
+
+
+def test_split_pkg_dot_conda():
+    pkg = "linux-64/python-3.7.6-py37djfa_0.conda"
+    res = split_pkg(pkg)
+
+    assert res == (
+        "linux-64",
+        "python",
+        "3.7.6",
+        "py37djfa_0",
+    )

conda_forge_artifact_validation/utils.py

@@ -29,9 +29,12 @@ def split_pkg(pkg):
     build : str
         The build string (e.g., `py37djfa_0`)
     """
-    if not pkg.endswith(".tar.bz2"):
-        raise RuntimeError("Can only process packages that end in .tar.bz2")
-    pkg = pkg[:-8]
+    if pkg.endswith(".tar.bz2"):
+        pkg = pkg[:-len(".tar.bz2")]
+    elif pkg.endswith(".conda"):
+        pkg = pkg[:-len(".conda")]
+    else:
+        raise RuntimeError("Can only process packages that end in .tar.bz2 or .conda!")
     plat, pkg_name = pkg.split(os.path.sep)
     name_ver, build = pkg_name.rsplit("-", 1)
     name, ver = name_ver.rsplit("-", 1)
@@ -92,9 +95,13 @@ def extract_subdir(path):
         pkg_dir, pkg = os.path.split(path)
 
         if pkg.endswith(".tar.bz2"):
-            pkg_nm = pkg[: -len(".tar.bz2")]
+            pkg_nm = pkg[:-len(".tar.bz2")]
+        elif pkg.endswith(".conda"):
+            pkg_nm = pkg[:-len(".conda")]
         else:
-            pkg_nm = pkg[: -len(".conda")]
+            raise RuntimeError(
+                "Can only process packages that end in .tar.bz2 or .conda!"
+            )
 
         conda_package_handling.api.extract(path)
 

conda_forge_artifact_validation/validate.py

@@ -73,9 +73,13 @@ def validate_file(path, validate_yamls, tmpdir=None, lock=None):
     _, output_name, _, _ = split_pkg(os.path.join("foo", pkg))
 
     if pkg.endswith(".tar.bz2"):
-        pkg_nm = pkg[: -len(".tar.bz2")]
+        pkg_nm = pkg[:-len(".tar.bz2")]
+    elif pkg.endswith(".conda"):
+        pkg_nm = pkg[:-len(".conda")]
     else:
-        pkg_nm = pkg[: -len(".conda")]
+        raise RuntimeError(
+            "Can only process packages that end in .tar.bz2 or .conda!"
+        )
 
     try:
         if lock is not None: