Stop injecting installed channels that do not exist (#274)

* do not inject installed channels that do not exist * add news * pre-commit * base_url can be None for <unknown> channels * better like this * amend news * simplify * Apply suggestions from code review Co-authored-by: Ken Odegard <[email protected]> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * debug Linux * use session, catch exception * timeout after 1s * move check to solver * pre-commit * get_session only available on conda 23.9 * debug test on CI (passes locally) * do not use current_repodata * deselect * pre-commit --------- Co-authored-by: Ken Odegard <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jannis Leidel <[email protected]>
conda · Sep 20, 2023 · 89859a4 · 89859a4
1 parent 7bf3ab4
commit 89859a4
Show file tree

Hide file tree

Showing 10 changed files with 149 additions and 26 deletions.
diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py
@@ -23,7 +23,7 @@
 from conda.base.constants import REPODATA_FN, UNKNOWN_CHANNEL, ChannelPriority, on_win
 from conda.base.context import context
 from conda.common.constants import NULL
-from conda.common.io import Spinner
+from conda.common.io import Spinner, timeout
 from conda.common.path import paths_equal
 from conda.common.url import join_url, percent_decode
 from conda.core.prefix_data import PrefixData
@@ -44,6 +44,7 @@
 from .index import LibMambaIndexHelper, _CachedLibMambaIndexHelper
 from .mamba_utils import init_api_context, mamba_version
 from .state import SolverInputState, SolverOutputState
+from .utils import is_channel_available
 
 log = logging.getLogger(f"conda.{__name__}")
 
@@ -171,21 +172,23 @@ def solve_final_state(
         else:
             IndexHelper = LibMambaIndexHelper
 
-        if os.getenv("CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED") or (
-            getattr(context, "_argparse_args", None) or {}
-        ).get("override_channels"):
-            # see https://github.com/conda/conda-libmamba-solver/issues/108
-            channels_from_installed = ()
-        else:
-            channels_from_installed = in_state.channels_from_installed()
-
-        all_channels = (
+        all_channels = [
             *conda_bld_channels,
             *self.channels,
             *in_state.channels_from_specs(),
-            *channels_from_installed,
-            *in_state.maybe_free_channel(),
-        )
+        ]
+        override = (getattr(context, "_argparse_args", None) or {}).get("override_channels")
+        if not os.getenv("CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED") and not override:
+            # see https://github.com/conda/conda-libmamba-solver/issues/108
+            all_urls = [url for c in all_channels for url in Channel(c).urls(False)]
+            installed_channels = in_state.channels_from_installed(seen=all_urls)
+            for channel in installed_channels:
+                # Only add to list if resource is available; check has timeout=1s
+                if timeout(1, is_channel_available, channel.base_url, default_return=False):
+                    all_channels.append(channel)
+        all_channels.extend(in_state.maybe_free_channel())
+
+        all_channels = tuple(all_channels)
         with Spinner(
             self._spinner_msg_metadata(all_channels, conda_bld_channels=conda_bld_channels),
             enabled=not context.verbosity and not context.quiet,
@@ -858,8 +861,19 @@ def _package_record_from_json_payload(
         if pkg_filename.startswith("__") and "/@/" in channel:
             return PackageRecord(**json.loads(json_payload))
 
-        channel_info = index.get_info(channel)
         kwargs = json.loads(json_payload)
+        try:
+            channel_info = index.get_info(channel)
+        except KeyError:
+            # this channel was never used to build the index, which
+            # means we obtained an already installed PackageRecord
+            # whose metadata contains a channel that doesn't exist
+            pd = PrefixData(self.prefix)
+            record = pd.get(kwargs["name"])
+            if record and record.fn == pkg_filename:
+                return record
+
+        # Otherwise, these are records from the index
         kwargs["fn"] = pkg_filename
         kwargs["channel"] = channel_info.channel
         kwargs["url"] = join_url(channel_info.full_url, pkg_filename)

diff --git a/conda_libmamba_solver/state.py b/conda_libmamba_solver/state.py
@@ -410,8 +410,8 @@ def channels_from_specs(self) -> Iterable[Channel]:
                     channel = Channel(spec.original_spec_str.split("::")[0])
                 yield channel
 
-    def channels_from_installed(self) -> Iterable[Channel]:
-        seen_urls = set()
+    def channels_from_installed(self, seen=None) -> Iterable[Channel]:
+        seen_urls = set(seen or [])
         # See https://github.com/conda/conda/issues/11790
         for record in self.installed.values():
             if record.channel.auth or record.channel.token:
@@ -422,10 +422,12 @@ def channels_from_installed(self) -> Iterable[Channel]:
                 # These "channels" are not really channels, more like
                 # metadata placeholders
                 continue
-            subdir_url = record.channel.subdir_url
-            if subdir_url not in seen_urls:
-                seen_urls.add(subdir_url)
-                yield record.channel
+            if record.channel.base_url is None:
+                continue
+            if record.channel.subdir_url in seen_urls:
+                continue
+            seen_urls.add(record.channel.subdir_url)
+            yield record.channel
 
     def maybe_free_channel(self) -> Iterable[Channel]:
         if context.restore_free_channel:

diff --git a/conda_libmamba_solver/utils.py b/conda_libmamba_solver/utils.py
@@ -1,11 +1,16 @@
 # Copyright (C) 2022 Anaconda, Inc
 # Copyright (C) 2023 conda
 # SPDX-License-Identifier: BSD-3-Clause
+from functools import lru_cache
 from logging import getLogger
+from pathlib import Path
 from urllib.parse import quote
 
+from conda.base.context import context
 from conda.common.compat import on_win
+from conda.common.path import url_to_path
 from conda.common.url import urlparse
+from conda.gateways.connection import session as gateway_session
 
 log = getLogger(f"conda.{__name__}")
 
@@ -31,3 +36,22 @@ def escape_channel_url(channel):
         parts = parts.replace(path=path)
         return str(parts)
     return channel
+
+
+@lru_cache(maxsize=None)
+def is_channel_available(channel_url) -> bool:
+    if context.offline:
+        # We don't know where the channel might be (even file:// might be a network share)
+        # so we play it safe and assume it's not available
+        return False
+    try:
+        if channel_url.startswith("file://"):
+            return Path(url_to_path(channel_url)).is_dir()
+        if hasattr(gateway_session, "get_session"):
+            session = gateway_session.get_session(channel_url)
+        else:
+            session = gateway_session.CondaSession()
+        return session.head(f"{channel_url}/noarch/repodata.json").ok
+    except Exception as exc:
+        log.debug("Failed to check if channel %s is available", channel_url, exc_info=exc)
+        return False
diff --git a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py b/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py
@@ -90,6 +90,8 @@
     ],
     # Added to test_modified_upstream.py
     "tests/test_priority.py": ["test_reorder_channel_priority"],
+    # Added to test_modified_upstream.py; this passes just by moving it to another test file
+    "tests/test_misc.py": ["test_explicit_missing_cache_entries"],
 }
 
 

diff --git a/dev/linux/bashrc.sh b/dev/linux/bashrc.sh
@@ -27,7 +27,7 @@ function recompile-mamba () {
   fi
 }
 
-sudo /opt/conda/condabin/conda install -y -p /opt/conda \
+sudo /opt/conda/condabin/conda install -y -p /opt/conda --repodata-fn repodata.json \
     --file /opt/conda-libmamba-solver-src/dev/requirements.txt \
     --file /opt/conda-libmamba-solver-src/tests/requirements.txt
 
@@ -44,7 +44,7 @@ if [ -d "/opt/mamba-src" ]; then
 fi
 
 cd /opt/conda-libmamba-solver-src
-sudo /opt/conda/bin/python -m pip install ./dev/collect_upstream_conda_tests/
+sudo /opt/conda/bin/python -m pip install ./dev/collect_upstream_conda_tests/ --no-deps
 sudo /opt/conda/bin/python -m pip install -e . --no-deps
 
 cd /opt/conda-src
@@ -53,4 +53,8 @@ source /opt/conda-src/dev/linux/bashrc.sh
 
 cd /opt/conda-libmamba-solver-src
 
-set +e
+set -x
+conda list -p /opt/conda
+conda info
+conda config --show-sources
+set +ex
diff --git a/dev/linux/upstream_integration.sh b/dev/linux/upstream_integration.sh
@@ -16,7 +16,7 @@ TEST_GROUP="${TEST_GROUP:-1}"
 sudo su root -c "/opt/conda/bin/conda install -yq conda-build"
 # make sure all test requirements are installed
 # CONDA LIBMAMBA SOLVER CHANGES
-sudo /opt/conda/bin/conda install --quiet -y --solver=classic \
+sudo /opt/conda/bin/conda install --quiet -y --solver=classic --repodata-fn repodata.json \
     --file "${CONDA_SRC}/tests/requirements.txt" \
     --file "${CONDA_LIBMAMBA_SOLVER_SRC}/dev/requirements.txt"
 sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC/dev/collect_upstream_conda_tests/"

diff --git a/dev/linux/upstream_unit.sh b/dev/linux/upstream_unit.sh
@@ -16,7 +16,7 @@ TEST_GROUP="${TEST_GROUP:-1}"
 eval "$(sudo /opt/conda/bin/python -m conda init --dev bash)"
 # make sure all test requirements are installed
 # CONDA LIBMAMBA SOLVER CHANGES
-sudo /opt/conda/bin/conda install --quiet -y --solver=classic \
+sudo /opt/conda/bin/conda install --quiet -y --solver=classic --repodata-fn repodata.json \
     --file "${CONDA_SRC}/tests/requirements.txt" \
     --file "${CONDA_LIBMAMBA_SOLVER_SRC}/dev/requirements.txt"
 sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC/dev/collect_upstream_conda_tests/"

diff --git a/news/274-ignore-unavailable-channels b/news/274-ignore-unavailable-channels
@@ -0,0 +1,19 @@
+### Enhancements
+
+* <news item>
+
+### Bug fixes
+
+* Do not inject those channels from installed packages that do not exist or are unavailable. (#262 via #274)
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>
diff --git a/tests/test_channels.py b/tests/test_channels.py
@@ -11,6 +11,8 @@
 import pytest
 from conda.common.compat import on_linux
 from conda.common.io import env_vars
+from conda.core.prefix_data import PrefixData
+from conda.models.channel import Channel
 from conda.testing.integration import _get_temp_prefix, make_temp_env
 from conda.testing.integration import run_command as conda_inprocess
 
@@ -62,6 +64,26 @@ def test_channels_prefixdata():
         )
 
 
+def test_channels_installed_unavailable():
+    "Ensure we don't fail if a channel coming ONLY from an installed pkg is unavailable"
+    with make_temp_env("xz", "--solver=libmamba", use_restricted_unicode=True) as prefix:
+        pd = PrefixData(prefix)
+        pd.load()
+        record = pd.get("xz")
+        assert record
+        record.channel = Channel.from_url("file:///nonexistent")
+
+        _, _, retcode = conda_inprocess(
+            "install",
+            prefix,
+            "zlib",
+            "--solver=libmamba",
+            "--dry-run",
+            use_exception_handler=True,
+        )
+        assert retcode == 0
+
+
 def _setup_channels_alias(prefix):
     write_env_config(
         prefix,

diff --git a/tests/test_modified_upstream.py b/tests/test_modified_upstream.py
@@ -17,8 +17,8 @@
 """
 
 import os
-import re
 import sys
+import warnings
 from pprint import pprint
 
 import pytest
@@ -31,6 +31,7 @@
 from conda.core.subdir_data import SubdirData
 from conda.exceptions import UnsatisfiableError
 from conda.gateways.subprocess import subprocess_call_with_clean_env
+from conda.misc import explicit
 from conda.models.match_spec import MatchSpec
 from conda.models.version import VersionOrder
 from conda.testing import (
@@ -60,6 +61,7 @@
     run_command,
 )
 from pytest import MonkeyPatch
+from pytest_mock import MockerFixture
 
 
 @pytest.mark.integration
@@ -1338,3 +1340,37 @@ def test_reorder_channel_priority(
         assert PrefixData(prefix).get(package1).channel.name == expected_channel
         # assert PrefixData(prefix).get(package2).channel.name == "conda-forge"
         # MODIFIED ^: Some packages do not change channels in libmamba
+
+
+def test_explicit_missing_cache_entries(
+    mocker: MockerFixture,
+    conda_cli: CondaCLIFixture,
+    tmp_env: TmpEnvFixture,
+):
+    """Test that explicit() raises and notifies if some of the specs were not found in the cache."""
+    from conda.core.package_cache_data import PackageCacheData
+
+    with tmp_env() as prefix:  # ensure writable env
+        if len(PackageCacheData.get_all_extracted_entries()) == 0:
+            # Package cache e.g. ./devenv/Darwin/x86_64/envs/devenv-3.9-c/pkgs/ can
+            # be empty in certain cases (Noted in OSX with Python 3.9, when
+            # Miniconda installs Python 3.10). Install a small package.
+            warnings.warn("test_explicit_missing_cache_entries: No packages in cache.")
+            out, err, retcode = conda_cli("install", "--prefix", prefix, "heapdict", "--yes")
+            assert retcode == 0, (out, err)  # MODIFIED
+
+        # Patching ProgressiveFetchExtract prevents trying to download a package from the url.
+        # Note that we cannot monkeypatch context.dry_run, because explicit() would exit early with that.
+        mocker.patch("conda.misc.ProgressiveFetchExtract")
+        print(PackageCacheData.get_all_extracted_entries()[0])  # MODIFIED
+        with pytest.raises(
+            AssertionError,
+            match="Missing package cache records for: pkgs/linux-64::foo==1.0.0=py_0",
+        ):
+            explicit(
+                [
+                    "http://test/pkgs/linux-64/foo-1.0.0-py_0.tar.bz2",  # does not exist
+                    PackageCacheData.get_all_extracted_entries()[0].url,  # exists
+                ],
+                prefix,
+            )