diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index ea0b4fcc..3c90eb43 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -23,7 +23,7 @@ from conda.base.constants import REPODATA_FN, UNKNOWN_CHANNEL, ChannelPriority, on_win from conda.base.context import context from conda.common.constants import NULL -from conda.common.io import Spinner +from conda.common.io import Spinner, timeout from conda.common.path import paths_equal from conda.common.url import join_url, percent_decode from conda.core.prefix_data import PrefixData @@ -44,6 +44,7 @@ from .index import LibMambaIndexHelper, _CachedLibMambaIndexHelper from .mamba_utils import init_api_context, mamba_version from .state import SolverInputState, SolverOutputState +from .utils import is_channel_available log = logging.getLogger(f"conda.{__name__}") @@ -171,21 +172,23 @@ def solve_final_state( else: IndexHelper = LibMambaIndexHelper - if os.getenv("CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED") or ( - getattr(context, "_argparse_args", None) or {} - ).get("override_channels"): - # see https://github.com/conda/conda-libmamba-solver/issues/108 - channels_from_installed = () - else: - channels_from_installed = in_state.channels_from_installed() - - all_channels = ( + all_channels = [ *conda_bld_channels, *self.channels, *in_state.channels_from_specs(), - *channels_from_installed, - *in_state.maybe_free_channel(), - ) + ] + override = (getattr(context, "_argparse_args", None) or {}).get("override_channels") + if not os.getenv("CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED") and not override: + # see https://github.com/conda/conda-libmamba-solver/issues/108 + all_urls = [url for c in all_channels for url in Channel(c).urls(False)] + installed_channels = in_state.channels_from_installed(seen=all_urls) + for channel in installed_channels: + # Only add to list if resource is available; check has timeout=1s + if timeout(1, is_channel_available, channel.base_url, default_return=False): + all_channels.append(channel) + all_channels.extend(in_state.maybe_free_channel()) + + all_channels = tuple(all_channels) with Spinner( self._spinner_msg_metadata(all_channels, conda_bld_channels=conda_bld_channels), enabled=not context.verbosity and not context.quiet, @@ -858,8 +861,19 @@ def _package_record_from_json_payload( if pkg_filename.startswith("__") and "/@/" in channel: return PackageRecord(**json.loads(json_payload)) - channel_info = index.get_info(channel) kwargs = json.loads(json_payload) + try: + channel_info = index.get_info(channel) + except KeyError: + # this channel was never used to build the index, which + # means we obtained an already installed PackageRecord + # whose metadata contains a channel that doesn't exist + pd = PrefixData(self.prefix) + record = pd.get(kwargs["name"]) + if record and record.fn == pkg_filename: + return record + + # Otherwise, these are records from the index kwargs["fn"] = pkg_filename kwargs["channel"] = channel_info.channel kwargs["url"] = join_url(channel_info.full_url, pkg_filename) diff --git a/conda_libmamba_solver/state.py b/conda_libmamba_solver/state.py index e5135b07..c394d8a9 100644 --- a/conda_libmamba_solver/state.py +++ b/conda_libmamba_solver/state.py @@ -410,8 +410,8 @@ def channels_from_specs(self) -> Iterable[Channel]: channel = Channel(spec.original_spec_str.split("::")[0]) yield channel - def channels_from_installed(self) -> Iterable[Channel]: - seen_urls = set() + def channels_from_installed(self, seen=None) -> Iterable[Channel]: + seen_urls = set(seen or []) # See https://github.com/conda/conda/issues/11790 for record in self.installed.values(): if record.channel.auth or record.channel.token: @@ -422,10 +422,12 @@ def channels_from_installed(self) -> Iterable[Channel]: # These "channels" are not really channels, more like # metadata placeholders continue - subdir_url = record.channel.subdir_url - if subdir_url not in seen_urls: - seen_urls.add(subdir_url) - yield record.channel + if record.channel.base_url is None: + continue + if record.channel.subdir_url in seen_urls: + continue + seen_urls.add(record.channel.subdir_url) + yield record.channel def maybe_free_channel(self) -> Iterable[Channel]: if context.restore_free_channel: diff --git a/conda_libmamba_solver/utils.py b/conda_libmamba_solver/utils.py index afb7e64b..92a153ee 100644 --- a/conda_libmamba_solver/utils.py +++ b/conda_libmamba_solver/utils.py @@ -1,11 +1,16 @@ # Copyright (C) 2022 Anaconda, Inc # Copyright (C) 2023 conda # SPDX-License-Identifier: BSD-3-Clause +from functools import lru_cache from logging import getLogger +from pathlib import Path from urllib.parse import quote +from conda.base.context import context from conda.common.compat import on_win +from conda.common.path import url_to_path from conda.common.url import urlparse +from conda.gateways.connection import session as gateway_session log = getLogger(f"conda.{__name__}") @@ -31,3 +36,22 @@ def escape_channel_url(channel): parts = parts.replace(path=path) return str(parts) return channel + + +@lru_cache(maxsize=None) +def is_channel_available(channel_url) -> bool: + if context.offline: + # We don't know where the channel might be (even file:// might be a network share) + # so we play it safe and assume it's not available + return False + try: + if channel_url.startswith("file://"): + return Path(url_to_path(channel_url)).is_dir() + if hasattr(gateway_session, "get_session"): + session = gateway_session.get_session(channel_url) + else: + session = gateway_session.CondaSession() + return session.head(f"{channel_url}/noarch/repodata.json").ok + except Exception as exc: + log.debug("Failed to check if channel %s is available", channel_url, exc_info=exc) + return False diff --git a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py b/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py index 78670a40..be97084d 100644 --- a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py +++ b/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py @@ -90,6 +90,8 @@ ], # Added to test_modified_upstream.py "tests/test_priority.py": ["test_reorder_channel_priority"], + # Added to test_modified_upstream.py; this passes just by moving it to another test file + "tests/test_misc.py": ["test_explicit_missing_cache_entries"], } diff --git a/dev/linux/bashrc.sh b/dev/linux/bashrc.sh index 4121d73a..fdceb01a 100644 --- a/dev/linux/bashrc.sh +++ b/dev/linux/bashrc.sh @@ -27,7 +27,7 @@ function recompile-mamba () { fi } -sudo /opt/conda/condabin/conda install -y -p /opt/conda \ +sudo /opt/conda/condabin/conda install -y -p /opt/conda --repodata-fn repodata.json \ --file /opt/conda-libmamba-solver-src/dev/requirements.txt \ --file /opt/conda-libmamba-solver-src/tests/requirements.txt @@ -44,7 +44,7 @@ if [ -d "/opt/mamba-src" ]; then fi cd /opt/conda-libmamba-solver-src -sudo /opt/conda/bin/python -m pip install ./dev/collect_upstream_conda_tests/ +sudo /opt/conda/bin/python -m pip install ./dev/collect_upstream_conda_tests/ --no-deps sudo /opt/conda/bin/python -m pip install -e . --no-deps cd /opt/conda-src @@ -53,4 +53,8 @@ source /opt/conda-src/dev/linux/bashrc.sh cd /opt/conda-libmamba-solver-src -set +e +set -x +conda list -p /opt/conda +conda info +conda config --show-sources +set +ex diff --git a/dev/linux/upstream_integration.sh b/dev/linux/upstream_integration.sh index 6b58c60f..1319bcf9 100755 --- a/dev/linux/upstream_integration.sh +++ b/dev/linux/upstream_integration.sh @@ -16,7 +16,7 @@ TEST_GROUP="${TEST_GROUP:-1}" sudo su root -c "/opt/conda/bin/conda install -yq conda-build" # make sure all test requirements are installed # CONDA LIBMAMBA SOLVER CHANGES -sudo /opt/conda/bin/conda install --quiet -y --solver=classic \ +sudo /opt/conda/bin/conda install --quiet -y --solver=classic --repodata-fn repodata.json \ --file "${CONDA_SRC}/tests/requirements.txt" \ --file "${CONDA_LIBMAMBA_SOLVER_SRC}/dev/requirements.txt" sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC/dev/collect_upstream_conda_tests/" diff --git a/dev/linux/upstream_unit.sh b/dev/linux/upstream_unit.sh index 139c48f5..c6928713 100755 --- a/dev/linux/upstream_unit.sh +++ b/dev/linux/upstream_unit.sh @@ -16,7 +16,7 @@ TEST_GROUP="${TEST_GROUP:-1}" eval "$(sudo /opt/conda/bin/python -m conda init --dev bash)" # make sure all test requirements are installed # CONDA LIBMAMBA SOLVER CHANGES -sudo /opt/conda/bin/conda install --quiet -y --solver=classic \ +sudo /opt/conda/bin/conda install --quiet -y --solver=classic --repodata-fn repodata.json \ --file "${CONDA_SRC}/tests/requirements.txt" \ --file "${CONDA_LIBMAMBA_SOLVER_SRC}/dev/requirements.txt" sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC/dev/collect_upstream_conda_tests/" diff --git a/news/274-ignore-unavailable-channels b/news/274-ignore-unavailable-channels new file mode 100644 index 00000000..3670827a --- /dev/null +++ b/news/274-ignore-unavailable-channels @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Do not inject those channels from installed packages that do not exist or are unavailable. (#262 via #274) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_channels.py b/tests/test_channels.py index c17c2225..c3f6bae4 100644 --- a/tests/test_channels.py +++ b/tests/test_channels.py @@ -11,6 +11,8 @@ import pytest from conda.common.compat import on_linux from conda.common.io import env_vars +from conda.core.prefix_data import PrefixData +from conda.models.channel import Channel from conda.testing.integration import _get_temp_prefix, make_temp_env from conda.testing.integration import run_command as conda_inprocess @@ -62,6 +64,26 @@ def test_channels_prefixdata(): ) +def test_channels_installed_unavailable(): + "Ensure we don't fail if a channel coming ONLY from an installed pkg is unavailable" + with make_temp_env("xz", "--solver=libmamba", use_restricted_unicode=True) as prefix: + pd = PrefixData(prefix) + pd.load() + record = pd.get("xz") + assert record + record.channel = Channel.from_url("file:///nonexistent") + + _, _, retcode = conda_inprocess( + "install", + prefix, + "zlib", + "--solver=libmamba", + "--dry-run", + use_exception_handler=True, + ) + assert retcode == 0 + + def _setup_channels_alias(prefix): write_env_config( prefix, diff --git a/tests/test_modified_upstream.py b/tests/test_modified_upstream.py index 417db390..c6be696e 100644 --- a/tests/test_modified_upstream.py +++ b/tests/test_modified_upstream.py @@ -17,8 +17,8 @@ """ import os -import re import sys +import warnings from pprint import pprint import pytest @@ -31,6 +31,7 @@ from conda.core.subdir_data import SubdirData from conda.exceptions import UnsatisfiableError from conda.gateways.subprocess import subprocess_call_with_clean_env +from conda.misc import explicit from conda.models.match_spec import MatchSpec from conda.models.version import VersionOrder from conda.testing import ( @@ -60,6 +61,7 @@ run_command, ) from pytest import MonkeyPatch +from pytest_mock import MockerFixture @pytest.mark.integration @@ -1338,3 +1340,37 @@ def test_reorder_channel_priority( assert PrefixData(prefix).get(package1).channel.name == expected_channel # assert PrefixData(prefix).get(package2).channel.name == "conda-forge" # MODIFIED ^: Some packages do not change channels in libmamba + + +def test_explicit_missing_cache_entries( + mocker: MockerFixture, + conda_cli: CondaCLIFixture, + tmp_env: TmpEnvFixture, +): + """Test that explicit() raises and notifies if some of the specs were not found in the cache.""" + from conda.core.package_cache_data import PackageCacheData + + with tmp_env() as prefix: # ensure writable env + if len(PackageCacheData.get_all_extracted_entries()) == 0: + # Package cache e.g. ./devenv/Darwin/x86_64/envs/devenv-3.9-c/pkgs/ can + # be empty in certain cases (Noted in OSX with Python 3.9, when + # Miniconda installs Python 3.10). Install a small package. + warnings.warn("test_explicit_missing_cache_entries: No packages in cache.") + out, err, retcode = conda_cli("install", "--prefix", prefix, "heapdict", "--yes") + assert retcode == 0, (out, err) # MODIFIED + + # Patching ProgressiveFetchExtract prevents trying to download a package from the url. + # Note that we cannot monkeypatch context.dry_run, because explicit() would exit early with that. + mocker.patch("conda.misc.ProgressiveFetchExtract") + print(PackageCacheData.get_all_extracted_entries()[0]) # MODIFIED + with pytest.raises( + AssertionError, + match="Missing package cache records for: pkgs/linux-64::foo==1.0.0=py_0", + ): + explicit( + [ + "http://test/pkgs/linux-64/foo-1.0.0-py_0.tar.bz2", # does not exist + PackageCacheData.get_all_extracted_entries()[0].url, # exists + ], + prefix, + )