From 520872800d312236a9d07946b305252bca37f544 Mon Sep 17 00:00:00 2001 From: Antoine Prouvost Date: Tue, 4 Jul 2023 12:04:05 +0200 Subject: [PATCH] Create Solver solution (#2584) * Replace MTransaction to_remove with PackageInfo * Add a solution class * Fix Solution for_each * Add Solution empty and size * Make MTransaction::filter const * Remove unused MTransaction::m_force_reinstall * Rename MSolver::set_flags * Refactor MSolver flags * Call solver.flag() once in MTransaction * Implement Solution as alias with free functions * Remove as many MSolver flag setters as possible * Make Soltion a struct to possibly add more members * Remove useless function call * Add Solution::Omit * Rename template typenames and parameters in Solution * Fix MSolver const parameter --- libmamba/CMakeLists.txt | 1 + libmamba/include/mamba/core/solution.hpp | 218 ++++++++++++++++++++ libmamba/include/mamba/core/solver.hpp | 37 ++-- libmamba/include/mamba/core/transaction.hpp | 9 +- libmamba/src/api/install.cpp | 10 +- libmamba/src/core/solver.cpp | 43 ++-- libmamba/src/core/transaction.cpp | 89 +++----- libmambapy/src/main.cpp | 10 +- 8 files changed, 319 insertions(+), 98 deletions(-) create mode 100644 libmamba/include/mamba/core/solution.hpp diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt index b3496c602b..15fb54210e 100644 --- a/libmamba/CMakeLists.txt +++ b/libmamba/CMakeLists.txt @@ -237,6 +237,7 @@ set(LIBMAMBA_PUBLIC_HEADERS ${LIBMAMBA_INCLUDE_DIR}/mamba/core/repo.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/run.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shell_init.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/solution.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/solver.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/subdirdata.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/thread_utils.hpp diff --git a/libmamba/include/mamba/core/solution.hpp b/libmamba/include/mamba/core/solution.hpp new file mode 100644 index 0000000000..327686f283 --- /dev/null +++ b/libmamba/include/mamba/core/solution.hpp @@ -0,0 +1,218 @@ +// Copyright (c) 2023, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#ifndef MAMBA_CORE_SOLUTION_HPP +#define MAMBA_CORE_SOLUTION_HPP + +#include +#include + +#include "package_info.hpp" + +namespace mamba +{ + namespace detail + { + template + inline constexpr bool is_any_of_v = std::disjunction_v...>; + } + + struct Solution + { + struct Omit + { + PackageInfo what; + }; + struct Upgrade + { + PackageInfo remove; + PackageInfo install; + }; + struct Downgrade + { + PackageInfo remove; + PackageInfo install; + }; + struct Change + { + PackageInfo remove; + PackageInfo install; + }; + struct Reinstall + { + PackageInfo what; + }; + struct Remove + { + PackageInfo remove; + }; + struct Install + { + PackageInfo install; + }; + + template + inline static constexpr bool has_remove_v = detail::is_any_of_v; + + template + inline static constexpr bool has_install_v = detail::is_any_of_v; + + using Action = std::variant; + using action_list = std::vector; + + action_list actions = {}; + }; + + template + void for_each_to_remove(Iter first, Iter last, UnaryFunc&& func); + template + void for_each_to_remove(Range&& actions, UnaryFunc&& func); + + template + void for_each_to_install(Iter first, Iter last, UnaryFunc&& func); + template + void for_each_to_install(Range&& actions, UnaryFunc&& func); + + template + void for_each_to_omit(Iter first, Iter last, UnaryFunc&& func); + template + void for_each_to_omit(Range&& actions, UnaryFunc&& func); +} + +#include + +namespace mamba +{ + /******************************** + * Implementation of Solution * + ********************************/ + + namespace detail + { + template + auto to_remove_ptr(Action& action) + { + using PackageInfoPtr = std::conditional_t, const PackageInfo*, PackageInfo*>; + return std::visit( + [](auto& a) -> PackageInfoPtr + { + using A = std::decay_t; + if constexpr (Solution::has_remove_v) + { + return &(a.remove); + } + else if constexpr (std::is_same_v) + { + return &(a.what); + } + return nullptr; + }, + action + ); + } + } + + template + void for_each_to_remove(Iter first, Iter last, UnaryFunc&& func) + { + for (; first != last; ++first) + { + if (auto* const ptr = detail::to_remove_ptr(*first)) + { + func(*ptr); + } + } + } + + template + void for_each_to_remove(Range&& actions, UnaryFunc&& func) + { + return for_each_to_remove(actions.begin(), actions.end(), std::forward(func)); + } + + namespace detail + { + template + auto to_install_ptr(Action& action) + { + using PackageInfoPtr = std::conditional_t, const PackageInfo*, PackageInfo*>; + return std::visit( + [](auto& a) -> PackageInfoPtr + { + using A = std::decay_t; + if constexpr (Solution::has_install_v) + { + return &(a.install); + } + else if constexpr (std::is_same_v) + { + return &(a.what); + } + return nullptr; + }, + action + ); + } + } + + template + void for_each_to_install(Iter first, Iter last, UnaryFunc&& func) + { + for (; first != last; ++first) + { + if (auto* const ptr = detail::to_install_ptr(*first)) + { + func(*ptr); + } + } + } + + template + void for_each_to_install(Range&& actions, UnaryFunc&& func) + { + return for_each_to_install(actions.begin(), actions.end(), std::forward(func)); + } + + namespace detail + { + template + auto to_omit_ptr(Action& action) + { + using PackageInfoPtr = std::conditional_t, const PackageInfo*, PackageInfo*>; + return std::visit( + [](auto& a) -> PackageInfoPtr + { + using A = std::decay_t; + if constexpr (std::is_same_v) + { + return &(a.what); + } + return nullptr; + }, + action + ); + } + } + + template + void for_each_to_omit(Iter first, Iter last, UnaryFunc&& func) + { + for (; first != last; ++first) + { + if (auto* const ptr = detail::to_omit_ptr(*first)) + { + func(*ptr); + } + } + } + + template + void for_each_to_omit(Range&& actions, UnaryFunc&& func) + { + return for_each_to_omit(actions.begin(), actions.end(), std::forward(func)); + } +} +#endif diff --git a/libmamba/include/mamba/core/solver.hpp b/libmamba/include/mamba/core/solver.hpp index 22cf156302..9f00b5b6c1 100644 --- a/libmamba/include/mamba/core/solver.hpp +++ b/libmamba/include/mamba/core/solver.hpp @@ -25,9 +25,9 @@ #include "match_spec.hpp" -#define MAMBA_NO_DEPS 0b0001 -#define MAMBA_ONLY_DEPS 0b0010 -#define MAMBA_FORCE_REINSTALL 0b0100 +#define PY_MAMBA_NO_DEPS 0b0001 +#define PY_MAMBA_ONLY_DEPS 0b0010 +#define PY_MAMBA_FORCE_REINSTALL 0b0100 extern "C" { @@ -60,6 +60,16 @@ namespace mamba { public: + struct Flags + { + /** Keep the dependencies of the install package in the solution. */ + bool keep_dependencies = true; + /** Keep the original required package in the solution. */ + bool keep_specs = true; + /** Force reinstallation of jobs. */ + bool force_reinstall = false; + }; + MSolver(MPool pool, std::vector> flags = {}); ~MSolver(); @@ -73,12 +83,17 @@ namespace mamba void add_constraint(const std::string& job); void add_pin(const std::string& pin); void add_pins(const std::vector& pins); - void set_flags(const std::vector>& flags); - void set_postsolve_flags(const std::vector>& flags); + + [[deprecated]] void py_set_postsolve_flags(const std::vector>& flags); + + void set_flags(const Flags& flags); // TODO temporary Itf meant to be passed in ctor + [[nodiscard]] auto flags() const -> const Flags&; + [[deprecated]] void py_set_libsolv_flags(const std::vector>& flags); + [[nodiscard]] bool try_solve(); void must_solve(); - [[nodiscard]] bool is_solved() const; + [[nodiscard]] std::string problems_to_str() const; [[nodiscard]] std::vector all_problems() const; [[nodiscard]] std::vector all_problems_structured() const; @@ -101,25 +116,23 @@ namespace mamba auto solver() -> solv::ObjSolver&; auto solver() const -> const solv::ObjSolver&; - bool only_deps = false; - bool no_deps = false; - bool force_reinstall = false; - private: - std::vector> m_flags; + std::vector> m_libsolv_flags; std::vector m_install_specs; std::vector m_remove_specs; std::vector m_neuter_specs; std::vector m_pinned_specs; - bool m_is_solved; // Order of m_pool and m_solver is critical since m_pool must outlive m_solver. MPool m_pool; // Temporary Pimpl all libsolv to keep it private std::unique_ptr m_solver; std::unique_ptr m_jobs; + Flags m_flags = {}; + bool m_is_solved; void add_reinstall_job(MatchSpec& ms, int job_flag); + void apply_libsolv_flags(); }; } // namespace mamba diff --git a/libmamba/include/mamba/core/transaction.hpp b/libmamba/include/mamba/core/transaction.hpp index f133ebd2ac..e1ec98b34c 100644 --- a/libmamba/include/mamba/core/transaction.hpp +++ b/libmamba/include/mamba/core/transaction.hpp @@ -18,6 +18,7 @@ #include "mamba_fs.hpp" #include "match_spec.hpp" #include "package_cache.hpp" +#include "package_info.hpp" #include "prefix_data.hpp" #include "solver.hpp" #include "transaction_context.hpp" @@ -86,8 +87,8 @@ namespace mamba TransactionContext m_transaction_context; MultiPackageCache m_multi_cache; const fs::u8path m_cache_path; - std::vector m_to_install; - std::vector m_to_remove; + std::vector m_to_install; + std::vector m_to_remove; History::UserRequest m_history_entry = History::UserRequest::prefilled(); // Temporarily using Pimpl for encapsulation @@ -95,10 +96,8 @@ namespace mamba std::vector m_requested_specs; - bool m_force_reinstall = false; - void init(); - bool filter(const solv::ObjSolvableViewConst& s); + bool filter(const solv::ObjSolvableViewConst& s) const; auto trans() -> solv::ObjTransaction&; auto trans() const -> const solv::ObjTransaction&; diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index c0c90c3063..b275a1b543 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -19,10 +19,10 @@ #include "mamba/core/channel.hpp" #include "mamba/core/env_lockfile.hpp" #include "mamba/core/environments_manager.hpp" +#include "mamba/core/fetch.hpp" #include "mamba/core/mamba_fs.hpp" #include "mamba/core/output.hpp" #include "mamba/core/package_cache.hpp" -#include "mamba/core/package_download.hpp" #include "mamba/core/pinning.hpp" #include "mamba/core/transaction.hpp" #include "mamba/core/util_string.hpp" @@ -512,9 +512,11 @@ namespace mamba } ); - solver.set_postsolve_flags({ { MAMBA_NO_DEPS, no_deps }, - { MAMBA_ONLY_DEPS, only_deps }, - { MAMBA_FORCE_REINSTALL, force_reinstall } }); + solver.set_flags({ + /* .keep_dependencies= */ !no_deps, + /* .keep_specs= */ !only_deps, + /* .force_reinstall= */ force_reinstall, + }); if (freeze_installed && !prefix_pkgs.empty()) { diff --git a/libmamba/src/core/solver.cpp b/libmamba/src/core/solver.cpp index 18bd825d9d..92acc0d575 100644 --- a/libmamba/src/core/solver.cpp +++ b/libmamba/src/core/solver.cpp @@ -27,12 +27,12 @@ namespace mamba { - MSolver::MSolver(MPool pool, const std::vector> flags) - : m_flags(std::move(flags)) - , m_is_solved(false) + MSolver::MSolver(MPool pool, std::vector> flags) + : m_libsolv_flags(std::move(flags)) , m_pool(std::move(pool)) , m_solver(nullptr) , m_jobs(std::make_unique()) + , m_is_solved(false) { // TODO should we lazyly create solver here? Should we what provides? m_pool.create_whatprovides(); @@ -153,7 +153,7 @@ namespace mamba } m_jobs->push_back(job_flag | SOLVER_SOLVABLE_PROVIDES, job_id); } - else if ((job_flag & SOLVER_INSTALL) && force_reinstall) + else if ((job_flag & SOLVER_INSTALL) && m_flags.force_reinstall) { add_reinstall_job(ms, job_flag); } @@ -243,29 +243,44 @@ namespace mamba } } - void MSolver::set_postsolve_flags(const std::vector>& flags) + void MSolver::py_set_postsolve_flags(const std::vector>& flags) { for (const auto& option : flags) { switch (option.first) { - case MAMBA_NO_DEPS: - no_deps = option.second; + case PY_MAMBA_NO_DEPS: + m_flags.keep_dependencies = !option.second; break; - case MAMBA_ONLY_DEPS: - only_deps = option.second; + case PY_MAMBA_ONLY_DEPS: + m_flags.keep_specs = !option.second; break; - case MAMBA_FORCE_REINSTALL: - force_reinstall = option.second; + case PY_MAMBA_FORCE_REINSTALL: + m_flags.force_reinstall = option.second; break; } } } - void MSolver::set_flags(const std::vector>& flags) + void MSolver::set_flags(const Flags& flags) + { + m_flags = flags; + } + + auto MSolver::flags() const -> const Flags& + { + return m_flags; + } + + void MSolver::py_set_libsolv_flags(const std::vector>& flags) + { + m_libsolv_flags = flags; + } + + void MSolver::apply_libsolv_flags() { // TODO use new API - for (const auto& option : flags) + for (const auto& option : m_libsolv_flags) { solver_set_flag(*this, option.first, option.second); } @@ -314,7 +329,7 @@ namespace mamba bool MSolver::try_solve() { m_solver = std::make_unique(m_pool.pool()); - set_flags(m_flags); + apply_libsolv_flags(); const bool success = solver().solve(m_pool.pool(), *m_jobs); m_is_solved = true; diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index b69ee2baa8..5461868df5 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -55,11 +55,6 @@ namespace mamba return std::move(pkginfo).value(); }; - nlohmann::json solvable_to_json(const MPool& pool, solv::ObjSolvableViewConst s) - { - return mk_pkginfo(pool, s).json_record(); - } - template auto make_pkg_info_from_explicit_match_specs(Range&& specs) { @@ -197,9 +192,10 @@ namespace mamba ); trans().order(pool); - if (solver.no_deps || solver.only_deps) + const auto& solver_flags = solver.flags(); + if (!solver_flags.keep_dependencies || !solver_flags.keep_specs) { - m_filter_type = solver.only_deps ? FilterType::keep_only : FilterType::ignore; + m_filter_type = !(solver_flags.keep_specs) ? FilterType::keep_only : FilterType::ignore; for (auto& s : solver.install_specs()) { m_filter_name_ids.insert(pool.add_string(s.name)); @@ -210,7 +206,7 @@ namespace mamba } } - if (solver.only_deps) + if (!solver_flags.keep_specs) { for (const solv::SolvableId r : trans().steps()) { @@ -251,8 +247,6 @@ namespace mamba m_history_entry.remove = to_string_vec(solver.remove_specs()); } - m_force_reinstall = solver.force_reinstall; - init(); // if no action required, don't even start logging them if (!empty()) @@ -424,25 +418,25 @@ namespace mamba case SOLVER_TRANSACTION_CHANGED: case SOLVER_TRANSACTION_REINSTALLED: { - m_to_remove.emplace_back(*s); + m_to_remove.emplace_back(mk_pkginfo(m_pool, *s)); // Packages that replace these one will show up under IGNORE // so we need to fetch them here if (auto maybe_newer = trans().step_newer(pool, s->id())) { - auto solvable = pool.get_solvable(*maybe_newer); - assert(solvable); - m_to_install.push_back(*solvable); + auto newer = pool.get_solvable(*maybe_newer); + assert(newer); + m_to_install.push_back(mk_pkginfo(m_pool, *newer)); } break; } case SOLVER_TRANSACTION_ERASE: { - m_to_remove.emplace_back(*s); + m_to_remove.emplace_back(mk_pkginfo(m_pool, *s)); break; } case SOLVER_TRANSACTION_INSTALL: { - m_to_install.emplace_back(*s); + m_to_install.emplace_back(mk_pkginfo(m_pool, *s)); break; } case SOLVER_TRANSACTION_IGNORE: @@ -456,7 +450,7 @@ namespace mamba ); } - bool MTransaction::filter(const solv::ObjSolvableViewConst& s) + bool MTransaction::filter(const solv::ObjSolvableViewConst& s) const { if (m_filter_type == FilterType::none) { @@ -486,9 +480,9 @@ namespace mamba for (auto s : m_to_install) { - if (s.name() == "python") + if (s.name == "python") { - new_py_ver = s.version(); + new_py_ver = s.version; LOG_INFO << "Found python version in packages to be installed " << new_py_ver; break; } @@ -709,33 +703,18 @@ namespace mamba auto MTransaction::to_conda() -> to_conda_type { - to_install_type to_install_structured; - to_remove_type to_remove_structured; - + to_remove_type to_remove_structured = {}; + to_remove_structured.reserve(m_to_remove.size()); for (auto s : m_to_remove) { - to_remove_structured.emplace_back( - solv::ObjRepoViewConst::of_solvable(s).name(), - s.file_name() - ); + to_remove_structured.emplace_back(s.channel, s.fn); } + to_install_type to_install_structured = {}; + to_install_structured.reserve(m_to_install.size()); for (auto s : m_to_install) { - std::string s_json = solvable_to_json(m_pool, s).dump(4); - - std::string chan_name; - if (auto str = s.channel(); !str.empty()) - { - chan_name = str; - } - else - { - // note this can and should be when e.g. installing from a tarball - chan_name = solv::ObjRepoViewConst::of_solvable(s).name(); - } - - to_install_structured.emplace_back(chan_name, s.file_name(), s_json); + to_install_structured.emplace_back(s.channel, s.fn, s.json_record().dump(4)); } to_specs_type specs; @@ -751,20 +730,20 @@ namespace mamba for (auto s : m_to_install) { - if (!need_pkg_download(mk_pkginfo(m_pool, s), m_multi_cache)) + if (!need_pkg_download(s, m_multi_cache)) { - to_link.push_back(solvable_to_json(m_pool, s)); + to_link.push_back(s.json_record()); } else { - to_fetch.push_back(solvable_to_json(m_pool, s)); - to_link.push_back(solvable_to_json(m_pool, s)); + to_fetch.push_back(s.json_record()); + to_link.push_back(s.json_record()); } } for (auto s : m_to_remove) { - to_unlink.push_back(solvable_to_json(m_pool, s)); + to_unlink.push_back(s.json_record()); } auto add_json = [](const auto& jlist, const char* s) @@ -804,25 +783,19 @@ namespace mamba for (auto& s : m_to_install) { - const auto s_url = solv::ObjRepoViewConst::of_solvable(s).url(); - if (ctx.experimental && ctx.verify_artifacts) { - const Channel& chan = m_pool.channel_context().make_channel(std::string(s_url)); - const auto& repo_checker = chan.repo_checker(m_multi_cache); - const auto pkg_info = mk_pkginfo(m_pool, s); - repo_checker.verify_package( - pkg_info.json_signable(), - nlohmann::json::parse(pkg_info.signatures) + const auto& repo_checker = m_pool.channel_context().make_channel(s.channel).repo_checker( + m_multi_cache ); + repo_checker.verify_package(s.json_signable(), nlohmann::json::parse(s.signatures)); - LOG_DEBUG << "'" << pkg_info.name << "' trusted from '" << s_url << "'"; + LOG_DEBUG << "'" << s.name << "' trusted from '" << s.channel << "'"; } - targets.emplace_back(std::make_unique( - mk_pkginfo(m_pool, s), - m_pool.channel_context() - )); + targets.emplace_back( + std::make_unique(s, m_pool.channel_context()) + ); DownloadTarget* download_target = targets.back()->target(m_multi_cache); if (download_target != nullptr) { diff --git a/libmambapy/src/main.cpp b/libmambapy/src/main.cpp index 5b2efc076e..d08d7d2567 100644 --- a/libmambapy/src/main.cpp +++ b/libmambapy/src/main.cpp @@ -226,8 +226,8 @@ PYBIND11_MODULE(bindings, m) .def("add_global_job", &MSolver::add_global_job) .def("add_constraint", &MSolver::add_constraint) .def("add_pin", &MSolver::add_pin) - .def("set_flags", &MSolver::set_flags) - .def("set_postsolve_flags", &MSolver::set_postsolve_flags) + .def("set_flags", &MSolver::py_set_libsolv_flags) + .def("set_postsolve_flags", &MSolver::py_set_postsolve_flags) .def("is_solved", &MSolver::is_solved) .def("problems_to_str", &MSolver::problems_to_str) .def("all_problems_to_str", &MSolver::all_problems_to_str) @@ -1116,9 +1116,9 @@ PYBIND11_MODULE(bindings, m) .value("SOLVER_RULE_STRICT_REPO_PRIORITY", SolverRuleinfo::SOLVER_RULE_STRICT_REPO_PRIORITY); // INSTALL FLAGS - m.attr("MAMBA_NO_DEPS") = MAMBA_NO_DEPS; - m.attr("MAMBA_ONLY_DEPS") = MAMBA_ONLY_DEPS; - m.attr("MAMBA_FORCE_REINSTALL") = MAMBA_FORCE_REINSTALL; + m.attr("MAMBA_NO_DEPS") = PY_MAMBA_NO_DEPS; + m.attr("MAMBA_ONLY_DEPS") = PY_MAMBA_ONLY_DEPS; + m.attr("MAMBA_FORCE_REINSTALL") = PY_MAMBA_FORCE_REINSTALL; // DOWNLOAD FLAGS m.attr("MAMBA_DOWNLOAD_FAILFAST") = MAMBA_DOWNLOAD_FAILFAST;