Merge pull request #1318 from wolfv/update_all_repoquery

add better update-all and repoquery, factor out load channels

libmamba/CMakeLists.txt

@@ -106,6 +106,7 @@ set(LIBMAMBA_SOURCES
     ${LIBMAMBA_SOURCE_DIR}/core/virtual_packages.cpp
     # API (high-level)
     ${LIBMAMBA_SOURCE_DIR}/api/c_api.cpp
+    ${LIBMAMBA_SOURCE_DIR}/api/channel_loader.cpp
     ${LIBMAMBA_SOURCE_DIR}/api/clean.cpp
     ${LIBMAMBA_SOURCE_DIR}/api/config.cpp
     ${LIBMAMBA_SOURCE_DIR}/api/configuration.cpp
@@ -114,6 +115,7 @@ set(LIBMAMBA_SOURCES
     ${LIBMAMBA_SOURCE_DIR}/api/install.cpp
     ${LIBMAMBA_SOURCE_DIR}/api/list.cpp
     ${LIBMAMBA_SOURCE_DIR}/api/remove.cpp
+    ${LIBMAMBA_SOURCE_DIR}/api/repoquery.cpp
     ${LIBMAMBA_SOURCE_DIR}/api/shell.cpp
     ${LIBMAMBA_SOURCE_DIR}/api/update.cpp
 )
@@ -158,6 +160,7 @@ set(LIBMAMBA_HEADERS
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/virtual_packages.hpp
     # API (high-level)
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/c_api.h
+    ${LIBMAMBA_INCLUDE_DIR}/mamba/api/channel_loader.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/clean.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/config.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/configuration.hpp
@@ -167,6 +170,7 @@ set(LIBMAMBA_HEADERS
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/install.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/list.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/remove.hpp
+    ${LIBMAMBA_INCLUDE_DIR}/mamba/api/repoquery.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/shell.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/update.hpp
 )

libmamba/include/mamba/api/channel_loader.hpp

@@ -0,0 +1,15 @@
+#include "mamba/core/channel.hpp"
+#include "mamba/core/pool.hpp"
+#include "mamba/core/repo.hpp"
+#include "mamba/core/subdirdata.hpp"
+#include "mamba/core/package_cache.hpp"
+
+namespace mamba
+{
+    namespace detail
+    {
+        MRepo create_repo_from_pkgs_dir(MPool& pool, const fs::path& pkgs_dir);
+    }
+
+    std::vector<MRepo> load_channels(MPool& pool, MultiPackageCache& package_caches, int is_retry);
+}

libmamba/include/mamba/api/install.hpp

@@ -34,16 +34,12 @@ namespace mamba
 
     namespace detail
     {
-        void load_tokens();
-
         void create_target_directory(const fs::path prefix);
 
         void create_empty_target(const fs::path& prefix);
 
         void file_specs_hook(std::vector<std::string>& file_specs);
 
-        MRepo create_repo_from_pkgs_dir(MPool& pool, const fs::path& pkgs_dir);
-
         bool download_explicit(const std::vector<PackageInfo>& pkgs, MultiPackageCache& pkg_caches);
 
         struct other_pkg_mgr_spec

libmamba/include/mamba/api/repoquery.hpp

@@ -0,0 +1,15 @@
+// Copyright (c) 2019, QuantStack and Mamba Contributors
+//
+// Distributed under the terms of the BSD 3-Clause License.
+//
+// The full license is in the file LICENSE, distributed with this software.
+
+#include "mamba/core/query.hpp"
+
+namespace mamba
+{
+    void repoquery(QueryType type,
+                   QueryResultFormat format,
+                   bool use_local,
+                   const std::string& query);
+}

libmamba/include/mamba/api/update.hpp

@@ -8,14 +8,15 @@
 #define MAMBA_API_UPDATE_HPP
 
 #include "mamba/core/mamba_fs.hpp"
+#include "mamba/core/query.hpp"
 
 #include <string>
 #include <vector>
 
 
 namespace mamba
 {
-    void update(bool update_all = false);
+    void update(bool update_all = false, bool prune = false);
 }
 
 #endif

libmamba/include/mamba/core/context.hpp

@@ -152,6 +152,10 @@ namespace mamba
         bool always_copy = false;
         bool always_softlink = false;
 
+        // solver options
+        bool allow_uninstall = true;
+        bool allow_downgrade = false;
+
         // add start menu shortcuts on Windows (not implemented on Linux / macOS)
         bool shortcuts = true;
 

libmamba/include/mamba/core/query.hpp

@@ -57,9 +57,17 @@ namespace mamba
 
     enum class QueryType
     {
-        Search,
-        Depends,
-        Whoneeds
+        kSEARCH,
+        kDEPENDS,
+        kWHONEEDS
+    };
+
+    enum class QueryResultFormat
+    {
+        kJSON,
+        kTREE,
+        kTABLE,
+        kPRETTY
     };
 
     class query_result

libmamba/include/mamba/core/solver.hpp

@@ -44,6 +44,7 @@ namespace mamba
         MSolver(MSolver&&) = delete;
         MSolver& operator=(MSolver&&) = delete;
 
+        void add_global_job(int job_flag);
         void add_jobs(const std::vector<std::string>& jobs, int job_flag);
         void add_constraint(const std::string& job);
         void add_pin(const std::string& pin);

libmamba/src/api/channel_loader.cpp

@@ -0,0 +1,153 @@
+#include "mamba/api/channel_loader.hpp"
+
+#include "mamba/core/channel.hpp"
+#include "mamba/core/pool.hpp"
+#include "mamba/core/repo.hpp"
+#include "mamba/core/subdirdata.hpp"
+
+
+namespace mamba
+{
+    namespace detail
+    {
+        MRepo create_repo_from_pkgs_dir(MPool& pool, const fs::path& pkgs_dir)
+        {
+            if (!fs::exists(pkgs_dir))
+            {
+                throw std::runtime_error("Specified pkgs_dir does not exist\n");
+            }
+            PrefixData prefix_data(pkgs_dir);
+            prefix_data.load();
+            for (const auto& entry : fs::directory_iterator(pkgs_dir))
+            {
+                fs::path repodata_record_json = entry.path() / "info" / "repodata_record.json";
+                if (!fs::exists(repodata_record_json))
+                {
+                    continue;
+                }
+                prefix_data.load_single_record(repodata_record_json);
+            }
+            return MRepo(pool, prefix_data);
+        }
+    }
+
+    std::vector<MRepo> load_channels(MPool& pool, MultiPackageCache& package_caches, int is_retry)
+    {
+        int RETRY_SUBDIR_FETCH = 1 << 0;
+
+        auto& ctx = Context::instance();
+
+        std::vector<std::string> channel_urls = ctx.channels;
+
+        std::vector<std::shared_ptr<MSubdirData>> subdirs;
+        MultiDownloadTarget multi_dl;
+
+        std::vector<std::pair<int, int>> priorities;
+        int max_prio = static_cast<int>(channel_urls.size());
+        std::string prev_channel_name;
+
+        if (ctx.experimental)
+        {
+            load_tokens();
+        }
+
+        for (auto channel : get_channels(channel_urls))
+        {
+            for (auto& [platform, url] : channel->platform_urls(true))
+            {
+                std::string repodata_full_url = concat(url, "/repodata.json");
+
+                auto sdir = std::make_shared<MSubdirData>(
+                    concat(channel->canonical_name(), "/", platform),
+                    repodata_full_url,
+                    cache_fn_url(repodata_full_url),
+                    package_caches,
+                    platform == "noarch");
+
+                sdir->load();
+                multi_dl.add(sdir->target());
+                subdirs.push_back(sdir);
+                if (ctx.channel_priority == ChannelPriority::kDisabled)
+                {
+                    priorities.push_back(std::make_pair(0, 0));
+                }
+                else
+                {
+                    // Consider 'flexible' and 'strict' the same way
+                    if (channel->name() != prev_channel_name)
+                    {
+                        max_prio--;
+                        prev_channel_name = channel->name();
+                    }
+                    priorities.push_back(std::make_pair(max_prio, 0));
+                }
+            }
+        }
+        // TODO load local channels even when offline
+        if (!ctx.offline)
+        {
+            multi_dl.download(true);
+        }
+
+        std::vector<MRepo> repos;
+        if (ctx.offline)
+        {
+            LOG_INFO << "Creating repo from pkgs_dir for offline";
+            for (const auto& c : ctx.pkgs_dirs)
+                repos.push_back(detail::create_repo_from_pkgs_dir(pool, c));
+        }
+
+        std::string prev_channel;
+        bool loading_failed = false;
+        for (std::size_t i = 0; i < subdirs.size(); ++i)
+        {
+            auto& subdir = subdirs[i];
+            if (!subdir->loaded())
+            {
+                if (ctx.offline || !mamba::ends_with(subdir->name(), "/noarch"))
+                {
+                    continue;
+                }
+                else
+                {
+                    throw std::runtime_error("Subdir " + subdir->name() + " not loaded!");
+                }
+            }
+
+            auto& prio = priorities[i];
+            try
+            {
+                MRepo repo = subdir->create_repo(pool);
+                repo.set_priority(prio.first, prio.second);
+                repos.push_back(repo);
+            }
+            catch (std::runtime_error& e)
+            {
+                if (is_retry & RETRY_SUBDIR_FETCH)
+                {
+                    std::stringstream ss;
+                    ss << "Could not load repodata.json for " << subdir->name() << " after retry."
+                       << "Please check repodata source. Exiting." << std::endl;
+                    throw std::runtime_error(ss.str());
+                }
+
+                LOG_WARNING << "Could not load repodata.json for " << subdir->name()
+                            << ". Deleting cache, and retrying.";
+                subdir->clear_cache();
+                loading_failed = true;
+            }
+        }
+
+        if (loading_failed)
+        {
+            if (!ctx.offline && !(is_retry & RETRY_SUBDIR_FETCH))
+            {
+                LOG_WARNING << "Encountered malformed repodata.json cache. Redownloading.";
+                return load_channels(pool, package_caches, is_retry | RETRY_SUBDIR_FETCH);
+            }
+            throw std::runtime_error("Could not load repodata. Cache corrupted?");
+        }
+        return repos;
+    }
+
+}

libmamba/src/api/configuration.cpp

@@ -790,6 +790,19 @@ namespace mamba
                    .set_env_var_names()
                    .description("If solve fails, try to fetch updated repodata"));
 
+        insert(Configurable("allow_uninstall", &ctx.allow_uninstall)
+                   .group("Solver")
+                   .set_rc_configurable()
+                   .set_env_var_names()
+                   .description(
+                       "Allow uninstall when installing or updating packages. Default is true."));
+
+        insert(Configurable("allow_downgrade", &ctx.allow_downgrade)
+                   .group("Solver")
+                   .set_rc_configurable()
+                   .set_env_var_names()
+                   .description("Allow downgrade when installing packages. Default is false."));
+
         // Extract, Link & Install
         insert(Configurable("download_threads", &ctx.download_threads)
                    .group("Extract, Link & Install")