From b9a2850ecb5e0849da940fb8336d2d50321eef29 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Mon, 19 Feb 2024 15:42:00 +0100 Subject: [PATCH] Enable verifying packages signatures --- libmamba/CMakeLists.txt | 2 + libmamba/include/mamba/core/context.hpp | 8 ++ .../include/mamba/core/repo_checker_store.hpp | 46 +++++++ libmamba/include/mamba/validation/errors.hpp | 11 ++ .../include/mamba/validation/repo_checker.hpp | 21 ++- .../mamba/validation/update_framework.hpp | 2 +- .../validation/update_framework_v0_6.hpp | 4 +- .../mamba/validation/update_framework_v1.hpp | 2 +- libmamba/src/api/configuration.cpp | 9 +- libmamba/src/core/pool.cpp | 10 +- libmamba/src/core/repo_checker_store.cpp | 83 ++++++++++++ libmamba/src/core/transaction.cpp | 71 ++++------ libmamba/src/solv-cpp/solvable.cpp | 20 +++ libmamba/src/solv-cpp/solvable.hpp | 9 +- libmamba/src/solver/libsolv/helpers.cpp | 128 +++++++++++++++++- libmamba/src/solver/libsolv/helpers.hpp | 6 +- libmamba/src/specs/package_info.cpp | 27 ++-- libmamba/src/validation/errors.cpp | 5 + libmamba/src/validation/repo_checker.cpp | 62 ++++++++- .../src/validation/update_framework_v0_6.cpp | 11 +- .../src/validation/update_framework_v1.cpp | 2 +- micromamba/src/common_options.cpp | 7 + 22 files changed, 458 insertions(+), 88 deletions(-) create mode 100644 libmamba/include/mamba/core/repo_checker_store.hpp create mode 100644 libmamba/src/core/repo_checker_store.cpp diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt index e86a0e4d49..e3935491d3 100644 --- a/libmamba/CMakeLists.txt +++ b/libmamba/CMakeLists.txt @@ -205,6 +205,7 @@ set( ${LIBMAMBA_SOURCE_DIR}/core/activation.cpp ${LIBMAMBA_SOURCE_DIR}/core/channel_context.cpp ${LIBMAMBA_SOURCE_DIR}/core/context.cpp + ${LIBMAMBA_SOURCE_DIR}/core/repo_checker_store.cpp ${LIBMAMBA_SOURCE_DIR}/core/download.cpp ${LIBMAMBA_SOURCE_DIR}/core/download_progress_bar.cpp ${LIBMAMBA_SOURCE_DIR}/core/environments_manager.cpp @@ -325,6 +326,7 @@ set( ${LIBMAMBA_INCLUDE_DIR}/mamba/core/channel_context.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/palette.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/context.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/repo_checker_store.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/download.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/download_progress_bar.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/environments_manager.hpp diff --git a/libmamba/include/mamba/core/context.hpp b/libmamba/include/mamba/core/context.hpp index d2993a03a2..d5f44c1e96 100644 --- a/libmamba/include/mamba/core/context.hpp +++ b/libmamba/include/mamba/core/context.hpp @@ -36,6 +36,14 @@ namespace mamba VerificationLevel safety_checks = VerificationLevel::Warn; bool extra_safety_checks = false; bool verify_artifacts = false; + // Leave this empty? + // Need to populate from server? from config?.... to think about it (TODO) + // if we just specify "channel0" it becomes "https://conda.anaconda.org/channel0" ... + // TODO test with multiple channels in there and check behavior: like uncommenting + // conda-forge as first channel + std::vector trusted_channels = { + /*"conda-forge", */ "http://127.0.0.1:8000/get/channel0" + }; }; diff --git a/libmamba/include/mamba/core/repo_checker_store.hpp b/libmamba/include/mamba/core/repo_checker_store.hpp new file mode 100644 index 0000000000..2e8290f464 --- /dev/null +++ b/libmamba/include/mamba/core/repo_checker_store.hpp @@ -0,0 +1,46 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#ifndef MAMBA_CORE_REPO_CHECKER_STORE_HPP +#define MAMBA_CORE_REPO_CHECKER_STORE_HPP + +#include +#include + +#include "mamba/specs/channel.hpp" +#include "mamba/validation/repo_checker.hpp" + +namespace mamba +{ + class Context; + class ChannelContext; + class MultiPackageCache; + + class RepoCheckerStore + { + public: + + using Channel = specs::Channel; + using RepoChecker = validation::RepoChecker; + using repo_checker_list = std::vector>; + + [[nodiscard]] static auto + make(const Context& ctx, ChannelContext& cc, MultiPackageCache& caches) -> RepoCheckerStore; + + explicit RepoCheckerStore(repo_checker_list checkers); + + [[nodiscard]] auto find_checker(const Channel& chan) const -> const RepoChecker*; + + [[nodiscard]] auto contains_checker(const Channel& chan) const -> bool; + + [[nodiscard]] auto at_checker(const Channel& chan) const -> const RepoChecker&; + + private: + + repo_checker_list m_repo_checkers = {}; + }; +} +#endif diff --git a/libmamba/include/mamba/validation/errors.hpp b/libmamba/include/mamba/validation/errors.hpp index f78feefa4e..b9ef5f4854 100644 --- a/libmamba/include/mamba/validation/errors.hpp +++ b/libmamba/include/mamba/validation/errors.hpp @@ -142,5 +142,16 @@ namespace mamba::validation index_error(); ~index_error() override = default; }; + + /** + * Error raised when the given signatures of a package are empty/invalid. + */ + class signatures_error : public trust_error + { + public: + + signatures_error(); + ~signatures_error() override = default; + }; } #endif diff --git a/libmamba/include/mamba/validation/repo_checker.hpp b/libmamba/include/mamba/validation/repo_checker.hpp index a95fa55e48..26b18377fd 100644 --- a/libmamba/include/mamba/validation/repo_checker.hpp +++ b/libmamba/include/mamba/validation/repo_checker.hpp @@ -9,6 +9,7 @@ #include #include +#include #include @@ -41,14 +42,23 @@ namespace mamba::validation * @param ref_path Path to the reference directory, hosting trusted root metadata * @param cache_path Path to the cache directory */ - RepoChecker(Context& context, std::string base_url, fs::u8path ref_path, fs::u8path cache_path = ""); + RepoChecker( + const Context& context, + std::string base_url, + fs::u8path ref_path, + fs::u8path cache_path = "" + ); + RepoChecker(RepoChecker&&) noexcept; ~RepoChecker(); + auto operator=(RepoChecker&&) noexcept -> RepoChecker&; + // Forwarding to a ``RepoIndexChecker`` implementation void verify_index(const nlohmann::json& j) const; void verify_index(const fs::u8path& p) const; void verify_package(const nlohmann::json& signed_data, const nlohmann::json& signatures) const; + void verify_package(const nlohmann::json& signed_data, std::string_view signatures) const; void generate_index_checker(); @@ -58,11 +68,14 @@ namespace mamba::validation private: + std::unique_ptr p_index_checker; + std::reference_wrapper m_context; + std::string m_base_url; - std::size_t m_root_version = 0; fs::u8path m_ref_path; fs::u8path m_cache_path; - Context& m_context; + + std::size_t m_root_version; auto initial_trusted_root() -> fs::u8path; auto ref_root() -> fs::u8path; @@ -70,8 +83,6 @@ namespace mamba::validation void persist_file(const fs::u8path& file_path); - std::unique_ptr p_index_checker; - auto get_root_role(const TimeRef& time_reference) -> std::unique_ptr; }; } diff --git a/libmamba/include/mamba/validation/update_framework.hpp b/libmamba/include/mamba/validation/update_framework.hpp index 57ee8abe87..2e163c7035 100644 --- a/libmamba/include/mamba/validation/update_framework.hpp +++ b/libmamba/include/mamba/validation/update_framework.hpp @@ -175,7 +175,7 @@ namespace mamba::validation auto possible_update_files() -> std::vector; virtual auto build_index_checker( - Context& context, + const Context& context, const TimeRef& time_reference, const std::string& url, const fs::u8path& cache_path diff --git a/libmamba/include/mamba/validation/update_framework_v0_6.hpp b/libmamba/include/mamba/validation/update_framework_v0_6.hpp index ba660c472c..3dd6ba35ce 100644 --- a/libmamba/include/mamba/validation/update_framework_v0_6.hpp +++ b/libmamba/include/mamba/validation/update_framework_v0_6.hpp @@ -74,7 +74,7 @@ namespace mamba::validation::v0_6 * Return a ``RepoIndexChecker`` implementation (derived class) from repository base URL. */ auto build_index_checker( - Context& context, + const Context& context, const TimeRef& time_reference, const std::string& url, const fs::u8path& cache_path @@ -131,7 +131,7 @@ namespace mamba::validation::v0_6 * Return a ``RepoIndexChecker`` implementation (derived class) from repository base URL. */ auto build_index_checker( - Context& context, + const Context& context, const TimeRef& time_reference, const std::string& url, const fs::u8path& cache_path diff --git a/libmamba/include/mamba/validation/update_framework_v1.hpp b/libmamba/include/mamba/validation/update_framework_v1.hpp index 3484857db8..9646fdb5c1 100644 --- a/libmamba/include/mamba/validation/update_framework_v1.hpp +++ b/libmamba/include/mamba/validation/update_framework_v1.hpp @@ -54,7 +54,7 @@ namespace mamba::validation::v1 [[nodiscard]] auto self_keys() const -> RoleFullKeys override; auto build_index_checker( - Context& context, + const Context& context, const TimeRef& time_reference, const std::string& url, const fs::u8path& cache_path diff --git a/libmamba/src/api/configuration.cpp b/libmamba/src/api/configuration.cpp index 9d5a197a7b..c55a6b18ae 100644 --- a/libmamba/src/api/configuration.cpp +++ b/libmamba/src/api/configuration.cpp @@ -1224,9 +1224,14 @@ namespace mamba insert(Configurable("experimental_repodata_parsing", &m_context.experimental_repodata_parsing) .group("Basic") - .description("Enable experimental parsing of repodata.json using nl::json") + .description( // + "Enable experimental parsing of `repodata.json` using simdjson.\n" + "Default is `true`. `false` means libsolv is used.\n" + "This feature may be still under active development and not stable yet.\n" + ) .set_rc_configurable() - .set_env_var_names()); + .set_env_var_names() + .set_post_merge_hook(detail::experimental_hook)); insert(Configurable("debug", &m_context.debug) .group("Basic") diff --git a/libmamba/src/core/pool.cpp b/libmamba/src/core/pool.cpp index a75ebba5e2..864ded6940 100644 --- a/libmamba/src/core/pool.cpp +++ b/libmamba/src/core/pool.cpp @@ -194,10 +194,16 @@ namespace mamba repo, path, std::string(url), - context().use_only_tar_bz2 + context().use_only_tar_bz2, + context().validation_params.verify_artifacts ); } - return solver::libsolv::libsolv_read_json(repo, path, context().use_only_tar_bz2) + return solver::libsolv::libsolv_read_json( + repo, + path, + context().use_only_tar_bz2, + context().validation_params.verify_artifacts + ) .transform( [&url](solv::ObjRepoView repo) { diff --git a/libmamba/src/core/repo_checker_store.cpp b/libmamba/src/core/repo_checker_store.cpp new file mode 100644 index 0000000000..a3c9cd81f8 --- /dev/null +++ b/libmamba/src/core/repo_checker_store.cpp @@ -0,0 +1,83 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#include "mamba/core/channel_context.hpp" +#include "mamba/core/context.hpp" +#include "mamba/core/output.hpp" +#include "mamba/core/package_cache.hpp" +#include "mamba/core/repo_checker_store.hpp" +#include "mamba/core/subdirdata.hpp" + +namespace mamba +{ + + auto RepoCheckerStore::make(const Context& ctx, ChannelContext& cc, MultiPackageCache& caches) + -> RepoCheckerStore + { + if (!ctx.validation_params.verify_artifacts) + { + return RepoCheckerStore({}); + } + + auto repo_checkers = repo_checker_list(); + repo_checkers.reserve(ctx.validation_params.trusted_channels.size()); + for (const auto& location : ctx.validation_params.trusted_channels) + { + for (auto& chan : cc.make_channel(location)) + { + // Parametrization + auto url = chan.url().str(specs::CondaURL::Credentials::Show); + auto url_id = cache_name_from_url(url); + // TODO make these configurable? + auto ref_path = ctx.prefix_params.root_prefix / "etc" / "trusted-repos" / url_id; + auto cache_path = caches.first_writable_path() / "cache" / url_id; + + LOG_INFO << "Creating RepoChecker with base_url: " << url + << ", ref_path: " << ref_path << ", and cache_path: " << cache_path; + + auto checker = RepoChecker(ctx, std::move(url), std::move(ref_path), cache_path); + + // Initialization + fs::create_directories(checker.cache_path()); + checker.generate_index_checker(); + + repo_checkers.emplace_back(std::move(chan), std::move(checker)); + } + } + return RepoCheckerStore(std::move(repo_checkers)); + } + + RepoCheckerStore::RepoCheckerStore(repo_checker_list checkers) + : m_repo_checkers(std::move(checkers)) + { + } + + auto RepoCheckerStore::find_checker(const Channel& chan) const -> const RepoChecker* + { + for (auto& [candidate_chan, checker] : m_repo_checkers) + { + if (candidate_chan.contains_equivalent(chan)) + { + return &checker; + } + } + return nullptr; + } + + auto RepoCheckerStore::contains_checker(const Channel& chan) const -> bool + { + return find_checker(chan) != nullptr; + } + + auto RepoCheckerStore::at_checker(const Channel& chan) const -> const RepoChecker& + { + if (auto ptr = find_checker(chan)) + { + return *ptr; + } + throw std::range_error("Checker not found"); + } +} diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index e2f2bcad93..cd46d016bb 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -25,6 +25,7 @@ #include "mamba/core/output.hpp" #include "mamba/core/package_fetcher.hpp" #include "mamba/core/pool.hpp" +#include "mamba/core/repo_checker_store.hpp" #include "mamba/core/solver.hpp" #include "mamba/core/thread_utils.hpp" #include "mamba/core/transaction.hpp" @@ -602,7 +603,7 @@ namespace mamba auto& channel_context = pool.channel_context(); auto& ctx = pool.context(); - if (ctx.experimental && ctx.validation_params.verify_artifacts) + if (ctx.validation_params.verify_artifacts) { LOG_INFO << "Content trust is enabled, package(s) signatures will be verified"; } @@ -610,46 +611,32 @@ namespace mamba solution.actions, [&](const auto& pkg) { - // The following was used for the The Update Framework (TUF) / package signing - // proof of concept. - // - // Due to uncertainties on how TUF would be implemented, this was left commented - // out as in was getting in the way of the Channel refactoring. - - // In channel.cpp, repo-checkers were instanciated with the folowing: - // const validation::RepoChecker& - // Channel::repo_checker(Context& context, MultiPackageCache& caches) const - // { - // if (p_repo_checker == nullptr) - // { - // p_repo_checker = std::make_unique( - // context, - // util::rsplit(base_url(), "/", 1).front(), - // context.prefix_params.root_prefix / "etc" / "trusted-repos" - // / util::cache_name_from_url(base_url()), - // caches.first_writable_path() / "cache" / - // util::cache_name_from_url(base_url()) - // ); - // - // fs::create_directories(p_repo_checker->cache_path()); - // p_repo_checker->generate_index_checker(); - // } - // - // return *p_repo_checker; - // } - - // Here, the repo-checker would be fetched the following way: - // if (ctx.experimental && ctx.validation_params.verify_artifacts) - // { - // const auto& repo_checker = channel_context.make_channel(pkg.channel) - // .repo_checker(ctx, multi_cache); - // repo_checker.verify_package( - // pkg.json_signable(), - // nlohmann::json::parse(pkg.signatures) - // ); - // - // LOG_DEBUG << "'" << pkg.name << "' trusted from '" << pkg.channel << "'"; - // } + if (ctx.validation_params.verify_artifacts) + { + LOG_INFO << "Creating RepoChecker..."; + auto repo_checker_store = RepoCheckerStore::make( + ctx, + channel_context, + multi_cache + ); + for (auto& chan : channel_context.make_channel(pkg.channel)) + { + auto repo_checker = repo_checker_store.find_checker(chan); + if (repo_checker) + { + LOG_INFO << "RepoChecker successfully created."; + repo_checker->verify_package( + pkg.json_signable(), + std::string_view(pkg.signatures) + ); + } + else + { + LOG_ERROR << "Could not create a valid RepoChecker."; + } + } + LOG_INFO << "'" << pkg.name << "' trusted from '" << pkg.channel << "'"; + } // FIXME: only do this for micromamba for now if (ctx.command_params.is_micromamba) @@ -669,7 +656,7 @@ namespace mamba } ); - if (ctx.experimental && ctx.validation_params.verify_artifacts) + if (ctx.validation_params.verify_artifacts) { auto out = Console::stream(); fmt::print( diff --git a/libmamba/src/solv-cpp/solvable.cpp b/libmamba/src/solv-cpp/solvable.cpp index b1ef04ea10..75b2da96fc 100644 --- a/libmamba/src/solv-cpp/solvable.cpp +++ b/libmamba/src/solv-cpp/solvable.cpp @@ -281,6 +281,26 @@ namespace mamba::solv return set_sha256(str.c_str()); } + auto ObjSolvableViewConst::signatures() const -> std::string_view + { + // NOTE This returns the package signatures json object alongside other package info + // in the following format: + // {"info":{},"signatures":{"public_key":{"signature":"metadata_signature"}}} + return ptr_to_strview( + ::solvable_lookup_str(const_cast<::Solvable*>(raw()), SOLVABLE_SIGNATUREDATA) + ); + } + + void ObjSolvableView::set_signatures(raw_str_view str) const + { + ::solvable_set_str(raw(), SOLVABLE_SIGNATUREDATA, str); + } + + void ObjSolvableView::set_signatures(const std::string& str) const + { + return set_signatures(str.c_str()); + } + auto ObjSolvableViewConst::size() const -> std::size_t { return ::solvable_lookup_num(const_cast<::Solvable*>(raw()), SOLVABLE_DOWNLOADSIZE, 0); diff --git a/libmamba/src/solv-cpp/solvable.hpp b/libmamba/src/solv-cpp/solvable.hpp index c9fc1a9364..065afca97a 100644 --- a/libmamba/src/solv-cpp/solvable.hpp +++ b/libmamba/src/solv-cpp/solvable.hpp @@ -56,6 +56,7 @@ namespace mamba::solv auto md5() const -> std::string_view; auto noarch() const -> std::string_view; auto sha256() const -> std::string_view; + auto signatures() const -> std::string_view; auto size() const -> std::size_t; auto timestamp() const -> std::size_t; @@ -190,7 +191,7 @@ namespace mamba::solv void set_noarch(const std::string& str) const; /** - * Set the sha256 has of the solvable file.. + * Set the sha256 hash of the solvable file. * * This is not used by libsolv and is purely for data storing. * @@ -200,6 +201,12 @@ namespace mamba::solv void set_sha256(raw_str_view str) const; void set_sha256(const std::string& str) const; + /** + * Set the signatures of the solvable file. + */ + void set_signatures(raw_str_view str) const; + void set_signatures(const std::string& str) const; + /** * Set the size of the solvable size. * diff --git a/libmamba/src/solver/libsolv/helpers.cpp b/libmamba/src/solver/libsolv/helpers.cpp index 3ae34706b3..bdb653a1f5 100644 --- a/libmamba/src/solver/libsolv/helpers.cpp +++ b/libmamba/src/solver/libsolv/helpers.cpp @@ -99,6 +99,7 @@ namespace mamba::solver::libsolv out.timestamp = s.timestamp(); out.md5 = s.md5(); out.sha256 = s.sha256(); + out.signatures = s.signatures(); const auto dep_to_str = [&pool](solv::DependencyId id) { return pool.dependency_to_string(id); }; @@ -140,6 +141,15 @@ namespace mamba::solver::libsolv return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); } + void + get_fake_signatures(simdjson::dom::parser& fake_parser, simdjson::dom::object& fake_signatures) + { + // Getting a fake simdjson::dom::object to simulate empty signatures + // A valid empty simdjson::dom::object is not handled in simdjson + const auto fake_signatures_json = R"( { "fake_signatures": "val" } )"_padded; + fake_signatures = fake_parser.parse(fake_signatures_json).get_object().value(); + } + [[nodiscard]] auto set_solvable( solv::ObjPool& pool, const std::string& repo_url_str, @@ -147,6 +157,7 @@ namespace mamba::solver::libsolv solv::ObjSolvableView solv, const std::string& filename, const simdjson::dom::element& pkg, + const simdjson::dom::object& signatures, const std::string& default_subdir ) -> bool { @@ -296,6 +307,32 @@ namespace mamba::solver::libsolv } } + // Setting signatures in solvable if they are available and `verify-artifacts` flag is + // enabled NOTE We need to use an intermediate nlohmann::json object to store signatures + // as simdjson objects are not conceived to be modified smoothly + // and we need an equivalent structure to how libsolv is storing the signatures + nlohmann::json glob_sigs, nested_sigs; + if (auto sigs = signatures[filename].get_object(); !sigs.error()) + { + for (auto dict : sigs) + { + for (auto nested_dict : dict.value.get_object()) + { + nested_sigs[dict.key]["signature"] = nested_dict.value; + } + glob_sigs["signatures"] = nested_sigs; + + solv.set_signatures(glob_sigs.dump()); + LOG_INFO << "Signatures for '" << filename + << "' are set in corresponding solvable."; + } + } + else + { + LOG_INFO << "No signatures available for '" << filename + << "'. Downloading without verifying artifacts."; + } + solv.add_self_provide(); return true; } @@ -306,7 +343,8 @@ namespace mamba::solver::libsolv const std::string& repo_url_str, const specs::CondaURL& repo_url, const std::string& default_subdir, - const simdjson::dom::object& packages + const simdjson::dom::object& packages, + const simdjson::dom::object& signatures ) { std::string filename = {}; @@ -321,6 +359,7 @@ namespace mamba::solver::libsolv solv, filename, pkg, + signatures, default_subdir ); if (parsed) @@ -334,14 +373,70 @@ namespace mamba::solver::libsolv } } } + + void set_repo_solvables_with_sigs( + solv::ObjPool& pool, + solv::ObjRepoView repo, + const std::string& repo_url_str, + const specs::CondaURL& repo_url, + const std::string& default_subdir, + const simdjson::dom::object& packages, + const simdjson::dom::object& repodata, + bool verify_artifacts + ) + { + if (auto signatures = repodata["signatures"].get_object(); + !signatures.error() && verify_artifacts) + { + set_repo_solvables( + pool, + repo, + repo_url_str, + repo_url, + default_subdir, + packages, + signatures.value() + ); + } + else + { + // NOTE We need to create a fake signatures json doc to get a valid + // simdjson::dom::object (otherwise we get a segfault because constructor yields to + // an invalid simdjson::dom::object) + simdjson::dom::parser fake_parser; + simdjson::dom::object fake_signatures; + get_fake_signatures(fake_parser, fake_signatures); + set_repo_solvables( + pool, + repo, + repo_url_str, + repo_url, + default_subdir, + packages, + fake_signatures + ); + } + } } - auto libsolv_read_json(solv::ObjRepoView repo, const fs::u8path& filename, bool only_tar_bz2) - -> expected_t + auto libsolv_read_json( + solv::ObjRepoView repo, + const fs::u8path& filename, + bool only_tar_bz2, + bool verify_artifacts + ) -> expected_t { LOG_INFO << "Reading repodata.json file " << filename << " for repo " << repo.name() << " using libsolv"; - const int flags = only_tar_bz2 ? CONDA_ADD_USE_ONLY_TAR_BZ2 : 0; + + int flags = only_tar_bz2 ? CONDA_ADD_USE_ONLY_TAR_BZ2 : 0; + if (verify_artifacts) + { + // cf. + // https://github.com/openSUSE/libsolv/commit/cc2da2e789f651b2d0d55fe31c258426bf9e984d + flags |= CONDA_ADD_WITH_SIGNATUREDATA; + } + const auto lock = LockFile(filename); return util::CFile::try_open(filename, "rb") @@ -368,7 +463,8 @@ namespace mamba::solver::libsolv solv::ObjRepoView repo, const fs::u8path& filename, const std::string& repo_url, - bool only_tar_bz2 + bool only_tar_bz2, + bool verify_artifacts ) -> expected_t { LOG_INFO << "Reading repodata.json file " << filename << " for repo " << repo.name() @@ -389,12 +485,30 @@ namespace mamba::solver::libsolv if (auto pkgs = repodata["packages"].get_object(); !pkgs.error()) { - set_repo_solvables(pool, repo, repo_url, parsed_url, default_subdir, pkgs.value()); + set_repo_solvables_with_sigs( + pool, + repo, + repo_url, + parsed_url, + default_subdir, + pkgs.value(), + repodata, + verify_artifacts + ); } if (auto pkgs = repodata["packages.conda"].get_object(); !pkgs.error() && !only_tar_bz2) { - set_repo_solvables(pool, repo, repo_url, parsed_url, default_subdir, pkgs.value()); + set_repo_solvables_with_sigs( + pool, + repo, + repo_url, + parsed_url, + default_subdir, + pkgs.value(), + repodata, + verify_artifacts + ); } return { repo }; diff --git a/libmamba/src/solver/libsolv/helpers.hpp b/libmamba/src/solver/libsolv/helpers.hpp index e523df6eac..f5f4a41b07 100644 --- a/libmamba/src/solver/libsolv/helpers.hpp +++ b/libmamba/src/solver/libsolv/helpers.hpp @@ -42,7 +42,8 @@ namespace mamba::solver::libsolv [[nodiscard]] auto libsolv_read_json( // solv::ObjRepoView repo, const fs::u8path& filename, - bool only_tar_bz2 + bool only_tar_bz2, + bool verify_artifacts ) -> expected_t; [[nodiscard]] auto mamba_read_json( @@ -50,7 +51,8 @@ namespace mamba::solver::libsolv solv::ObjRepoView repo, const fs::u8path& filename, const std::string& repo_url, - bool only_tar_bz2 + bool only_tar_bz2, + bool verify_artifacts ) -> expected_t; [[nodiscard]] auto read_solv( diff --git a/libmamba/src/specs/package_info.cpp b/libmamba/src/specs/package_info.cpp index 96db20b248..6e7663c997 100644 --- a/libmamba/src/specs/package_info.cpp +++ b/libmamba/src/specs/package_info.cpp @@ -147,10 +147,6 @@ namespace mamba::specs j["timestamp"] = timestamp; j["build"] = build_string; j["build_number"] = build_number; - if (noarch != NoArchType::No) - { - j["noarch"] = noarch; - } j["license"] = license; j["md5"] = md5; j["sha256"] = sha256; @@ -167,14 +163,12 @@ namespace mamba::specs { j["depends"] = depends; } - if (constrains.empty()) - { - if (!contains(defaulted_keys, "constrains")) - { - j["constrains"] = nlohmann::json::array(); - } - } - else + + // NOTE `constrains` is not included in server side (i.e Quetz) + // If it is later (or is included within signed metadata even as + // an empty array on conda side for example) + // => do the same as "depends" above + if (!constrains.empty()) { j["constrains"] = constrains; } @@ -279,6 +273,10 @@ namespace mamba::specs { return invoke_field_string(*this, &PackageInfo::timestamp); } + if (field_name == "signatures") + { + return invoke_field_string(*this, &PackageInfo::signatures); + } throw std::invalid_argument(fmt::format(R"(Invalid field "{}")", field_name)); } @@ -347,6 +345,10 @@ namespace mamba::specs { j["sha256"] = pkg.sha256; } + if (!pkg.signatures.empty()) + { + j["signatures"] = pkg.signatures; + } if (pkg.depends.empty()) { j["depends"] = nlohmann::json::array(); @@ -388,6 +390,7 @@ namespace mamba::specs pkg.license = j.value("license", ""); pkg.md5 = j.value("md5", ""); pkg.sha256 = j.value("sha256", ""); + pkg.signatures = j.value("signatures", ""); if (auto it = j.find("track_features"); it != j.end()) { if (it->is_string() && !it->get().empty()) diff --git a/libmamba/src/validation/errors.cpp b/libmamba/src/validation/errors.cpp index ad15661450..cfecf84c57 100644 --- a/libmamba/src/validation/errors.cpp +++ b/libmamba/src/validation/errors.cpp @@ -68,4 +68,9 @@ namespace mamba::validation : trust_error("Invalid package index metadata") { } + + signatures_error::signatures_error() + : trust_error("Invalid package signatures") + { + } } diff --git a/libmamba/src/validation/repo_checker.cpp b/libmamba/src/validation/repo_checker.cpp index e784ad243d..e84d6c5cc9 100644 --- a/libmamba/src/validation/repo_checker.cpp +++ b/libmamba/src/validation/repo_checker.cpp @@ -19,16 +19,26 @@ namespace mamba::validation { - RepoChecker::RepoChecker(Context& context, std::string base_url, fs::u8path ref_path, fs::u8path cache_path) - : m_base_url(std::move(base_url)) + RepoChecker::RepoChecker( + const Context& context, + std::string base_url, + fs::u8path ref_path, + fs::u8path cache_path + ) + : m_context(context) + , m_base_url(std::move(base_url)) , m_ref_path(std::move(ref_path)) , m_cache_path(std::move(cache_path)) - , m_context(context) { + m_root_version = 0; } + RepoChecker::RepoChecker(RepoChecker&&) noexcept = default; + RepoChecker::~RepoChecker() = default; + auto RepoChecker::operator=(RepoChecker&&) noexcept -> RepoChecker& = default; + auto RepoChecker::cache_path() -> const fs::u8path& { return m_cache_path; @@ -36,7 +46,7 @@ namespace mamba::validation void RepoChecker::generate_index_checker() { - if (p_index_checker == nullptr) + if (!p_index_checker) { // TUF spec 5.1 - Record fixed update start time // Expiration computations will be done against @@ -58,18 +68,54 @@ namespace mamba::validation void RepoChecker::verify_index(const nlohmann::json& j) const { - p_index_checker->verify_index(j); + if (p_index_checker) + { + p_index_checker->verify_index(j); + } + else + { + LOG_ERROR << "Index checker not valid."; + } } void RepoChecker::verify_index(const fs::u8path& p) const { - p_index_checker->verify_index(p); + if (p_index_checker) + { + p_index_checker->verify_index(p); + } + else + { + LOG_ERROR << "Index checker not valid."; + } } void RepoChecker::verify_package(const nlohmann::json& signed_data, const nlohmann::json& signatures) const { - p_index_checker->verify_package(signed_data, signatures); + if (p_index_checker) + { + p_index_checker->verify_package(signed_data, signatures); + } + else + { + LOG_ERROR << "Index checker not valid."; + } + } + + void + RepoChecker::verify_package(const nlohmann::json& signed_data, std::string_view signatures) const + { + if (signatures.empty()) + { + LOG_ERROR << "The given package signatures are empty"; + throw signatures_error(); + } + else + { + LOG_INFO << "Verifying package..."; + verify_package(signed_data, nlohmann::json::parse(signatures)); + } } auto RepoChecker::root_version() -> std::size_t @@ -138,10 +184,12 @@ namespace mamba::validation if (v0_6::SpecImpl().is_compatible(trusted_root)) { + LOG_INFO << "Getting 'root' role, using v0.6"; updated_root = std::make_unique(trusted_root); } else if (v1::SpecImpl().is_compatible(trusted_root)) { + LOG_INFO << "Getting 'root' role, using v1"; updated_root = std::make_unique(trusted_root); } else diff --git a/libmamba/src/validation/update_framework_v0_6.cpp b/libmamba/src/validation/update_framework_v0_6.cpp index c6a64bee37..cda5d35cca 100644 --- a/libmamba/src/validation/update_framework_v0_6.cpp +++ b/libmamba/src/validation/update_framework_v0_6.cpp @@ -188,7 +188,7 @@ namespace mamba::validation::v0_6 } auto RootImpl::build_index_checker( - Context& context, + const Context& context, const TimeRef& time_reference, const std::string& base_url, const fs::u8path& cache_path @@ -348,7 +348,7 @@ namespace mamba::validation::v0_6 } auto KeyMgrRole::build_index_checker( - Context& context, + const Context& context, const TimeRef& time_reference, const std::string& base_url, const fs::u8path& cache_path @@ -580,7 +580,12 @@ namespace mamba::validation::v0_6 auto PkgMgrRole::pkg_signatures(const nlohmann::json& j) const -> std::set { - auto sigs = j.get>>(); + // Libsolv's `repodata.json` parsing returns the signatures alongside other package info + // But, we are here only interested in the signatures + // In the case of parsing using mamba/simdjson, the solvable signatures are set to have the + // same format + auto j_sig = j["signatures"]; + auto sigs = j_sig.get>>(); std::set unique_sigs; for (auto& s : sigs) diff --git a/libmamba/src/validation/update_framework_v1.cpp b/libmamba/src/validation/update_framework_v1.cpp index 613bc088a5..0980d7ee74 100644 --- a/libmamba/src/validation/update_framework_v1.cpp +++ b/libmamba/src/validation/update_framework_v1.cpp @@ -115,7 +115,7 @@ namespace mamba::validation::v1 } auto RootImpl::build_index_checker( - Context&, + const Context&, const TimeRef& /*time_reference*/, const std::string& /*url*/, const fs::u8path& /*cache_path*/ diff --git a/micromamba/src/common_options.cpp b/micromamba/src/common_options.cpp index d7299899d3..3ab62d4bf4 100644 --- a/micromamba/src/common_options.cpp +++ b/micromamba/src/common_options.cpp @@ -434,6 +434,13 @@ init_install_options(CLI::App* subcom, Configuration& config) auto& av = config.at("verify_artifacts"); subcom->add_flag("--verify-artifacts", av.get_cli_config(), av.description()); + auto& repo_parsing = config.at("experimental_repodata_parsing"); + subcom->add_flag( + "--exp-repodata-parsing, !--no-exp-repodata-parsing", + repo_parsing.get_cli_config(), + repo_parsing.description() + ); + auto& platform = config.at("platform"); subcom->add_option("--platform", platform.get_cli_config(), platform.description());