Skip to content

Commit

Permalink
[OCI - Mirrors] Add tests and doc (#3307)
Browse files Browse the repository at this point in the history
* Add doc and micromamba python test

* Add some tests for mirrors

* Use another channel name to avoid interfering with conda-forge

* Add test parameter to use libsolv parser

* Remove dry-run from tests
  • Loading branch information
Hind-M authored Jun 12, 2024
1 parent 4604d39 commit 7f8d564
Show file tree
Hide file tree
Showing 8 changed files with 287 additions and 62 deletions.
41 changes: 39 additions & 2 deletions docs/source/developer_zone/changes-2.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -136,5 +136,42 @@ The main changes are:

- Improved downloaders.

.. TODO OCI registry
.. TODO Mirrors
Mirrors and OCI registries
--------------------------
In the perspective of ensuring continuous and faster access when downloading packages, we now support mirroring channels.

Furthermore, we support fetching packages from `OCI registries <https://github.com/opencontainers/distribution-spec/blob/v1.0/spec.md#definitions>`_
in order to provide an alternative to hosting on https://conda.anaconda.org/conda-forge/.

Specifying a mirror can be done in the rc file as follows:

.. code::
$ cat ~/.mambarc
# Specify a mirror (can be a list of mirrors) for conda-forge channel
mirrored_channels:
conda-forge: ["oci://ghcr.io/channel-mirrors/conda-forge"]
# ``repodata_use_zst`` isn't considered when fetching from oci registries
# since compressed repodata is handled internally
# (if present, compressed repodata is necessarily fetched)
# Setting ``repodata_use_zst`` to ``false`` avoids useless requests with
# zst extension in repodata filename
repodata_use_zst: false
Then, you can for instance create a new environment ``pandoc_from_oci`` where ``pandoc`` can be fetched from the specified mirror and installed:

.. code::
$ micromamba create -n pandoc_from_oci pandoc -c conda-forge
Listing packages in the created ``pandoc_from_oci`` environment:

.. code::
$ micromamba list -n pandoc_from_oci
Name Version Build Channel
───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
pandoc 3.2 ha770c72_0 https://pkg-containers.githubusercontent.com/ghcr1/blobs/pandoc-3.2-ha770c72_0.conda
2 changes: 1 addition & 1 deletion docs/source/developer_zone/dev_environment.rst
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ Running commands manually
The CI files in ``.github/workflow`` provide an alternative way of developing Mamba.

Install development dependencies
*******************************
********************************

.. code:: bash
Expand Down
2 changes: 1 addition & 1 deletion libmamba/include/mamba/download/request.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ namespace mamba::download

// This class is used to create strong alias on
// string_view. This helps to avoid error-prone
// calls to functionsthat accept many arguments
// calls to functions that accept many arguments
// of the same type
template <int I>
class string_view_alias
Expand Down
8 changes: 4 additions & 4 deletions libmamba/src/download/mirror_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ namespace mamba::download
* OCIMirror implementation *
****************************/

namespace
namespace utils
{
std::pair<std::string, std::string> split_path_tag(const std::string& path)
{
Expand Down Expand Up @@ -161,7 +161,7 @@ namespace mamba::download
// NB: This method can be executed by many threads in parallel. Therefore,
// data should not be captured in lambda used for building the request, as
// inserting a new ArtifactData object may relocate preexisting ones.
auto [split_path, split_tag] = split_path_tag(url_path);
auto [split_path, split_tag] = utils::split_path_tag(url_path);

// TODO we are getting here a new token for every artifact/path
// => we should handle this differently to use the same token
Expand Down Expand Up @@ -226,7 +226,7 @@ namespace mamba::download
req.on_success = [data](const Success& success) -> expected_t<void>
{
const Buffer& buf = std::get<Buffer>(success.content);
auto j = parse_json_nothrow(buf.value);
auto j = utils::parse_json_nothrow(buf.value);
if (j.contains("token"))
{
data->token = j["token"].get<std::string>();
Expand Down Expand Up @@ -259,7 +259,7 @@ namespace mamba::download
req.on_success = [data](const Success& success) -> expected_t<void>
{
const Buffer& buf = std::get<Buffer>(success.content);
auto j = parse_json_nothrow(buf.value);
auto j = utils::parse_json_nothrow(buf.value);
if (j.contains("layers"))
{
std::string digest;
Expand Down
2 changes: 1 addition & 1 deletion libmamba/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ set(
src/validation/test_update_framework_v1.cpp
# Implementation of downloaders and mirrors
src/download/test_downloader.cpp
src/download/test_mirror.cpp
# Core tests
../longpath.manifest
src/core/test_activation.cpp
Expand All @@ -75,7 +76,6 @@ set(
src/core/test_environments_manager.cpp
src/core/test_history.cpp
src/core/test_lockfile.cpp
src/core/test_mirror.cpp
src/core/test_pinning.cpp
src/core/test_output.cpp
src/core/test_progress_bar.cpp
Expand Down
53 changes: 0 additions & 53 deletions libmamba/tests/src/core/test_mirror.cpp

This file was deleted.

199 changes: 199 additions & 0 deletions libmamba/tests/src/download/test_mirror.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
// Copyright (c) 2023, QuantStack and Mamba Contributors
//
// Distributed under the terms of the BSD 3-Clause License.
//
// The full license is in the file LICENSE, distributed with this software.

#include <typeinfo>

#include <doctest/doctest.h>

#include "../src/download/mirror_impl.hpp"

namespace mamba::download
{
namespace utils
{
std::pair<std::string, std::string> split_path_tag(const std::string& path);

TEST_SUITE("split_path_tag")
{
TEST_CASE("tar_bz2_extension")
{
auto [split_path, split_tag] = split_path_tag("xtensor-0.23.10-h2acdbc0_0.tar.bz2");
CHECK_EQ(split_path, "xtensor");
CHECK_EQ(split_tag, "0.23.10-h2acdbc0-0");
}

TEST_CASE("multiple_parts")
{
auto [split_path, split_tag] = split_path_tag("x-tensor-10.23.10-h2acdbc0_0.tar.bz2");
CHECK_EQ(split_path, "x-tensor");
CHECK_EQ(split_tag, "10.23.10-h2acdbc0-0");
}

TEST_CASE("more_multiple_parts")
{
auto [split_path, split_tag] = split_path_tag("x-tens-or-10.23.10-h2acdbc0_0.tar.bz2");
CHECK_EQ(split_path, "x-tens-or");
CHECK_EQ(split_tag, "10.23.10-h2acdbc0-0");
}

TEST_CASE("json_extension")
{
auto [split_path, split_tag] = split_path_tag("xtensor-0.23.10-h2acdbc0_0.json");
CHECK_EQ(split_path, "xtensor-0.23.10-h2acdbc0_0.json");
CHECK_EQ(split_tag, "latest");
}

TEST_CASE("not_enough_parts")
{
CHECK_THROWS_AS(split_path_tag("xtensor.tar.bz2"), std::runtime_error);
}
}
}

TEST_SUITE("mirrors")
{
TEST_CASE("PassThroughMirror")
{
std::unique_ptr<Mirror> mir = make_mirror("");
CHECK_EQ(typeid(*mir), typeid(PassThroughMirror));

Mirror::request_generator_list req_gen = mir->get_request_generators("", "");
CHECK_EQ(req_gen.size(), 1);

Request req_repodata("some_request_name", MirrorName("mirror_name"), "linux-64/repodata.json");
MirrorRequest mir_req = req_gen[0](req_repodata, nullptr);

CHECK_EQ(mir_req.name, "some_request_name");
CHECK_EQ(mir_req.url, "linux-64/repodata.json");
}

TEST_CASE("HTTPMirror")
{
SUBCASE("https")
{
std::unique_ptr<Mirror> mir = make_mirror("https://conda.anaconda.org/conda-forge");
CHECK_EQ(typeid(*mir), typeid(HTTPMirror));

Mirror::request_generator_list req_gen = mir->get_request_generators("", "");
CHECK_EQ(req_gen.size(), 1);

Request req_repodata(
"repodata_request",
MirrorName("mirror_name"),
"linux-64/repodata.json"
);
MirrorRequest mir_req = req_gen[0](req_repodata, nullptr);

CHECK_EQ(mir_req.name, "repodata_request");
CHECK_EQ(mir_req.url, "https://conda.anaconda.org/conda-forge/linux-64/repodata.json");
}

SUBCASE("http")
{
std::unique_ptr<Mirror> mir = make_mirror("http://conda.anaconda.org/conda-forge");
CHECK_EQ(typeid(*mir), typeid(HTTPMirror));

Mirror::request_generator_list req_gen = mir->get_request_generators("", "");
CHECK_EQ(req_gen.size(), 1);

Request req_repodata(
"repodata_request",
MirrorName("mirror_name"),
"linux-64/repodata.json"
);
MirrorRequest mir_req = req_gen[0](req_repodata, nullptr);

CHECK_EQ(mir_req.name, "repodata_request");
CHECK_EQ(mir_req.url, "http://conda.anaconda.org/conda-forge/linux-64/repodata.json");
}

SUBCASE("file")
{
std::unique_ptr<Mirror> mir = make_mirror("file://channel_path");
CHECK_EQ(typeid(*mir), typeid(HTTPMirror));

Mirror::request_generator_list req_gen = mir->get_request_generators("", "");
CHECK_EQ(req_gen.size(), 1);

Request req_repodata(
"repodata_request",
MirrorName("mirror_name"),
"linux-64/repodata.json"
);
MirrorRequest mir_req = req_gen[0](req_repodata, nullptr);

CHECK_EQ(mir_req.name, "repodata_request");
CHECK_EQ(mir_req.url, "file://channel_path/linux-64/repodata.json");
}
}

TEST_CASE("OCIMirror")
{
SUBCASE("Request repodata.json")
{
std::unique_ptr<Mirror> mir = make_mirror("oci://ghcr.io/channel-mirrors/conda-forge");
CHECK_EQ(typeid(*mir), typeid(OCIMirror));

Mirror::request_generator_list req_gen = mir->get_request_generators(
"linux-64/repodata.json",
""
);
CHECK_EQ(req_gen.size(), 3);

Request req_repodata(
"repodata_request",
MirrorName("mirror_name"),
"linux-64/repodata.json"
);
MirrorRequest mir_req = req_gen[0](req_repodata, nullptr);

CHECK_EQ(mir_req.name, "repodata_request");
CHECK_EQ(
mir_req.url,
"https://ghcr.io/token?scope=repository:channel-mirrors/conda-forge/linux-64/repodata.json:pull"
);

// Empty token leads to throwing an exception
CHECK_THROWS_AS(req_gen[1](req_repodata, nullptr), std::invalid_argument);
CHECK_THROWS_AS(req_gen[2](req_repodata, nullptr), std::invalid_argument);
}

SUBCASE("Request spec with sha")
{
std::unique_ptr<Mirror> mir = make_mirror("oci://ghcr.io/channel-mirrors/conda-forge");
CHECK_EQ(typeid(*mir), typeid(OCIMirror));

Mirror::request_generator_list req_gen = mir->get_request_generators(
"linux-64/pandoc-3.2-ha770c72_0.conda",
"418348076c1a39170efb0bdc8a584ddd11e9ed0ff58ccd905488d3f165ca98ba"
);
CHECK_EQ(req_gen.size(), 2);

Request req_spec(
"pandoc_request",
MirrorName("mirror_name"),
"linux-64/pandoc-3.2-ha770c72_0.conda"
);
MirrorRequest mir_req = req_gen[0](req_spec, nullptr);

CHECK_EQ(mir_req.name, "pandoc_request");
CHECK_EQ(
mir_req.url,
"https://ghcr.io/token?scope=repository:channel-mirrors/conda-forge/linux-64/pandoc:pull"
);

// Empty token leads to throwing an exception
CHECK_THROWS_AS(req_gen[1](req_spec, nullptr), std::invalid_argument);
}
}

TEST_CASE("nullptr")
{
std::unique_ptr<Mirror> mir = make_mirror("ghcr.io/channel-mirrors/conda-forge");
CHECK_EQ(mir, nullptr);
}
}
}
Loading

0 comments on commit 7f8d564

Please sign in to comment.