From 907f5decf5b54dfa848924bb75efc76099a79157 Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Fri, 27 Nov 2020 20:36:53 +0300 Subject: [PATCH 1/8] containerd docker hub registry mirror support --- .gitlab-ci.yml | 1 + .../containerd/templates/config.toml.j2 | 2 +- tests/files/_docker_hub_registry_mirror.yml | 14 ++++++++++++++ tests/scripts/testcases_run.sh | 14 +++++++------- 4 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 tests/files/_docker_hub_registry_mirror.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d235ddbd8c..4b0afab9646 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ variables: MAGIC: "ci check this" TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID" CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml" + CI_TEST_REGISTRY_MIRROR: "./tests/files/_docker_hub_registry_mirror.yml" GS_ACCESS_KEY_ID: $GS_KEY GS_SECRET_ACCESS_KEY: $GS_SECRET CONTAINER_ENGINE: docker diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 index ceccaa2fc4a..671af29e48d 100644 --- a/roles/container-engine/containerd/templates/config.toml.j2 +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -62,7 +62,7 @@ disabled_plugins = ["restart"] [plugins.cri.registry.mirrors] {% for registry, addr in containerd_config.registries.items() %} [plugins.cri.registry.mirrors."{{ registry }}"] - endpoint = ["{{ addr }}"] + endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"] {% endfor %} {% endif %} diff --git a/tests/files/_docker_hub_registry_mirror.yml b/tests/files/_docker_hub_registry_mirror.yml new file mode 100644 index 00000000000..21391396ac3 --- /dev/null +++ b/tests/files/_docker_hub_registry_mirror.yml @@ -0,0 +1,14 @@ +docker_registry_mirrors: + - "https://mirror.gcr.io" + +containerd_config: + grpc: + max_recv_message_size: 16777216 + max_send_message_size: 16777216 + debug: + level: "" + registries: + "docker.io": + - "https://mirror.gcr.io" + - "https://registry-1.docker.io" + max_container_log_line_size: -1 diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh index 3acbfb232d1..e6e1abc8c21 100755 --- a/tests/scripts/testcases_run.sh +++ b/tests/scripts/testcases_run.sh @@ -51,20 +51,20 @@ if [ "${MITOGEN_ENABLE}" = "true" ]; then fi # Create cluster -ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" cluster.yml +ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml # Repeat deployment if testing upgrade if [ "${UPGRADE_TEST}" != "false" ]; then test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml" test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml" git checkout "${CI_BUILD_REF}" - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" $PLAYBOOK + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK fi # Test control plane recovery if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml fi # Tests Cases @@ -88,7 +88,7 @@ ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} --limit "all:!fake_ ## Idempotency checks 1/5 (repeat deployment) if [ "${IDEMPOT_CHECK}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" cluster.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml fi ## Idempotency checks 2/5 (Advanced DNS checks) @@ -98,12 +98,12 @@ fi ## Idempotency checks 3/5 (reset deployment) if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e reset_confirmation=yes -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" reset.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml fi ## Idempotency checks 4/5 (redeploy after reset) if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" cluster.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml fi ## Idempotency checks 5/5 (Advanced DNS checks) From b4f1f36a9856f7788bb81859fed0ffe7c19941cc Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Fri, 27 Nov 2020 20:49:09 +0300 Subject: [PATCH 2/8] add docs --- docs/containerd.md | 31 +++++++++++++++++++ .../sample/group_vars/all/containerd.yml | 6 +++- 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 docs/containerd.md diff --git a/docs/containerd.md b/docs/containerd.md new file mode 100644 index 00000000000..9a0d402873e --- /dev/null +++ b/docs/containerd.md @@ -0,0 +1,31 @@ +# conrainerd + +[containerd] An industry-standard container runtime with an emphasis on simplicity, robustness and portability +Kubespray supports basic functionality for using containerd as the default container runtime in a cluster. + +_To use the containerd container runtime set the following variables:_ + +## k8s-cluster.yml + +```yaml +container_manager: crio +``` + +## Containerd config + +Example: define registry mirror for docker hub + +``` +containerd_config: + grpc: + max_recv_message_size: 16777216 + max_send_message_size: 16777216 + debug: + level: "" + registries: + "docker.io": + - "https://mirror.gcr.io" + - "https://registry-1.docker.io" +``` + +[containerd] https://containerd.io diff --git a/inventory/sample/group_vars/all/containerd.yml b/inventory/sample/group_vars/all/containerd.yml index 2fc66b636d4..49e330adf6f 100644 --- a/inventory/sample/group_vars/all/containerd.yml +++ b/inventory/sample/group_vars/all/containerd.yml @@ -1,6 +1,8 @@ --- # Please see roles/container-engine/containerd/defaults/main.yml for more configuration options +# Example: define registry mirror for docker hub + # containerd_config: # grpc: # max_recv_message_size: 16777216 @@ -8,7 +10,9 @@ # debug: # level: "" # registries: -# "docker.io": "https://registry-1.docker.io" +# "docker.io": +# - "https://mirror.gcr.io" +# - "https://registry-1.docker.io" # max_container_log_line_size: -1 # metrics: # address: "" From 5cb41ac6d964731a96ed98228b34b16b659d5093 Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Fri, 27 Nov 2020 20:51:10 +0300 Subject: [PATCH 3/8] fix typo --- docs/containerd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/containerd.md b/docs/containerd.md index 9a0d402873e..f7ee9a6ef80 100644 --- a/docs/containerd.md +++ b/docs/containerd.md @@ -8,7 +8,7 @@ _To use the containerd container runtime set the following variables:_ ## k8s-cluster.yml ```yaml -container_manager: crio +container_manager: containerd ``` ## Containerd config From f5e728cf93358d58acd7df396e9be0be7c44ff08 Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Fri, 27 Nov 2020 20:53:44 +0300 Subject: [PATCH 4/8] fix yamllint --- tests/files/_docker_hub_registry_mirror.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/files/_docker_hub_registry_mirror.yml b/tests/files/_docker_hub_registry_mirror.yml index 21391396ac3..3dadb08ff67 100644 --- a/tests/files/_docker_hub_registry_mirror.yml +++ b/tests/files/_docker_hub_registry_mirror.yml @@ -1,3 +1,4 @@ +--- docker_registry_mirrors: - "https://mirror.gcr.io" @@ -9,6 +10,6 @@ containerd_config: level: "" registries: "docker.io": - - "https://mirror.gcr.io" - - "https://registry-1.docker.io" + - "https://mirror.gcr.io" + - "https://registry-1.docker.io" max_container_log_line_size: -1 From b9a08cfe248f040ac684951e9bca20aa5eca2e9d Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Fri, 27 Nov 2020 20:57:54 +0300 Subject: [PATCH 5/8] fix indent in sample and ansible-playbook param in testcases_run --- inventory/sample/group_vars/all/containerd.yml | 4 ++-- tests/scripts/testcases_run.sh | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/inventory/sample/group_vars/all/containerd.yml b/inventory/sample/group_vars/all/containerd.yml index 49e330adf6f..0f1e97749e1 100644 --- a/inventory/sample/group_vars/all/containerd.yml +++ b/inventory/sample/group_vars/all/containerd.yml @@ -11,8 +11,8 @@ # level: "" # registries: # "docker.io": -# - "https://mirror.gcr.io" -# - "https://registry-1.docker.io" +# - "https://mirror.gcr.io" +# - "https://registry-1.docker.io" # max_container_log_line_size: -1 # metrics: # address: "" diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh index e6e1abc8c21..ebed014e905 100755 --- a/tests/scripts/testcases_run.sh +++ b/tests/scripts/testcases_run.sh @@ -51,20 +51,20 @@ if [ "${MITOGEN_ENABLE}" = "true" ]; then fi # Create cluster -ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml +ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml # Repeat deployment if testing upgrade if [ "${UPGRADE_TEST}" != "false" ]; then test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml" test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml" git checkout "${CI_BUILD_REF}" - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK fi # Test control plane recovery if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml fi # Tests Cases @@ -88,7 +88,7 @@ ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} --limit "all:!fake_ ## Idempotency checks 1/5 (repeat deployment) if [ "${IDEMPOT_CHECK}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml fi ## Idempotency checks 2/5 (Advanced DNS checks) @@ -98,12 +98,12 @@ fi ## Idempotency checks 3/5 (reset deployment) if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml fi ## Idempotency checks 4/5 (redeploy after reset) if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e@${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml fi ## Idempotency checks 5/5 (Advanced DNS checks) From 2faed4d62a058f05a9629b2a1cd5bc07e8c17200 Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Fri, 27 Nov 2020 21:01:59 +0300 Subject: [PATCH 6/8] fix md --- docs/containerd.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/containerd.md b/docs/containerd.md index f7ee9a6ef80..58fd44d8f68 100644 --- a/docs/containerd.md +++ b/docs/containerd.md @@ -15,7 +15,7 @@ container_manager: containerd Example: define registry mirror for docker hub -``` +```yaml containerd_config: grpc: max_recv_message_size: 16777216 @@ -28,4 +28,4 @@ containerd_config: - "https://registry-1.docker.io" ``` -[containerd] https://containerd.io +[containerd]: https://containerd.io/ From d21a47647dd5829c1be3df22be20bee92f0a3302 Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Fri, 27 Nov 2020 21:34:08 +0300 Subject: [PATCH 7/8] mv common vars to tests/common/_docker_hub_registry_mirror.yml --- .gitlab-ci.yml | 2 +- tests/{files => common}/_docker_hub_registry_mirror.yml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/{files => common}/_docker_hub_registry_mirror.yml (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4b0afab9646..a4f0ceed81a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ variables: MAGIC: "ci check this" TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID" CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml" - CI_TEST_REGISTRY_MIRROR: "./tests/files/_docker_hub_registry_mirror.yml" + CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml" GS_ACCESS_KEY_ID: $GS_KEY GS_SECRET_ACCESS_KEY: $GS_SECRET CONTAINER_ENGINE: docker diff --git a/tests/files/_docker_hub_registry_mirror.yml b/tests/common/_docker_hub_registry_mirror.yml similarity index 100% rename from tests/files/_docker_hub_registry_mirror.yml rename to tests/common/_docker_hub_registry_mirror.yml From 657d56d55d2747c7db70568a5bc5caf4240cecf1 Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Sat, 28 Nov 2020 02:46:55 +0300 Subject: [PATCH 8/8] checkout vars to upgrade tests --- tests/scripts/testcases_run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh index ebed014e905..9f9870b5767 100755 --- a/tests/scripts/testcases_run.sh +++ b/tests/scripts/testcases_run.sh @@ -42,6 +42,7 @@ fi test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout "$KUBESPRAY_VERSION" # Checkout the CI vars file so it is available test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml +test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" ${CI_TEST_REGISTRY_MIRROR} # Install mitogen ansible plugin if [ "${MITOGEN_ENABLE}" = "true" ]; then