diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eaa0e6d7b..04159736d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,9 @@ `systemd` cgroupDriver for Kubelet and containerd (PR[#3377](https://github.com/scality/metalk8s/pull/3377)) +- Allow to manually deploy a second registry container + (PR[#3400](https://github.com/scality/metalk8s/pull/3400)) + ### Breaking changes - [#2199](https://github.com/scality/metalk8s/issues/2199) - Prometheus label diff --git a/docs/operation/index.rst b/docs/operation/index.rst index 3a51e3b5f8..a08c430b6e 100644 --- a/docs/operation/index.rst +++ b/docs/operation/index.rst @@ -20,6 +20,7 @@ do not have a working MetalK8s_ setup. solutions changing_node_hostname metalk8s-utils + registry_ha listening_processes troubleshooting/index sosreport diff --git a/docs/operation/registry_ha.rst b/docs/operation/registry_ha.rst new file mode 100644 index 0000000000..5146f3d675 --- /dev/null +++ b/docs/operation/registry_ha.rst @@ -0,0 +1,80 @@ +Registry HA +=========== + +To be able to run fully offline, MetalK8s comes with its own registry serving +all necessary images used by its containers. +This registry container sits on the Bootstrap node. + +With a highly available registry, container images are served by multiple +nodes, which means the Bootstrap node can be lost without impacting the +cluster. +It allows pods to be scheduled, even if the needed images are not cached +locally. + +.. note:: + + This procedure only talk about registry HA as Bootstrap HA is not + supported for the moment, so it's only a part of the Bootstrap + functionnaly. Check this ticket for more informations + https://github.com/scality/metalk8s/issues/2002 + +Prepare the node +---------------- + +To configure a node to host a registry, a ``repository`` pod must be scheduled +on it. +This node must be part of the MetalK8s cluster and no specific roles or +taints are needed. + +All ISOs listed in the ``archives`` section of +``/etc/metalk8s/bootstrap.yaml`` and ``/etc/metalk8s/solutions.yaml`` +must be copied from the Bootstrap node to the target node at exactly the same +location. + +Deploy the registry +------------------- + +Connect to the node where you want to deploy the registry and run the +following salt states + +- Prepare all the MetalK8s ISOs + + .. parsed-literal:: + + root@node-1 $ salt-call state.sls \\ + metalk8s.archives.mounted \\ + saltenv=metalk8s-|version| + +- If you have some solutions, prepare the solutions ISOs + + .. parsed-literal:: + + root@node-1 $ salt-call state.sls \\ + metalk8s.solutions.available \\ + saltenv=metalk8s-|version| + +- Deploy the registry container + + .. parsed-literal:: + + root@node-1 $ salt-call state.sls \\ + metalk8s.repo.installed \\ + saltenv=metalk8s-|version| + + +Reconfigure the container engines +--------------------------------- + +Containerd must be reconfigured to add the freshly deployed registry to its +endpoints and so it can still pull images in case the Bootstrap node's one is +down. + +From the Bootstrap node, run (replace ```` with the +actual Bootstrap node name): + +.. parsed-literal:: + + root@bootstrap $ kubectl exec -n kube-system -c salt-master \\ + --kubeconfig=/etc/kubernetes/admin.conf \\ + salt-master- -- salt '*' state.sls \\ + metalk8s.container-engine saltenv=metalk8s-|version| diff --git a/eve/main.yml b/eve/main.yml index ea0f192464..71fccc6ce3 100644 --- a/eve/main.yml +++ b/eve/main.yml @@ -269,9 +269,10 @@ models: ARCHIVE_DIRECTORY: "%(prop:builddir)s/build" ARCHIVE: "metalk8s.iso" DEST: '' + NODE: bootstrap command: > scp -F ssh_config "$ARCHIVE_DIRECTORY/$ARCHIVE" - bootstrap:"$DEST" + $NODE:"$DEST" workdir: *terraform_workdir haltOnFailure: true - ShellCommand: &create_mountpoint @@ -456,7 +457,7 @@ models: name: Run fast tests on Bastion env: &_env_bastion_fast_tests <<: *_env_bastion_tests - PYTEST_FILTERS: "post and ci and not multinode and not slow" + PYTEST_FILTERS: "post and ci and not multinode and not slow and not registry_ha" - SetPropertyFromCommand: &set_bootstrap_cp_ip_ssh name: Set the bootstrap node control plane IP as a property property: bootstrap_control_plane_ip @@ -647,7 +648,7 @@ models: doStepIf: "%(prop:install_solution:-false)s" name: Copy Solution archive to bootstrap <<: *copy_iso_bootstrap_ssh - env: + env: &_env_copy_solution_archive_bootstrap_ssh <<: *_env_copy_iso_bootstrap_ssh ARCHIVE: "%(prop:solution_archive:-example-solution-1.0.0.iso)s" - ShellCommand: &import_solution @@ -2267,10 +2268,12 @@ stages: workdir: *terraform_workdir haltOnFailure: true - ShellCommand: *set_bootstrap_minion_id_ssh - - ShellCommand: + - ShellCommand: &create_archive_directory name: Create /archives directory + env: + NODE: "bootstrap" command: > - ssh -F ssh_config bootstrap ' + ssh -F ssh_config $NODE ' sudo mkdir /archives && sudo chown $USER: /archives ' workdir: *terraform_workdir @@ -2317,6 +2320,23 @@ stages: - ShellCommand: *wait_for_solution_operator # }}} - ShellCommand: *wait_pods_stable_ssh + - ShellCommand: + <<: *create_archive_directory + env: + NODE: node-1 + - ShellCommand: + <<: *copy_iso_bootstrap_ssh + name: Copy archive to node-1 + env: + <<: *_env_copy_iso_bootstrap_ssh + NODE: node-1 + DEST: /archives/metalk8s.iso + - ShellCommand: + <<: *copy_solution_archive_bootstrap_ssh + name: Copy Solution archive to node-1 + env: + <<: *_env_copy_solution_archive_bootstrap_ssh + NODE: node-1 - ShellCommand: &multi_node_fast_tests <<: *bastion_tests name: Run fast tests on Bastion @@ -2408,6 +2428,12 @@ stages: PYTEST_FILTERS: "install and ci and multinodes" - ShellCommand: *provision_volumes_on_node1 - ShellCommand: *wait_pods_stable_ssh + - ShellCommand: + <<: *copy_iso_bootstrap_ssh + name: Copy archive to node-1 + env: + <<: *_env_copy_iso_bootstrap_ssh + NODE: node-1 - ShellCommand: *multi_node_fast_tests - ShellCommand: *multi_node_slow_tests - SetPropertyFromCommand: diff --git a/salt/_modules/metalk8s_kubernetes_utils.py b/salt/_modules/metalk8s_kubernetes_utils.py index 6a72272bcf..ba667c2548 100644 --- a/salt/_modules/metalk8s_kubernetes_utils.py +++ b/salt/_modules/metalk8s_kubernetes_utils.py @@ -180,17 +180,19 @@ def get_service_endpoints(service, namespace, kubeconfig): raise CommandExecutionError(error_tpl.format(service, namespace)) from exc try: - # Extract hostname, ip and node_name - result = { - k: v - for k, v in endpoint["subsets"][0]["addresses"][0].items() - if k in ["hostname", "ip", "node_name"] - } - - # Add ports info to result dict - result["ports"] = { - port["name"]: port["port"] for port in endpoint["subsets"][0]["ports"] - } + result = [] + + for address in endpoint["subsets"][0]["addresses"]: + # Extract hostname, ip and node_name + res_ep = { + k: v for k, v in address.items() if k in ["hostname", "ip", "node_name"] + } + + # Add ports info to result dict + res_ep["ports"] = { + port["name"]: port["port"] for port in endpoint["subsets"][0]["ports"] + } + result.append(res_ep) except (AttributeError, IndexError, KeyError, TypeError) as exc: raise CommandExecutionError(error_tpl.format(service, namespace)) from exc diff --git a/salt/_pillar/metalk8s_endpoints.py b/salt/_pillar/metalk8s_endpoints.py index 4b202fd29d..3c226ec51b 100644 --- a/salt/_pillar/metalk8s_endpoints.py +++ b/salt/_pillar/metalk8s_endpoints.py @@ -30,19 +30,28 @@ def ext_pillar(minion_id, pillar, kubeconfig): # pylint: disable=unused-argumen else: endpoints = {} + errors = [] for namespace, services in services.items(): for service in services: + service_endpoints = [] try: service_endpoints = __salt__[ "metalk8s_kubernetes.get_service_endpoints" ](service, namespace, kubeconfig) except CommandExecutionError as exc: - service_endpoints = __utils__["pillar_utils.errors_to_dict"]( - str(exc) - ) + errors.append(str(exc)) + + # NOTE: This is needed for downgrade as this pillar + # is used for downgrade + # To be removed in 2.11 + if len(service_endpoints) == 1: + service_endpoints = service_endpoints[0] + endpoints.update({service: service_endpoints}) - __utils__["pillar_utils.promote_errors"](endpoints, service) + + if errors: + endpoints.update(__utils__["pillar_utils.errors_to_dict"](errors)) result = {"metalk8s": {"endpoints": endpoints}} diff --git a/salt/metalk8s/container-engine/containerd/installed.sls b/salt/metalk8s/container-engine/containerd/installed.sls index e0f2092ff1..8083625f6c 100644 --- a/salt/metalk8s/container-engine/containerd/installed.sls +++ b/salt/metalk8s/container-engine/containerd/installed.sls @@ -5,8 +5,14 @@ {%- from "metalk8s/map.jinja" import networks with context %} {%- from "metalk8s/map.jinja" import proxies with context %} -{%- set registry_ip = metalk8s.endpoints['repositories'].ip %} -{%- set registry_port = metalk8s.endpoints['repositories'].ports.http %} +{%- set registry_eps = [] %} +{%- set pillar_endpoints = metalk8s.endpoints.repositories %} +{%- if not pillar_endpoints | is_list %} + {%- set pillar_endpoints = [pillar_endpoints] %} +{%- endif %} +{%- for ep in pillar_endpoints %} + {%- do registry_eps.append('"http://' ~ ep.ip ~ ":" ~ ep.ports.http ~ '"') %} +{%- endfor %} include: - metalk8s.repo @@ -102,7 +108,7 @@ Configure registry IP in containerd conf: version = 2 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ repo.registry_endpoint }}"] - endpoint = ["http://{{ registry_ip }}:{{ registry_port }}"] + endpoint = [{{ registry_eps | join(",") }}] [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] runtime_type = "io.containerd.runc.v2" diff --git a/salt/metalk8s/orchestrate/apiserver.sls b/salt/metalk8s/orchestrate/apiserver.sls index 492a668c24..f6761a4960 100644 --- a/salt/metalk8s/orchestrate/apiserver.sls +++ b/salt/metalk8s/orchestrate/apiserver.sls @@ -16,8 +16,7 @@ Check pillar on {{ node }}: - tgt: {{ node }} - kwarg: keys: - - metalk8s.endpoints.repositories.ip - - metalk8s.endpoints.repositories.ports.http + - metalk8s.endpoints.repositories # We cannot raise when using `salt.function` as we need to return # `False` to have a failed state # https://github.com/saltstack/salt/issues/55503 diff --git a/salt/metalk8s/orchestrate/bootstrap/init.sls b/salt/metalk8s/orchestrate/bootstrap/init.sls index d327da02a1..fdfc9dd02c 100644 --- a/salt/metalk8s/orchestrate/bootstrap/init.sls +++ b/salt/metalk8s/orchestrate/bootstrap/init.sls @@ -57,12 +57,12 @@ Get metalk8s:control_plane_ip grain: 'api': 4507, }, }, - 'repositories': { + 'repositories': [{ 'ip': bootstrap_grains['control_plane_ip'], 'ports': { 'http': 8080, }, - }, + }], }, }, } diff --git a/salt/metalk8s/orchestrate/deploy_node.sls b/salt/metalk8s/orchestrate/deploy_node.sls index 0438605fb3..f0e7d66568 100644 --- a/salt/metalk8s/orchestrate/deploy_node.sls +++ b/salt/metalk8s/orchestrate/deploy_node.sls @@ -145,9 +145,8 @@ Check pillar before salt-minion configuration: - tgt: {{ node_name }} - kwarg: keys: - - metalk8s.endpoints.salt-master.ip - - metalk8s.endpoints.repositories.ip - - metalk8s.endpoints.repositories.ports.http + - metalk8s.endpoints.salt-master + - metalk8s.endpoints.repositories # We cannot raise when using `salt.function` as we need to return # `False` to have a failed state # https://github.com/saltstack/salt/issues/55503 @@ -195,9 +194,8 @@ Check pillar before etcd deployment: - tgt: {{ node_name }} - kwarg: keys: - - metalk8s.endpoints.salt-master.ip - - metalk8s.endpoints.repositories.ip - - metalk8s.endpoints.repositories.ports.http + - metalk8s.endpoints.salt-master + - metalk8s.endpoints.repositories # We cannot raise when using `salt.function` as we need to return # `False` to have a failed state # https://github.com/saltstack/salt/issues/55503 @@ -250,9 +248,8 @@ Check pillar before highstate: - tgt: {{ node_name }} - kwarg: keys: - - metalk8s.endpoints.salt-master.ip - - metalk8s.endpoints.repositories.ip - - metalk8s.endpoints.repositories.ports.http + - metalk8s.endpoints.salt-master + - metalk8s.endpoints.repositories # We cannot raise when using `salt.function` as we need to return # `False` to have a failed state # https://github.com/saltstack/salt/issues/55503 diff --git a/salt/metalk8s/orchestrate/etcd.sls b/salt/metalk8s/orchestrate/etcd.sls index bb9a856b86..b62640d1fe 100644 --- a/salt/metalk8s/orchestrate/etcd.sls +++ b/salt/metalk8s/orchestrate/etcd.sls @@ -24,8 +24,7 @@ Check pillar on {{ node }}: - tgt: {{ node }} - kwarg: keys: - - metalk8s.endpoints.repositories.ip - - metalk8s.endpoints.repositories.ports.http + - metalk8s.endpoints.repositories # We cannot raise when using `salt.function` as we need to return # `False` to have a failed state # https://github.com/saltstack/salt/issues/55503 diff --git a/salt/metalk8s/orchestrate/upgrade/init.sls b/salt/metalk8s/orchestrate/upgrade/init.sls index f3678f467f..a705ba172e 100644 --- a/salt/metalk8s/orchestrate/upgrade/init.sls +++ b/salt/metalk8s/orchestrate/upgrade/init.sls @@ -35,8 +35,7 @@ Check pillar on {{ node }} before installing apiserver-proxy: - tgt: {{ node }} - kwarg: keys: - - metalk8s.endpoints.repositories.ip - - metalk8s.endpoints.repositories.ports.http + - metalk8s.endpoints.repositories # We cannot raise when using `salt.function` as we need to return # `False` to have a failed state # https://github.com/saltstack/salt/issues/55503 diff --git a/salt/metalk8s/repo/redhat.sls b/salt/metalk8s/repo/redhat.sls index 635665b0b0..12897de33e 100644 --- a/salt/metalk8s/repo/redhat.sls +++ b/salt/metalk8s/repo/redhat.sls @@ -3,8 +3,27 @@ {%- from "metalk8s/map.jinja" import repo with context %} {%- from "metalk8s/map.jinja" import package_exclude_list with context %} -{%- set repo_host = pillar.metalk8s.endpoints['repositories'].ip %} -{%- set repo_port = pillar.metalk8s.endpoints['repositories'].ports.http %} +{%- set repo_base_urls = [] %} +{%- if repo.local_mode %} + {%- do repo_base_urls.append("file://" ~ + salt.metalk8s.get_archives()[saltenv].path ~ "/" ~ + repo.relative_path ~ "/" ~ + grains['os_family'].lower() ~ "/" ~ + "$metalk8s_osmajorrelease") %} + +{%- else %} + {%- set pillar_endpoints = metalk8s.endpoints.repositories %} + {%- if not pillar_endpoints | is_list %} + {%- set pillar_endpoints = [pillar_endpoints] %} + {%- endif %} + {%- for endpoint in pillar_endpoints %} + {%- do repo_base_urls.append("http://" ~ + endpoint.ip ~ ":" ~ endpoint.ports.http ~ + "/" ~ saltenv ~ "/" ~ + grains['os_family'].lower() ~ "/" ~ + "$metalk8s_osmajorrelease") %} + {%- endfor %} +{%- endif %} Set metalk8s_osmajorrelease in yum vars: file.managed: @@ -41,29 +60,25 @@ Ensure yum versionlock plugin is enabled: - pkg: Install yum-plugin-versionlock {%- for repo_name, repo_config in repo.repositories.items() %} - {%- if repo.local_mode %} - {%- set repo_base_url = "file://" ~ - salt.metalk8s.get_archives()[saltenv].path ~ "/" ~ - repo.relative_path ~ "/" ~ - grains['os_family'].lower() ~ "/" ~ - "$metalk8s_osmajorrelease" %} - {%- else %} - {%- set repo_base_url = "http://" ~ repo_host ~ ':' ~ repo_port ~ - "/" ~ saltenv ~ "/" ~ - grains['os_family'].lower() ~ "/" ~ - "$metalk8s_osmajorrelease" %} - {%- endif %} - {%- set repo_url = repo_base_url ~ "/" ~ repo_name ~ - "-el$metalk8s_osmajorrelease" %} + {%- set repo_urls = [] %} + {%- for base_url in repo_base_urls %} + {%- do repo_urls.append(base_url ~ "/" ~ + repo_name ~ "-el$metalk8s_osmajorrelease") %} + {%- endfor %} {%- set gpg_keys = [] %} {%- for gpgkey in repo_config.gpgkeys %} - {%- do gpg_keys.append(repo_url ~ "/" ~ gpgkey) %} + {# NOTE: We only use the first repo URL for GPG key since these are the same, + and we do not want to depend on both repos to import this key. + When installing a package, if the package's GPG key is not already imported + on the host, it will try to import **all** GPG keys listed and will fail if + unable to download one of them #} + {%- do gpg_keys.append(repo_urls[0] ~ "/" ~ gpgkey) %} {%- endfor %} Configure {{ repo_name }} repository: pkgrepo.managed: - name: {{ repo_name }} - humanname: {{ repo_config.humanname }} - - baseurl: {{ repo_url }} + - baseurl: {{ repo_urls | join(' ') }} - gpgcheck: {{ repo_config.gpgcheck }} {%- if gpg_keys %} - gpgkey: "{{ gpg_keys | join (' ') }}" diff --git a/salt/metalk8s/salt/minion/configured.sls b/salt/metalk8s/salt/minion/configured.sls index 5ca981f9a1..f3204d72e4 100644 --- a/salt/metalk8s/salt/minion/configured.sls +++ b/salt/metalk8s/salt/minion/configured.sls @@ -16,6 +16,7 @@ Configure salt minion: - backup: false - defaults: debug: {{ metalk8s.debug }} + {#- NOTE: We only consider a single master for the moment #} master_hostname: {{ metalk8s.endpoints['salt-master'].ip }} minion_id: {{ grains.id }} saltenv: {{ saltenv }} diff --git a/salt/metalk8s/salt/minion/installed.sls b/salt/metalk8s/salt/minion/installed.sls index 03831f64c5..4ddba523ad 100644 --- a/salt/metalk8s/salt/minion/installed.sls +++ b/salt/metalk8s/salt/minion/installed.sls @@ -4,6 +4,13 @@ include : - .dependencies - .restart +# Make sure `genisoimage` is installed on every minions as it's used +# in some MetalK8s custom execution modules +Install genisoimage: + {{ pkg_installed('genisoimage') }}: + - require: + - test: Repositories configured + # Make sure `python36-psutil` is installed on every minions as it's used # in some MetalK8s custom execution modules Install python psutil: diff --git a/salt/metalk8s/solutions/available.sls b/salt/metalk8s/solutions/available.sls index 0af6795666..70aae9fea9 100644 --- a/salt/metalk8s/solutions/available.sls +++ b/salt/metalk8s/solutions/available.sls @@ -66,6 +66,7 @@ Expose container images for Solution {{ display_name }}: - source: {{ mount_path }}/registry-config.inc.j2 - name: {{ repo.config.directory }}/{{ solution.id }}-registry-config.inc - template: jinja + - makedirs: true - defaults: repository: {{ solution.id }} registry_root: {{ mount_path }}/images diff --git a/salt/tests/unit/modules/files/test_metalk8s_kubernetes_utils.yaml b/salt/tests/unit/modules/files/test_metalk8s_kubernetes_utils.yaml index ed454113b9..61239eb020 100644 --- a/salt/tests/unit/modules/files/test_metalk8s_kubernetes_utils.yaml +++ b/salt/tests/unit/modules/files/test_metalk8s_kubernetes_utils.yaml @@ -165,25 +165,94 @@ get_service_endpoints: port: 4507 protocol: TCP result: - ip: 10.11.12.13 - node_name: my-node - hostname: null - ports: - requestserver: 4506 - publisher: 4505 - api: 4507 + - ip: 10.11.12.13 + node_name: my-node + hostname: null + ports: + requestserver: 4506 + publisher: 4505 + api: 4507 + + # 2. Multiple addresses for an endpoint + - obj: + api_version: v1 + kind: Endpoints + metadata: + annotations: + endpoints.kubernetes.io/last-change-trigger-time: '2021-05-25T07:36:46Z' + cluster_name: null + creation_timestamp: 2021-05-11 09:49:51+00:00 + deletion_grace_period_seconds: null + deletion_timestamp: null + finalizers: null + generate_name: null + generation: null + labels: + app: repositories + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: repositories + app.kubernetes.io/part-of: metalk8s + heritage: salt + metalk8s.scality.com/version: 2.10.0-dev + service.kubernetes.io/headless: '' + name: repositories + namespace: kube-system + owner_references: null + resource_version: '2882986' + self_link: null + uid: f101166a-5175-41b0-bc9c-899da5b61685 + subsets: + - addresses: + - hostname: null + ip: 10.11.12.13 + node_name: my-node + target_ref: + api_version: null + field_path: null + kind: Pod + name: repositories-my-node + namespace: kube-system + resource_version: '2871130' + uid: c24f3ddc-c24a-4eb8-9080-e57468ce6429 + - hostname: null + ip: 10.21.22.23 + node_name: my-second-node + target_ref: + api_version: null + field_path: null + kind: Pod + name: repositories-my-second-node + namespace: kube-system + resource_version: '2882985' + uid: 9d6eea84-211d-47f8-b048-1778febf468e + not_ready_addresses: null + ports: + - name: http + port: 8080 + protocol: TCP + result: + - ip: 10.11.12.13 + node_name: my-node + hostname: null + ports: + http: 8080 + - ip: 10.21.22.23 + node_name: my-second-node + hostname: null + ports: + http: 8080 - # 2. Error when getting the Endpoint object + # 3. Error when getting the Endpoint object - obj: False raises: True result: "Unable to get kubernetes endpoints for my_service in namespace my_namespace" - # 3. Endpoint object does not exists + # 4. Endpoint object does not exist - obj: null raises: True result: "Unable to get kubernetes endpoints for my_service in namespace my_namespace" - # 4. Endpoint do not have any addresses + # 5. Endpoint does not have any address - obj: api_version: v1 kind: Endpoints diff --git a/scripts/bootstrap.sh.in b/scripts/bootstrap.sh.in index 616f5e9ad7..d244d9db1f 100755 --- a/scripts/bootstrap.sh.in +++ b/scripts/bootstrap.sh.in @@ -77,10 +77,10 @@ orchestrate_bootstrap() { " 'repo': {'local_mode': True}," " 'metalk8s': {" " 'endpoints': {" - " 'repositories': {" + " 'repositories': [{" " 'ip': $control_plane_ip," " 'ports': {'http': 8080}" - " }," + " }]," " 'salt-master': {'ip': $control_plane_ip}" " }" " }" diff --git a/tests/post/features/registry.feature b/tests/post/features/registry.feature new file mode 100644 index 0000000000..299317647a --- /dev/null +++ b/tests/post/features/registry.feature @@ -0,0 +1,23 @@ +@post @ci @local @registry +Feature: Registry is up and running + Scenario: Pull container image from registry + Given the Kubernetes API is available + And pods with label 'app.kubernetes.io/name=repositories' are 'Ready' + When we pull metalk8s utils image from node 'bootstrap' + Then pull succeeds + + @registry_ha + Scenario: Pull container image from registry (HA) + Given the Kubernetes API is available + And we are on a multi node cluster + And we have 1 running pod labeled 'app.kubernetes.io/name=repositories' in namespace 'kube-system' + When we set up repositories on 'node-1' + Then we have 1 running pod labeled 'app.kubernetes.io/name=repositories' in namespace 'kube-system' on node 'node-1' + When we pull metalk8s utils image from node 'bootstrap' + Then pull succeeds + When we stop repositories on node 'bootstrap' + And we pull metalk8s utils image from node 'bootstrap' + Then pull succeeds + When we stop repositories on node 'node-1' + And we pull metalk8s utils image from node 'bootstrap' + Then pull fails diff --git a/tests/post/steps/test_registry.py b/tests/post/steps/test_registry.py new file mode 100644 index 0000000000..3bfe300617 --- /dev/null +++ b/tests/post/steps/test_registry.py @@ -0,0 +1,173 @@ +import json +import os +import os.path + +import pytest +from pytest_bdd import given, when, scenario, then, parsers +import testinfra + +from tests import kube_utils, utils + +# Fixture {{{ + + +@pytest.fixture(scope="function") +def context(): + return {} + + +@pytest.fixture +def teardown(context): + yield + if "repo_to_restore" in context: + for host, filename in context["repo_to_restore"].items(): + with host.sudo(): + host.check_output( + "mv {} /etc/kubernetes/manifests/repositories.yaml".format(filename) + ) + + if "file_to_remove" in context: + for host, filename in context["file_to_remove"].items(): + with host.sudo(): + host.check_output("rm -f {}".format(filename)) + + +# }}} +# Scenario {{{ + + +@scenario("../features/registry.feature", "Pull container image from registry") +def test_pull_registry(host): + pass + + +@scenario("../features/registry.feature", "Pull container image from registry (HA)") +def test_pull_registry_ha(host, teardown): + pass + + +# }}} +# Given {{{ + + +@given("we are on a multi node cluster") +def check_multi_node(k8s_client): + nodes = k8s_client.list_node() + + if len(nodes.items) == 1: + pytest.skip("We skip single node cluster for this test") + + +# }}} +# When {{{ + + +@when(parsers.parse("we pull metalk8s utils image from node '{node_name}'")) +def pull_metalk8s_utils(host, context, ssh_config, utils_image, node_name): + if ssh_config: + node = testinfra.get_host(node_name, ssh_config=ssh_config) + else: + node = host + + with node.sudo(): + context["pull_ret"] = node.run("crictl pull {}".format(utils_image)) + + +@when(parsers.parse("we set up repositories on '{node_name}'")) +def set_up_repo(context, host, ssh_config, version, node_name): + node = testinfra.get_host(node_name, ssh_config=ssh_config) + bootstrap = testinfra.get_host("bootstrap", ssh_config=ssh_config) + + with bootstrap.sudo(): + cmd_ret = bootstrap.run("ls /etc/metalk8s/solutions.yaml") + + with node.sudo(): + node.check_output( + "salt-call --retcode-passthrough state.sls " + "metalk8s.archives.mounted " + "saltenv=metalk8s-{}".format(version) + ) + + # Only run solution state if we have some solutions + if cmd_ret.rc == 0: + node.check_output( + "salt-call --retcode-passthrough state.sls " + "metalk8s.solutions.available " + "saltenv=metalk8s-{}".format(version) + ) + + node.check_output( + "salt-call --retcode-passthrough state.sls " + "metalk8s.repo.installed " + "saltenv=metalk8s-{}".format(version) + ) + + command = [ + "salt", + "'*'", + "state.sls", + "metalk8s.container-engine", + "saltenv=metalk8s-{}".format(version), + ] + + utils.run_salt_command(host, command, ssh_config) + + context.setdefault("file_to_remove", {})[ + node + ] = "/etc/kubernetes/manifests/repositories.yaml" + + +@when(parsers.parse("we stop repositories on node '{node_name}'")) +def stop_repo_pod(context, ssh_config, k8s_client, node_name): + node = testinfra.get_host(node_name, ssh_config=ssh_config) + + with node.sudo(): + cmd_ret = node.check_output("salt-call --out json --local temp.dir") + + tmp_dir = json.loads(cmd_ret)["local"] + + with node.sudo(): + node.check_output( + "mv /etc/kubernetes/manifests/repositories.yaml {}".format(tmp_dir) + ) + + context.setdefault("repo_to_restore", {})[node] = os.path.join( + tmp_dir, "repositories.yaml" + ) + + def _wait_repo_stopped(): + pods = kube_utils.get_pods( + k8s_client, + ssh_config, + "app.kubernetes.io/name=repositories", + node_name, + namespace="kube-system", + ) + assert not pods + + utils.retry(_wait_repo_stopped, times=24, wait=5) + + +# }}} +# Then {{{ + + +@then("pull succeeds") +def pull_succeeds(context): + assert context["pull_ret"].rc == 0, "cmd: {}\nstdout:{}\nstderr:{}".format( + context["pull_ret"].command, + context["pull_ret"].stdout, + context["pull_ret"].stderr, + ) + + +@then("pull fails") +def pull_fails(context): + assert context["pull_ret"].rc == 1, "cmd: {}\nstdout:{}\nstderr:{}".format( + context["pull_ret"].command, + context["pull_ret"].stdout, + context["pull_ret"].stderr, + ) + + +# }}}