From a012e5feb1b30b5e1e82c4003f13e4ae7dd4111d Mon Sep 17 00:00:00 2001 From: Venktesh Shivam Patel Date: Thu, 16 May 2024 11:34:50 +0100 Subject: [PATCH] add files for upgrade tests (#5553) --- tests/Makefile | 7 + tests/conftest.py | 6 + .../upgrade-test-resources/deployment.yaml | 19 +++ .../data/upgrade-test-resources/ingress.yaml | 21 +++ tests/data/upgrade-test-resources/ns.yaml | 4 + tests/data/upgrade-test-resources/secret.yaml | 8 ++ .../data/upgrade-test-resources/service.yaml | 12 ++ .../tcp-deployment.yaml | 21 +++ .../transport-server.yaml | 14 ++ .../virtual-server.yaml | 16 +++ tests/suite/test_upgrade_resources.py | 128 ++++++++++++++++++ tests/suite/utils/custom_resources_utils.py | 19 +++ 12 files changed, 275 insertions(+) create mode 100644 tests/data/upgrade-test-resources/deployment.yaml create mode 100644 tests/data/upgrade-test-resources/ingress.yaml create mode 100644 tests/data/upgrade-test-resources/ns.yaml create mode 100644 tests/data/upgrade-test-resources/secret.yaml create mode 100644 tests/data/upgrade-test-resources/service.yaml create mode 100644 tests/data/upgrade-test-resources/tcp-deployment.yaml create mode 100644 tests/data/upgrade-test-resources/transport-server.yaml create mode 100644 tests/data/upgrade-test-resources/virtual-server.yaml create mode 100644 tests/suite/test_upgrade_resources.py diff --git a/tests/Makefile b/tests/Makefile index 11c557a0dd..28c3f58bf1 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -91,3 +91,10 @@ image-load: ## Load the image into the Kind K8S cluster test-lint: ## Run Python linting tools isort . black . + + +.PHONY: upgrade-resources +upgrade-resources: +## Create and delete resources for upgrade tests e.g. `make upgrade-resources PYTEST_ARGS="create OR delete"` + pip install -r ../tests/requirements.txt --no-deps + pytest -v -s -m $(PYTEST_ARGS) diff --git a/tests/conftest.py b/tests/conftest.py index 403a39806b..d8edec86da 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -120,6 +120,12 @@ def pytest_addoption(parser) -> None: default=os.environ.get("AZURE_AD_AUTOMATION"), help="Azure active directory secret for JWKs", ) + parser.addoption( + "--num", + action="store", + default="1", + help="Number of resources to deploy for upgrade tests", + ) # import fixtures into pytest global namespace diff --git a/tests/data/upgrade-test-resources/deployment.yaml b/tests/data/upgrade-test-resources/deployment.yaml new file mode 100644 index 0000000000..f2b28d35d1 --- /dev/null +++ b/tests/data/upgrade-test-resources/deployment.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend +spec: + replicas: 2 + selector: + matchLabels: + app: backend + template: + metadata: + labels: + app: backend + spec: + containers: + - name: backend + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 diff --git a/tests/data/upgrade-test-resources/ingress.yaml b/tests/data/upgrade-test-resources/ingress.yaml new file mode 100644 index 0000000000..b5cc519b1d --- /dev/null +++ b/tests/data/upgrade-test-resources/ingress.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress +spec: + ingressClassName: nginx + tls: + - hosts: + - cafe.example.com + secretName: secret + rules: + - host: cafe.example.com + http: + paths: + - path: /backend + pathType: Prefix + backend: + service: + name: backend-svc + port: + number: 80 diff --git a/tests/data/upgrade-test-resources/ns.yaml b/tests/data/upgrade-test-resources/ns.yaml new file mode 100644 index 0000000000..115c384d62 --- /dev/null +++ b/tests/data/upgrade-test-resources/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ns diff --git a/tests/data/upgrade-test-resources/secret.yaml b/tests/data/upgrade-test-resources/secret.yaml new file mode 100644 index 0000000000..8061900506 --- /dev/null +++ b/tests/data/upgrade-test-resources/secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: test-secret +type: kubernetes.io/tls +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ4RENDQXRpZ0F3SUJBZ0lKQU9jbHdCelprYmlhTUEwR0NTcUdTSWIzRFFFQkJRVUFNRmd4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFaE1COEdBMVVFQ2hNWVNXNTBaWEp1WlhRZ1YybGtaMmwwY3lCUQpkSGtnVEhSa01Sa3dGd1lEVlFRREV4QmpZV1psTG1WNFlXMXdiR1V1WTI5dE1CNFhEVEUzTURnek1URXdNVGN5Ck1Gb1hEVEU0TURnek1URXdNVGN5TUZvd1dERUxNQWtHQTFVRUJoTUNWVk14Q3pBSkJnTlZCQWdUQWtOQk1TRXcKSHdZRFZRUUtFeGhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXhHVEFYQmdOVkJBTVRFR05oWm1VdQpaWGhoYlhCc1pTNWpiMjB3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRQzdIT0xJCm5oZjE1aUcxOE16RXBzN0lvZmxHQmovMi9NVjA0OWtBS0hrTnZOem1XaXRXWDV2QU1yRkF5THY0dXBIWDI5b0IKa3l6YUhlYyt2TlFibEh1bStINUtkUWZXWHFXNkJ6UnVhMzBreEkrcG91cnhpNy9jaDJORS92djBhRGtvaTJ0RAovOUI2aHAyVkoxWXFJdm9hQ2wwSmFYWDd0WEc4SGU1S1BSZzBYMm1Mblcwa29tay9ZVGRPbS9xOVRjUDRnUmhrCms3bUhJMDlSME5vNUhTbURydmVBWFEyY3lGWVJQVUNjWkNPd0h6UUdVUVB1UU0wNVArWUNnVlNRcElKWFV0b0kKbGdEMHhZZUw4UU1rZjZ0TWpYcXpTVVRhQlhzNkRKU2x1YWN1aHpkV212UnFPUVNNYlVpZ3dVUEZCRDVLRUFIcwozM0hHeVZ5dkI4cVlrYUczQWdNQkFBR2pnYnd3Z2Jrd0hRWURWUjBPQkJZRUZOdTQvMTdpSituRGxPMkoyVisvCitqM2x5SzVZTUlHSkJnTlZIU01FZ1lFd2Y0QVUyN2ovWHVJbjZjT1U3WW5aWDcvNlBlWElybGloWEtSYU1GZ3gKQ3pBSkJnTlZCQVlUQWxWVE1Rc3dDUVlEVlFRSUV3SkRRVEVoTUI4R0ExVUVDaE1ZU1c1MFpYSnVaWFFnVjJsawpaMmwwY3lCUWRIa2dUSFJrTVJrd0Z3WURWUVFERXhCallXWmxMbVY0WVcxd2JHVXVZMjl0Z2drQTV5WEFITm1SCnVKb3dEQVlEVlIwVEJBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRVUZBQU9DQVFFQUtGUHJBcXA3a3lzTDVGNnMKWFhWdXZkZzAyc0srUlpzb2F3QWVxbHlSRmpJeUlQL2VTajBhQjQwQmNOcWFyRHhwNjhBd1pZNG4yQk9EVmo5WgphOFlvV1YyOFpwamloaThxNnBPSElOa0MrOXpCY1hsZ2lvVUZBTERCcXFPTXFUZkw1cjNGejNUTGN1clozajhuCnUzL2hRVHNXZG5TZENWbmN0aXhaUHJ5cnhJSFlWSERiVHF4ZWdTQUN6WkU1MHMwdlRpMFJkNUkrcVdubVpIUloKL0hLNVZnNWlNS2E1clBPRTFaT2M3L2VnVjZ6R2p4THJiNEdlQ2JyTjBBb01tazNpL2d2K2kzL0N6aTlXOVhNNApwa2hQSjJUcEtMSjVaOHgxUVhjUW5Dem5yOEdtL2FuVzV4b3lDdWhjZzlXMlVSYzRKVTZ1UXh0WU9tczYrc0RxCjBBN1Y3UT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdXh6aXlKNFg5ZVlodGZETXhLYk95S0g1UmdZLzl2ekZkT1BaQUNoNURiemM1bG9yClZsK2J3REt4UU1pNytMcVIxOXZhQVpNczJoM25QcnpVRzVSN3B2aCtTblVIMWw2bHVnYzBibXQ5Sk1TUHFhTHEKOFl1LzNJZGpSUDc3OUdnNUtJdHJRLy9RZW9hZGxTZFdLaUw2R2dwZENXbDErN1Z4dkIzdVNqMFlORjlwaTUxdApKS0pwUDJFM1RwdjZ2VTNEK0lFWVpKTzVoeU5QVWREYU9SMHBnNjczZ0YwTm5NaFdFVDFBbkdRanNCODBCbEVECjdrRE5PVC9tQW9GVWtLU0NWMUxhQ0pZQTlNV0hpL0VESkgrclRJMTZzMGxFMmdWN09neVVwYm1uTG9jM1ZwcjAKYWprRWpHMUlvTUZEeFFRK1NoQUI3Tjl4eHNsY3J3ZkttSkdodHdJREFRQUJBb0lCQUVuZHpXbUZmOUFEV2F1Sgp0RXl0elZSSEhURVhwb2pLb09qVVNnWlY4L1FJYXV4RkRIYThwNi9vVXpGUURXVFR3bCtFMnp0ajdvRHM3UzFIClBqVGxHU3VCVGRuMitYRVhURFYwUXE2VW9JS3pWa09SblU1ZDdSQVNJbzVLV3d6UldEODVTczg5WGdBQXhKVHQKUW9hLzZCdi9tMXJyMXpmWEdWODZNYWY5Rm1FVjI5bmpCcHZ3cUpzOFlUaFU5VE5BRUdnbkxBWGFJYTJZSUJTSgozcVhJd1RrdFBHUUZyZUowdnBiOVlaTkRxWk0rMmI1K3BBVEVXclpyeTQxTlpvdzhNUnVOYnoxVU5sWXkyazZKCjlXclUxUDVYM2l3dEZmcXA3TzREakNOMFR2Y1Y1Nm5QczJoYldDeHp4RmxFalJwNW5KQ3Bsdi9yNCt4eHJVRWQKd0NjU0x3RUNnWUVBOGVDUGM2S2JpSXVFUWxwVjcxTXRQMjdBZzhkMzN5aWZKajAvQ3R3RlF6dlhWcHZCSHFRagphUE1ZaEswZ3o0MzFHMGtqVUpTeUU2STJYVWhFU3gxZUw2TU5sVS9xc29Uc0JaZEV3N0cvMFRhdW1TQUJWemZPCkIvckdtbEZkZitxWVpiR3pIdGtvbVJWb1JqVWtPcGRJL3RnL1ZlSmJ3K1RIS2dYS01NU2V5cjhDZ1lFQXhnbTkKWXN5dVZVUGlEdDhuL1JXeEJwZ1Zqazk0M3BFOVFjK2FVK0VPcGROK2NNQjkvaGJiWUpMZEZ2WUUwd2s1cEQ1SQpibktscjZycndEVUJKNWY2TXM0L3gyS3JISjlCQ3VNT3JyTEVTVm42ZG9NdDhWdEtpQkVLTVFuSnIzYmM4UDV3CnpvVmNhRGl1dCtISjcySG83cG5QQTFhaS9QV1ZwM2pZUTlCSXZ3a0NnWUJRNzkzUXlmYlZxQ25uc2liVFlMZmgKWkFRVGxLbXVDUC9JWWZJNGhndFV4aTkya2NQN3B0MGFmMDRUQjRQVk1DRjJzZkNaUkVpYWZVdEh4Nmppb2I4awpuYUVyOTRRSG5LY0Y3K3BZdWFBQU9CWVFzejcvbW5MZEJMTjBiQW1uaGk3Y3lLdXhoT1VxNUpqeDlWSmNNTWVDClQ0WlNETjY4SEUvdzVlTVVrcGE0TFFLQmdGM0piUXhtUE1XYW9XdERtYytNdjBxTktlQThtTlJtMmlqWnBZL0YKek1jUnN4YTR3ckpicHNkRXBqbmlod1Jlb1JLOGdGYjJLcXRYK2RBTUNpRHpJNFYrRWN4ZVdRVDBFcnlTTFhqawpwbnJLaHdnck5jM1EyeW8zVDZsTHBsMVhvR2p0UndVM09UME9Zd2dvZ1JiQ09xc000bklGVEtrWnNTY2YzdU8yCnQwenBBb0dCQU5Fc1V4Yit6UHpKbEgwL2hOTlhteisvNGx4aXlJb1ZQQXgzaEtBemtjOGtkZzhoS25XWkZhK0YKTjZMOXZjTDhNbnRjNHpmVDdDanNxWGJ2cGUzRUFOeGk4TWRSZzd1Z093L2NiZWdLVklZY3hTdXdRKzYrNUZYOApKRzlhbGxzdXovTk40b2RpMzRvblpEVmRZcURnY2xaZnZucXZKMHZWSzc1VXhQZ3F3aUJQCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/tests/data/upgrade-test-resources/service.yaml b/tests/data/upgrade-test-resources/service.yaml new file mode 100644 index 0000000000..7f651dc024 --- /dev/null +++ b/tests/data/upgrade-test-resources/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: backend-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: backend diff --git a/tests/data/upgrade-test-resources/tcp-deployment.yaml b/tests/data/upgrade-test-resources/tcp-deployment.yaml new file mode 100644 index 0000000000..ddb8798e43 --- /dev/null +++ b/tests/data/upgrade-test-resources/tcp-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tcp +spec: + replicas: 2 + selector: + matchLabels: + app: tcp + template: + metadata: + labels: + app: tcp + spec: + containers: + - name: tcp-service + image: ghcr.io/nginxinc/kic-test-tcp-server:0.2.1 + ports: + - containerPort: 3333 + name: tcp-server + protocol: TCP diff --git a/tests/data/upgrade-test-resources/transport-server.yaml b/tests/data/upgrade-test-resources/transport-server.yaml new file mode 100644 index 0000000000..7f3a4d8c9b --- /dev/null +++ b/tests/data/upgrade-test-resources/transport-server.yaml @@ -0,0 +1,14 @@ +apiVersion: k8s.nginx.org/v1 +kind: TransportServer +metadata: + name: transport-server +spec: + listener: + name: tcp-server + protocol: TCP + upstreams: + - name: tcp-app + service: tcp-service + port: 3333 + action: + pass: tcp-app diff --git a/tests/data/upgrade-test-resources/virtual-server.yaml b/tests/data/upgrade-test-resources/virtual-server.yaml new file mode 100644 index 0000000000..255377034e --- /dev/null +++ b/tests/data/upgrade-test-resources/virtual-server.yaml @@ -0,0 +1,16 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: vs +spec: + host: vs.example.com + tls: + secret: secret + upstreams: + - name: backend + service: backend-svc + port: 80 + routes: + - path: "/backend" + action: + pass: backend diff --git a/tests/suite/test_upgrade_resources.py b/tests/suite/test_upgrade_resources.py new file mode 100644 index 0000000000..2852e57b1d --- /dev/null +++ b/tests/suite/test_upgrade_resources.py @@ -0,0 +1,128 @@ +import os +import tempfile + +import pytest +import yaml +from settings import DEPLOYMENTS, TEST_DATA +from suite.utils.custom_resources_utils import create_resource_from_manifest, read_custom_resource +from suite.utils.resources_utils import ( + create_ingress, + create_items_from_yaml, + create_namespace, + delete_namespace, + wait_before_test, +) +from suite.utils.vs_vsr_resources_utils import create_virtual_server + +tcp_deployment = f"{TEST_DATA}/upgrade-test-resources/tcp-deployment.yaml" +deployment = f"{TEST_DATA}/upgrade-test-resources/deployment.yaml" +service = f"{TEST_DATA}/upgrade-test-resources/service.yaml" +ns = f"{TEST_DATA}/upgrade-test-resources/ns.yaml" +ingress = f"{TEST_DATA}/upgrade-test-resources/ingress.yaml" +vs = f"{TEST_DATA}/upgrade-test-resources/virtual-server.yaml" +ts = f"{TEST_DATA}/upgrade-test-resources/transport-server.yaml" +secret = f"{TEST_DATA}/upgrade-test-resources/secret.yaml" + +# Below test class only deployes resources for upgrade testing, IC deployment should be done manually via helm. + + +class TestUpgrade: + @pytest.mark.create + def test_create(self, request, kube_apis): + count = int(request.config.getoption("--num")) + + for i in range(1, count + 1): + with open(ns) as f: + doc = yaml.safe_load(f) + doc["metadata"]["name"] = f"ns-{i}" + with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp: + temp.write(yaml.safe_dump(doc) + "---\n") + namespace = create_namespace(kube_apis.v1, doc) + os.remove(temp.name) + + with open(deployment) as f: + doc = yaml.safe_load(f) + doc["metadata"]["name"] = f"backend-{i}" + doc["spec"]["selector"]["matchLabels"]["app"] = f"backend-{i}" + doc["spec"]["template"]["metadata"]["labels"]["app"] = f"backend-{i}" + doc["metadata"]["name"] = f"backend-{i}" + with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp: + temp.write(yaml.safe_dump(doc) + "---\n") + create_items_from_yaml(kube_apis, temp.name, namespace) + os.remove(temp.name) + + with open(service) as f: + doc = yaml.safe_load(f) + doc["metadata"]["name"] = f"backend-svc-{i}" + doc["spec"]["selector"]["app"] = f"backend-{i}" + with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp: + temp.write(yaml.safe_dump(doc) + "---\n") + create_items_from_yaml(kube_apis, temp.name, namespace) + os.remove(temp.name) + + with open(secret) as f: + doc = yaml.safe_load(f) + doc["metadata"]["name"] = f"secret-{i}" + with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp: + temp.write(yaml.safe_dump(doc) + "---\n") + create_items_from_yaml(kube_apis, temp.name, namespace) + os.remove(temp.name) + + # VirtualServer + with open(vs) as f: + doc = yaml.safe_load(f) + doc["metadata"]["name"] = f"vs-{i}" + doc["spec"]["host"] = f"vs-{i}.example.com" + doc["spec"]["tls"]["secret"] = f"secret-{i}" + doc["spec"]["upstreams"][0]["name"] = f"backend-{i}" + doc["spec"]["upstreams"][0]["service"] = f"backend-svc-{i}" + doc["spec"]["routes"][0]["action"]["pass"] = f"backend-{i}" + create_virtual_server(kube_apis.custom_objects, doc, namespace) + + # Ingress + with open(ingress) as f: + doc = yaml.safe_load(f) + doc["metadata"]["name"] = f"ingress-{i}" + doc["spec"]["tls"][0]["hosts"][0] = f"ingress-{i}.example.com" + doc["spec"]["tls"][0]["secretName"] = f"secret-{i}" + doc["spec"]["rules"][0]["host"] = f"ingress-{i}.example.com" + doc["spec"]["rules"][0]["http"]["paths"][0]["path"] = f"/backend-{i}" + doc["spec"]["rules"][0]["http"]["paths"][0]["backend"]["service"]["name"] = f"backend-svc-{i}" + create_ingress(kube_apis.networking_v1, namespace, doc) + + # TransportServer + with open(tcp_deployment) as f: + doc = yaml.safe_load(f) + doc["metadata"]["name"] = f"tcp-{i}" + doc["spec"]["selector"]["matchLabels"]["app"] = f"tcp-{i}" + doc["spec"]["template"]["metadata"]["labels"]["app"] = f"tcp-{i}" + with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp: + temp.write(yaml.safe_dump(doc) + "---\n") + create_items_from_yaml(kube_apis, temp.name, namespace) + os.remove(temp.name) + + with open(service) as f: + doc = yaml.safe_load(f) + doc["metadata"]["name"] = f"tcp-svc-{i}" + doc["spec"]["selector"]["app"] = f"tcp-{i}" + with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp: + temp.write(yaml.safe_dump(doc) + "---\n") + create_items_from_yaml(kube_apis, temp.name, namespace) + os.remove(temp.name) + + with open(ts) as f: + doc = yaml.safe_load(f) + doc["metadata"]["name"] = f"ts-{i}" + doc["spec"]["listener"]["name"] = "dns-tcp" + doc["spec"]["upstreams"][0]["name"] = f"tcp-{i}" + doc["spec"]["upstreams"][0]["service"] = f"tcp-svc-{i}" + doc["spec"]["upstreams"][0]["port"] = 5353 + doc["spec"]["action"]["pass"] = f"tcp-{i}" + create_resource_from_manifest(kube_apis.custom_objects, doc, namespace, "transportservers") + + @pytest.mark.delete + def test_delete(self, request, kube_apis): + count = int(request.config.getoption("--num")) + # delete namespaces + for i in range(1, count + 1): + delete_namespace(kube_apis.v1, f"ns-{i}") diff --git a/tests/suite/utils/custom_resources_utils.py b/tests/suite/utils/custom_resources_utils.py index e315228b5a..8c99c588f8 100644 --- a/tests/suite/utils/custom_resources_utils.py +++ b/tests/suite/utils/custom_resources_utils.py @@ -101,6 +101,25 @@ def is_dnsendpoint_present(custom_objects: CustomObjectsApi, name, namespace) -> return True +def create_resource_from_manifest(custom_objects: CustomObjectsApi, body, namespace, plural) -> None: + """ + Create a Resource based on manifest. + + :param custom_objects: CustomObjectsApi + :param body: manifest body + :param namespace: namespace where the resource will be created + :param plural: the plural of the resource + """ + try: + print("Create a Custom Resource: " + body["kind"]) + group, version = body["apiVersion"].split("/") + custom_objects.create_namespaced_custom_object(group, version, namespace, plural, body) + print(f"Custom resource {body['kind']} created with name '{body['metadata']['name']}'") + except ApiException as ex: + logging.exception(f"Exception: {ex} occurred while creating {body['kind']}: {body['metadata']['name']}") + raise ex + + def read_custom_resource_v1alpha1(custom_objects: CustomObjectsApi, namespace, plural, name) -> object: """ Get CRD information (kubectl describe output)