add files for upgrade tests (#5553)

tests/Makefile

@@ -91,3 +91,10 @@ image-load: ## Load the image into the Kind K8S cluster
 test-lint: ## Run Python linting tools
 	isort .
 	black .
+
+
+.PHONY: upgrade-resources
+upgrade-resources:
+## Create and delete resources for upgrade tests e.g. `make upgrade-resources PYTEST_ARGS="create OR delete"`
+	pip install -r ../tests/requirements.txt --no-deps
+	pytest -v -s -m $(PYTEST_ARGS)

tests/conftest.py

@@ -120,6 +120,12 @@ def pytest_addoption(parser) -> None:
         default=os.environ.get("AZURE_AD_AUTOMATION"),
         help="Azure active directory secret for JWKs",
     )
+    parser.addoption(
+        "--num",
+        action="store",
+        default="1",
+        help="Number of resources to deploy for upgrade tests",
+    )
 
 
 # import fixtures into pytest global namespace

tests/data/upgrade-test-resources/deployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backend
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: backend
+  template:
+    metadata:
+      labels:
+        app: backend
+    spec:
+      containers:
+      - name: backend
+        image: nginxdemos/nginx-hello:plain-text
+        ports:
+        - containerPort: 8080

tests/data/upgrade-test-resources/ingress.yaml

@@ -0,0 +1,21 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ingress
+spec:
+  ingressClassName: nginx
+  tls:
+  - hosts:
+    - cafe.example.com
+    secretName: secret
+  rules:
+  - host: cafe.example.com
+    http:
+      paths:
+      - path: /backend
+        pathType: Prefix
+        backend:
+          service:
+            name: backend-svc
+            port:
+              number: 80

tests/data/upgrade-test-resources/ns.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ns

tests/data/upgrade-test-resources/secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: test-secret
+type: kubernetes.io/tls
+data:
+  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ4RENDQXRpZ0F3SUJBZ0lKQU9jbHdCelprYmlhTUEwR0NTcUdTSWIzRFFFQkJRVUFNRmd4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFaE1COEdBMVVFQ2hNWVNXNTBaWEp1WlhRZ1YybGtaMmwwY3lCUQpkSGtnVEhSa01Sa3dGd1lEVlFRREV4QmpZV1psTG1WNFlXMXdiR1V1WTI5dE1CNFhEVEUzTURnek1URXdNVGN5Ck1Gb1hEVEU0TURnek1URXdNVGN5TUZvd1dERUxNQWtHQTFVRUJoTUNWVk14Q3pBSkJnTlZCQWdUQWtOQk1TRXcKSHdZRFZRUUtFeGhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXhHVEFYQmdOVkJBTVRFR05oWm1VdQpaWGhoYlhCc1pTNWpiMjB3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRQzdIT0xJCm5oZjE1aUcxOE16RXBzN0lvZmxHQmovMi9NVjA0OWtBS0hrTnZOem1XaXRXWDV2QU1yRkF5THY0dXBIWDI5b0IKa3l6YUhlYyt2TlFibEh1bStINUtkUWZXWHFXNkJ6UnVhMzBreEkrcG91cnhpNy9jaDJORS92djBhRGtvaTJ0RAovOUI2aHAyVkoxWXFJdm9hQ2wwSmFYWDd0WEc4SGU1S1BSZzBYMm1Mblcwa29tay9ZVGRPbS9xOVRjUDRnUmhrCms3bUhJMDlSME5vNUhTbURydmVBWFEyY3lGWVJQVUNjWkNPd0h6UUdVUVB1UU0wNVArWUNnVlNRcElKWFV0b0kKbGdEMHhZZUw4UU1rZjZ0TWpYcXpTVVRhQlhzNkRKU2x1YWN1aHpkV212UnFPUVNNYlVpZ3dVUEZCRDVLRUFIcwozM0hHeVZ5dkI4cVlrYUczQWdNQkFBR2pnYnd3Z2Jrd0hRWURWUjBPQkJZRUZOdTQvMTdpSituRGxPMkoyVisvCitqM2x5SzVZTUlHSkJnTlZIU01FZ1lFd2Y0QVUyN2ovWHVJbjZjT1U3WW5aWDcvNlBlWElybGloWEtSYU1GZ3gKQ3pBSkJnTlZCQVlUQWxWVE1Rc3dDUVlEVlFRSUV3SkRRVEVoTUI4R0ExVUVDaE1ZU1c1MFpYSnVaWFFnVjJsawpaMmwwY3lCUWRIa2dUSFJrTVJrd0Z3WURWUVFERXhCallXWmxMbVY0WVcxd2JHVXVZMjl0Z2drQTV5WEFITm1SCnVKb3dEQVlEVlIwVEJBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRVUZBQU9DQVFFQUtGUHJBcXA3a3lzTDVGNnMKWFhWdXZkZzAyc0srUlpzb2F3QWVxbHlSRmpJeUlQL2VTajBhQjQwQmNOcWFyRHhwNjhBd1pZNG4yQk9EVmo5WgphOFlvV1YyOFpwamloaThxNnBPSElOa0MrOXpCY1hsZ2lvVUZBTERCcXFPTXFUZkw1cjNGejNUTGN1clozajhuCnUzL2hRVHNXZG5TZENWbmN0aXhaUHJ5cnhJSFlWSERiVHF4ZWdTQUN6WkU1MHMwdlRpMFJkNUkrcVdubVpIUloKL0hLNVZnNWlNS2E1clBPRTFaT2M3L2VnVjZ6R2p4THJiNEdlQ2JyTjBBb01tazNpL2d2K2kzL0N6aTlXOVhNNApwa2hQSjJUcEtMSjVaOHgxUVhjUW5Dem5yOEdtL2FuVzV4b3lDdWhjZzlXMlVSYzRKVTZ1UXh0WU9tczYrc0RxCjBBN1Y3UT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+  tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdXh6aXlKNFg5ZVlodGZETXhLYk95S0g1UmdZLzl2ekZkT1BaQUNoNURiemM1bG9yClZsK2J3REt4UU1pNytMcVIxOXZhQVpNczJoM25QcnpVRzVSN3B2aCtTblVIMWw2bHVnYzBibXQ5Sk1TUHFhTHEKOFl1LzNJZGpSUDc3OUdnNUtJdHJRLy9RZW9hZGxTZFdLaUw2R2dwZENXbDErN1Z4dkIzdVNqMFlORjlwaTUxdApKS0pwUDJFM1RwdjZ2VTNEK0lFWVpKTzVoeU5QVWREYU9SMHBnNjczZ0YwTm5NaFdFVDFBbkdRanNCODBCbEVECjdrRE5PVC9tQW9GVWtLU0NWMUxhQ0pZQTlNV0hpL0VESkgrclRJMTZzMGxFMmdWN09neVVwYm1uTG9jM1ZwcjAKYWprRWpHMUlvTUZEeFFRK1NoQUI3Tjl4eHNsY3J3ZkttSkdodHdJREFRQUJBb0lCQUVuZHpXbUZmOUFEV2F1Sgp0RXl0elZSSEhURVhwb2pLb09qVVNnWlY4L1FJYXV4RkRIYThwNi9vVXpGUURXVFR3bCtFMnp0ajdvRHM3UzFIClBqVGxHU3VCVGRuMitYRVhURFYwUXE2VW9JS3pWa09SblU1ZDdSQVNJbzVLV3d6UldEODVTczg5WGdBQXhKVHQKUW9hLzZCdi9tMXJyMXpmWEdWODZNYWY5Rm1FVjI5bmpCcHZ3cUpzOFlUaFU5VE5BRUdnbkxBWGFJYTJZSUJTSgozcVhJd1RrdFBHUUZyZUowdnBiOVlaTkRxWk0rMmI1K3BBVEVXclpyeTQxTlpvdzhNUnVOYnoxVU5sWXkyazZKCjlXclUxUDVYM2l3dEZmcXA3TzREakNOMFR2Y1Y1Nm5QczJoYldDeHp4RmxFalJwNW5KQ3Bsdi9yNCt4eHJVRWQKd0NjU0x3RUNnWUVBOGVDUGM2S2JpSXVFUWxwVjcxTXRQMjdBZzhkMzN5aWZKajAvQ3R3RlF6dlhWcHZCSHFRagphUE1ZaEswZ3o0MzFHMGtqVUpTeUU2STJYVWhFU3gxZUw2TU5sVS9xc29Uc0JaZEV3N0cvMFRhdW1TQUJWemZPCkIvckdtbEZkZitxWVpiR3pIdGtvbVJWb1JqVWtPcGRJL3RnL1ZlSmJ3K1RIS2dYS01NU2V5cjhDZ1lFQXhnbTkKWXN5dVZVUGlEdDhuL1JXeEJwZ1Zqazk0M3BFOVFjK2FVK0VPcGROK2NNQjkvaGJiWUpMZEZ2WUUwd2s1cEQ1SQpibktscjZycndEVUJKNWY2TXM0L3gyS3JISjlCQ3VNT3JyTEVTVm42ZG9NdDhWdEtpQkVLTVFuSnIzYmM4UDV3CnpvVmNhRGl1dCtISjcySG83cG5QQTFhaS9QV1ZwM2pZUTlCSXZ3a0NnWUJRNzkzUXlmYlZxQ25uc2liVFlMZmgKWkFRVGxLbXVDUC9JWWZJNGhndFV4aTkya2NQN3B0MGFmMDRUQjRQVk1DRjJzZkNaUkVpYWZVdEh4Nmppb2I4awpuYUVyOTRRSG5LY0Y3K3BZdWFBQU9CWVFzejcvbW5MZEJMTjBiQW1uaGk3Y3lLdXhoT1VxNUpqeDlWSmNNTWVDClQ0WlNETjY4SEUvdzVlTVVrcGE0TFFLQmdGM0piUXhtUE1XYW9XdERtYytNdjBxTktlQThtTlJtMmlqWnBZL0YKek1jUnN4YTR3ckpicHNkRXBqbmlod1Jlb1JLOGdGYjJLcXRYK2RBTUNpRHpJNFYrRWN4ZVdRVDBFcnlTTFhqawpwbnJLaHdnck5jM1EyeW8zVDZsTHBsMVhvR2p0UndVM09UME9Zd2dvZ1JiQ09xc000bklGVEtrWnNTY2YzdU8yCnQwenBBb0dCQU5Fc1V4Yit6UHpKbEgwL2hOTlhteisvNGx4aXlJb1ZQQXgzaEtBemtjOGtkZzhoS25XWkZhK0YKTjZMOXZjTDhNbnRjNHpmVDdDanNxWGJ2cGUzRUFOeGk4TWRSZzd1Z093L2NiZWdLVklZY3hTdXdRKzYrNUZYOApKRzlhbGxzdXovTk40b2RpMzRvblpEVmRZcURnY2xaZnZucXZKMHZWSzc1VXhQZ3F3aUJQCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==

tests/data/upgrade-test-resources/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: backend-svc
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: backend

tests/data/upgrade-test-resources/tcp-deployment.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tcp
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: tcp
+  template:
+    metadata:
+      labels:
+        app: tcp
+    spec:
+      containers:
+        - name: tcp-service
+          image: ghcr.io/nginxinc/kic-test-tcp-server:0.2.1
+          ports:
+            - containerPort: 3333
+              name: tcp-server
+              protocol: TCP

tests/data/upgrade-test-resources/transport-server.yaml

@@ -0,0 +1,14 @@
+apiVersion: k8s.nginx.org/v1
+kind: TransportServer
+metadata:
+  name: transport-server
+spec:
+  listener:
+    name: tcp-server
+    protocol: TCP
+  upstreams:
+    - name: tcp-app
+      service: tcp-service
+      port: 3333
+  action:
+    pass: tcp-app

tests/data/upgrade-test-resources/virtual-server.yaml

@@ -0,0 +1,16 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+  name: vs
+spec:
+  host: vs.example.com
+  tls:
+    secret: secret
+  upstreams:
+  - name: backend
+    service: backend-svc
+    port: 80
+  routes:
+  - path: "/backend"
+    action:
+      pass: backend

tests/suite/test_upgrade_resources.py

@@ -0,0 +1,128 @@
+import os
+import tempfile
+
+import pytest
+import yaml
+from settings import DEPLOYMENTS, TEST_DATA
+from suite.utils.custom_resources_utils import create_resource_from_manifest, read_custom_resource
+from suite.utils.resources_utils import (
+    create_ingress,
+    create_items_from_yaml,
+    create_namespace,
+    delete_namespace,
+    wait_before_test,
+)
+from suite.utils.vs_vsr_resources_utils import create_virtual_server
+
+tcp_deployment = f"{TEST_DATA}/upgrade-test-resources/tcp-deployment.yaml"
+deployment = f"{TEST_DATA}/upgrade-test-resources/deployment.yaml"
+service = f"{TEST_DATA}/upgrade-test-resources/service.yaml"
+ns = f"{TEST_DATA}/upgrade-test-resources/ns.yaml"
+ingress = f"{TEST_DATA}/upgrade-test-resources/ingress.yaml"
+vs = f"{TEST_DATA}/upgrade-test-resources/virtual-server.yaml"
+ts = f"{TEST_DATA}/upgrade-test-resources/transport-server.yaml"
+secret = f"{TEST_DATA}/upgrade-test-resources/secret.yaml"
+
+# Below test class only deployes resources for upgrade testing, IC deployment should be done manually via helm.
+
+
+class TestUpgrade:
+    @pytest.mark.create
+    def test_create(self, request, kube_apis):
+        count = int(request.config.getoption("--num"))
+
+        for i in range(1, count + 1):
+            with open(ns) as f:
+                doc = yaml.safe_load(f)
+                doc["metadata"]["name"] = f"ns-{i}"
+                with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp:
+                    temp.write(yaml.safe_dump(doc) + "---\n")
+                namespace = create_namespace(kube_apis.v1, doc)
+                os.remove(temp.name)
+
+            with open(deployment) as f:
+                doc = yaml.safe_load(f)
+                doc["metadata"]["name"] = f"backend-{i}"
+                doc["spec"]["selector"]["matchLabels"]["app"] = f"backend-{i}"
+                doc["spec"]["template"]["metadata"]["labels"]["app"] = f"backend-{i}"
+                doc["metadata"]["name"] = f"backend-{i}"
+                with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp:
+                    temp.write(yaml.safe_dump(doc) + "---\n")
+                create_items_from_yaml(kube_apis, temp.name, namespace)
+                os.remove(temp.name)
+
+            with open(service) as f:
+                doc = yaml.safe_load(f)
+                doc["metadata"]["name"] = f"backend-svc-{i}"
+                doc["spec"]["selector"]["app"] = f"backend-{i}"
+                with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp:
+                    temp.write(yaml.safe_dump(doc) + "---\n")
+                create_items_from_yaml(kube_apis, temp.name, namespace)
+                os.remove(temp.name)
+
+            with open(secret) as f:
+                doc = yaml.safe_load(f)
+                doc["metadata"]["name"] = f"secret-{i}"
+                with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp:
+                    temp.write(yaml.safe_dump(doc) + "---\n")
+                create_items_from_yaml(kube_apis, temp.name, namespace)
+                os.remove(temp.name)
+
+            # VirtualServer
+            with open(vs) as f:
+                doc = yaml.safe_load(f)
+                doc["metadata"]["name"] = f"vs-{i}"
+                doc["spec"]["host"] = f"vs-{i}.example.com"
+                doc["spec"]["tls"]["secret"] = f"secret-{i}"
+                doc["spec"]["upstreams"][0]["name"] = f"backend-{i}"
+                doc["spec"]["upstreams"][0]["service"] = f"backend-svc-{i}"
+                doc["spec"]["routes"][0]["action"]["pass"] = f"backend-{i}"
+                create_virtual_server(kube_apis.custom_objects, doc, namespace)
+
+            # Ingress
+            with open(ingress) as f:
+                doc = yaml.safe_load(f)
+                doc["metadata"]["name"] = f"ingress-{i}"
+                doc["spec"]["tls"][0]["hosts"][0] = f"ingress-{i}.example.com"
+                doc["spec"]["tls"][0]["secretName"] = f"secret-{i}"
+                doc["spec"]["rules"][0]["host"] = f"ingress-{i}.example.com"
+                doc["spec"]["rules"][0]["http"]["paths"][0]["path"] = f"/backend-{i}"
+                doc["spec"]["rules"][0]["http"]["paths"][0]["backend"]["service"]["name"] = f"backend-svc-{i}"
+                create_ingress(kube_apis.networking_v1, namespace, doc)
+
+            # TransportServer
+            with open(tcp_deployment) as f:
+                doc = yaml.safe_load(f)
+                doc["metadata"]["name"] = f"tcp-{i}"
+                doc["spec"]["selector"]["matchLabels"]["app"] = f"tcp-{i}"
+                doc["spec"]["template"]["metadata"]["labels"]["app"] = f"tcp-{i}"
+                with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp:
+                    temp.write(yaml.safe_dump(doc) + "---\n")
+                create_items_from_yaml(kube_apis, temp.name, namespace)
+                os.remove(temp.name)
+
+            with open(service) as f:
+                doc = yaml.safe_load(f)
+                doc["metadata"]["name"] = f"tcp-svc-{i}"
+                doc["spec"]["selector"]["app"] = f"tcp-{i}"
+                with tempfile.NamedTemporaryFile(mode="w+", suffix=".yml", delete=False) as temp:
+                    temp.write(yaml.safe_dump(doc) + "---\n")
+                create_items_from_yaml(kube_apis, temp.name, namespace)
+                os.remove(temp.name)
+
+            with open(ts) as f:
+                doc = yaml.safe_load(f)
+                doc["metadata"]["name"] = f"ts-{i}"
+                doc["spec"]["listener"]["name"] = "dns-tcp"
+                doc["spec"]["upstreams"][0]["name"] = f"tcp-{i}"
+                doc["spec"]["upstreams"][0]["service"] = f"tcp-svc-{i}"
+                doc["spec"]["upstreams"][0]["port"] = 5353
+                doc["spec"]["action"]["pass"] = f"tcp-{i}"
+                create_resource_from_manifest(kube_apis.custom_objects, doc, namespace, "transportservers")
+
+    @pytest.mark.delete
+    def test_delete(self, request, kube_apis):
+        count = int(request.config.getoption("--num"))
+        # delete namespaces
+        for i in range(1, count + 1):
+            delete_namespace(kube_apis.v1, f"ns-{i}")

tests/suite/utils/custom_resources_utils.py

@@ -101,6 +101,25 @@ def is_dnsendpoint_present(custom_objects: CustomObjectsApi, name, namespace) ->
     return True
 
 
+def create_resource_from_manifest(custom_objects: CustomObjectsApi, body, namespace, plural) -> None:
+    """
+    Create a Resource based on manifest.
+
+    :param custom_objects: CustomObjectsApi
+    :param body: manifest body
+    :param namespace: namespace where the resource will be created
+    :param plural: the plural of the resource
+    """
+    try:
+        print("Create a Custom Resource: " + body["kind"])
+        group, version = body["apiVersion"].split("/")
+        custom_objects.create_namespaced_custom_object(group, version, namespace, plural, body)
+        print(f"Custom resource {body['kind']} created with name '{body['metadata']['name']}'")
+    except ApiException as ex:
+        logging.exception(f"Exception: {ex} occurred while creating {body['kind']}: {body['metadata']['name']}")
+        raise ex
+
+
 def read_custom_resource_v1alpha1(custom_objects: CustomObjectsApi, namespace, plural, name) -> object:
     """
     Get CRD information (kubectl describe output)