Kops - migrate presubmit network plugin jobs to kubetest2

config/jobs/kubernetes/kops/build_jobs.py

@@ -19,7 +19,7 @@
 import boto3 # pylint: disable=import-error
 import jinja2 # pylint: disable=import-error
 
-job_template = """
+periodic_template = """
 - name: {{job_name}}
   cron: '{{cron}}'
   labels:
@@ -91,6 +91,72 @@
           memory: 3Gi
 """
 
+presubmit_template = """
+  - name: {{job_name}}
+    branches:
+    - master
+    run_if_changed: '{{run_if_changed}}'
+    skip_report: {{skip_report}}
+    labels:
+      preset-service-account: "true"
+      preset-aws-ssh: "true"
+      preset-aws-credential: "true"
+      preset-bazel-scratch-dir: "true"
+      preset-bazel-remote-cache-enabled: "true"
+      preset-dind-enabled: "true"
+    decorate: true
+    decoration_config:
+      timeout: {{job_timeout}}
+    path_alias: k8s.io/kops
+    spec:
+      containers:
+      - image: gcr.io/k8s-testimages/kubekins-e2e:v20210327-170ffe2-master
+        imagePullPolicy: Always
+        command:
+        - runner.sh
+        args:
+        - bash
+        - -c
+        - |
+            make test-e2e-install
+            kubetest2 kops \\
+            -v 2 \\
+            --up --build --down \\
+            --cloud-provider=aws \\
+            --create-args="{{create_args}}" \\
+            --kubernetes-version={{k8s_deploy_url}} \\
+            {%- if terraform_version %}
+            --terraform-version={{terraform_version}} \\
+            {%- endif %}
+            --test=kops \\
+            -- \\
+            --ginkgo-args="--debug" \\
+            --test-args="-test.timeout={{test_timeout}} -num-nodes=0" \\
+            {%- if test_package_bucket %}
+            --test-package-bucket={{test_package_bucket}} \\
+            {%- endif %}
+            {%- if test_package_dir %}
+            --test-package-dir={{test_package_dir}} \\
+            {%- endif %}
+            --test-package-marker={{marker}} \\
+            --parallel={{test_parallelism}} \\
+            {%- if focus_regex %}
+            --focus-regex="{{focus_regex}}" \\
+            {%- endif %}
+            --skip-regex="{{skip_regex}}"
+        securityContext:
+          privileged: true
+        env:
+        - name: KUBE_SSH_KEY_PATH
+          value: /etc/aws-ssh/aws-ssh-private
+        - name: KUBE_SSH_USER
+          value: {{kops_ssh_user}}
+        resources:
+          requests:
+            cpu: "2"
+            memory: "6Gi"
+"""
+
 # We support rapid focus on a few tests of high concern
 # This should be used for temporary tests we are evaluating,
 # and ideally linked to a bug, and removed once the bug is fixed
@@ -159,6 +225,42 @@ def should_skip_newer_k8s(k8s_version, kops_version):
         return True
     return float(k8s_version) > float(kops_version)
 
+def k8s_version_info(k8s_version):
+    test_package_bucket = ''
+    test_package_dir = ''
+    if k8s_version == 'latest':
+        marker = 'latest.txt'
+        k8s_deploy_url = "https://storage.googleapis.com/kubernetes-release/release/latest.txt"
+    elif k8s_version == 'ci':
+        marker = 'latest.txt'
+        k8s_deploy_url = "https://storage.googleapis.com/kubernetes-release-dev/ci/latest.txt"
+        test_package_bucket = 'kubernetes-release-dev'
+        test_package_dir = 'ci'
+    elif k8s_version == 'stable':
+        marker = 'stable.txt'
+        k8s_deploy_url = "https://storage.googleapis.com/kubernetes-release/release/stable.txt"
+    elif k8s_version:
+        marker = f"stable-{k8s_version}.txt"
+        k8s_deploy_url = f"https://storage.googleapis.com/kubernetes-release/release/stable-{k8s_version}.txt" # pylint: disable=line-too-long
+    else:
+        raise Exception('missing required k8s_version')
+    return marker, k8s_deploy_url, test_package_bucket, test_package_dir
+
+def create_args(kops_channel, networking, container_runtime, extra_flags, kops_image):
+    args = f"--channel={kops_channel} --networking=" + (networking or "kubenet")
+    if container_runtime:
+        args += f" --container-runtime={container_runtime}"
+
+    image_overridden = False
+    if extra_flags:
+        for arg in extra_flags:
+            if "--image=" in arg:
+                image_overridden = True
+            args = args + " " + arg
+    if not image_overridden:
+        args = f"--image='{kops_image}' {args}"
+    return args.strip()
+
 def latest_aws_image(owner, name):
     client = boto3.client('ec2', region_name='us-east-1')
     response = client.describe_images(
@@ -207,7 +309,7 @@ def latest_aws_image(owner, name):
 # Build Test #
 ##############
 
-# Returns a string representing the prow job YAML and the number of job invocations per week
+# Returns a string representing the periodic prow job and the number of job invocations per week
 def build_test(cloud='aws',
                distro='u2004',
                networking=None,
@@ -247,40 +349,8 @@ def build_test(cloud='aws',
     kops_image = distro_images[distro]
     kops_ssh_user = distros_ssh_user[distro]
 
-    test_package_bucket = ''
-    test_package_dir = ''
-    if k8s_version == 'latest':
-        marker = 'latest.txt'
-        k8s_deploy_url = "https://storage.googleapis.com/kubernetes-release/release/latest.txt"
-    elif k8s_version == 'ci':
-        marker = 'latest.txt'
-        k8s_deploy_url = "https://storage.googleapis.com/kubernetes-release-dev/ci/latest.txt"
-        test_package_bucket = 'kubernetes-release-dev'
-        test_package_dir = 'ci'
-    elif k8s_version == 'stable':
-        marker = 'stable.txt'
-        k8s_deploy_url = "https://storage.googleapis.com/kubernetes-release/release/stable.txt"
-    elif k8s_version:
-        marker = f"stable-{k8s_version}.txt"
-        k8s_deploy_url = f"https://storage.googleapis.com/kubernetes-release/release/stable-{k8s_version}.txt" # pylint: disable=line-too-long
-    else:
-        raise Exception('missing required k8s_version')
-
-    create_args = f"--channel={kops_channel} --networking=" + (networking or "kubenet")
-
-    if container_runtime:
-        create_args += f" --container-runtime={container_runtime}"
-
-    image_overridden = False
-    if extra_flags:
-        for arg in extra_flags:
-            if "--image=" in arg:
-                image_overridden = True
-            create_args = create_args + " " + arg
-    if not image_overridden:
-        create_args = f"--image='{kops_image}' {create_args}"
-
-    create_args = create_args.strip()
+    marker, k8s_deploy_url, test_package_bucket, test_package_dir = k8s_version_info(k8s_version)
+    args = create_args(kops_channel, networking, container_runtime, extra_flags, kops_image)
 
     skip_regex = r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[HPA\]|Dashboard|RuntimeClass|RuntimeHandler|Services.*functioning.*NodePort|Services.*rejected.*endpoints|Services.*affinity' # pylint: disable=line-too-long
     if networking == "cilium":
@@ -312,12 +382,12 @@ def build_test(cloud='aws',
 
     cron, runs_per_week = build_cron(tab, runs_per_day)
 
-    tmpl = jinja2.Template(job_template)
+    tmpl = jinja2.Template(periodic_template)
     job = tmpl.render(
         job_name=job_name,
         cron=cron,
         kops_ssh_user=kops_ssh_user,
-        create_args=create_args,
+        create_args=args,
         k8s_deploy_url=k8s_deploy_url,
         kops_deploy_url=kops_deploy_url,
         test_parallelism=str(test_parallelism),
@@ -374,6 +444,91 @@ def build_test(cloud='aws',
         output += f"  {line}\n"
     return output, runs_per_week
 
+# Returns a string representing a presubmit prow job YAML
+def presubmit_test(cloud='aws',
+                   distro='u2004',
+                   networking=None,
+                   container_runtime='docker',
+                   k8s_version='latest',
+                   kops_channel='alpha',
+                   name=None,
+                   tab_name=None,
+                   feature_flags=(),
+                   extra_flags=None,
+                   extra_dashboards=None,
+                   test_parallelism=25,
+                   test_timeout_minutes=60,
+                   skip_override=None,
+                   focus_regex=None,
+                   run_if_changed=None,
+                   skip_report=False):
+    # pylint: disable=too-many-statements,too-many-branches,too-many-arguments
+
+    kops_image = distro_images[distro]
+    kops_ssh_user = distros_ssh_user[distro]
+
+    marker, k8s_deploy_url, test_package_bucket, test_package_dir = k8s_version_info(k8s_version)
+    args = create_args(kops_channel, networking, container_runtime, extra_flags, kops_image)
+
+    tmpl = jinja2.Template(presubmit_template)
+    job = tmpl.render(
+        job_name=name,
+        kops_ssh_user=kops_ssh_user,
+        create_args=args,
+        k8s_deploy_url=k8s_deploy_url,
+        test_parallelism=str(test_parallelism),
+        job_timeout=str(test_timeout_minutes + 30) + 'm',
+        test_timeout=str(test_timeout_minutes) + 'm',
+        marker=marker,
+        skip_regex=skip_override,
+        kops_feature_flags=','.join(feature_flags),
+        test_package_bucket=test_package_bucket,
+        test_package_dir=test_package_dir,
+        focus_regex=focus_regex,
+        run_if_changed=run_if_changed,
+        skip_report='true' if skip_report else 'false',
+    )
+
+    spec = {
+        'cloud': cloud,
+        'networking': networking,
+        'distro': distro,
+        'k8s_version': k8s_version,
+        'container_runtime': container_runtime,
+        'kops_channel': kops_channel,
+    }
+    if feature_flags:
+        spec['feature_flags'] = ','.join(feature_flags)
+    if extra_flags:
+        spec['extra_flags'] = ' '.join(extra_flags)
+    jsonspec = json.dumps(spec, sort_keys=True)
+
+    dashboards = [
+        'presubmits-kops',
+        'kops-presubmits',
+        'sig-cluster-lifecycle-kops',
+        'kops-kubetest2',
+        f"kops-distro-{distro}",
+        f"kops-k8s-{k8s_version or 'latest'}",
+    ]
+    if extra_dashboards:
+        dashboards.extend(extra_dashboards)
+
+    annotations = {
+        'testgrid-dashboards': ', '.join(sorted(dashboards)),
+        'testgrid-days-of-results': '90',
+        'testgrid-tab-name': tab_name,
+    }
+    for (k, v) in spec.items():
+        annotations[f"test.kops.k8s.io/{k}"] = v or ""
+
+    extra = yaml.dump({'annotations': annotations}, width=9999, default_flow_style=False)
+
+    output = f"\n# {jsonspec}{job}\n"
+    for line in extra.splitlines():
+        output += f"    {line}\n"
+    return output
+
 ####################
 # Grid Definitions #
 ####################
@@ -702,6 +857,52 @@ def generate_pipeline():
         )
     return results
 
+########################################
+# kops-presubmits-network-plugins.yaml #
+########################################
+def generate_presubmit_network_plugins():
+    plugins = {
+        'amazonvpc': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.amazon-vpc-routed-eni\/|pkg\/model\/(firewall|components\/kubeproxy|iam\/iam_builder).go|nodeup\/pkg\/model\/(context|kubelet).go|upup\/pkg\/fi\/cloudup\/defaults.go)', # pylint: disable=line-too-long
+        'calico': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.projectcalico\.org\/|pkg\/model\/(firewall.go|pki.go|iam\/iam_builder.go)|nodeup\/pkg\/model\/networking\/calico.go)', # pylint: disable=line-too-long
+        'canal': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.projectcalico\.org\.canal\/|nodeup\/pkg\/model\/networking\/(flannel|canal).go)', # pylint: disable=line-too-long
+        'cilium': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.cilium\.io\/|pkg\/model\/(firewall|components\/cilium|iam\/iam_builder).go|nodeup\/pkg\/model\/(context|networking\/cilium).go|upup\/pkg\/fi\/cloudup\/template_functions.go)', # pylint: disable=line-too-long
+        'flannel': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.flannel\/|nodeup\/pkg\/model\/(sysctls|networking\/flannel).go|upup\/pkg\/fi\/cloudup\/template_functions.go)', # pylint: disable=line-too-long
+        'kuberouter': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.kuberouter\/|upup\/pkg\/fi\/cloudup\/template_functions.go)', # pylint: disable=line-too-long
+        'weave': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.weave\/|upup\/pkg\/fi\/cloudup\/template_functions.go)' # pylint: disable=line-too-long
+    }
+    results = []
+    skip_base = r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[HPA\]|Dashboard|RuntimeClass|RuntimeHandler' # pylint: disable=line-too-long
+    for plugin, run_if_changed in plugins.items():
+        networking_arg = plugin
+        skip_regex = skip_base
+        if plugin == 'cilium':
+            skip_regex += r'|should.set.TCP.CLOSE_WAIT'
+        else:
+            skip_regex += r'|Services.*functioning.*NodePort'
+        if plugin in ['calico', 'canal', 'weave', 'cilium']:
+            skip_regex += r'|Services.*rejected.*endpoints'
+        if plugin == 'kuberouter':
+            skip_regex += r'|load-balancer|hairpin|affinity\stimeout|service\.kubernetes\.io|CLOSE_WAIT' # pylint: disable=line-too-long
+            networking_arg = 'kube-router'
+        if plugin in ['canal', 'flannel']:
+            skip_regex += r'|up\sand\sdown|headless|service-proxy-name'
+        results.append(
+            presubmit_test(
+                container_runtime='containerd',
+                k8s_version='stable',
+                kops_channel='alpha',
+                name=f"pull-kops-e2e-cni-{plugin}",
+                tab_name=f"e2e-{plugin}",
+                networking=networking_arg,
+                extra_flags=['--node-size=t3.large'],
+                extra_dashboards=['kops-network-plugins'],
+                skip_override=skip_regex,
+                run_if_changed=run_if_changed,
+                skip_report=False,
+            )
+        )
+    return results
+
 ########################
 # YAML File Generation #
 ########################
@@ -714,6 +915,10 @@ def generate_pipeline():
     'kops-periodics-pipeline.yaml': generate_pipeline,
 }
 
+presubmits_files = {
+    'kops-presubmits-network-plugins.yaml': generate_presubmit_network_plugins,
+}
+
 def main():
     for filename, generate_func in periodics_files.items():
         print(f"Generating {filename}")
@@ -724,11 +929,24 @@ def main():
             output.append(res[0])
             runs_per_week += res[1]
             job_count += 1
-        output.insert(0, "# Test jobs generated by build_grid.py (do not manually edit)\n")
+        output.insert(0, "# Test jobs generated by build_jobs.py (do not manually edit)\n")
         output.insert(1, f"# {job_count} jobs, total of {runs_per_week} runs per week\n")
         output.insert(2, "periodics:\n")
         with open(filename, 'w') as fd:
             fd.write(''.join(output))
+    for filename, generate_func in presubmits_files.items():
+        print(f"Generating {filename}")
+        output = []
+        job_count = 0
+        for res in generate_func():
+            output.append(res)
+            job_count += 1
+        output.insert(0, "# Test jobs generated by build_jobs.py (do not manually edit)\n")
+        output.insert(1, f"# {job_count} jobs\n")
+        output.insert(2, "presubmits:\n")
+        output.insert(3, "  kubernetes/kops:\n")
+        with open(filename, 'w') as fd:
+            fd.write(''.join(output))
 
 if __name__ == "__main__":
     main()

config/jobs/kubernetes/kops/kops-periodics-distros.yaml

@@ -1,4 +1,4 @@
-# Test jobs generated by build_grid.py (do not manually edit)
+# Test jobs generated by build_jobs.py (do not manually edit)
 # 10 jobs, total of 210 runs per week
 periodics:
 

config/jobs/kubernetes/kops/kops-periodics-grid.yaml

@@ -1,4 +1,4 @@
-# Test jobs generated by build_grid.py (do not manually edit)
+# Test jobs generated by build_jobs.py (do not manually edit)
 # 560 jobs, total of 1040 runs per week
 periodics:
 

config/jobs/kubernetes/kops/kops-periodics-misc2.yaml

@@ -1,4 +1,4 @@
-# Test jobs generated by build_grid.py (do not manually edit)
+# Test jobs generated by build_jobs.py (do not manually edit)
 # 12 jobs, total of 462 runs per week
 periodics:
 

config/jobs/kubernetes/kops/kops-periodics-network-plugins.yaml

@@ -1,4 +1,4 @@
-# Test jobs generated by build_grid.py (do not manually edit)
+# Test jobs generated by build_jobs.py (do not manually edit)
 # 8 jobs, total of 168 runs per week
 periodics: