From 21aba10e08374db3a8728b5da2255e552f9e8996 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 21 May 2024 16:00:49 +0200
Subject: [PATCH 01/11] Use alternate self-sufficient shellcheck precommit

This pre-commit does not require prerequisite on the host, making it
easier to run in CI workflows.
---
 .gitlab-ci.yml            |  1 -
 .gitlab-ci/shellcheck.yml | 16 ----------------
 .pre-commit-config.yaml   |  6 +++---
 3 files changed, 3 insertions(+), 20 deletions(-)
 delete mode 100644 .gitlab-ci/shellcheck.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 13c314dfa7b..0c642985c19 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -77,7 +77,6 @@ ci-authorized:
 include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml
-  - .gitlab-ci/shellcheck.yml
   - .gitlab-ci/terraform.yml
   - .gitlab-ci/packet.yml
   - .gitlab-ci/vagrant.yml
diff --git a/.gitlab-ci/shellcheck.yml b/.gitlab-ci/shellcheck.yml
deleted file mode 100644
index 307e121c5c4..00000000000
--- a/.gitlab-ci/shellcheck.yml
+++ /dev/null
@@ -1,16 +0,0 @@
----
-shellcheck:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  variables:
-    SHELLCHECK_VERSION: v0.7.1
-  before_script:
-    - ./tests/scripts/rebase.sh
-    - curl --silent --location "https://github.com/koalaman/shellcheck/releases/download/"${SHELLCHECK_VERSION}"/shellcheck-"${SHELLCHECK_VERSION}".linux.x86_64.tar.xz" | tar -xJv
-    - cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/
-    - shellcheck --version
-  script:
-    # Run shellcheck for all *.sh
-    - find . -name '*.sh' -not -path './.git/*' | xargs shellcheck --severity error
-  except: ['triggers', 'master']
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 337b484f199..ec011eaa9c6 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,11 +27,11 @@ repos:
         args: [-r, "~MD013,~MD029"]
         exclude: "^.git"
 
-  - repo: https://github.com/jumanjihouse/pre-commit-hooks
-    rev: 3.0.0
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.10.0.1
     hooks:
       - id: shellcheck
-        args: [--severity, "error"]
+        args: ["--severity=error"]
         exclude: "^.git"
         files: "\\.sh$"
 

From f6d9ff4196d3b0ad7e5e3bf72d6446b97a658c4a Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 21 May 2024 20:57:06 +0200
Subject: [PATCH 02/11] Switch to upstream ansible-lint pre-commit hook

This way, the hook is self contained and does not depend on a previous
virtualenv installation.
---
 .pre-commit-config.yaml | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ec011eaa9c6..81a2ebf4513 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -35,16 +35,18 @@ repos:
         exclude: "^.git"
         files: "\\.sh$"
 
-  - repo: local
+  - repo: https://github.com/ansible/ansible-lint
+    rev: v24.5.0
     hooks:
       - id: ansible-lint
-        name: ansible-lint
-        entry: ansible-lint -v
-        language: python
-        pass_filenames: false
         additional_dependencies:
-          - .[community]
+          - ansible==9.5.1
+          - jsonschema==4.22.0
+          - jmespath==1.0.1
+          - netaddr==1.2.1
 
+  - repo: local
+    hooks:
       - id: ansible-syntax-check
         name: ansible-syntax-check
         entry: env ANSIBLE_INVENTORY=inventory/local-tests.cfg ANSIBLE_REMOTE_USER=root ANSIBLE_BECOME="true" ANSIBLE_BECOME_USER=root ANSIBLE_VERBOSITY="3" ansible-playbook --syntax-check

From 0e449ca75e10d00a84ffec34251e956ae66303c4 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 21 May 2024 20:59:51 +0200
Subject: [PATCH 03/11] pre-commit: fix hooks dependencies

- ansible-syntax-check
- tox-inventory-builder
- jinja-syntax-check
---
 .pre-commit-config.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 81a2ebf4513..0a4b32681c5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -52,12 +52,16 @@ repos:
         entry: env ANSIBLE_INVENTORY=inventory/local-tests.cfg ANSIBLE_REMOTE_USER=root ANSIBLE_BECOME="true" ANSIBLE_BECOME_USER=root ANSIBLE_VERBOSITY="3" ansible-playbook --syntax-check
         language: python
         files: "^cluster.yml|^upgrade-cluster.yml|^reset.yml|^extra_playbooks/upgrade-only-k8s.yml"
+        additional_dependencies:
+          - ansible==9.5.1
 
       - id: tox-inventory-builder
         name: tox-inventory-builder
         entry: bash -c "cd contrib/inventory_builder && tox"
         language: python
         pass_filenames: false
+        additional_dependencies:
+          - tox==4.15.0
 
       - id: check-readme-versions
         name: check-readme-versions
@@ -84,4 +88,4 @@ repos:
         types:
           - jinja
         additional_dependencies:
-          - Jinja2
+          - jinja2

From 77bfb53455865ea739abcd68178821c2e488fdfc Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 21 May 2024 21:31:32 +0200
Subject: [PATCH 04/11] Fix ci-matrix pre-commit hook

- Remove dependency of pydblite which fails to setup on recent pythons
- Discard shell script and put everything into pre-commit
---
 .gitlab-ci/lint.yml                     |  3 ++-
 .pre-commit-config.yaml                 |  8 ++++--
 tests/scripts/md-table/main.py          | 36 ++++++++++++-------------
 tests/scripts/md-table/requirements.txt |  4 ---
 tests/scripts/md-table/test.sh          | 11 --------
 5 files changed, 26 insertions(+), 36 deletions(-)
 delete mode 100644 tests/scripts/md-table/requirements.txt
 delete mode 100755 tests/scripts/md-table/test.sh

diff --git a/.gitlab-ci/lint.yml b/.gitlab-ci/lint.yml
index 2f96fec5fa2..658bd35aa32 100644
--- a/.gitlab-ci/lint.yml
+++ b/.gitlab-ci/lint.yml
@@ -123,4 +123,5 @@ ci-matrix:
   tags: [light]
   image: python:3
   script:
-    - tests/scripts/md-table/test.sh
+    - tests/scripts/md-table/main.py
+    - git diff --exit-code
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0a4b32681c5..e4116c1fae0 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -77,9 +77,13 @@ repos:
 
       - id: ci-matrix
         name: ci-matrix
-        entry: tests/scripts/md-table/test.sh
-        language: script
+        entry: tests/scripts/md-table/main.py
+        language: python
         pass_filenames: false
+        additional_dependencies:
+          - jinja2
+          - pathlib
+          - pyaml
 
       - id: jinja-syntax-check
         name: jinja-syntax-check
diff --git a/tests/scripts/md-table/main.py b/tests/scripts/md-table/main.py
index 9e000058564..09d5506e9bb 100755
--- a/tests/scripts/md-table/main.py
+++ b/tests/scripts/md-table/main.py
@@ -4,7 +4,6 @@
 import glob
 from pathlib import Path
 import yaml
-from pydblite import Base
 import re
 import jinja2
 import sys
@@ -14,6 +13,7 @@
 
 parser = argparse.ArgumentParser(description='Generate a Markdown table representing the CI test coverage')
 parser.add_argument('--dir', default='tests/files/', help='folder with test yml files')
+parser.add_argument('--output', default='docs/developers/ci.md', help='output file')
 
 
 args = parser.parse_args()
@@ -24,25 +24,26 @@
 # Data represents CI coverage data matrix
 class Data:
     def __init__(self):
-        self.db = Base(':memory:')
-        self.db.create('container_manager', 'network_plugin', 'operating_system')
+        self.container_managers = set()
+        self.network_plugins = set()
+        self.os = set()
+        self.combination = set()
 
 
-    def set(self, container_manager, network_plugin, operating_system):
-        self.db.insert(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
-        self.db.commit()
-    def exists(self, container_manager, network_plugin, operating_system):
-        return len((self.db("container_manager") == container_manager) & (self.db("network_plugin") == network_plugin) & (self.db("operating_system") == operating_system)) > 0
+    def set(self, container_manager, network_plugin, os):
+        self.container_managers.add(container_manager)
+        self.network_plugins.add(network_plugin)
+        self.os.add(os)
+        self.combination.add(container_manager+network_plugin+os)
+
+    def exists(self, container_manager, network_plugin, os):
+        return (container_manager+network_plugin+os) in self.combination
 
     def jinja(self):
         template = env.get_template('table.md.j2')
-        container_engines = list(self.db.get_unique_ids('container_manager'))
-        network_plugins = list(self.db.get_unique_ids("network_plugin"))
-        operating_systems = list(self.db.get_unique_ids("operating_system"))
-
-        container_engines.sort()
-        network_plugins.sort()
-        operating_systems.sort()
+        container_engines = sorted(self.container_managers)
+        network_plugins = sorted(self.network_plugins)
+        operating_systems = sorted(self.os)
 
         return template.render(
             container_engines=container_engines,
@@ -91,6 +92,5 @@ def markdown(self):
     network_plugin = y.get('kube_network_plugin', 'calico')
     x = re.match(r"^[a-z-]+_([a-z0-9]+).*", f.name)
     operating_system = x.group(1)
-    data.set(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
-#print(data.markdown())
-print(data.jinja())
+    data.set(container_manager=container_manager, network_plugin=network_plugin, os=operating_system)
+print(data.jinja(), file=open(args.output, 'w'))
diff --git a/tests/scripts/md-table/requirements.txt b/tests/scripts/md-table/requirements.txt
deleted file mode 100644
index 6d4aca36aa5..00000000000
--- a/tests/scripts/md-table/requirements.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-jinja2
-pathlib ; python_version < '3.10'
-pyaml
-pydblite
diff --git a/tests/scripts/md-table/test.sh b/tests/scripts/md-table/test.sh
deleted file mode 100755
index cf9df90856d..00000000000
--- a/tests/scripts/md-table/test.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-echo "Install requirements..."
-pip install -r ./tests/scripts/md-table/requirements.txt
-
-echo "Generate current file..."
-./tests/scripts/md-table/main.py > tmp.md
-
-echo "Compare docs/developers/ci.md with actual tests in tests/files/*.yml ..."
-cmp docs/developers/ci.md tmp.md

From d50f61eae5c8ef78d081ec078914198e95e3c161 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 21 May 2024 20:17:05 +0200
Subject: [PATCH 05/11] pre-commit: apply autofixes hooks and fix the rest
 manually

- markdownlint (manual fix)
- end-of-file-fixer
- requirements-txt-fixer
- trailing-whitespace
---
 contrib/terraform/nifcloud/README.md                      | 1 +
 contrib/terraform/upcloud/cluster-settings.tfvars         | 2 +-
 .../terraform/upcloud/modules/kubernetes-cluster/main.tf  | 2 +-
 .../upcloud/modules/kubernetes-cluster/variables.tf       | 2 +-
 contrib/terraform/upcloud/sample-inventory/cluster.tfvars | 2 +-
 docs/operations/recover-control-plane.md                  | 1 -
 inventory/sample/group_vars/etcd.yml                      | 2 +-
 requirements.txt                                          | 2 +-
 roles/container-engine/containerd/defaults/main.yml       | 2 +-
 .../container-engine/containerd/templates/config.toml.j2  | 1 -
 roles/etcd/defaults/main.yml                              | 2 +-
 .../gcp_pd/templates/gcp-pd-csi-controller.yml.j2         | 2 +-
 .../csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2    | 2 +-
 .../gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2        | 2 +-
 .../gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2           | 2 +-
 .../templates/vsphere-csi-controller-config.yml.j2        | 2 +-
 ...l-huawei-cloud-controller-manager-role-bindings.yml.j2 | 2 +-
 .../external-huawei-cloud-controller-manager-roles.yml.j2 | 2 +-
 .../templates/local-path-storage-cm.yml.j2                | 1 -
 .../templates/local-path-storage-cr.yml.j2                | 2 +-
 roles/kubernetes-apps/metallb/defaults/main.yml           | 2 +-
 .../templates/nfd-rolebinding.yaml.j2                     | 1 -
 .../templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2  | 2 +-
 .../templates/cm-scheduler-plugins.yaml.j2                | 2 +-
 .../templates/deploy-scheduler-plugins.yaml.j2            | 2 +-
 .../scheduler_plugins/templates/namespace.yaml.j2         | 2 +-
 ...worktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 | 2 +-
 .../templates/rbac-scheduler-plugins.yaml.j2              | 2 +-
 .../templates/sa-scheduler-plugins.yaml.j2                | 2 +-
 .../templates/scheduling.x-k8s.io_elasticquotas.yaml.j2   | 2 +-
 .../templates/scheduling.x-k8s.io_podgroups.yaml.j2       | 2 +-
 .../topology.node.k8s.io_noderesourcetopologies.yaml.j2   | 2 +-
 .../control-plane/templates/apiserver-tracing.yaml.j2     | 2 +-
 .../node/templates/kubelet-config.v1beta1.yaml.j2         | 2 +-
 .../network_plugin/calico/templates/calico-config.yml.j2  | 1 -
 .../network_plugin/cilium/templates/cilium/config.yml.j2  | 2 +-
 roles/network_plugin/cilium/templates/cilium/cr.yml.j2    | 2 +-
 .../network_plugin/cilium/templates/hubble/config.yml.j2  | 8 ++++----
 .../network_plugin/cilium/templates/hubble/service.yml.j2 | 1 -
 .../kube-ovn/templates/cni-kube-ovn-crd.yml.j2            | 2 +-
 scale.yml                                                 | 2 +-
 scripts/openstack-cleanup/main.py                         | 2 +-
 tests/files/vagrant_ubuntu20-flannel-collection.rb        | 2 +-
 tests/requirements.txt                                    | 4 ++--
 44 files changed, 42 insertions(+), 47 deletions(-)

diff --git a/contrib/terraform/nifcloud/README.md b/contrib/terraform/nifcloud/README.md
index 8c46df402f5..a6dcf014855 100644
--- a/contrib/terraform/nifcloud/README.md
+++ b/contrib/terraform/nifcloud/README.md
@@ -72,6 +72,7 @@ The setup looks like following
 
   ```bash
   ./generate-inventory.sh > sample-inventory/inventory.ini
+  ```
 
 * Export Variables:
 
diff --git a/contrib/terraform/upcloud/cluster-settings.tfvars b/contrib/terraform/upcloud/cluster-settings.tfvars
index 45a374900f7..eae1551e221 100644
--- a/contrib/terraform/upcloud/cluster-settings.tfvars
+++ b/contrib/terraform/upcloud/cluster-settings.tfvars
@@ -146,4 +146,4 @@ server_groups = {
   #   ]
   #   anti_affinity_policy = "yes"
   # }
-}
\ No newline at end of file
+}
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
index 2adeb44678f..9639bdeae87 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
@@ -558,4 +558,4 @@ resource "upcloud_server_group" "server_groups" {
   anti_affinity_policy = each.value.anti_affinity_policy
   labels               = {}
   members              = [for server in each.value.servers : merge(upcloud_server.master, upcloud_server.worker)[server].id]
-}
\ No newline at end of file
+}
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
index ad2cc70f020..87e5e537063 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
@@ -106,4 +106,4 @@ variable "server_groups" {
     anti_affinity_policy = string
     servers              = list(string)
   }))
-}
\ No newline at end of file
+}
diff --git a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
index 58536674f68..d1546004bcc 100644
--- a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
@@ -146,4 +146,4 @@ server_groups = {
   #   ]
   #   anti_affinity_policy = "yes"
   # }
-}
\ No newline at end of file
+}
diff --git a/docs/operations/recover-control-plane.md b/docs/operations/recover-control-plane.md
index 7cda08afb2b..c54808b41cb 100644
--- a/docs/operations/recover-control-plane.md
+++ b/docs/operations/recover-control-plane.md
@@ -8,7 +8,6 @@ Examples of what broken means in this context:
 * One or more bare metal node(s) suffer from unrecoverable hardware failure
 * One or more node(s) fail during patching or upgrading
 * Etcd database corruption
-  
 * Other node related failures leaving your control plane degraded or nonfunctional
 
 __Note that you need at least one functional node to be able to recover using this method.__
diff --git a/inventory/sample/group_vars/etcd.yml b/inventory/sample/group_vars/etcd.yml
index 68beeb62b2f..66bbc0d4888 100644
--- a/inventory/sample/group_vars/etcd.yml
+++ b/inventory/sample/group_vars/etcd.yml
@@ -32,4 +32,4 @@
 # etcd_experimental_enable_distributed_tracing: false
 # etcd_experimental_distributed_tracing_sample_rate: 100
 # etcd_experimental_distributed_tracing_address: "localhost:4317"
-# etcd_experimental_distributed_tracing_service_name: etcd
\ No newline at end of file
+# etcd_experimental_distributed_tracing_service_name: etcd
diff --git a/requirements.txt b/requirements.txt
index 028f272f38e..c1e7ce973f3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,9 +2,9 @@ ansible==9.5.1
 cryptography==42.0.7
 jinja2==3.1.4
 jmespath==1.0.1
+jsonschema==4.22.0
 MarkupSafe==2.1.5
 netaddr==1.2.1
 pbr==6.0.0
 ruamel.yaml==0.18.6
 ruamel.yaml.clib==0.2.8
-jsonschema==4.22.0
diff --git a/roles/container-engine/containerd/defaults/main.yml b/roles/container-engine/containerd/defaults/main.yml
index a6b24843c8b..291e96e347f 100644
--- a/roles/container-engine/containerd/defaults/main.yml
+++ b/roles/container-engine/containerd/defaults/main.yml
@@ -116,4 +116,4 @@ containerd_tracing_enabled: false
 containerd_tracing_endpoint: "0.0.0.0:4317"
 containerd_tracing_protocol: "grpc"
 containerd_tracing_sampling_ratio: 1.0
-containerd_tracing_service_name: "containerd"
\ No newline at end of file
+containerd_tracing_service_name: "containerd"
diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2
index 8a1ee8aa53c..fea6f7f10db 100644
--- a/roles/container-engine/containerd/templates/config.toml.j2
+++ b/roles/container-engine/containerd/templates/config.toml.j2
@@ -107,4 +107,3 @@ oom_score = {{ containerd_oom_score }}
     sampling_ratio = {{ containerd_tracing_sampling_ratio }}
     service_name = "{{ containerd_tracing_service_name }}"
 {% endif %}
-
diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml
index 814caed8ebb..fee6903df5f 100644
--- a/roles/etcd/defaults/main.yml
+++ b/roles/etcd/defaults/main.yml
@@ -124,4 +124,4 @@ unsafe_show_logs: false
 etcd_experimental_enable_distributed_tracing: false
 etcd_experimental_distributed_tracing_sample_rate: 100
 etcd_experimental_distributed_tracing_address: "localhost:4317"
-etcd_experimental_distributed_tracing_service_name: etcd
\ No newline at end of file
+etcd_experimental_distributed_tracing_service_name: etcd
diff --git a/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2 b/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2
index 61157d8fc60..6bdaff67726 100644
--- a/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2
@@ -162,4 +162,4 @@ metadata:
   name: pd.csi.storage.gke.io
 spec:
   attachRequired: true
-  podInfoOnMount: false
\ No newline at end of file
+  podInfoOnMount: false
diff --git a/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2 b/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2
index 9aad6206936..2992d7ff9a2 100644
--- a/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2
@@ -109,4 +109,4 @@ spec:
       # See "special case". This will tolerate everything. Node component should
       # be scheduled on all nodes.
       tolerations:
-      - operator: Exists
\ No newline at end of file
+      - operator: Exists
diff --git a/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2 b/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2
index 57a8675e451..fa2e5a81f1f 100644
--- a/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2
@@ -6,4 +6,4 @@ provisioner: pd.csi.storage.gke.io
 parameters:
   type: pd-balanced
   replication-type: regional-pd
-volumeBindingMode: WaitForFirstConsumer
\ No newline at end of file
+volumeBindingMode: WaitForFirstConsumer
diff --git a/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2 b/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2
index e9bedaf83c5..dc530162573 100644
--- a/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2
@@ -5,4 +5,4 @@ metadata:
 provisioner: pd.csi.storage.gke.io
 parameters:
   type: pd-balanced
-volumeBindingMode: WaitForFirstConsumer
\ No newline at end of file
+volumeBindingMode: WaitForFirstConsumer
diff --git a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2
index fb52d107e47..274889604c4 100644
--- a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2
@@ -18,7 +18,7 @@ data:
   "max-pvscsi-targets-per-vm": "true"
   "multi-vcenter-csi-topology": "true"
   "csi-internal-generated-cluster-id": "true"
-  "listview-tasks": "true" 
+  "listview-tasks": "true"
 {% if vsphere_csi_controller is version('v2.7.0', '>=') %}
   "improved-csi-idempotency": "true"
   "improved-volume-topology": "true"
diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2
index 3c893f3faff..65dfefaa4df 100644
--- a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2
+++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2
@@ -9,4 +9,4 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: cloud-controller-manager
-    namespace: kube-system
\ No newline at end of file
+    namespace: kube-system
diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2
index d2710e960b5..ccb7c0bfa7e 100644
--- a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2
+++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2
@@ -110,4 +110,4 @@ rules:
       - list
       - watch
     apiGroups:
-      - discovery.k8s.io
\ No newline at end of file
+      - discovery.k8s.io
diff --git a/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2
index 9cd7fd3ff3b..a0b0bc1238f 100644
--- a/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2
@@ -32,4 +32,3 @@ data:
       - name: helper-pod
         image: "{{ local_path_provisioner_helper_image_repo }}:{{ local_path_provisioner_helper_image_tag }}"
         imagePullPolicy: IfNotPresent
-
diff --git a/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2
index 299db6eba86..2b53ba72d6e 100644
--- a/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2
@@ -15,4 +15,4 @@ rules:
     verbs: [ "create", "patch" ]
   - apiGroups: [ "storage.k8s.io" ]
     resources: [ "storageclasses" ]
-    verbs: [ "get", "list", "watch" ]
\ No newline at end of file
+    verbs: [ "get", "list", "watch" ]
diff --git a/roles/kubernetes-apps/metallb/defaults/main.yml b/roles/kubernetes-apps/metallb/defaults/main.yml
index 02f4e3cae94..c83b293d9f2 100644
--- a/roles/kubernetes-apps/metallb/defaults/main.yml
+++ b/roles/kubernetes-apps/metallb/defaults/main.yml
@@ -13,4 +13,4 @@ metallb_speaker_tolerations:
     key: node-role.kubernetes.io/control-plane
     operator: Exists
 metallb_controller_tolerations: []
-metallb_loadbalancer_class: ""
\ No newline at end of file
+metallb_loadbalancer_class: ""
diff --git a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2 b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2
index 5493087619c..04a5786967a 100644
--- a/roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2
+++ b/roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2
@@ -11,4 +11,3 @@ subjects:
 - kind: ServiceAccount
   name: {{ node_feature_discovery_worker_sa_name }}
   namespace: {{ node_feature_discovery_namespace }}
-
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2
index 757a3b12d39..10c30c79976 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2
@@ -194,4 +194,4 @@ spec:
             type: object
         type: object
     served: true
-    storage: true
\ No newline at end of file
+    storage: true
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2
index 7e022e8895b..4b5e0248a02 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2
@@ -25,4 +25,4 @@ data:
 {% if scheduler_plugins_plugin_config is defined and scheduler_plugins_plugin_config | length != 0 %}
       pluginConfig:
 {{ scheduler_plugins_plugin_config | to_nice_yaml(indent=2, width=256) | indent(6, true) }}
-{% endif %}
\ No newline at end of file
+{% endif %}
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2
index 114698a9419..1ded700969e 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2
@@ -71,4 +71,4 @@ spec:
       volumes:
       - name: scheduler-config
         configMap:
-          name: scheduler-config
\ No newline at end of file
+          name: scheduler-config
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2
index d54ae66fd80..41b0806fef6 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2
@@ -4,4 +4,4 @@ kind: Namespace
 metadata:
   name: {{ scheduler_plugins_namespace }}
   labels:
-    name: {{ scheduler_plugins_namespace }}
\ No newline at end of file
+    name: {{ scheduler_plugins_namespace }}
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2
index e33157c0f2f..7e562f84792 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2
@@ -145,4 +145,4 @@ spec:
             type: object
         type: object
     served: true
-    storage: true
\ No newline at end of file
+    storage: true
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2
index aa6f211d7de..8e86f6bff2d 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2
@@ -137,4 +137,4 @@ subjects:
   namespace: {{ scheduler_plugins_namespace }}
 - kind: ServiceAccount
   name: scheduler-plugins-controller
-  namespace: {{ scheduler_plugins_namespace }}
\ No newline at end of file
+  namespace: {{ scheduler_plugins_namespace }}
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2
index 6c25e18090c..7cefdb18472 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2
@@ -8,4 +8,4 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: scheduler-plugins-controller
-  namespace: {{ scheduler_plugins_namespace }}
\ No newline at end of file
+  namespace: {{ scheduler_plugins_namespace }}
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2
index d63f57209f4..e8f64c35497 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2
@@ -79,4 +79,4 @@ spec:
     served: true
     storage: true
     subresources:
-      status: {}
\ No newline at end of file
+      status: {}
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2
index 3767cf96291..a0790dc71cb 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2
@@ -94,4 +94,4 @@ spec:
     served: true
     storage: true
     subresources:
-      status: {}
\ No newline at end of file
+      status: {}
diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2
index d83ef0b9b5c..567432f3f59 100644
--- a/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2
+++ b/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2
@@ -150,4 +150,4 @@ spec:
         - zones
         type: object
     served: true
-    storage: true
\ No newline at end of file
+    storage: true
diff --git a/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2
index 98decde86b8..7301a354d5e 100644
--- a/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2
+++ b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2
@@ -1,4 +1,4 @@
 apiVersion: apiserver.config.k8s.io/v1beta1
 kind: TracingConfiguration
 endpoint: {{ kube_apiserver_tracing_endpoint }}
-samplingRatePerMillion: {{ kube_apiserver_tracing_sampling_rate_per_million }}
\ No newline at end of file
+samplingRatePerMillion: {{ kube_apiserver_tracing_sampling_rate_per_million }}
diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
index 1a664a0edf7..d45ede27214 100644
--- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
+++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
@@ -174,4 +174,4 @@ topologyManagerScope: {{ kubelet_topology_manager_scope }}
 tracing:
   endpoint: {{ kubelet_tracing_endpoint }}
   samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }}
-{% endif %}
\ No newline at end of file
+{% endif %}
diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2
index 26983ecaed4..d949af1ec6b 100644
--- a/roles/network_plugin/calico/templates/calico-config.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-config.yml.j2
@@ -102,4 +102,3 @@ data:
         }
       ]
     }
-
diff --git a/roles/network_plugin/cilium/templates/cilium/config.yml.j2 b/roles/network_plugin/cilium/templates/cilium/config.yml.j2
index d294c6e291d..bdb07212bc5 100644
--- a/roles/network_plugin/cilium/templates/cilium/config.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/config.yml.j2
@@ -134,7 +134,7 @@ data:
   ## DSR setting
   bpf-lb-mode: "{{ cilium_loadbalancer_mode }}"
 
-  # l2 
+  # l2
   enable-l2-announcements: "{{ cilium_l2announcements }}"
 
   # Enable Bandwidth Manager
diff --git a/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
index a4395b24203..833076de140 100644
--- a/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
@@ -140,7 +140,7 @@ rules:
   verbs:
   - list
   - watch
-{% if cilium_version %} 
+{% if cilium_version %}
 - apiGroups:
   - coordination.k8s.io
   resources:
diff --git a/roles/network_plugin/cilium/templates/hubble/config.yml.j2 b/roles/network_plugin/cilium/templates/hubble/config.yml.j2
index 888db41242d..f3af7174110 100644
--- a/roles/network_plugin/cilium/templates/hubble/config.yml.j2
+++ b/roles/network_plugin/cilium/templates/hubble/config.yml.j2
@@ -12,10 +12,10 @@ data:
     peer-service: "hubble-peer.kube-system.svc.{{ dns_domain }}:443"
     listen-address: :4245
     metrics-listen-address: ":9966"
-    dial-timeout: 
-    retry-timeout: 
-    sort-buffer-len-max: 
-    sort-buffer-drain-timeout: 
+    dial-timeout:
+    retry-timeout:
+    sort-buffer-len-max:
+    sort-buffer-drain-timeout:
     tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt
     tls-client-key-file: /var/lib/hubble-relay/tls/client.key
     tls-server-cert-file: /var/lib/hubble-relay/tls/server.crt
diff --git a/roles/network_plugin/cilium/templates/hubble/service.yml.j2 b/roles/network_plugin/cilium/templates/hubble/service.yml.j2
index 0f862a9c5ee..48e90b82518 100644
--- a/roles/network_plugin/cilium/templates/hubble/service.yml.j2
+++ b/roles/network_plugin/cilium/templates/hubble/service.yml.j2
@@ -102,4 +102,3 @@ spec:
     protocol: TCP
     targetPort: 4244
   internalTrafficPolicy: Local
-
diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
index 379381d681b..8040cc77bd3 100644
--- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
+++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
@@ -1530,4 +1530,4 @@ spec:
       subresources:
         status: {}
   conversion:
-    strategy: None
\ No newline at end of file
+    strategy: None
diff --git a/scale.yml b/scale.yml
index b78fc69fdcf..3eebaee892c 100644
--- a/scale.yml
+++ b/scale.yml
@@ -1,3 +1,3 @@
 ---
 - name: Scale the cluster
-  ansible.builtin.import_playbook: playbooks/scale.yml
\ No newline at end of file
+  ansible.builtin.import_playbook: playbooks/scale.yml
diff --git a/scripts/openstack-cleanup/main.py b/scripts/openstack-cleanup/main.py
index 2ddccc067f5..ba3d4586e5f 100755
--- a/scripts/openstack-cleanup/main.py
+++ b/scripts/openstack-cleanup/main.py
@@ -61,7 +61,7 @@ def main():
 
         for ip in conn.network.ips():
             fn_if_old(conn.network.delete_ip, ip)
-                
+
         # After removing unnecessary subnet from router, retry to delete ports
         map_if_old(conn.network.delete_port,
                    conn.network.ports())
diff --git a/tests/files/vagrant_ubuntu20-flannel-collection.rb b/tests/files/vagrant_ubuntu20-flannel-collection.rb
index c739f58a2ab..55daa19e3eb 100644
--- a/tests/files/vagrant_ubuntu20-flannel-collection.rb
+++ b/tests/files/vagrant_ubuntu20-flannel-collection.rb
@@ -6,4 +6,4 @@
 # Checking for box update can trigger API rate limiting
 # https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html
 $box_check_update = false
-$vm_cpus = 2
\ No newline at end of file
+$vm_cpus = 2
diff --git a/tests/requirements.txt b/tests/requirements.txt
index a9e1e7e3f21..43eef23a3d9 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -5,8 +5,8 @@ ara[server]==1.7.1
 dopy==0.3.7
 molecule==24.2.1
 molecule-plugins[vagrant]==23.5.3
-python-vagrant==1.0.0
 pytest-testinfra==10.1.0
+python-vagrant==1.0.0
 tox==4.15.0
-yamllint==1.35.1
 tzdata==2024.1
+yamllint==1.35.1

From 7f7b65d388bb9344f60fcb27929594fd397d23c4 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Wed, 22 May 2024 10:31:18 +0200
Subject: [PATCH 06/11] Convert check_typo to pre-commit + use maintained
 version

client9/misspell is unmaintained, and has been forked by the golangci
team, see https://github.com/client9/misspell/issues/197#issuecomment-1596318684.

They haven't yet added a pre-commit config, so use my fork with the
pre-commit hook config until the pull request is merged.
---
 .pre-commit-config.yaml     |  7 +++++++
 tests/scripts/check_typo.sh | 12 ------------
 2 files changed, 7 insertions(+), 12 deletions(-)
 delete mode 100755 tests/scripts/check_typo.sh

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e4116c1fae0..cbafa4bb30c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -45,6 +45,13 @@ repos:
           - jmespath==1.0.1
           - netaddr==1.2.1
 
+  - repo: https://github.com/VannTen/misspell
+    # Waiting on https://github.com/golangci/misspell/pull/19 to get merged
+    rev: 8592a4e
+    hooks:
+      - id: misspell
+        exclude: "OWNERS_ALIASES$"
+
   - repo: local
     hooks:
       - id: ansible-syntax-check
diff --git a/tests/scripts/check_typo.sh b/tests/scripts/check_typo.sh
deleted file mode 100755
index 522d4b2d60c..00000000000
--- a/tests/scripts/check_typo.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# cd to the root directory of kubespray
-cd $(dirname $0)/../../
-
-rm ./misspell*
-
-set -e
-wget https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_linux_64bit.tar.gz
-tar -zxvf ./misspell_0.3.4_linux_64bit.tar.gz
-chmod 755 ./misspell
-git ls-files | grep -v OWNERS_ALIASES | xargs ./misspell -error

From 19851bb07c79ebec4a81d12ae9748f10f5dbc03f Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Wed, 22 May 2024 13:40:41 +0200
Subject: [PATCH 07/11] collection-build-install convert to pre-commit

---
 .pre-commit-config.yaml                   | 8 ++++++++
 tests/scripts/collection-build-install.sh | 7 +++++++
 2 files changed, 15 insertions(+)
 create mode 100755 tests/scripts/collection-build-install.sh

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index cbafa4bb30c..5b2f78f7aef 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -76,6 +76,14 @@ repos:
         language: script
         pass_filenames: false
 
+      - id: collection-build-install
+        name: Build and install kubernetes-sigs.kubespray Ansible collection
+        language: python
+        additional_dependencies:
+          - ansible-core>=2.16.4
+        entry: tests/scripts/collection-build-install.sh
+        pass_filenames: false
+
       - id: generate-docs-sidebar
         name: generate-docs-sidebar
         entry: scripts/gen_docs_sidebar.sh
diff --git a/tests/scripts/collection-build-install.sh b/tests/scripts/collection-build-install.sh
new file mode 100755
index 00000000000..6419850d7a7
--- /dev/null
+++ b/tests/scripts/collection-build-install.sh
@@ -0,0 +1,7 @@
+#!/bin/sh -e
+export ANSIBLE_COLLECTIONS_PATH="./ansible_collections"
+ansible-galaxy collection build --force
+ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
+ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
+test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
+test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml

From bc21433a05759431e4904e27f9ceaf56f95a99e5 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Wed, 22 May 2024 15:53:06 +0200
Subject: [PATCH 08/11] Run pre-commit hooks in dynamic pipeline

Use gitlab dynamic child pipelines feature to have one source of truth
for the pre-commit jobs, the pre-commit config file.

Use one cache per pre-commit. This should reduce the "fetching cache"
time steps in gitlab-ci, since each job will have a separate cache with
only its hook installed.
---
 .gitlab-ci/lint.yml                    | 21 +++++++++++++++++++++
 .gitlab-ci/pre-commit-dynamic-stub.yml | 17 +++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 .gitlab-ci/pre-commit-dynamic-stub.yml

diff --git a/.gitlab-ci/lint.yml b/.gitlab-ci/lint.yml
index 658bd35aa32..a407fd38445 100644
--- a/.gitlab-ci/lint.yml
+++ b/.gitlab-ci/lint.yml
@@ -1,4 +1,25 @@
 ---
+generate-pre-commit:
+  image: 'mikefarah/yq@sha256:bcb889a1f9bdb0613c8a054542d02360c2b1b35521041be3e1bd8fbd0534d411'
+  stage: build
+  before_script: []
+  script:
+    - >
+      yq -r < .pre-commit-config.yaml '.repos[].hooks[].id' |
+      sed 's/^/      - /' |
+      cat .gitlab-ci/pre-commit-dynamic-stub.yml - > pre-commit-generated.yml
+  artifacts:
+    paths:
+      - pre-commit-generated.yml
+
+run-pre-commit:
+  stage: unit-tests
+  trigger:
+    include:
+      - artifact: pre-commit-generated.yml
+        job: generate-pre-commit
+    strategy: depend
+
 yamllint:
   extends: .job
   stage: unit-tests
diff --git a/.gitlab-ci/pre-commit-dynamic-stub.yml b/.gitlab-ci/pre-commit-dynamic-stub.yml
new file mode 100644
index 00000000000..55f81b513a2
--- /dev/null
+++ b/.gitlab-ci/pre-commit-dynamic-stub.yml
@@ -0,0 +1,17 @@
+---
+# stub pipeline for dynamic generation
+pre-commit:
+  tags:
+  - light
+  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
+  variables:
+    PRE_COMMIT_HOME: /pre-commit-cache
+  script:
+  - pre-commit run -a $HOOK_ID
+  cache:
+    key: pre-commit-$HOOK_ID
+    paths:
+    - /pre-commit-cache
+  parallel:
+    matrix:
+    - HOOK_ID:

From 0faa805525a1c1889c749ae0c1f6d83a65ceb2e2 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Wed, 22 May 2024 15:44:40 +0200
Subject: [PATCH 09/11] Remove gitlab-ci job done in pre-commit

---
 .gitlab-ci/lint.yml | 110 +-------------------------------------------
 1 file changed, 1 insertion(+), 109 deletions(-)

diff --git a/.gitlab-ci/lint.yml b/.gitlab-ci/lint.yml
index a407fd38445..00f381f4ee4 100644
--- a/.gitlab-ci/lint.yml
+++ b/.gitlab-ci/lint.yml
@@ -20,16 +20,6 @@ run-pre-commit:
         job: generate-pre-commit
     strategy: depend
 
-yamllint:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  variables:
-    LANG: C.UTF-8
-  script:
-    - yamllint --strict .
-  except: ['triggers', 'master']
-
 vagrant-validate:
   extends: .job
   stage: unit-tests
@@ -40,109 +30,11 @@ vagrant-validate:
     - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
 
-ansible-lint:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  script:
-    - ansible-lint -v
-  except: ['triggers', 'master']
-
-jinja-syntax-check:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  script:
-    - "find -name '*.j2' -exec tests/scripts/check-templates.py {} +"
-  except: ['triggers', 'master']
-
-syntax-check:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  variables:
-    ANSIBLE_INVENTORY: inventory/local-tests.cfg
-    ANSIBLE_REMOTE_USER: root
-    ANSIBLE_BECOME: "true"
-    ANSIBLE_BECOME_USER: root
-    ANSIBLE_VERBOSITY: "3"
-  script:
-    - ansible-playbook --syntax-check cluster.yml
-    - ansible-playbook --syntax-check playbooks/cluster.yml
-    - ansible-playbook --syntax-check upgrade-cluster.yml
-    - ansible-playbook --syntax-check playbooks/upgrade_cluster.yml
-    - ansible-playbook --syntax-check reset.yml
-    - ansible-playbook --syntax-check playbooks/reset.yml
-    - ansible-playbook --syntax-check extra_playbooks/upgrade-only-k8s.yml
-  except: ['triggers', 'master']
-
-collection-build-install-sanity-check:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  variables:
-    ANSIBLE_COLLECTIONS_PATH: "./ansible_collections"
-  script:
-    - ansible-galaxy collection build
-    - ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
-    - ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
-    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
-    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml
-  except: ['triggers', 'master']
-
-tox-inventory-builder:
-  stage: unit-tests
-  tags: [light]
-  extends: .job
-  before_script:
-    - ./tests/scripts/rebase.sh
-  script:
-    - pip3 install tox
-    - cd contrib/inventory_builder && tox
-  except: ['triggers', 'master']
-
-markdownlint:
-  stage: unit-tests
-  tags: [light]
-  image: node
-  before_script:
-    - npm install -g markdownlint-cli@0.22.0
-  script:
-    - markdownlint $(find . -name '*.md' | grep -vF './.git') --ignore docs/_sidebar.md --ignore contrib/dind/README.md
-
-generate-sidebar:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  script:
-    - scripts/gen_docs_sidebar.sh
-    - git diff --exit-code
-
-check-readme-versions:
-  stage: unit-tests
-  tags: [light]
-  image: python:3
-  script:
-    - tests/scripts/check_readme_versions.sh
 
+# TODO: convert to pre-commit hook
 check-galaxy-version:
   stage: unit-tests
   tags: [light]
   image: python:3
   script:
     - tests/scripts/check_galaxy_version.sh
-
-check-typo:
-  stage: unit-tests
-  tags: [light]
-  image: python:3
-  script:
-    - tests/scripts/check_typo.sh
-
-ci-matrix:
-  stage: unit-tests
-  tags: [light]
-  image: python:3
-  script:
-    - tests/scripts/md-table/main.py
-    - git diff --exit-code

From ff481446075f834161eda09560bc515e30c38148 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 28 May 2024 10:08:30 +0200
Subject: [PATCH 10/11] pre-commit: adjust mardownlint default, md fixes

Use a style file as recommended by upstream. This makes for only one
source of truth.
Conserve previous upstream default for MD007 (upstream default changed
here https://github.com/markdownlint/markdownlint/pull/373)
---
 .markdownlint.yaml                       | 3 ---
 .md_style.rb                             | 4 ++++
 .mdlrc                                   | 1 +
 .pre-commit-config.yaml                  | 3 +--
 docs/cloud_providers/openstack.md        | 1 -
 docs/operations/recover-control-plane.md | 1 -
 6 files changed, 6 insertions(+), 7 deletions(-)
 delete mode 100644 .markdownlint.yaml
 create mode 100644 .md_style.rb
 create mode 100644 .mdlrc

diff --git a/.markdownlint.yaml b/.markdownlint.yaml
deleted file mode 100644
index 8ece4c7613e..00000000000
--- a/.markdownlint.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-MD013: false
-MD029: false
diff --git a/.md_style.rb b/.md_style.rb
new file mode 100644
index 00000000000..73adf8ae47e
--- /dev/null
+++ b/.md_style.rb
@@ -0,0 +1,4 @@
+all
+exclude_rule 'MD013'
+exclude_rule 'MD029'
+rule 'MD007', :indent => 2
diff --git a/.mdlrc b/.mdlrc
new file mode 100644
index 00000000000..8ca55a8cee5
--- /dev/null
+++ b/.mdlrc
@@ -0,0 +1 @@
+style "#{File.dirname(__FILE__)}/.md_style.rb"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5b2f78f7aef..d79a716ad40 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -24,8 +24,7 @@ repos:
     rev: v0.11.0
     hooks:
       - id: markdownlint
-        args: [-r, "~MD013,~MD029"]
-        exclude: "^.git"
+        exclude: "^.github|(^docs/_sidebar\\.md$)"
 
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.10.0.1
diff --git a/docs/cloud_providers/openstack.md b/docs/cloud_providers/openstack.md
index 6f53da53444..1506be370c4 100644
--- a/docs/cloud_providers/openstack.md
+++ b/docs/cloud_providers/openstack.md
@@ -1,4 +1,3 @@
-
 # OpenStack
 
 ## Known compatible public clouds
diff --git a/docs/operations/recover-control-plane.md b/docs/operations/recover-control-plane.md
index c54808b41cb..d54aa13f58a 100644
--- a/docs/operations/recover-control-plane.md
+++ b/docs/operations/recover-control-plane.md
@@ -1,4 +1,3 @@
-
 # Recovering the control plane
 
 To recover from broken nodes in the control plane use the "recover\-control\-plane.yml" playbook.

From 37d824fd2d42a335373f51b4f6efa2e6e029682b Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 28 May 2024 11:27:06 +0200
Subject: [PATCH 11/11] Update pre-commit hooks

---
 .pre-commit-config.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index d79a716ad40..5bba7e7eac9 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -15,13 +15,13 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.27.1
+    rev: v1.35.1
     hooks:
       - id: yamllint
         args: [--strict]
 
   - repo: https://github.com/markdownlint/markdownlint
-    rev: v0.11.0
+    rev: v0.12.0
     hooks:
       - id: markdownlint
         exclude: "^.github|(^docs/_sidebar\\.md$)"