From 064b878b51b56f7c9a9150fba51310c76361e6ef Mon Sep 17 00:00:00 2001 From: Ugur Ozturk Date: Sun, 14 Jan 2024 21:06:55 +0100 Subject: [PATCH 1/3] [apiserver-kubelet/tracing]: add distributed tracing config flags Signed-off-by: Ugur Ozturk --- .../control-plane/defaults/main/main.yml | 7 +++++++ .../control-plane/tasks/kubeadm-setup.yml | 14 ++++++++++++++ .../templates/apiserver-tracing.yaml.j2 | 8 ++++++++ .../templates/kubeadm-config.v1beta3.yaml.j2 | 10 ++++++++++ roles/kubernetes/node/defaults/main.yml | 8 ++++++++ .../node/templates/kubelet-config.v1beta1.yaml.j2 | 5 +++++ 6 files changed, 52 insertions(+) create mode 100644 roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 19503817069..190a9e77c5a 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -235,3 +235,10 @@ kubeadm_upgrade_auto_cert_renewal: true # Bash alias of kubectl to interact with Kubernetes cluster much easier # kubectl_alias: k + +## Enable distributed tracing for kube-apiserver +## For the clusters with version lower than v1.27; you have to enable the feature by adding 'APIServerTracing=true' to the `kube_apiserver_feature_gates` list +## When upgrading the cluster to v1.27; you should remove it from the `kube_apiserver_feature_gates` list +kube_apiserver_tracing: false +kube_apiserver_tracing_endpoint: 0.0.0.0:4317 +kube_apiserver_tacing_samplingRatePerMillion: 100 diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index dcad832ba89..1f4ff20a3c6 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -68,6 +68,20 @@ mode: 0640 when: kubernetes_audit_webhook | default(false) +- name: Create apiserver tracing config directory + file: + path: "{{ kube_config_dir }}/tracing" + state: directory + mode: 0640 + when: kube_apiserver_tracing + +- name: Write apiserver tracing config yaml + template: + src: apiserver-tracing.yaml.j2 + dest: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml" + mode: 0640 + when: kube_apiserver_tracing + # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint. - name: Set kubeadm_config_api_fqdn define set_fact: diff --git a/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 new file mode 100644 index 00000000000..f93f966d5e2 --- /dev/null +++ b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 @@ -0,0 +1,8 @@ +{% if kube_major_version == "v1.26" %} +apiVersion: apiserver.config.k8s.io/v1alpha1 +{% else %} +apiVersion: apiserver.config.k8s.io/v1beta1 +{% endif %} +kind: TracingConfiguration +endpoint: {{ kube_apiserver_tracing_endpoint }} +samplingRatePerMillion: {{ kube_apiserver_tacing_samplingRatePerMillion }} \ No newline at end of file diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index cbb22182315..b11fb33431e 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -227,6 +227,9 @@ apiServer: {% if kubelet_rotate_server_certificates %} kubelet-certificate-authority: {{ kube_cert_dir }}/ca.crt {% endif %} +{% if kube_apiserver_tracing %} + tracing-config-file: {{ kube_config_dir }}/tracing/apiserver-tracing.yaml +{% endif %} {% if kubernetes_audit or kube_token_auth | default(true) or kube_webhook_token_auth | default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] ) or apiserver_extra_volumes or ssl_ca_dirs | length %} extraVolumes: {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %} @@ -267,6 +270,13 @@ apiServer: readOnly: false pathType: DirectoryOrCreate {% endif %} +{% if kube_apiserver_tracing %} + - name: tracing + hostPath: {{ kube_config_dir }}/tracing + mountPath: {{ kube_config_dir }}/tracing + readOnly: true + pathType: DirectoryOrCreate +{% endif %} {% for volume in apiserver_extra_volumes %} - name: {{ volume.name }} hostPath: {{ volume.hostPath }} diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 0522187b95a..39e047228e6 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -253,3 +253,11 @@ kube_proxy_ipvs_modules: conntrack_modules: - nf_conntrack - nf_conntrack_ipv4 + + +## Enable distributed tracing for kubelet +## For the clusters with version lower than v1.27; you have to enable the feature by adding 'KubeletTracing=true' to the `kubelet_feature_gates` list +## When upgrading the cluster to v1.27; you should remove it from the `kubelet_feature_gates` list +kubelet_tracing: false +kubelet_tracing_endpoint: 0.0.0.0:4317 +kubelet_tracing_samplingRatePerMillion: 100 \ No newline at end of file diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 index f54d1f8b3ee..a8687776f1a 100644 --- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -168,3 +168,8 @@ topologyManagerPolicy: {{ kubelet_topology_manager_policy }} {% if kubelet_topology_manager_scope is defined %} topologyManagerScope: {{ kubelet_topology_manager_scope }} {% endif %} +{% if kubelet_tracing %} +tracing: + endpoint: {{ kubelet_tracing_endpoint }} + samplingRatePerMillion: {{ kubelet_tracing_samplingRatePerMillion }} +{% endif %} \ No newline at end of file From 5f7ce1496afc8524e68979c0920ef43ee35f6494 Mon Sep 17 00:00:00 2001 From: Ugur Ozturk Date: Tue, 23 Jan 2024 23:52:32 +0100 Subject: [PATCH 2/3] [apiserver-kubelet/tracing]: add distributed tracing config flags - fix Signed-off-by: Ugur Ozturk --- roles/kubernetes/control-plane/defaults/main/main.yml | 4 +--- .../control-plane/templates/apiserver-tracing.yaml.j2 | 2 +- roles/kubernetes/node/defaults/main.yml | 4 +--- .../kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 190a9e77c5a..a2558ec050b 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -237,8 +237,6 @@ kubeadm_upgrade_auto_cert_renewal: true # kubectl_alias: k ## Enable distributed tracing for kube-apiserver -## For the clusters with version lower than v1.27; you have to enable the feature by adding 'APIServerTracing=true' to the `kube_apiserver_feature_gates` list -## When upgrading the cluster to v1.27; you should remove it from the `kube_apiserver_feature_gates` list kube_apiserver_tracing: false kube_apiserver_tracing_endpoint: 0.0.0.0:4317 -kube_apiserver_tacing_samplingRatePerMillion: 100 +kube_apiserver_tracing_sampling_rate_per_million: 100 diff --git a/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 index f93f966d5e2..e84fe21c480 100644 --- a/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 @@ -5,4 +5,4 @@ apiVersion: apiserver.config.k8s.io/v1beta1 {% endif %} kind: TracingConfiguration endpoint: {{ kube_apiserver_tracing_endpoint }} -samplingRatePerMillion: {{ kube_apiserver_tacing_samplingRatePerMillion }} \ No newline at end of file +samplingRatePerMillion: {{ kube_apiserver_tracing_sampling_rate_per_million }} \ No newline at end of file diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 39e047228e6..b35304173b6 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -256,8 +256,6 @@ conntrack_modules: ## Enable distributed tracing for kubelet -## For the clusters with version lower than v1.27; you have to enable the feature by adding 'KubeletTracing=true' to the `kubelet_feature_gates` list -## When upgrading the cluster to v1.27; you should remove it from the `kubelet_feature_gates` list kubelet_tracing: false kubelet_tracing_endpoint: 0.0.0.0:4317 -kubelet_tracing_samplingRatePerMillion: 100 \ No newline at end of file +kubelet_tracing_sampling_rate_per_million: 100 \ No newline at end of file diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 index a8687776f1a..04350c774c5 100644 --- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -171,5 +171,5 @@ topologyManagerScope: {{ kubelet_topology_manager_scope }} {% if kubelet_tracing %} tracing: endpoint: {{ kubelet_tracing_endpoint }} - samplingRatePerMillion: {{ kubelet_tracing_samplingRatePerMillion }} + samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }} {% endif %} \ No newline at end of file From f39e955554833a0e384f4a2acc1d43fffe23926d Mon Sep 17 00:00:00 2001 From: Ugur Ozturk Date: Tue, 23 Jan 2024 23:57:31 +0100 Subject: [PATCH 3/3] [apiserver-kubelet/tracing]: add distributed tracing config flags - fix Signed-off-by: Ugur Ozturk --- .../control-plane/templates/apiserver-tracing.yaml.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 index e84fe21c480..98decde86b8 100644 --- a/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 @@ -1,8 +1,4 @@ -{% if kube_major_version == "v1.26" %} -apiVersion: apiserver.config.k8s.io/v1alpha1 -{% else %} apiVersion: apiserver.config.k8s.io/v1beta1 -{% endif %} kind: TracingConfiguration endpoint: {{ kube_apiserver_tracing_endpoint }} samplingRatePerMillion: {{ kube_apiserver_tracing_sampling_rate_per_million }} \ No newline at end of file