feat: Neutron OVS+uwsgi support (#1160)

relate to #42
Note: ovn can't properly use uWSGI for now due to: https://bugs.launchpad.net/neutron/+bug/1912359

Reviewed-by: Mohammed Naser <[email protected]>

.charts.yml

@@ -127,14 +127,13 @@ charts:
     repository: *openstack_helm_infra_repository
     dependencies: *openstack_helm_dependencies
   - name: neutron
-    version: 0.3.29
+    version: 0.3.41
     repository: *openstack_helm_repository
     dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
-          - 902767
-          - 914886
+          - 916862
   - name: node-feature-discovery
     version: 0.15.4
     repository:

charts/neutron/Chart.yaml

@@ -9,4 +9,4 @@ name: neutron
 sources:
 - https://opendev.org/openstack/neutron
 - https://opendev.org/openstack/openstack-helm
-version: 0.3.29
+version: 0.3.41

charts/neutron/templates/bin/_health-probe.py.tpl

@@ -315,16 +315,20 @@ if __name__ == "__main__":
     data = {}
     if os.path.isfile(pidfile):
         with open(pidfile,'r') as f:
-            data = json.load(f)
-        if check_pid_running(data['pid']):
-            if data['exit_count'] > 1:
-                # Third time in, kill the previous process
-                os.kill(int(data['pid']), signal.SIGTERM)
-            else:
-                data['exit_count'] = data['exit_count'] + 1
-                with open(pidfile, 'w') as f:
-                    json.dump(data, f)
-                sys.exit(0)
+            file_content = f.read().strip()
+            if file_content:
+                data = json.loads(file_content)
+
+    if 'pid' in data and check_pid_running(data['pid']):
+        if 'exit_count' in data and data['exit_count'] > 1:
+            # Third time in, kill the previous process
+            os.kill(int(data['pid']), signal.SIGTERM)
+        else:
+            data['exit_count'] = data.get('exit_count', 0) + 1
+            with open(pidfile, 'w') as f:
+                json.dump(data, f)
+            sys.exit(0)
+
     data['pid'] = os.getpid()
     data['exit_count'] = 0
     with open(pidfile, 'w') as f:

charts/neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl

@@ -174,6 +174,21 @@ function bind_dpdk_nic {
   fi
 }
 
+function ensure_vf_state {
+  iface=${1}
+  vf_string=${2}
+  check_string=${3}
+  expected=${4}
+
+  # wait for the vf really get the needed state
+  for i in 0 1 2 4 8 16 32; do
+    sleep ${i};
+    if [ "$(ip link show ${iface} | grep "${vf_string} " | grep -Eo "${check_string}")" == "${expected}" ]; then
+      break;
+    fi;
+  done
+}
+
 function process_dpdk_nics {
   target_driver=$(get_dpdk_config_value ${DPDK_CONFIG} '.driver')
   # loop over all nics
@@ -195,11 +210,14 @@ function process_dpdk_nics {
       if [ -n "${vf_index}" ]; then
         vf_string="vf ${vf_index}"
         ip link set ${iface} ${vf_string} trust on
+        ensure_vf_state "${iface}" "${vf_string}" "trust o(n|ff)" "trust on"
 
         # NOTE: To ensure proper toggle of spoofchk,
         # turn it on then off.
         ip link set ${iface} ${vf_string} spoofchk on
+        ensure_vf_state "${iface}" "${vf_string}" "spoof checking o(n|ff)" "spoof checking on"
         ip link set ${iface} ${vf_string} spoofchk off
+        ensure_vf_state "${iface}" "${vf_string}" "spoof checking o(n|ff)" "spoof checking off"
       fi
     fi
 
@@ -291,11 +309,14 @@ function process_dpdk_bonds {
         if [ -n "${vf_index}" ]; then
           vf_string="vf ${vf_index}"
           ip link set ${iface} ${vf_string} trust on
+          ensure_vf_state "${iface}" "${vf_string}" "trust o(n|ff)" "trust on"
 
           # NOTE: To ensure proper toggle of spoofchk,
           # turn it on then off.
           ip link set ${iface} ${vf_string} spoofchk on
+          ensure_vf_state "${iface}" "${vf_string}" "spoof checking o(n|ff)" "spoof checking on"
           ip link set ${iface} ${vf_string} spoofchk off
+          ensure_vf_state "${iface}" "${vf_string}" "spoof checking o(n|ff)" "spoof checking off"
         fi
       fi
 
@@ -406,12 +427,16 @@ for bmap in `sed 's/[{}"]//g' /tmp/auto_bridge_add | tr "," "\n"`
 do
   bridge=${bmap%:*}
   iface=${bmap#*:}
-  ovs-vsctl --no-wait --may-exist add-br $bridge
-  if [ -n "$iface" ] && [ "$iface" != "null" ]
+  if [[ "${DPDK_ENABLED}" == "true" ]]; then
+    ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-br $bridge -- set bridge $bridge datapath_type=netdev
+  else
+    ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-br $bridge
+  fi
+  if [ -n "$iface" ] && [ "$iface" != "null" ] && ( ip link show $iface 1>/dev/null 2>&1 );
   then
-    ovs-vsctl --no-wait --may-exist add-port $bridge $iface
+    ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-port $bridge $iface
     migrate_ip_from_nic $iface $bridge
-    if [[ $(get_dpdk_config_value ${DPDK_CONFIG} '.enabled') != "true" ]]; then
+    if [[ "${DPDK_ENABLED}" != "true" ]]; then
       ip link set dev $iface up
     fi
   fi

charts/neutron/templates/bin/_neutron-rpc-server.sh.tpl

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+COMMAND="${@:-start}"
+
+function start () {
+  exec neutron-rpc-server \
+        --config-file /etc/neutron/neutron.conf \
+{{- if ( has "ovn" .Values.network.backend ) }}
+        --config-file /tmp/pod-shared/ovn.ini \
+{{- end }}
+{{- if .Values.conf.plugins.taas.taas.enabled }}
+        --config-file /etc/neutron/taas_plugin.ini \
+{{- end }}
+{{- if ( has "sriov" .Values.network.backend ) }}
+        --config-file /etc/neutron/plugins/ml2/sriov_agent.ini \
+{{- end }}
+{{- if .Values.conf.plugins.l2gateway }}
+        --config-file /etc/neutron/l2gw_plugin.ini \
+{{- end }}
+{{- if ( has "tungstenfabric" .Values.network.backend ) }}
+        --config-file /etc/neutron/plugins/tungstenfabric/tf_plugin.ini
+{{- else }}
+        --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
+{{- end }}
+}
+
+function stop () {
+  kill -TERM 1
+}
+
+$COMMAND

charts/neutron/templates/bin/_neutron-server.sh.tpl

@@ -18,6 +18,17 @@ set -ex
 COMMAND="${@:-start}"
 
 function start () {
+# (ricolin): Currently ovn have issue with uWSGI,
+# let's keep using non-uWSGI way until this bug fixed:
+# https://bugs.launchpad.net/neutron/+bug/1912359
+{{- if ( has "ovn" .Values.network.backend ) }}
+  start_ovn
+{{- else }}
+  exec uwsgi --ini /etc/neutron/neutron-api-uwsgi.ini
+{{- end }}
+}
+
+function start_ovn () {
   exec neutron-server \
         --config-file /etc/neutron/neutron.conf \
 {{- if ( has "ovn" .Values.network.backend ) }}

charts/neutron/templates/configmap-bin.yaml

@@ -91,6 +91,8 @@ data:
 {{- end }}
   neutron-server.sh: |
 {{ tuple "bin/_neutron-server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  neutron-rpc-server.sh: |
+{{ tuple "bin/_neutron-rpc-server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   neutron-ironic-agent.sh: |
 {{ tuple "bin/_neutron-ironic-agent.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   neutron-netns-cleanup-cron.sh: |

charts/neutron/templates/configmap-etc.yaml

@@ -120,18 +120,11 @@ limitations under the License.
 {{- $_ := tuple "load_balancer" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set $envAll.Values.conf.neutron.octavia "base_url" -}}
 {{- end }}
 
-{{/*
-nova_metadata_ip can go away when Newton is no longer supported, otherwise
-just set it along with nova_metadata_host.
-*/}}
-{{- if empty $envAll.Values.conf.metadata_agent.DEFAULT.nova_metadata_ip -}}
-{{- $_ := tuple "compute_metadata" "public" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | set $envAll.Values.conf.metadata_agent.DEFAULT "nova_metadata_ip" -}}
-{{- end -}}
 {{- if empty $envAll.Values.conf.metadata_agent.DEFAULT.nova_metadata_host -}}
-{{- $_ := tuple "compute_metadata" "public" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | set $envAll.Values.conf.metadata_agent.DEFAULT "nova_metadata_host" -}}
+{{- $_ := tuple "compute_metadata" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | set $envAll.Values.conf.metadata_agent.DEFAULT "nova_metadata_host" -}}
 {{- end -}}
 {{- if empty $envAll.Values.conf.metadata_agent.DEFAULT.nova_metadata_port -}}
-{{- $_ := set $envAll.Values.conf.metadata_agent.DEFAULT "nova_metadata_port" 80 -}}
+{{- $_ := tuple "compute_metadata" "internal" "metadata" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | set $envAll.Values.conf.metadata_agent.DEFAULT "nova_metadata_port" }}
 {{- end -}}
 {{- if empty $envAll.Values.conf.metadata_agent.cache.memcache_servers -}}
 {{- $_ := tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" | set $envAll.Values.conf.metadata_agent.cache "memcache_servers" -}}
@@ -196,6 +189,14 @@ just set it along with nova_metadata_host.
 {{- if empty .Values.conf.neutron.DEFAULT.bind_port -}}
 {{- $_ := tuple "network" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | set .Values.conf.neutron.DEFAULT "bind_port" -}}
 {{- end -}}
+{{- if empty .Values.conf.neutron_api_uwsgi.uwsgi.processes -}}
+{{- $_ := set .Values.conf.neutron_api_uwsgi.uwsgi "processes" .Values.conf.neutron.DEFAULT.api_workers -}}
+{{- end -}}
+{{- if empty (index .Values.conf.neutron_api_uwsgi.uwsgi "http-socket") -}}
+{{- $http_socket_port := tuple "network" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | toString }}
+{{- $http_socket := printf "0.0.0.0:%s" $http_socket_port }}
+{{- $_ := set .Values.conf.neutron_api_uwsgi.uwsgi "http-socket" $http_socket -}}
+{{- end -}}
 
 {{- if and (empty .Values.conf.logging.handler_fluent) (has "fluent" .Values.conf.logging.handlers.keys) -}}
 {{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
@@ -291,6 +292,7 @@ data:
   rally_tests.yaml: {{ toYaml $envAll.Values.conf.rally_tests.tests | b64enc }}
   api-paste.ini: {{ include "helm-toolkit.utils.to_ini" $envAll.Values.conf.paste | b64enc }}
   policy.yaml: {{ toYaml $envAll.Values.conf.policy | b64enc }}
+  neutron-api-uwsgi.ini: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.neutron_api_uwsgi | b64enc }}
   neutron.conf: {{ include "helm-toolkit.utils.to_oslo_conf" $envAll.Values.conf.neutron | b64enc }}
   logging.conf: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.logging | b64enc }}
   api_audit_map.conf: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.api_audit_map | b64enc }}

charts/neutron/templates/daemonset-bagpipe-bgp.yaml

@@ -54,6 +54,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_bagpipe_bgp" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_bagpipe_bgp" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}

charts/neutron/templates/daemonset-bgp-dragent.yaml

@@ -53,6 +53,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_bgp_dragent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_bgp_dragent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}

charts/neutron/templates/daemonset-dhcp-agent.yaml

@@ -75,6 +75,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_dhcp_agent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
 {{ dict "envAll" $envAll "podName" "neutron-dhcp-agent-default" "containerNames" (list "neutron-dhcp-agent" "neutron-dhcp-agent-init" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_dhcp_agent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}

charts/neutron/templates/daemonset-l2gw-agent.yaml

@@ -77,6 +77,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_l2gw_agent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_l2gw_agent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}

charts/neutron/templates/daemonset-l3-agent.yaml

@@ -76,6 +76,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_l3_agent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
 {{ dict "envAll" $envAll "podName" "neutron-l3-agent-default" "containerNames" (list "neutron-l3-agent" "init"  "neutron-l3-agent-init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_l3_agent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}

charts/neutron/templates/daemonset-lb-agent.yaml

@@ -52,6 +52,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_lb_agent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_lb_agent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}

charts/neutron/templates/daemonset-metadata-agent.yaml

@@ -72,6 +72,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_metadata_agent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
 {{ dict "envAll" $envAll "podName" "neutron-metadata-agent-default" "containerNames" (list "neutron-metadata-agent" "neutron-metadata-agent-init" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_metadata_agent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}

charts/neutron/templates/daemonset-netns-cleanup-cron.yaml

@@ -44,6 +44,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_netns_cleanup_cron" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
 {{ dict "envAll" $envAll "podName" "neutron-netns-cleanup-cron-default" "containerNames" (list "neutron-netns-cleanup-cron" "init" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_netns_cleanup_cron" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}

charts/neutron/templates/daemonset-ovn-metadata-agent.yaml

@@ -72,6 +72,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_ovn_metadata_agent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
 {{ dict "envAll" $envAll "podName" "neutron-ovn-metadata-agent-default" "containerNames" (list "neutron-ovn-metadata-agent" "neutron-ovn-metadata-agent-init" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_ovn_metadata_agent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
@@ -143,8 +144,8 @@ spec:
           command:
             - /tmp/neutron-ovn-metadata-agent.sh
           volumeMounts:
-            - name: run
-              mountPath: /run
+            - name: run-openvswitch
+              mountPath: /run/openvswitch
             - name: pod-tmp
               mountPath: /tmp
             - name: pod-var-neutron
@@ -217,9 +218,6 @@ spec:
           emptyDir: {}
         - name: pod-var-neutron
           emptyDir: {}
-        - name: run
-          hostPath:
-            path: /run
         - name: run-openvswitch
           hostPath:
             path: /run/openvswitch

charts/neutron/templates/daemonset-ovs-agent.yaml

@@ -55,6 +55,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_ovs_agent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
 {{ dict "envAll" $envAll "podName" "$configMapName" "containerNames" (list "neutron-ovs-agent" "init" "neutron-openvswitch-agent-kernel-modules" "neutron-ovs-agent-init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_ovs_agent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}

charts/neutron/templates/daemonset-sriov-agent.yaml

@@ -58,6 +58,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_sriov_agent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
 {{ dict "envAll" $envAll "podName" "neutron-sriov-agent-default" "containerNames" (list "neutron-sriov-agent-init" "init" "neutron-sriov-agent") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_sriov_agent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}

charts/neutron/templates/deployment-ironic-agent.yaml

@@ -46,6 +46,7 @@ spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ tuple "neutron_ironic_agent" . | include "helm-toolkit.snippets.custom_pod_annotations" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "neutron_ironic_agent" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}