fix(CSI-241): conflict in metrics between node and controller (#325)

### TL;DR

Updated metrics configuration for CSI WekaFS Plugin to support separate ports for different components.

### What changed?

- Modified the prometheus.io/port annotation in controllerserver-statefulset.yaml to include separate ports for controller, provisioner, resizer, and snapshotter.
- Updated the metrics port configuration in both controllerserver-statefulset.yaml and nodeserver-daemonset.yaml to use component-specific ports.
- Added new metrics port configurations in values.yaml:
  - controllerPort: 9090
  - provisionerPort: 9091
  - resizerPort: 9092
  - snapshotterPort: 9093
  - nodePort: 9094

### How to test?

1. Deploy the CSI WekaFS Plugin with metrics enabled.
2. Verify that the prometheus.io/port annotation in the controller pod includes all specified ports.
3. Check that the controller and node server pods are exposing metrics on their respective ports.
4. Attempt to scrape metrics from each component using the newly configured ports.

### Why make this change?

This change allows for more granular control over metrics collection for different components of the CSI WekaFS Plugin. By using separate ports for each component, it becomes easier to monitor and troubleshoot specific parts of the plugin, enhancing observability and debugging capabilities.

Additional idea behind this change is avoiding port conflicts when running CSI plugin in hostNetwork mode (required for NFS)
---

charts/csi-wekafsplugin/templates/controllerserver-statefulset.yaml

@@ -23,7 +23,7 @@ spec:
       annotations:
         prometheus.io/scrape: 'true'
         prometheus.io/path: '/metrics'
-        prometheus.io/port: '{{ .Values.metrics.port | default 9090 }}'
+        prometheus.io/port: '{{ .Values.metrics.controllerPort | default 9090 }},{{ .Values.metrics.provisionerPort | default 9091 }},{{ .Values.metrics.resizerPort | default 9092 }},{{ .Values.metrics.snapshotterPort | default 9093 }}'
     {{- end }}
     spec:
       {{- if .Values.nodeSelector }}
@@ -192,7 +192,7 @@ spec:
           {{- end }}
           {{- if .Values.metrics.enabled }}
             - "--enablemetrics"
-            - "--metricsport={{ .Values.metrics.port | default 9090 }}"
+            - "--metricsport={{ .Values.metrics.controllerPort | default 9090 }}"
           {{- end }}
           {{- if .Values.pluginConfig.allowInsecureHttps }}
             - "--allowinsecurehttps"
@@ -229,7 +229,7 @@ spec:
               name: healthz
               protocol: TCP
           {{- if .Values.metrics.enabled }}
-            - containerPort: {{ .Values.metrics.port }}
+            - containerPort: {{ .Values.metrics.controllerPort | default 9090 }}
               name: metrics
               protocol: TCP
           {{- end }}

charts/csi-wekafsplugin/templates/nodeserver-daemonset.yaml

@@ -17,7 +17,7 @@ spec:
       annotations:
         prometheus.io/scrape: 'true'
         prometheus.io/path: '/metrics'
-        prometheus.io/port: '{{ .Values.metrics.port | default 9090 }}'
+        prometheus.io/port: '{{ .Values.metrics.nodePort | default 9090 }}'
     {{- end }}
     spec:
       {{- if (eq .Values.selinuxSupport "mixed")}}
@@ -86,7 +86,7 @@ spec:
           {{- end }}
           {{- if .Values.metrics.enabled }}
             - "--enablemetrics"
-            - "--metricsport={{ .Values.metrics.port | default 9090 }}"
+            - "--metricsport={{ .Values.metrics.nodePort | default 9090 }}"
           {{- end }}
           {{- if .Values.pluginConfig.allowInsecureHttps }}
             - "--allowinsecurehttps"
@@ -120,7 +120,7 @@ spec:
               name: healthz
               protocol: TCP
           {{- if .Values.metrics.enabled }}
-            - containerPort: {{ .Values.metrics.port }}
+            - containerPort: {{ .Values.metrics.nodePort }}
               name: metrics
               protocol: TCP
           {{- end }}

charts/csi-wekafsplugin/values.yaml

@@ -96,14 +96,16 @@ kubeletPath: "/var/lib/kubelet"
 metrics:
   # -- Enable Prometheus Metrics
   enabled: true
-  # -- Metrics port
-  port: 9090
+  # -- Metrics port for Controller Server
+  controllerPort: 9090
   # -- Provisioner metrics port
   provisionerPort: 9091
   # -- Resizer metrics port
   resizerPort: 9092
   # -- Snapshotter metrics port
   snapshotterPort: 9093
+  # -- Metrics port for Node Serer
+  nodePort: 9094
 # -- Tracing URL (For Jaeger tracing engine / OpenTelemetry), optional
 # @ignore
 tracingUrl: ""