Presets for only logsCollection and hostMetrics

charts/opentelemetry-collector/README.md

@@ -72,41 +72,9 @@ config:
 ```yaml
 mode: daemonset
 
-config:
-  receivers:
-    hostmetrics:
-      scrapers:
-        cpu:
-        load:
-        memory:
-        disk:
-  service:
-    pipelines:
-      metrics:
-        receivers:
-          - hostmestrics
-          - otlp
-          - prometheus
-
-extraEnvs:
-- name: HOST_PROC
-  value: /hostfs/proc
-- name: HOST_SYS
-  value: /hostfs/sys
-- name: HOST_ETC
-  value: /hostfs/etc
-- name: HOST_VAR
-  value: /hostfs/var
-- name: HOST_RUN
-  value: /hostfs/run
-- name: HOST_DEV
-  value: /hostfs/dev
-extraHostPathMounts:
-- name: hostfs
-  hostPath: /
-  mountPath: /hostfs
-  readOnly: true
-  mountPropagation: HostToContainer
+presets:
+  hostMetrics:
+    enabled: true
 ```
 
 ### Configuration for Kubernetes container logs
@@ -118,28 +86,16 @@ This feature is disabled by default. It has the following requirements:
 - It requires the [contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib) version
 of the collector image.
 
-To enable this feature, set the  `containerLogs.enabled` property to `true` and replace the collector image.
+To enable this feature, set the  `presets.logsCollection.enabled` property to `true`.
 Here is an example `values.yaml`:
 
 ```yaml
 mode: daemonset
 
-containerLogs:
-  enabled: true
-
-config:
-  service:
-    pipelines:
-      logs:
-        receivers:
-          - otlp
-          - filelog
-
-image:
-  repository: otel/opentelemetry-collector-contrib
-
-command:
-  name: otelcol-contrib
+presets:
+  logsCollection:
+    enabled: true
+    includeCollectorLogs: true
 ```
 
 The way this feature works is it adds a `filelog` receiver on the `logs` pipeline. This receiver is preconfigured
@@ -166,30 +122,20 @@ It also clears the `filelog` receiver's `exclude` property, for collector logs t
 ```yaml
 mode: daemonset
 
-containerLogs:
-  enabled: true
+presets:
+  logsCollection:
+    enabled: true
+    includeCollectorLogs: true
 
 config:
   exporters:
     otlphttp:
       endpoint: https://example.com:55681
-  receivers:
-    filelog:
-      exclude: []
   service:
     pipelines:
       logs:
-        receivers:
-          - otlp
-          - filelog
         exporters:
           - otlphttp
-
-image:
-  repository: otel/opentelemetry-collector-contrib
-
-command:
-  name: otelcol-contrib
 ```
 
 ### CRDs

charts/opentelemetry-collector/UPGRADING.md

@@ -1,5 +1,32 @@
 # Upgrade guidelines
 
+## 0.23.0 to 0.24.0
+
+[Remove containerLogs in favor of presets.logsCollection]()
+
+The ability to enable logs collection from the collector has been moved from `containerLogs.enabled` to `presets.logsCollection.enabled`. If you are currently using `containerLogs.enabled`, you should instead use the preset:
+
+```yaml
+presets:
+  logsCollection:
+    enabled: true
+```
+
+If you are using `containerLogs.enabled` and also enabling collection of the collector logs you can use `includeCollectorLogs`
+
+```yaml
+presets:
+  logsCollection:
+    enabled: true
+    includeCollectorLogs: true
+```
+
+You no longer need to update `config.service.pipelines.logs` to include the filelog receiver yourself as the preset will automatically update the defau logs pipeline to include the filelog receiver.
+
+The filelog's preset configuration can modified by `config.receivers`, but preset configuration cannot be removed.  If you need to remove any filelog receiver configuration generated by the preset you should not use the preset.  Instead, configure the filelog receiver manually in `config.receivers` and set any other necessary fields in the values.yaml to modify k8s as needed.
+
+See the [daemonset-collector-logs example](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector/examples/daemonset-collector-logs) to see an example of the preset in action.
+
 ## 0.18.0 to 0.19.0
 
 [Remove agentCollector and standaloneCollector settings](https://github.com/open-telemetry/opentelemetry-helm-charts/pull/216)

charts/opentelemetry-collector/ci/daemonset-container-logs-values.yaml

@@ -1,18 +1,10 @@
 mode: daemonset
 
-containerLogs:
-  enabled: true
+presets:
+  logsCollection:
+    enabled: true
+    includeCollectorLogs: true
 
-config:
-  receivers:
-    filelog:
-      exclude: []
-  service:
-    pipelines:
-      logs:
-        receivers:
-          - otlp
-          - filelog
 resources:
   limits:
     cpu: 100m

charts/opentelemetry-collector/examples/daemonset-collector-logs/values.yaml

@@ -1,15 +1,6 @@
 mode: daemonset
 
-containerLogs:
-  enabled: true
-
-config:
-  receivers:
-    filelog:
-      exclude: []
-  service:
-    pipelines:
-      logs:
-        receivers:
-          - otlp
-          - filelog
+presets:
+  logsCollection:
+    enabled: true
+    includeCollectorLogs: true

charts/opentelemetry-collector/examples/daemonset-hostmetrics/rendered/configmap-agent.yaml

@@ -26,11 +26,16 @@ data:
         spike_limit_mib: 512
     receivers:
       hostmetrics:
+        collection_interval: 10s
         scrapers:
           cpu: null
           disk: null
+          filesystem: null
           load: null
           memory: null
+          network: null
+          paging: null
+          processes: null
       jaeger:
         protocols:
           grpc:
@@ -75,9 +80,9 @@ data:
           - memory_limiter
           - batch
           receivers:
-          - hostmestrics
           - otlp
           - prometheus
+          - hostmetrics
         traces:
           exporters:
           - logging

charts/opentelemetry-collector/examples/daemonset-hostmetrics/rendered/daemonset.yaml

@@ -110,6 +110,6 @@ spec:
               - key: relay
                 path: relay.yaml
         - name: hostfs
-          hostPath:
+          hostPath: 
             path: /
       hostNetwork: false

charts/opentelemetry-collector/examples/daemonset-hostmetrics/values.yaml

@@ -1,35 +1,5 @@
 mode: daemonset
-config:
-  receivers:
-    hostmetrics:
-      scrapers:
-        cpu:
-        load:
-        memory:
-        disk:
-  service:
-    pipelines:
-      metrics: 
-        receivers:
-          - hostmestrics
-          - otlp
-          - prometheus
-extraEnvs:
-- name: HOST_PROC
-  value: /hostfs/proc
-- name: HOST_SYS
-  value: /hostfs/sys
-- name: HOST_ETC
-  value: /hostfs/etc
-- name: HOST_VAR
-  value: /hostfs/var
-- name: HOST_RUN
-  value: /hostfs/run
-- name: HOST_DEV
-  value: /hostfs/dev
-extraHostPathMounts:
-- name: hostfs
-  hostPath: /
-  mountPath: /hostfs
-  readOnly: true
-  mountPropagation: HostToContainer
+
+presets:
+  hostMetrics:
+    enabled: true

charts/opentelemetry-collector/templates/NOTES.txt

@@ -17,3 +17,11 @@
 {{- if not .Values.configMap.create }}
 [WARNING] "configMap" wil not be created and "config" will not take effect.
 {{ end }}
+
+{{- if (eq (toString .Values.secretMounts) "true") }}
+{{ fail "[ERROR] 'containerLogs' should not be used.  Use 'presets.logsCollection' instead. See https://github.com/open-telemetry/opentelemetry-helm-charts/blob/main/charts/opentelemetry-collector/UPGRADING.md for instructions on how to migrate." }}
+{{ end }}
+
+{{- if and .Values.presets.logsCollection.enabled (not (eq .Values.mode "daemonset")) }}
+[WARNING] "presets.logsCollection.enabled" only takes effect if "mode" is set to "daemonset"
+{{ end }}

charts/opentelemetry-collector/templates/_config.tpl

@@ -1,3 +1,8 @@
+
+{{- define "opentelemetry-collector.var_dump" -}}
+{{- . | mustToPrettyJson | printf "\nThe JSON output of the dumped var is: \n%s" | fail }}
+{{- end -}}
+
 {{/*
 Default memory limiter configuration for OpenTelemetry Collector based on k8s resource limits.
 */}}
@@ -42,7 +47,12 @@ Build config file for daemonset OpenTelemetry Collector
 {{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }}
 {{- $config := include "opentelemetry-collector.baseConfig" $data | fromYaml }}
 {{- $config := include "opentelemetry-collector.ballastConfig" $data | fromYaml | mustMergeOverwrite $config }}
-{{- $config := mustMergeOverwrite (include "opentelemetry-collector.daemonset.containerLogsConfig" $data | fromYaml) $config }}
+{{- if .Values.presets.logsCollection.enabled }}
+{{- $config = (include "opentelemetry-collector.applyLogsCollectionConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
+{{- if .Values.presets.hostMetrics.enabled }}
+{{- $config = (include "opentelemetry-collector.applyHostMetricsConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 
@@ -53,6 +63,9 @@ Build config file for deployment OpenTelemetry Collector
 {{- $values := deepCopy .Values }}
 {{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }}
 {{- $config := include "opentelemetry-collector.baseConfig" $data | fromYaml }}
+{{- if .Values.presets.hostMetrics.enabled }}
+{{- $config = (include "opentelemetry-collector.applyHostMetricsConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 
@@ -111,13 +124,45 @@ Get otel memory_limiter ballast_size_mib value based on 40% of resources.memory.
 {{- div (mul (include "opentelemetry-collector.convertMemToMib" .) 40) 100 }}
 {{- end -}}
 
-{{- define "opentelemetry-collector.daemonset.containerLogsConfig" -}}
-{{- if .Values.containerLogs.enabled }}
+{{- define "opentelemetry-collector.applyHostMetricsConfig" -}}
+{{- $config := mustMergeOverwrite (include "opentelemetry-collector.hostMetricsConfig" .Values | fromYaml) .config }}
+{{- $newList := dict "service" (dict "pipelines" (dict "metrics" (dict "receivers" (append .config.service.pipelines.metrics.receivers "hostmetrics" | uniq))))}}
+{{- $config := mustMergeOverwrite $config $newList  }}
+{{- $config | toYaml }}
+{{- end }}
+
+{{- define "opentelemetry-collector.hostMetricsConfig" -}}
+receivers:
+  hostmetrics:
+    collection_interval: 10s
+    scrapers:
+      cpu:
+      disk:
+      filesystem:
+      memory:
+      network:
+      load:
+      paging:
+      processes:
+{{- end }}
+
+{{- define "opentelemetry-collector.applyLogsCollectionConfig" -}}
+{{- $config := mustMergeOverwrite (include "opentelemetry-collector.logsCollectionConfig" .Values | fromYaml) .config }}
+{{- $newList := dict "service" (dict "pipelines" (dict "logs" (dict "receivers" (append .config.service.pipelines.logs.receivers "filelog" | uniq))))}}
+{{- $config := mustMergeOverwrite $config $newList  }}
+{{- $config | toYaml }}
+{{- end }}
+
+{{- define "opentelemetry-collector.logsCollectionConfig" -}}
 receivers:
   filelog:
     include: [ /var/log/pods/*/*/*.log ]
+    {{- if .Values.presets.logsCollection.includeCollectorLogs }}
+    exclude: []
+    {{- else }}
     # Exclude collector container's logs. The file format is /var/log/pods/<namespace_name>_<pod_name>_<pod_uid>/<container_name>/<run_id>.log
     exclude: [ /var/log/pods/{{ .Release.Namespace }}_{{ include "opentelemetry-collector.fullname" . }}*_*/{{ .Chart.Name }}/*.log ]
+    {{- end }}
     start_at: beginning
     include_file_path: true
     include_file_name: false
@@ -184,13 +229,6 @@ receivers:
       - type: move
         from: attributes.log
         to: body
-service:
-  pipelines:
-    logs:
-      receivers:
-        - filelog
-        - otlp
-{{- end }}
 {{- end }}
 
 {{/* Build the list of port for deployment service */}}