diff --git a/NOTICE.txt b/NOTICE.txt index cca066349fa3..6e60972d6745 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12268,11 +12268,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.6.5 +Version: v0.6.6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.6/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 6732fbc60061..e4943105c6a4 100644 --- a/go.mod +++ b/go.mod @@ -200,7 +200,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.6.5 + github.com/elastic/elastic-agent-autodiscover v0.6.6 github.com/elastic/elastic-agent-libs v0.7.2 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.9.1 @@ -405,7 +405,6 @@ replace ( github.com/docker/go-plugins-helpers => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 - github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c github.com/godror/godror => github.com/godror/godror v0.33.2 // updating to v0.24.2 caused a breaking change @@ -415,7 +414,6 @@ replace ( github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19 github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15 - ) // Exclude this version because the version has an invalid checksum. diff --git a/go.sum b/go.sum index 5b4d63c39652..c83841bdcb59 100644 --- a/go.sum +++ b/go.sum @@ -652,8 +652,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.6.5 h1:5DeMpuNc8c/tN6HN0A4A2uOFTNFHSg7xrKApzfhvF1U= -github.com/elastic/elastic-agent-autodiscover v0.6.5/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= +github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A= +github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= github.com/elastic/elastic-agent-client/v7 v7.6.0/go.mod h1:GlUKrbVd/O1CRAZonpBeN3J0RlVqP6VGcrBjFWca+aM= github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o= diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index 0998a275ea4f..7c74c82268d1 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -50,15 +50,13 @@ type Enabled struct { type PluginConfig []map[string]config.C -func defaultKubernetesAnnotatorConfig() kubeAnnotatorConfig { - return kubeAnnotatorConfig{ - SyncPeriod: 10 * time.Minute, - CleanupTimeout: 60 * time.Second, - DefaultMatchers: Enabled{true}, - DefaultIndexers: Enabled{true}, - Scope: "node", - AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), - } +func (k *kubeAnnotatorConfig) InitDefaults() { + k.SyncPeriod = 10 * time.Minute + k.CleanupTimeout = 60 * time.Second + k.DefaultMatchers = Enabled{true} + k.DefaultIndexers = Enabled{true} + k.Scope = "node" + k.AddResourceMetadata = metadata.GetDefaultResourceMetadataConfig() } func (k *kubeAnnotatorConfig) Validate() error { @@ -83,7 +81,7 @@ func (k *kubeAnnotatorConfig) Validate() error { err := matcherCfg.Unpack(&logsPathMatcher) if err != nil { - return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %s", err) + return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %w", err) } if logsPathMatcher.LogsPath == "" { return fmt.Errorf("invalid logs_path matcher configuration: when resource_type is defined, logs_path must be set as well") diff --git a/libbeat/processors/add_kubernetes_metadata/config_test.go b/libbeat/processors/add_kubernetes_metadata/config_test.go index e94089f388a9..3857eb148fa6 100644 --- a/libbeat/processors/add_kubernetes_metadata/config_test.go +++ b/libbeat/processors/add_kubernetes_metadata/config_test.go @@ -50,7 +50,7 @@ func TestConfigValidate(t *testing.T) { for _, test := range tests { cfg := config.MustNewConfigFrom(test.cfg) - c := defaultKubernetesAnnotatorConfig() + var c kubeAnnotatorConfig err := cfg.Unpack(&c) if test.error { @@ -116,16 +116,16 @@ func TestConfigValidate_LogsPatchMatcher(t *testing.T) { for _, test := range tests { cfg, _ := config.NewConfigFrom(test.matcherConfig) - c := defaultKubernetesAnnotatorConfig() - c.DefaultMatchers = Enabled{false} + var c kubeAnnotatorConfig - err := cfg.Unpack(&c) + _ = cfg.Unpack(&c) + c.DefaultMatchers = Enabled{false} c.Matchers = PluginConfig{ { test.matcherName: *cfg, }, } - err = c.Validate() + err := c.Validate() if test.error { require.NotNil(t, err) } else { diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index a8667aef0a8a..954a59ab3f12 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -123,8 +123,7 @@ func New(cfg *config.C) (beat.Processor, error) { } func newProcessorConfig(cfg *config.C, register *Register) (kubeAnnotatorConfig, error) { - config := defaultKubernetesAnnotatorConfig() - + var config kubeAnnotatorConfig err := cfg.Unpack(&config) if err != nil { return config, fmt.Errorf("fail to unpack the kubernetes configuration: %w", err) diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 6b0bbc023f3c..9ff079faa3be 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -232,11 +232,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -276,13 +283,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 3588aaec9769..d6b8b9e9475d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -504,11 +504,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -548,13 +555,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index dcd59309119d..23f5ce8dea62 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -18,11 +18,18 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -62,13 +69,21 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index 44ef19c97862..1c56e57b167f 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -16,15 +16,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 26728fccdaea..60b3360ab891 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -161,7 +161,7 @@ func NewResourceMetadataEnricher( return &nilEnricher{} } - // GetPodMetaGen requires cfg of type Config + // commonMetaConfig stores the metadata configuration of the resource itself commonMetaConfig := metadata.Config{} if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { logp.Err("Error initializing Kubernetes metadata enricher: %s", err) @@ -206,7 +206,7 @@ func NewResourceMetadataEnricher( // update func(m map[string]mapstr.M, r kubernetes.Resource) { accessor, _ := meta.Accessor(r) - id := join(accessor.GetNamespace(), accessor.GetName()) //nolint:all + id := join(accessor.GetNamespace(), accessor.GetName()) switch r := r.(type) { case *kubernetes.Pod: @@ -308,6 +308,14 @@ func NewContainerMetadataEnricher( return &nilEnricher{} } + // commonMetaConfig stores the metadata configuration of the resource itself + commonMetaConfig := metadata.Config{} + if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { + logp.Err("Error initializing Kubernetes metadata enricher: %s", err) + return &nilEnricher{} + } + cfg, _ := conf.NewConfigFrom(&commonMetaConfig) + // Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to // in order to be able to retrieve 2nd layer Owner metadata like in case of: // Deployment -> Replicaset -> Pod @@ -331,13 +339,6 @@ func NewContainerMetadataEnricher( } } - commonMetaConfig := metadata.Config{} - if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { - logp.Err("Error initializing Kubernetes metadata enricher: %s", err) - return &nilEnricher{} - } - cfg, _ := conf.NewConfigFrom(&commonMetaConfig) - metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, config.AddResourceMetadata) enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, @@ -508,6 +509,7 @@ func GetConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) { SyncPeriod: time.Minute * 10, AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), } + if err := base.Module().UnpackConfig(&config); err != nil { return nil, errors.New("error unpacking configs") } diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 23bd210a8357..12bbeee26ca5 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -19,15 +19,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 46e951ccb56e..a22db4f7f8cf 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -901,11 +901,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -945,13 +952,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: